LLVM  14.0.0git
MemorySanitizer.cpp
Go to the documentation of this file.
1 //===- MemorySanitizer.cpp - detector of uninitialized reads --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file is a part of MemorySanitizer, a detector of uninitialized
11 /// reads.
12 ///
13 /// The algorithm of the tool is similar to Memcheck
14 /// (http://goo.gl/QKbem). We associate a few shadow bits with every
15 /// byte of the application memory, poison the shadow of the malloc-ed
16 /// or alloca-ed memory, load the shadow bits on every memory read,
17 /// propagate the shadow bits through some of the arithmetic
18 /// instruction (including MOV), store the shadow bits on every memory
19 /// write, report a bug on some other instructions (e.g. JMP) if the
20 /// associated shadow is poisoned.
21 ///
22 /// But there are differences too. The first and the major one:
23 /// compiler instrumentation instead of binary instrumentation. This
24 /// gives us much better register allocation, possible compiler
25 /// optimizations and a fast start-up. But this brings the major issue
26 /// as well: msan needs to see all program events, including system
27 /// calls and reads/writes in system libraries, so we either need to
28 /// compile *everything* with msan or use a binary translation
29 /// component (e.g. DynamoRIO) to instrument pre-built libraries.
30 /// Another difference from Memcheck is that we use 8 shadow bits per
31 /// byte of application memory and use a direct shadow mapping. This
32 /// greatly simplifies the instrumentation code and avoids races on
33 /// shadow updates (Memcheck is single-threaded so races are not a
34 /// concern there. Memcheck uses 2 shadow bits per byte with a slow
35 /// path storage that uses 8 bits per byte).
36 ///
37 /// The default value of shadow is 0, which means "clean" (not poisoned).
38 ///
39 /// Every module initializer should call __msan_init to ensure that the
40 /// shadow memory is ready. On error, __msan_warning is called. Since
41 /// parameters and return values may be passed via registers, we have a
42 /// specialized thread-local shadow for return values
43 /// (__msan_retval_tls) and parameters (__msan_param_tls).
44 ///
45 /// Origin tracking.
46 ///
47 /// MemorySanitizer can track origins (allocation points) of all uninitialized
48 /// values. This behavior is controlled with a flag (msan-track-origins) and is
49 /// disabled by default.
50 ///
51 /// Origins are 4-byte values created and interpreted by the runtime library.
52 /// They are stored in a second shadow mapping, one 4-byte value for 4 bytes
53 /// of application memory. Propagation of origins is basically a bunch of
54 /// "select" instructions that pick the origin of a dirty argument, if an
55 /// instruction has one.
56 ///
57 /// Every 4 aligned, consecutive bytes of application memory have one origin
58 /// value associated with them. If these bytes contain uninitialized data
59 /// coming from 2 different allocations, the last store wins. Because of this,
60 /// MemorySanitizer reports can show unrelated origins, but this is unlikely in
61 /// practice.
62 ///
63 /// Origins are meaningless for fully initialized values, so MemorySanitizer
64 /// avoids storing origin to memory when a fully initialized value is stored.
65 /// This way it avoids needless overwriting origin of the 4-byte region on
66 /// a short (i.e. 1 byte) clean store, and it is also good for performance.
67 ///
68 /// Atomic handling.
69 ///
70 /// Ideally, every atomic store of application value should update the
71 /// corresponding shadow location in an atomic way. Unfortunately, atomic store
72 /// of two disjoint locations can not be done without severe slowdown.
73 ///
74 /// Therefore, we implement an approximation that may err on the safe side.
75 /// In this implementation, every atomically accessed location in the program
76 /// may only change from (partially) uninitialized to fully initialized, but
77 /// not the other way around. We load the shadow _after_ the application load,
78 /// and we store the shadow _before_ the app store. Also, we always store clean
79 /// shadow (if the application store is atomic). This way, if the store-load
80 /// pair constitutes a happens-before arc, shadow store and load are correctly
81 /// ordered such that the load will get either the value that was stored, or
82 /// some later value (which is always clean).
83 ///
84 /// This does not work very well with Compare-And-Swap (CAS) and
85 /// Read-Modify-Write (RMW) operations. To follow the above logic, CAS and RMW
86 /// must store the new shadow before the app operation, and load the shadow
87 /// after the app operation. Computers don't work this way. Current
88 /// implementation ignores the load aspect of CAS/RMW, always returning a clean
89 /// value. It implements the store part as a simple atomic store by storing a
90 /// clean shadow.
91 ///
92 /// Instrumenting inline assembly.
93 ///
94 /// For inline assembly code LLVM has little idea about which memory locations
95 /// become initialized depending on the arguments. It can be possible to figure
96 /// out which arguments are meant to point to inputs and outputs, but the
97 /// actual semantics can be only visible at runtime. In the Linux kernel it's
98 /// also possible that the arguments only indicate the offset for a base taken
99 /// from a segment register, so it's dangerous to treat any asm() arguments as
100 /// pointers. We take a conservative approach generating calls to
101 /// __msan_instrument_asm_store(ptr, size)
102 /// , which defer the memory unpoisoning to the runtime library.
103 /// The latter can perform more complex address checks to figure out whether
104 /// it's safe to touch the shadow memory.
105 /// Like with atomic operations, we call __msan_instrument_asm_store() before
106 /// the assembly call, so that changes to the shadow memory will be seen by
107 /// other threads together with main memory initialization.
108 ///
109 /// KernelMemorySanitizer (KMSAN) implementation.
110 ///
111 /// The major differences between KMSAN and MSan instrumentation are:
112 /// - KMSAN always tracks the origins and implies msan-keep-going=true;
113 /// - KMSAN allocates shadow and origin memory for each page separately, so
114 /// there are no explicit accesses to shadow and origin in the
115 /// instrumentation.
116 /// Shadow and origin values for a particular X-byte memory location
117 /// (X=1,2,4,8) are accessed through pointers obtained via the
118 /// __msan_metadata_ptr_for_load_X(ptr)
119 /// __msan_metadata_ptr_for_store_X(ptr)
120 /// functions. The corresponding functions check that the X-byte accesses
121 /// are possible and returns the pointers to shadow and origin memory.
122 /// Arbitrary sized accesses are handled with:
123 /// __msan_metadata_ptr_for_load_n(ptr, size)
124 /// __msan_metadata_ptr_for_store_n(ptr, size);
125 /// - TLS variables are stored in a single per-task struct. A call to a
126 /// function __msan_get_context_state() returning a pointer to that struct
127 /// is inserted into every instrumented function before the entry block;
128 /// - __msan_warning() takes a 32-bit origin parameter;
129 /// - local variables are poisoned with __msan_poison_alloca() upon function
130 /// entry and unpoisoned with __msan_unpoison_alloca() before leaving the
131 /// function;
132 /// - the pass doesn't declare any global variables or add global constructors
133 /// to the translation unit.
134 ///
135 /// Also, KMSAN currently ignores uninitialized memory passed into inline asm
136 /// calls, making sure we're on the safe side wrt. possible false positives.
137 ///
138 /// KernelMemorySanitizer only supports X86_64 at the moment.
139 ///
140 //
141 // FIXME: This sanitizer does not yet handle scalable vectors
142 //
143 //===----------------------------------------------------------------------===//
144 
146 #include "llvm/ADT/APInt.h"
147 #include "llvm/ADT/ArrayRef.h"
149 #include "llvm/ADT/SmallSet.h"
150 #include "llvm/ADT/SmallString.h"
151 #include "llvm/ADT/SmallVector.h"
152 #include "llvm/ADT/StringExtras.h"
153 #include "llvm/ADT/StringRef.h"
154 #include "llvm/ADT/Triple.h"
157 #include "llvm/IR/Argument.h"
158 #include "llvm/IR/Attributes.h"
159 #include "llvm/IR/BasicBlock.h"
160 #include "llvm/IR/CallingConv.h"
161 #include "llvm/IR/Constant.h"
162 #include "llvm/IR/Constants.h"
163 #include "llvm/IR/DataLayout.h"
164 #include "llvm/IR/DerivedTypes.h"
165 #include "llvm/IR/Function.h"
166 #include "llvm/IR/GlobalValue.h"
167 #include "llvm/IR/GlobalVariable.h"
168 #include "llvm/IR/IRBuilder.h"
169 #include "llvm/IR/InlineAsm.h"
170 #include "llvm/IR/InstVisitor.h"
171 #include "llvm/IR/InstrTypes.h"
172 #include "llvm/IR/Instruction.h"
173 #include "llvm/IR/Instructions.h"
174 #include "llvm/IR/IntrinsicInst.h"
175 #include "llvm/IR/Intrinsics.h"
176 #include "llvm/IR/IntrinsicsX86.h"
177 #include "llvm/IR/LLVMContext.h"
178 #include "llvm/IR/MDBuilder.h"
179 #include "llvm/IR/Module.h"
180 #include "llvm/IR/Type.h"
181 #include "llvm/IR/Value.h"
182 #include "llvm/IR/ValueMap.h"
183 #include "llvm/InitializePasses.h"
184 #include "llvm/Pass.h"
186 #include "llvm/Support/Casting.h"
188 #include "llvm/Support/Compiler.h"
189 #include "llvm/Support/Debug.h"
191 #include "llvm/Support/MathExtras.h"
197 #include <algorithm>
198 #include <cassert>
199 #include <cstddef>
200 #include <cstdint>
201 #include <memory>
202 #include <string>
203 #include <tuple>
204 
205 using namespace llvm;
206 
207 #define DEBUG_TYPE "msan"
208 
209 static const unsigned kOriginSize = 4;
210 static const Align kMinOriginAlignment = Align(4);
211 static const Align kShadowTLSAlignment = Align(8);
212 
213 // These constants must be kept in sync with the ones in msan.h.
214 static const unsigned kParamTLSSize = 800;
215 static const unsigned kRetvalTLSSize = 800;
216 
217 // Accesses sizes are powers of two: 1, 2, 4, 8.
218 static const size_t kNumberOfAccessSizes = 4;
219 
220 /// Track origins of uninitialized values.
221 ///
222 /// Adds a section to MemorySanitizer report that points to the allocation
223 /// (stack or heap) the uninitialized bits came from originally.
224 static cl::opt<int> ClTrackOrigins("msan-track-origins",
225  cl::desc("Track origins (allocation sites) of poisoned memory"),
226  cl::Hidden, cl::init(0));
227 
228 static cl::opt<bool> ClKeepGoing("msan-keep-going",
229  cl::desc("keep going after reporting a UMR"),
230  cl::Hidden, cl::init(false));
231 
232 static cl::opt<bool> ClPoisonStack("msan-poison-stack",
233  cl::desc("poison uninitialized stack variables"),
234  cl::Hidden, cl::init(true));
235 
236 static cl::opt<bool> ClPoisonStackWithCall("msan-poison-stack-with-call",
237  cl::desc("poison uninitialized stack variables with a call"),
238  cl::Hidden, cl::init(false));
239 
240 static cl::opt<int> ClPoisonStackPattern("msan-poison-stack-pattern",
241  cl::desc("poison uninitialized stack variables with the given pattern"),
242  cl::Hidden, cl::init(0xff));
243 
244 static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
245  cl::desc("poison undef temps"),
246  cl::Hidden, cl::init(true));
247 
248 static cl::opt<bool> ClHandleICmp("msan-handle-icmp",
249  cl::desc("propagate shadow through ICmpEQ and ICmpNE"),
250  cl::Hidden, cl::init(true));
251 
252 static cl::opt<bool> ClHandleICmpExact("msan-handle-icmp-exact",
253  cl::desc("exact handling of relational integer ICmp"),
254  cl::Hidden, cl::init(false));
255 
257  "msan-handle-lifetime-intrinsics",
258  cl::desc(
259  "when possible, poison scoped variables at the beginning of the scope "
260  "(slower, but more precise)"),
261  cl::Hidden, cl::init(true));
262 
263 // When compiling the Linux kernel, we sometimes see false positives related to
264 // MSan being unable to understand that inline assembly calls may initialize
265 // local variables.
266 // This flag makes the compiler conservatively unpoison every memory location
267 // passed into an assembly call. Note that this may cause false positives.
268 // Because it's impossible to figure out the array sizes, we can only unpoison
269 // the first sizeof(type) bytes for each type* pointer.
270 // The instrumentation is only enabled in KMSAN builds, and only if
271 // -msan-handle-asm-conservative is on. This is done because we may want to
272 // quickly disable assembly instrumentation when it breaks.
274  "msan-handle-asm-conservative",
275  cl::desc("conservative handling of inline assembly"), cl::Hidden,
276  cl::init(true));
277 
278 // This flag controls whether we check the shadow of the address
279 // operand of load or store. Such bugs are very rare, since load from
280 // a garbage address typically results in SEGV, but still happen
281 // (e.g. only lower bits of address are garbage, or the access happens
282 // early at program startup where malloc-ed memory is more likely to
283 // be zeroed. As of 2012-08-28 this flag adds 20% slowdown.
284 static cl::opt<bool> ClCheckAccessAddress("msan-check-access-address",
285  cl::desc("report accesses through a pointer which has poisoned shadow"),
286  cl::Hidden, cl::init(true));
287 
289  "msan-eager-checks",
290  cl::desc("check arguments and return values at function call boundaries"),
291  cl::Hidden, cl::init(false));
292 
293 static cl::opt<bool> ClDumpStrictInstructions("msan-dump-strict-instructions",
294  cl::desc("print out instructions with default strict semantics"),
295  cl::Hidden, cl::init(false));
296 
298  "msan-instrumentation-with-call-threshold",
299  cl::desc(
300  "If the function being instrumented requires more than "
301  "this number of checks and origin stores, use callbacks instead of "
302  "inline checks (-1 means never use callbacks)."),
303  cl::Hidden, cl::init(3500));
304 
305 static cl::opt<bool>
306  ClEnableKmsan("msan-kernel",
307  cl::desc("Enable KernelMemorySanitizer instrumentation"),
308  cl::Hidden, cl::init(false));
309 
310 // This is an experiment to enable handling of cases where shadow is a non-zero
311 // compile-time constant. For some unexplainable reason they were silently
312 // ignored in the instrumentation.
313 static cl::opt<bool> ClCheckConstantShadow("msan-check-constant-shadow",
314  cl::desc("Insert checks for constant shadow values"),
315  cl::Hidden, cl::init(false));
316 
317 // This is off by default because of a bug in gold:
318 // https://sourceware.org/bugzilla/show_bug.cgi?id=19002
319 static cl::opt<bool> ClWithComdat("msan-with-comdat",
320  cl::desc("Place MSan constructors in comdat sections"),
321  cl::Hidden, cl::init(false));
322 
323 // These options allow to specify custom memory map parameters
324 // See MemoryMapParams for details.
325 static cl::opt<uint64_t> ClAndMask("msan-and-mask",
326  cl::desc("Define custom MSan AndMask"),
327  cl::Hidden, cl::init(0));
328 
329 static cl::opt<uint64_t> ClXorMask("msan-xor-mask",
330  cl::desc("Define custom MSan XorMask"),
331  cl::Hidden, cl::init(0));
332 
333 static cl::opt<uint64_t> ClShadowBase("msan-shadow-base",
334  cl::desc("Define custom MSan ShadowBase"),
335  cl::Hidden, cl::init(0));
336 
337 static cl::opt<uint64_t> ClOriginBase("msan-origin-base",
338  cl::desc("Define custom MSan OriginBase"),
339  cl::Hidden, cl::init(0));
340 
341 const char kMsanModuleCtorName[] = "msan.module_ctor";
342 const char kMsanInitName[] = "__msan_init";
343 
344 namespace {
345 
346 // Memory map parameters used in application-to-shadow address calculation.
347 // Offset = (Addr & ~AndMask) ^ XorMask
348 // Shadow = ShadowBase + Offset
349 // Origin = OriginBase + Offset
350 struct MemoryMapParams {
351  uint64_t AndMask;
352  uint64_t XorMask;
353  uint64_t ShadowBase;
354  uint64_t OriginBase;
355 };
356 
357 struct PlatformMemoryMapParams {
358  const MemoryMapParams *bits32;
359  const MemoryMapParams *bits64;
360 };
361 
362 } // end anonymous namespace
363 
364 // i386 Linux
365 static const MemoryMapParams Linux_I386_MemoryMapParams = {
366  0x000080000000, // AndMask
367  0, // XorMask (not used)
368  0, // ShadowBase (not used)
369  0x000040000000, // OriginBase
370 };
371 
372 // x86_64 Linux
373 static const MemoryMapParams Linux_X86_64_MemoryMapParams = {
374 #ifdef MSAN_LINUX_X86_64_OLD_MAPPING
375  0x400000000000, // AndMask
376  0, // XorMask (not used)
377  0, // ShadowBase (not used)
378  0x200000000000, // OriginBase
379 #else
380  0, // AndMask (not used)
381  0x500000000000, // XorMask
382  0, // ShadowBase (not used)
383  0x100000000000, // OriginBase
384 #endif
385 };
386 
387 // mips64 Linux
388 static const MemoryMapParams Linux_MIPS64_MemoryMapParams = {
389  0, // AndMask (not used)
390  0x008000000000, // XorMask
391  0, // ShadowBase (not used)
392  0x002000000000, // OriginBase
393 };
394 
395 // ppc64 Linux
396 static const MemoryMapParams Linux_PowerPC64_MemoryMapParams = {
397  0xE00000000000, // AndMask
398  0x100000000000, // XorMask
399  0x080000000000, // ShadowBase
400  0x1C0000000000, // OriginBase
401 };
402 
403 // s390x Linux
404 static const MemoryMapParams Linux_S390X_MemoryMapParams = {
405  0xC00000000000, // AndMask
406  0, // XorMask (not used)
407  0x080000000000, // ShadowBase
408  0x1C0000000000, // OriginBase
409 };
410 
411 // aarch64 Linux
412 static const MemoryMapParams Linux_AArch64_MemoryMapParams = {
413  0, // AndMask (not used)
414  0x06000000000, // XorMask
415  0, // ShadowBase (not used)
416  0x01000000000, // OriginBase
417 };
418 
419 // i386 FreeBSD
420 static const MemoryMapParams FreeBSD_I386_MemoryMapParams = {
421  0x000180000000, // AndMask
422  0x000040000000, // XorMask
423  0x000020000000, // ShadowBase
424  0x000700000000, // OriginBase
425 };
426 
427 // x86_64 FreeBSD
428 static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams = {
429  0xc00000000000, // AndMask
430  0x200000000000, // XorMask
431  0x100000000000, // ShadowBase
432  0x380000000000, // OriginBase
433 };
434 
435 // x86_64 NetBSD
436 static const MemoryMapParams NetBSD_X86_64_MemoryMapParams = {
437  0, // AndMask
438  0x500000000000, // XorMask
439  0, // ShadowBase
440  0x100000000000, // OriginBase
441 };
442 
443 static const PlatformMemoryMapParams Linux_X86_MemoryMapParams = {
446 };
447 
448 static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams = {
449  nullptr,
451 };
452 
453 static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams = {
454  nullptr,
456 };
457 
458 static const PlatformMemoryMapParams Linux_S390_MemoryMapParams = {
459  nullptr,
461 };
462 
463 static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams = {
464  nullptr,
466 };
467 
468 static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams = {
471 };
472 
473 static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams = {
474  nullptr,
476 };
477 
478 namespace {
479 
480 /// Instrument functions of a module to detect uninitialized reads.
481 ///
482 /// Instantiating MemorySanitizer inserts the msan runtime library API function
483 /// declarations into the module if they don't exist already. Instantiating
484 /// ensures the __msan_init function is in the list of global constructors for
485 /// the module.
486 class MemorySanitizer {
487 public:
488  MemorySanitizer(Module &M, MemorySanitizerOptions Options)
489  : CompileKernel(Options.Kernel), TrackOrigins(Options.TrackOrigins),
490  Recover(Options.Recover) {
491  initializeModule(M);
492  }
493 
494  // MSan cannot be moved or copied because of MapParams.
495  MemorySanitizer(MemorySanitizer &&) = delete;
496  MemorySanitizer &operator=(MemorySanitizer &&) = delete;
497  MemorySanitizer(const MemorySanitizer &) = delete;
498  MemorySanitizer &operator=(const MemorySanitizer &) = delete;
499 
500  bool sanitizeFunction(Function &F, TargetLibraryInfo &TLI);
501 
502 private:
503  friend struct MemorySanitizerVisitor;
504  friend struct VarArgAMD64Helper;
505  friend struct VarArgMIPS64Helper;
506  friend struct VarArgAArch64Helper;
507  friend struct VarArgPowerPC64Helper;
508  friend struct VarArgSystemZHelper;
509 
510  void initializeModule(Module &M);
511  void initializeCallbacks(Module &M);
512  void createKernelApi(Module &M);
513  void createUserspaceApi(Module &M);
514 
515  /// True if we're compiling the Linux kernel.
516  bool CompileKernel;
517  /// Track origins (allocation points) of uninitialized values.
518  int TrackOrigins;
519  bool Recover;
520 
521  LLVMContext *C;
522  Type *IntptrTy;
523  Type *OriginTy;
524 
525  // XxxTLS variables represent the per-thread state in MSan and per-task state
526  // in KMSAN.
527  // For the userspace these point to thread-local globals. In the kernel land
528  // they point to the members of a per-task struct obtained via a call to
529  // __msan_get_context_state().
530 
531  /// Thread-local shadow storage for function parameters.
532  Value *ParamTLS;
533 
534  /// Thread-local origin storage for function parameters.
535  Value *ParamOriginTLS;
536 
537  /// Thread-local shadow storage for function return value.
538  Value *RetvalTLS;
539 
540  /// Thread-local origin storage for function return value.
541  Value *RetvalOriginTLS;
542 
543  /// Thread-local shadow storage for in-register va_arg function
544  /// parameters (x86_64-specific).
545  Value *VAArgTLS;
546 
547  /// Thread-local shadow storage for in-register va_arg function
548  /// parameters (x86_64-specific).
549  Value *VAArgOriginTLS;
550 
551  /// Thread-local shadow storage for va_arg overflow area
552  /// (x86_64-specific).
553  Value *VAArgOverflowSizeTLS;
554 
555  /// Are the instrumentation callbacks set up?
556  bool CallbacksInitialized = false;
557 
558  /// The run-time callback to print a warning.
559  FunctionCallee WarningFn;
560 
561  // These arrays are indexed by log2(AccessSize).
562  FunctionCallee MaybeWarningFn[kNumberOfAccessSizes];
563  FunctionCallee MaybeStoreOriginFn[kNumberOfAccessSizes];
564 
565  /// Run-time helper that generates a new origin value for a stack
566  /// allocation.
567  FunctionCallee MsanSetAllocaOrigin4Fn;
568 
569  /// Run-time helper that poisons stack on function entry.
570  FunctionCallee MsanPoisonStackFn;
571 
572  /// Run-time helper that records a store (or any event) of an
573  /// uninitialized value and returns an updated origin id encoding this info.
574  FunctionCallee MsanChainOriginFn;
575 
576  /// Run-time helper that paints an origin over a region.
577  FunctionCallee MsanSetOriginFn;
578 
579  /// MSan runtime replacements for memmove, memcpy and memset.
580  FunctionCallee MemmoveFn, MemcpyFn, MemsetFn;
581 
582  /// KMSAN callback for task-local function argument shadow.
583  StructType *MsanContextStateTy;
584  FunctionCallee MsanGetContextStateFn;
585 
586  /// Functions for poisoning/unpoisoning local variables
587  FunctionCallee MsanPoisonAllocaFn, MsanUnpoisonAllocaFn;
588 
589  /// Each of the MsanMetadataPtrXxx functions returns a pair of shadow/origin
590  /// pointers.
591  FunctionCallee MsanMetadataPtrForLoadN, MsanMetadataPtrForStoreN;
592  FunctionCallee MsanMetadataPtrForLoad_1_8[4];
593  FunctionCallee MsanMetadataPtrForStore_1_8[4];
594  FunctionCallee MsanInstrumentAsmStoreFn;
595 
596  /// Helper to choose between different MsanMetadataPtrXxx().
597  FunctionCallee getKmsanShadowOriginAccessFn(bool isStore, int size);
598 
599  /// Memory map parameters used in application-to-shadow calculation.
600  const MemoryMapParams *MapParams;
601 
602  /// Custom memory map parameters used when -msan-shadow-base or
603  // -msan-origin-base is provided.
604  MemoryMapParams CustomMapParams;
605 
606  MDNode *ColdCallWeights;
607 
608  /// Branch weights for origin store.
609  MDNode *OriginStoreWeights;
610 };
611 
612 void insertModuleCtor(Module &M) {
615  /*InitArgTypes=*/{},
616  /*InitArgs=*/{},
617  // This callback is invoked when the functions are created the first
618  // time. Hook them into the global ctors list in that case:
619  [&](Function *Ctor, FunctionCallee) {
620  if (!ClWithComdat) {
621  appendToGlobalCtors(M, Ctor, 0);
622  return;
623  }
624  Comdat *MsanCtorComdat = M.getOrInsertComdat(kMsanModuleCtorName);
625  Ctor->setComdat(MsanCtorComdat);
626  appendToGlobalCtors(M, Ctor, 0, Ctor);
627  });
628 }
629 
630 /// A legacy function pass for msan instrumentation.
631 ///
632 /// Instruments functions to detect uninitialized reads.
633 struct MemorySanitizerLegacyPass : public FunctionPass {
634  // Pass identification, replacement for typeid.
635  static char ID;
636 
637  MemorySanitizerLegacyPass(MemorySanitizerOptions Options = {})
640  }
641  StringRef getPassName() const override { return "MemorySanitizerLegacyPass"; }
642 
643  void getAnalysisUsage(AnalysisUsage &AU) const override {
645  }
646 
647  bool runOnFunction(Function &F) override {
648  return MSan->sanitizeFunction(
649  F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F));
650  }
651  bool doInitialization(Module &M) override;
652 
655 };
656 
657 template <class T> T getOptOrDefault(const cl::opt<T> &Opt, T Default) {
658  return (Opt.getNumOccurrences() > 0) ? Opt : Default;
659 }
660 
661 } // end anonymous namespace
662 
664  : Kernel(getOptOrDefault(ClEnableKmsan, K)),
665  TrackOrigins(getOptOrDefault(ClTrackOrigins, Kernel ? 2 : TO)),
666  Recover(getOptOrDefault(ClKeepGoing, Kernel || R)) {}
667 
670  MemorySanitizer Msan(*F.getParent(), Options);
671  if (Msan.sanitizeFunction(F, FAM.getResult<TargetLibraryAnalysis>(F)))
672  return PreservedAnalyses::none();
673  return PreservedAnalyses::all();
674 }
675 
678  if (Options.Kernel)
679  return PreservedAnalyses::all();
680  insertModuleCtor(M);
681  return PreservedAnalyses::none();
682 }
683 
685  raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
687  OS, MapClassName2PassName);
688  OS << "<";
689  if (Options.Recover)
690  OS << "recover;";
691  if (Options.Kernel)
692  OS << "kernel;";
693  OS << "track-origins=" << Options.TrackOrigins;
694  OS << ">";
695 }
696 
698 
699 INITIALIZE_PASS_BEGIN(MemorySanitizerLegacyPass, "msan",
700  "MemorySanitizer: detects uninitialized reads.", false,
701  false)
703 INITIALIZE_PASS_END(MemorySanitizerLegacyPass, "msan",
704  "MemorySanitizer: detects uninitialized reads.", false,
705  false)
706 
707 FunctionPass *
709  return new MemorySanitizerLegacyPass(Options);
710 }
711 
712 /// Create a non-const global initialized with the given string.
713 ///
714 /// Creates a writable global for Str so that we can pass it to the
715 /// run-time lib. Runtime uses first 4 bytes of the string to store the
716 /// frame ID, so the string needs to be mutable.
718  StringRef Str) {
719  Constant *StrConst = ConstantDataArray::getString(M.getContext(), Str);
720  return new GlobalVariable(M, StrConst->getType(), /*isConstant=*/false,
721  GlobalValue::PrivateLinkage, StrConst, "");
722 }
723 
724 /// Create KMSAN API callbacks.
725 void MemorySanitizer::createKernelApi(Module &M) {
726  IRBuilder<> IRB(*C);
727 
728  // These will be initialized in insertKmsanPrologue().
729  RetvalTLS = nullptr;
730  RetvalOriginTLS = nullptr;
731  ParamTLS = nullptr;
732  ParamOriginTLS = nullptr;
733  VAArgTLS = nullptr;
734  VAArgOriginTLS = nullptr;
735  VAArgOverflowSizeTLS = nullptr;
736 
737  WarningFn = M.getOrInsertFunction("__msan_warning", IRB.getVoidTy(),
738  IRB.getInt32Ty());
739  // Requests the per-task context state (kmsan_context_state*) from the
740  // runtime library.
741  MsanContextStateTy = StructType::get(
742  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
743  ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8),
744  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8),
745  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8), /* va_arg_origin */
746  IRB.getInt64Ty(), ArrayType::get(OriginTy, kParamTLSSize / 4), OriginTy,
747  OriginTy);
748  MsanGetContextStateFn = M.getOrInsertFunction(
749  "__msan_get_context_state", PointerType::get(MsanContextStateTy, 0));
750 
751  Type *RetTy = StructType::get(PointerType::get(IRB.getInt8Ty(), 0),
752  PointerType::get(IRB.getInt32Ty(), 0));
753 
754  for (int ind = 0, size = 1; ind < 4; ind++, size <<= 1) {
755  std::string name_load =
756  "__msan_metadata_ptr_for_load_" + std::to_string(size);
757  std::string name_store =
758  "__msan_metadata_ptr_for_store_" + std::to_string(size);
759  MsanMetadataPtrForLoad_1_8[ind] = M.getOrInsertFunction(
760  name_load, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
761  MsanMetadataPtrForStore_1_8[ind] = M.getOrInsertFunction(
762  name_store, RetTy, PointerType::get(IRB.getInt8Ty(), 0));
763  }
764 
765  MsanMetadataPtrForLoadN = M.getOrInsertFunction(
766  "__msan_metadata_ptr_for_load_n", RetTy,
767  PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
768  MsanMetadataPtrForStoreN = M.getOrInsertFunction(
769  "__msan_metadata_ptr_for_store_n", RetTy,
770  PointerType::get(IRB.getInt8Ty(), 0), IRB.getInt64Ty());
771 
772  // Functions for poisoning and unpoisoning memory.
773  MsanPoisonAllocaFn =
774  M.getOrInsertFunction("__msan_poison_alloca", IRB.getVoidTy(),
775  IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
776  MsanUnpoisonAllocaFn = M.getOrInsertFunction(
777  "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
778 }
779 
781  return M.getOrInsertGlobal(Name, Ty, [&] {
782  return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,
783  nullptr, Name, nullptr,
785  });
786 }
787 
788 /// Insert declarations for userspace-specific functions and globals.
789 void MemorySanitizer::createUserspaceApi(Module &M) {
790  IRBuilder<> IRB(*C);
791 
792  // Create the callback.
793  // FIXME: this function should have "Cold" calling conv,
794  // which is not yet implemented.
795  StringRef WarningFnName = Recover ? "__msan_warning_with_origin"
796  : "__msan_warning_with_origin_noreturn";
797  WarningFn =
798  M.getOrInsertFunction(WarningFnName, IRB.getVoidTy(), IRB.getInt32Ty());
799 
800  // Create the global TLS variables.
801  RetvalTLS =
802  getOrInsertGlobal(M, "__msan_retval_tls",
803  ArrayType::get(IRB.getInt64Ty(), kRetvalTLSSize / 8));
804 
805  RetvalOriginTLS = getOrInsertGlobal(M, "__msan_retval_origin_tls", OriginTy);
806 
807  ParamTLS =
808  getOrInsertGlobal(M, "__msan_param_tls",
809  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
810 
811  ParamOriginTLS =
812  getOrInsertGlobal(M, "__msan_param_origin_tls",
813  ArrayType::get(OriginTy, kParamTLSSize / 4));
814 
815  VAArgTLS =
816  getOrInsertGlobal(M, "__msan_va_arg_tls",
817  ArrayType::get(IRB.getInt64Ty(), kParamTLSSize / 8));
818 
819  VAArgOriginTLS =
820  getOrInsertGlobal(M, "__msan_va_arg_origin_tls",
821  ArrayType::get(OriginTy, kParamTLSSize / 4));
822 
823  VAArgOverflowSizeTLS =
824  getOrInsertGlobal(M, "__msan_va_arg_overflow_size_tls", IRB.getInt64Ty());
825 
826  for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes;
827  AccessSizeIndex++) {
828  unsigned AccessSize = 1 << AccessSizeIndex;
829  std::string FunctionName = "__msan_maybe_warning_" + itostr(AccessSize);
830  SmallVector<std::pair<unsigned, Attribute>, 2> MaybeWarningFnAttrs;
831  MaybeWarningFnAttrs.push_back(std::make_pair(
832  AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt)));
833  MaybeWarningFnAttrs.push_back(std::make_pair(
834  AttributeList::FirstArgIndex + 1, Attribute::get(*C, Attribute::ZExt)));
835  MaybeWarningFn[AccessSizeIndex] = M.getOrInsertFunction(
836  FunctionName, AttributeList::get(*C, MaybeWarningFnAttrs),
837  IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
838 
839  FunctionName = "__msan_maybe_store_origin_" + itostr(AccessSize);
840  SmallVector<std::pair<unsigned, Attribute>, 2> MaybeStoreOriginFnAttrs;
841  MaybeStoreOriginFnAttrs.push_back(std::make_pair(
842  AttributeList::FirstArgIndex, Attribute::get(*C, Attribute::ZExt)));
843  MaybeStoreOriginFnAttrs.push_back(std::make_pair(
844  AttributeList::FirstArgIndex + 2, Attribute::get(*C, Attribute::ZExt)));
845  MaybeStoreOriginFn[AccessSizeIndex] = M.getOrInsertFunction(
846  FunctionName, AttributeList::get(*C, MaybeStoreOriginFnAttrs),
847  IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt8PtrTy(),
848  IRB.getInt32Ty());
849  }
850 
851  MsanSetAllocaOrigin4Fn = M.getOrInsertFunction(
852  "__msan_set_alloca_origin4", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy,
853  IRB.getInt8PtrTy(), IntptrTy);
854  MsanPoisonStackFn =
855  M.getOrInsertFunction("__msan_poison_stack", IRB.getVoidTy(),
856  IRB.getInt8PtrTy(), IntptrTy);
857 }
858 
859 /// Insert extern declaration of runtime-provided functions and globals.
860 void MemorySanitizer::initializeCallbacks(Module &M) {
861  // Only do this once.
862  if (CallbacksInitialized)
863  return;
864 
865  IRBuilder<> IRB(*C);
866  // Initialize callbacks that are common for kernel and userspace
867  // instrumentation.
868  MsanChainOriginFn = M.getOrInsertFunction(
869  "__msan_chain_origin", IRB.getInt32Ty(), IRB.getInt32Ty());
870  MsanSetOriginFn =
871  M.getOrInsertFunction("__msan_set_origin", IRB.getVoidTy(),
872  IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
873  MemmoveFn = M.getOrInsertFunction(
874  "__msan_memmove", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
875  IRB.getInt8PtrTy(), IntptrTy);
876  MemcpyFn = M.getOrInsertFunction(
877  "__msan_memcpy", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
878  IntptrTy);
879  MemsetFn = M.getOrInsertFunction(
880  "__msan_memset", IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(),
881  IntptrTy);
882 
883  MsanInstrumentAsmStoreFn =
884  M.getOrInsertFunction("__msan_instrument_asm_store", IRB.getVoidTy(),
885  PointerType::get(IRB.getInt8Ty(), 0), IntptrTy);
886 
887  if (CompileKernel) {
888  createKernelApi(M);
889  } else {
890  createUserspaceApi(M);
891  }
892  CallbacksInitialized = true;
893 }
894 
895 FunctionCallee MemorySanitizer::getKmsanShadowOriginAccessFn(bool isStore,
896  int size) {
897  FunctionCallee *Fns =
898  isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
899  switch (size) {
900  case 1:
901  return Fns[0];
902  case 2:
903  return Fns[1];
904  case 4:
905  return Fns[2];
906  case 8:
907  return Fns[3];
908  default:
909  return nullptr;
910  }
911 }
912 
913 /// Module-level initialization.
914 ///
915 /// inserts a call to __msan_init to the module's constructor list.
916 void MemorySanitizer::initializeModule(Module &M) {
917  auto &DL = M.getDataLayout();
918 
919  bool ShadowPassed = ClShadowBase.getNumOccurrences() > 0;
920  bool OriginPassed = ClOriginBase.getNumOccurrences() > 0;
921  // Check the overrides first
922  if (ShadowPassed || OriginPassed) {
923  CustomMapParams.AndMask = ClAndMask;
924  CustomMapParams.XorMask = ClXorMask;
925  CustomMapParams.ShadowBase = ClShadowBase;
926  CustomMapParams.OriginBase = ClOriginBase;
927  MapParams = &CustomMapParams;
928  } else {
929  Triple TargetTriple(M.getTargetTriple());
930  switch (TargetTriple.getOS()) {
931  case Triple::FreeBSD:
932  switch (TargetTriple.getArch()) {
933  case Triple::x86_64:
934  MapParams = FreeBSD_X86_MemoryMapParams.bits64;
935  break;
936  case Triple::x86:
937  MapParams = FreeBSD_X86_MemoryMapParams.bits32;
938  break;
939  default:
940  report_fatal_error("unsupported architecture");
941  }
942  break;
943  case Triple::NetBSD:
944  switch (TargetTriple.getArch()) {
945  case Triple::x86_64:
946  MapParams = NetBSD_X86_MemoryMapParams.bits64;
947  break;
948  default:
949  report_fatal_error("unsupported architecture");
950  }
951  break;
952  case Triple::Linux:
953  switch (TargetTriple.getArch()) {
954  case Triple::x86_64:
955  MapParams = Linux_X86_MemoryMapParams.bits64;
956  break;
957  case Triple::x86:
958  MapParams = Linux_X86_MemoryMapParams.bits32;
959  break;
960  case Triple::mips64:
961  case Triple::mips64el:
962  MapParams = Linux_MIPS_MemoryMapParams.bits64;
963  break;
964  case Triple::ppc64:
965  case Triple::ppc64le:
966  MapParams = Linux_PowerPC_MemoryMapParams.bits64;
967  break;
968  case Triple::systemz:
969  MapParams = Linux_S390_MemoryMapParams.bits64;
970  break;
971  case Triple::aarch64:
972  case Triple::aarch64_be:
973  MapParams = Linux_ARM_MemoryMapParams.bits64;
974  break;
975  default:
976  report_fatal_error("unsupported architecture");
977  }
978  break;
979  default:
980  report_fatal_error("unsupported operating system");
981  }
982  }
983 
984  C = &(M.getContext());
985  IRBuilder<> IRB(*C);
986  IntptrTy = IRB.getIntPtrTy(DL);
987  OriginTy = IRB.getInt32Ty();
988 
989  ColdCallWeights = MDBuilder(*C).createBranchWeights(1, 1000);
990  OriginStoreWeights = MDBuilder(*C).createBranchWeights(1, 1000);
991 
992  if (!CompileKernel) {
993  if (TrackOrigins)
994  M.getOrInsertGlobal("__msan_track_origins", IRB.getInt32Ty(), [&] {
995  return new GlobalVariable(
996  M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
997  IRB.getInt32(TrackOrigins), "__msan_track_origins");
998  });
999 
1000  if (Recover)
1001  M.getOrInsertGlobal("__msan_keep_going", IRB.getInt32Ty(), [&] {
1002  return new GlobalVariable(M, IRB.getInt32Ty(), true,
1003  GlobalValue::WeakODRLinkage,
1004  IRB.getInt32(Recover), "__msan_keep_going");
1005  });
1006 }
1007 }
1008 
1009 bool MemorySanitizerLegacyPass::doInitialization(Module &M) {
1010  if (!Options.Kernel)
1011  insertModuleCtor(M);
1012  MSan.emplace(M, Options);
1013  return true;
1014 }
1015 
1016 namespace {
1017 
1018 /// A helper class that handles instrumentation of VarArg
1019 /// functions on a particular platform.
1020 ///
1021 /// Implementations are expected to insert the instrumentation
1022 /// necessary to propagate argument shadow through VarArg function
1023 /// calls. Visit* methods are called during an InstVisitor pass over
1024 /// the function, and should avoid creating new basic blocks. A new
1025 /// instance of this class is created for each instrumented function.
1026 struct VarArgHelper {
1027  virtual ~VarArgHelper() = default;
1028 
1029  /// Visit a CallBase.
1030  virtual void visitCallBase(CallBase &CB, IRBuilder<> &IRB) = 0;
1031 
1032  /// Visit a va_start call.
1033  virtual void visitVAStartInst(VAStartInst &I) = 0;
1034 
1035  /// Visit a va_copy call.
1036  virtual void visitVACopyInst(VACopyInst &I) = 0;
1037 
1038  /// Finalize function instrumentation.
1039  ///
1040  /// This method is called after visiting all interesting (see above)
1041  /// instructions in a function.
1042  virtual void finalizeInstrumentation() = 0;
1043 };
1044 
1045 struct MemorySanitizerVisitor;
1046 
1047 } // end anonymous namespace
1048 
1049 static VarArgHelper *CreateVarArgHelper(Function &Func, MemorySanitizer &Msan,
1050  MemorySanitizerVisitor &Visitor);
1051 
1052 static unsigned TypeSizeToSizeIndex(unsigned TypeSize) {
1053  if (TypeSize <= 8) return 0;
1054  return Log2_32_Ceil((TypeSize + 7) / 8);
1055 }
1056 
1057 namespace {
1058 
1059 /// This class does all the work for a given function. Store and Load
1060 /// instructions store and load corresponding shadow and origin
1061 /// values. Most instructions propagate shadow from arguments to their
1062 /// return values. Certain instructions (most importantly, BranchInst)
1063 /// test their argument shadow and print reports (with a runtime call) if it's
1064 /// non-zero.
1065 struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
1066  Function &F;
1067  MemorySanitizer &MS;
1068  SmallVector<PHINode *, 16> ShadowPHINodes, OriginPHINodes;
1069  ValueMap<Value*, Value*> ShadowMap, OriginMap;
1070  std::unique_ptr<VarArgHelper> VAHelper;
1071  const TargetLibraryInfo *TLI;
1072  Instruction *FnPrologueEnd;
1073 
1074  // The following flags disable parts of MSan instrumentation based on
1075  // exclusion list contents and command-line options.
1076  bool InsertChecks;
1077  bool PropagateShadow;
1078  bool PoisonStack;
1079  bool PoisonUndef;
1080 
1081  struct ShadowOriginAndInsertPoint {
1082  Value *Shadow;
1083  Value *Origin;
1084  Instruction *OrigIns;
1085 
1086  ShadowOriginAndInsertPoint(Value *S, Value *O, Instruction *I)
1087  : Shadow(S), Origin(O), OrigIns(I) {}
1088  };
1089  SmallVector<ShadowOriginAndInsertPoint, 16> InstrumentationList;
1090  bool InstrumentLifetimeStart = ClHandleLifetimeIntrinsics;
1091  SmallSet<AllocaInst *, 16> AllocaSet;
1093  SmallVector<StoreInst *, 16> StoreList;
1094 
1095  MemorySanitizerVisitor(Function &F, MemorySanitizer &MS,
1096  const TargetLibraryInfo &TLI)
1097  : F(F), MS(MS), VAHelper(CreateVarArgHelper(F, MS, *this)), TLI(&TLI) {
1098  bool SanitizeFunction = F.hasFnAttribute(Attribute::SanitizeMemory);
1099  InsertChecks = SanitizeFunction;
1100  PropagateShadow = SanitizeFunction;
1101  PoisonStack = SanitizeFunction && ClPoisonStack;
1102  PoisonUndef = SanitizeFunction && ClPoisonUndef;
1103 
1104  // In the presence of unreachable blocks, we may see Phi nodes with
1105  // incoming nodes from such blocks. Since InstVisitor skips unreachable
1106  // blocks, such nodes will not have any shadow value associated with them.
1107  // It's easier to remove unreachable blocks than deal with missing shadow.
1109 
1110  MS.initializeCallbacks(*F.getParent());
1111  FnPrologueEnd = IRBuilder<>(F.getEntryBlock().getFirstNonPHI())
1112  .CreateIntrinsic(Intrinsic::donothing, {}, {});
1113 
1114  if (MS.CompileKernel) {
1115  IRBuilder<> IRB(FnPrologueEnd);
1116  insertKmsanPrologue(IRB);
1117  }
1118 
1119  LLVM_DEBUG(if (!InsertChecks) dbgs()
1120  << "MemorySanitizer is not inserting checks into '"
1121  << F.getName() << "'\n");
1122  }
1123 
1124  bool isInPrologue(Instruction &I) {
1125  return I.getParent() == FnPrologueEnd->getParent() &&
1126  (&I == FnPrologueEnd || I.comesBefore(FnPrologueEnd));
1127  }
1128 
1129  Value *updateOrigin(Value *V, IRBuilder<> &IRB) {
1130  if (MS.TrackOrigins <= 1) return V;
1131  return IRB.CreateCall(MS.MsanChainOriginFn, V);
1132  }
1133 
1134  Value *originToIntptr(IRBuilder<> &IRB, Value *Origin) {
1135  const DataLayout &DL = F.getParent()->getDataLayout();
1136  unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1137  if (IntptrSize == kOriginSize) return Origin;
1138  assert(IntptrSize == kOriginSize * 2);
1139  Origin = IRB.CreateIntCast(Origin, MS.IntptrTy, /* isSigned */ false);
1140  return IRB.CreateOr(Origin, IRB.CreateShl(Origin, kOriginSize * 8));
1141  }
1142 
1143  /// Fill memory range with the given origin value.
1144  void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *OriginPtr,
1145  unsigned Size, Align Alignment) {
1146  const DataLayout &DL = F.getParent()->getDataLayout();
1147  const Align IntptrAlignment = DL.getABITypeAlign(MS.IntptrTy);
1148  unsigned IntptrSize = DL.getTypeStoreSize(MS.IntptrTy);
1149  assert(IntptrAlignment >= kMinOriginAlignment);
1150  assert(IntptrSize >= kOriginSize);
1151 
1152  unsigned Ofs = 0;
1153  Align CurrentAlignment = Alignment;
1154  if (Alignment >= IntptrAlignment && IntptrSize > kOriginSize) {
1155  Value *IntptrOrigin = originToIntptr(IRB, Origin);
1156  Value *IntptrOriginPtr =
1157  IRB.CreatePointerCast(OriginPtr, PointerType::get(MS.IntptrTy, 0));
1158  for (unsigned i = 0; i < Size / IntptrSize; ++i) {
1159  Value *Ptr = i ? IRB.CreateConstGEP1_32(MS.IntptrTy, IntptrOriginPtr, i)
1160  : IntptrOriginPtr;
1161  IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
1162  Ofs += IntptrSize / kOriginSize;
1163  CurrentAlignment = IntptrAlignment;
1164  }
1165  }
1166 
1167  for (unsigned i = Ofs; i < (Size + kOriginSize - 1) / kOriginSize; ++i) {
1168  Value *GEP =
1169  i ? IRB.CreateConstGEP1_32(MS.OriginTy, OriginPtr, i) : OriginPtr;
1170  IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
1171  CurrentAlignment = kMinOriginAlignment;
1172  }
1173  }
1174 
1175  void storeOrigin(IRBuilder<> &IRB, Value *Addr, Value *Shadow, Value *Origin,
1176  Value *OriginPtr, Align Alignment, bool AsCall) {
1177  const DataLayout &DL = F.getParent()->getDataLayout();
1178  const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1179  unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
1180  Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1181  if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1182  if (ClCheckConstantShadow && !ConstantShadow->isZeroValue())
1183  paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1184  OriginAlignment);
1185  return;
1186  }
1187 
1188  unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1189  unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1190  if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1191  FunctionCallee Fn = MS.MaybeStoreOriginFn[SizeIndex];
1192  Value *ConvertedShadow2 =
1193  IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1194  CallBase *CB = IRB.CreateCall(
1195  Fn, {ConvertedShadow2,
1196  IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()), Origin});
1197  CB->addParamAttr(0, Attribute::ZExt);
1198  CB->addParamAttr(2, Attribute::ZExt);
1199  } else {
1200  Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1202  Cmp, &*IRB.GetInsertPoint(), false, MS.OriginStoreWeights);
1203  IRBuilder<> IRBNew(CheckTerm);
1204  paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1205  OriginAlignment);
1206  }
1207  }
1208 
1209  void materializeStores(bool InstrumentWithCalls) {
1210  for (StoreInst *SI : StoreList) {
1211  IRBuilder<> IRB(SI);
1212  Value *Val = SI->getValueOperand();
1213  Value *Addr = SI->getPointerOperand();
1214  Value *Shadow = SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1215  Value *ShadowPtr, *OriginPtr;
1216  Type *ShadowTy = Shadow->getType();
1217  const Align Alignment = assumeAligned(SI->getAlignment());
1218  const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1219  std::tie(ShadowPtr, OriginPtr) =
1220  getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ true);
1221 
1222  StoreInst *NewSI = IRB.CreateAlignedStore(Shadow, ShadowPtr, Alignment);
1223  LLVM_DEBUG(dbgs() << " STORE: " << *NewSI << "\n");
1224  (void)NewSI;
1225 
1226  if (SI->isAtomic())
1227  SI->setOrdering(addReleaseOrdering(SI->getOrdering()));
1228 
1229  if (MS.TrackOrigins && !SI->isAtomic())
1230  storeOrigin(IRB, Addr, Shadow, getOrigin(Val), OriginPtr,
1231  OriginAlignment, InstrumentWithCalls);
1232  }
1233  }
1234 
1235  /// Helper function to insert a warning at IRB's current insert point.
1236  void insertWarningFn(IRBuilder<> &IRB, Value *Origin) {
1237  if (!Origin)
1238  Origin = (Value *)IRB.getInt32(0);
1239  assert(Origin->getType()->isIntegerTy());
1240  IRB.CreateCall(MS.WarningFn, Origin)->setCannotMerge();
1241  // FIXME: Insert UnreachableInst if !MS.Recover?
1242  // This may invalidate some of the following checks and needs to be done
1243  // at the very end.
1244  }
1245 
1246  void materializeOneCheck(Instruction *OrigIns, Value *Shadow, Value *Origin,
1247  bool AsCall) {
1248  IRBuilder<> IRB(OrigIns);
1249  LLVM_DEBUG(dbgs() << " SHAD0 : " << *Shadow << "\n");
1250  Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1251  LLVM_DEBUG(dbgs() << " SHAD1 : " << *ConvertedShadow << "\n");
1252 
1253  if (auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1254  if (ClCheckConstantShadow && !ConstantShadow->isZeroValue()) {
1255  insertWarningFn(IRB, Origin);
1256  }
1257  return;
1258  }
1259 
1260  const DataLayout &DL = OrigIns->getModule()->getDataLayout();
1261 
1262  unsigned TypeSizeInBits = DL.getTypeSizeInBits(ConvertedShadow->getType());
1263  unsigned SizeIndex = TypeSizeToSizeIndex(TypeSizeInBits);
1264  if (AsCall && SizeIndex < kNumberOfAccessSizes && !MS.CompileKernel) {
1265  FunctionCallee Fn = MS.MaybeWarningFn[SizeIndex];
1266  Value *ConvertedShadow2 =
1267  IRB.CreateZExt(ConvertedShadow, IRB.getIntNTy(8 * (1 << SizeIndex)));
1268  CallBase *CB = IRB.CreateCall(
1269  Fn, {ConvertedShadow2,
1270  MS.TrackOrigins && Origin ? Origin : (Value *)IRB.getInt32(0)});
1271  CB->addParamAttr(0, Attribute::ZExt);
1272  CB->addParamAttr(1, Attribute::ZExt);
1273  } else {
1274  Value *Cmp = convertToBool(ConvertedShadow, IRB, "_mscmp");
1276  Cmp, OrigIns,
1277  /* Unreachable */ !MS.Recover, MS.ColdCallWeights);
1278 
1279  IRB.SetInsertPoint(CheckTerm);
1280  insertWarningFn(IRB, Origin);
1281  LLVM_DEBUG(dbgs() << " CHECK: " << *Cmp << "\n");
1282  }
1283  }
1284 
1285  void materializeChecks(bool InstrumentWithCalls) {
1286  for (const auto &ShadowData : InstrumentationList) {
1287  Instruction *OrigIns = ShadowData.OrigIns;
1288  Value *Shadow = ShadowData.Shadow;
1289  Value *Origin = ShadowData.Origin;
1290  materializeOneCheck(OrigIns, Shadow, Origin, InstrumentWithCalls);
1291  }
1292  LLVM_DEBUG(dbgs() << "DONE:\n" << F);
1293  }
1294 
1295  // Returns the last instruction in the new prologue
1296  void insertKmsanPrologue(IRBuilder<> &IRB) {
1297  Value *ContextState = IRB.CreateCall(MS.MsanGetContextStateFn, {});
1298  Constant *Zero = IRB.getInt32(0);
1299  MS.ParamTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1300  {Zero, IRB.getInt32(0)}, "param_shadow");
1301  MS.RetvalTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1302  {Zero, IRB.getInt32(1)}, "retval_shadow");
1303  MS.VAArgTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1304  {Zero, IRB.getInt32(2)}, "va_arg_shadow");
1305  MS.VAArgOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1306  {Zero, IRB.getInt32(3)}, "va_arg_origin");
1307  MS.VAArgOverflowSizeTLS =
1308  IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1309  {Zero, IRB.getInt32(4)}, "va_arg_overflow_size");
1310  MS.ParamOriginTLS = IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1311  {Zero, IRB.getInt32(5)}, "param_origin");
1312  MS.RetvalOriginTLS =
1313  IRB.CreateGEP(MS.MsanContextStateTy, ContextState,
1314  {Zero, IRB.getInt32(6)}, "retval_origin");
1315  }
1316 
1317  /// Add MemorySanitizer instrumentation to a function.
1318  bool runOnFunction() {
1319  // Iterate all BBs in depth-first order and create shadow instructions
1320  // for all instructions (where applicable).
1321  // For PHI nodes we create dummy shadow PHIs which will be finalized later.
1322  for (BasicBlock *BB : depth_first(FnPrologueEnd->getParent()))
1323  visit(*BB);
1324 
1325  // Finalize PHI nodes.
1326  for (PHINode *PN : ShadowPHINodes) {
1327  PHINode *PNS = cast<PHINode>(getShadow(PN));
1328  PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1329  size_t NumValues = PN->getNumIncomingValues();
1330  for (size_t v = 0; v < NumValues; v++) {
1331  PNS->addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1332  if (PNO) PNO->addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1333  }
1334  }
1335 
1336  VAHelper->finalizeInstrumentation();
1337 
1338  // Poison llvm.lifetime.start intrinsics, if we haven't fallen back to
1339  // instrumenting only allocas.
1340  if (InstrumentLifetimeStart) {
1341  for (auto Item : LifetimeStartList) {
1342  instrumentAlloca(*Item.second, Item.first);
1343  AllocaSet.erase(Item.second);
1344  }
1345  }
1346  // Poison the allocas for which we didn't instrument the corresponding
1347  // lifetime intrinsics.
1348  for (AllocaInst *AI : AllocaSet)
1349  instrumentAlloca(*AI);
1350 
1351  bool InstrumentWithCalls = ClInstrumentationWithCallThreshold >= 0 &&
1352  InstrumentationList.size() + StoreList.size() >
1354 
1355  // Insert shadow value checks.
1356  materializeChecks(InstrumentWithCalls);
1357 
1358  // Delayed instrumentation of StoreInst.
1359  // This may not add new address checks.
1360  materializeStores(InstrumentWithCalls);
1361 
1362  return true;
1363  }
1364 
1365  /// Compute the shadow type that corresponds to a given Value.
1366  Type *getShadowTy(Value *V) {
1367  return getShadowTy(V->getType());
1368  }
1369 
1370  /// Compute the shadow type that corresponds to a given Type.
1371  Type *getShadowTy(Type *OrigTy) {
1372  if (!OrigTy->isSized()) {
1373  return nullptr;
1374  }
1375  // For integer type, shadow is the same as the original type.
1376  // This may return weird-sized types like i1.
1377  if (IntegerType *IT = dyn_cast<IntegerType>(OrigTy))
1378  return IT;
1379  const DataLayout &DL = F.getParent()->getDataLayout();
1380  if (VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1381  uint32_t EltSize = DL.getTypeSizeInBits(VT->getElementType());
1382  return FixedVectorType::get(IntegerType::get(*MS.C, EltSize),
1383  cast<FixedVectorType>(VT)->getNumElements());
1384  }
1385  if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1386  return ArrayType::get(getShadowTy(AT->getElementType()),
1387  AT->getNumElements());
1388  }
1389  if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1390  SmallVector<Type*, 4> Elements;
1391  for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1392  Elements.push_back(getShadowTy(ST->getElementType(i)));
1393  StructType *Res = StructType::get(*MS.C, Elements, ST->isPacked());
1394  LLVM_DEBUG(dbgs() << "getShadowTy: " << *ST << " ===> " << *Res << "\n");
1395  return Res;
1396  }
1397  uint32_t TypeSize = DL.getTypeSizeInBits(OrigTy);
1398  return IntegerType::get(*MS.C, TypeSize);
1399  }
1400 
1401  /// Flatten a vector type.
1402  Type *getShadowTyNoVec(Type *ty) {
1403  if (VectorType *vt = dyn_cast<VectorType>(ty))
1404  return IntegerType::get(*MS.C,
1405  vt->getPrimitiveSizeInBits().getFixedSize());
1406  return ty;
1407  }
1408 
1409  /// Extract combined shadow of struct elements as a bool
1410  Value *collapseStructShadow(StructType *Struct, Value *Shadow,
1411  IRBuilder<> &IRB) {
1412  Value *FalseVal = IRB.getIntN(/* width */ 1, /* value */ 0);
1413  Value *Aggregator = FalseVal;
1414 
1415  for (unsigned Idx = 0; Idx < Struct->getNumElements(); Idx++) {
1416  // Combine by ORing together each element's bool shadow
1417  Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1418  Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1419  Value *ShadowBool = convertToBool(ShadowInner, IRB);
1420 
1421  if (Aggregator != FalseVal)
1422  Aggregator = IRB.CreateOr(Aggregator, ShadowBool);
1423  else
1424  Aggregator = ShadowBool;
1425  }
1426 
1427  return Aggregator;
1428  }
1429 
1430  // Extract combined shadow of array elements
1431  Value *collapseArrayShadow(ArrayType *Array, Value *Shadow,
1432  IRBuilder<> &IRB) {
1433  if (!Array->getNumElements())
1434  return IRB.getIntN(/* width */ 1, /* value */ 0);
1435 
1436  Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
1437  Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1438 
1439  for (unsigned Idx = 1; Idx < Array->getNumElements(); Idx++) {
1440  Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1441  Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1442  Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1443  }
1444  return Aggregator;
1445  }
1446 
1447  /// Convert a shadow value to it's flattened variant. The resulting
1448  /// shadow may not necessarily have the same bit width as the input
1449  /// value, but it will always be comparable to zero.
1450  Value *convertShadowToScalar(Value *V, IRBuilder<> &IRB) {
1451  if (StructType *Struct = dyn_cast<StructType>(V->getType()))
1452  return collapseStructShadow(Struct, V, IRB);
1453  if (ArrayType *Array = dyn_cast<ArrayType>(V->getType()))
1454  return collapseArrayShadow(Array, V, IRB);
1455  Type *Ty = V->getType();
1456  Type *NoVecTy = getShadowTyNoVec(Ty);
1457  if (Ty == NoVecTy) return V;
1458  return IRB.CreateBitCast(V, NoVecTy);
1459  }
1460 
1461  // Convert a scalar value to an i1 by comparing with 0
1462  Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &name = "") {
1463  Type *VTy = V->getType();
1464  assert(VTy->isIntegerTy());
1465  if (VTy->getIntegerBitWidth() == 1)
1466  // Just converting a bool to a bool, so do nothing.
1467  return V;
1468  return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), name);
1469  }
1470 
1471  /// Compute the integer shadow offset that corresponds to a given
1472  /// application address.
1473  ///
1474  /// Offset = (Addr & ~AndMask) ^ XorMask
1475  Value *getShadowPtrOffset(Value *Addr, IRBuilder<> &IRB) {
1476  Value *OffsetLong = IRB.CreatePointerCast(Addr, MS.IntptrTy);
1477 
1478  uint64_t AndMask = MS.MapParams->AndMask;
1479  if (AndMask)
1480  OffsetLong =
1481  IRB.CreateAnd(OffsetLong, ConstantInt::get(MS.IntptrTy, ~AndMask));
1482 
1483  uint64_t XorMask = MS.MapParams->XorMask;
1484  if (XorMask)
1485  OffsetLong =
1486  IRB.CreateXor(OffsetLong, ConstantInt::get(MS.IntptrTy, XorMask));
1487  return OffsetLong;
1488  }
1489 
1490  /// Compute the shadow and origin addresses corresponding to a given
1491  /// application address.
1492  ///
1493  /// Shadow = ShadowBase + Offset
1494  /// Origin = (OriginBase + Offset) & ~3ULL
1495  std::pair<Value *, Value *>
1496  getShadowOriginPtrUserspace(Value *Addr, IRBuilder<> &IRB, Type *ShadowTy,
1497  MaybeAlign Alignment) {
1498  Value *ShadowOffset = getShadowPtrOffset(Addr, IRB);
1499  Value *ShadowLong = ShadowOffset;
1500  uint64_t ShadowBase = MS.MapParams->ShadowBase;
1501  if (ShadowBase != 0) {
1502  ShadowLong =
1503  IRB.CreateAdd(ShadowLong,
1504  ConstantInt::get(MS.IntptrTy, ShadowBase));
1505  }
1506  Value *ShadowPtr =
1507  IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1508  Value *OriginPtr = nullptr;
1509  if (MS.TrackOrigins) {
1510  Value *OriginLong = ShadowOffset;
1511  uint64_t OriginBase = MS.MapParams->OriginBase;
1512  if (OriginBase != 0)
1513  OriginLong = IRB.CreateAdd(OriginLong,
1514  ConstantInt::get(MS.IntptrTy, OriginBase));
1515  if (!Alignment || *Alignment < kMinOriginAlignment) {
1517  OriginLong =
1518  IRB.CreateAnd(OriginLong, ConstantInt::get(MS.IntptrTy, ~Mask));
1519  }
1520  OriginPtr =
1521  IRB.CreateIntToPtr(OriginLong, PointerType::get(MS.OriginTy, 0));
1522  }
1523  return std::make_pair(ShadowPtr, OriginPtr);
1524  }
1525 
1526  std::pair<Value *, Value *> getShadowOriginPtrKernel(Value *Addr,
1527  IRBuilder<> &IRB,
1528  Type *ShadowTy,
1529  bool isStore) {
1530  Value *ShadowOriginPtrs;
1531  const DataLayout &DL = F.getParent()->getDataLayout();
1532  int Size = DL.getTypeStoreSize(ShadowTy);
1533 
1534  FunctionCallee Getter = MS.getKmsanShadowOriginAccessFn(isStore, Size);
1535  Value *AddrCast =
1537  if (Getter) {
1538  ShadowOriginPtrs = IRB.CreateCall(Getter, AddrCast);
1539  } else {
1540  Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
1541  ShadowOriginPtrs = IRB.CreateCall(isStore ? MS.MsanMetadataPtrForStoreN
1542  : MS.MsanMetadataPtrForLoadN,
1543  {AddrCast, SizeVal});
1544  }
1545  Value *ShadowPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 0);
1546  ShadowPtr = IRB.CreatePointerCast(ShadowPtr, PointerType::get(ShadowTy, 0));
1547  Value *OriginPtr = IRB.CreateExtractValue(ShadowOriginPtrs, 1);
1548 
1549  return std::make_pair(ShadowPtr, OriginPtr);
1550  }
1551 
1552  std::pair<Value *, Value *> getShadowOriginPtr(Value *Addr, IRBuilder<> &IRB,
1553  Type *ShadowTy,
1554  MaybeAlign Alignment,
1555  bool isStore) {
1556  if (MS.CompileKernel)
1557  return getShadowOriginPtrKernel(Addr, IRB, ShadowTy, isStore);
1558  return getShadowOriginPtrUserspace(Addr, IRB, ShadowTy, Alignment);
1559  }
1560 
1561  /// Compute the shadow address for a given function argument.
1562  ///
1563  /// Shadow = ParamTLS+ArgOffset.
1564  Value *getShadowPtrForArgument(Value *A, IRBuilder<> &IRB,
1565  int ArgOffset) {
1566  Value *Base = IRB.CreatePointerCast(MS.ParamTLS, MS.IntptrTy);
1567  if (ArgOffset)
1568  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1569  return IRB.CreateIntToPtr(Base, PointerType::get(getShadowTy(A), 0),
1570  "_msarg");
1571  }
1572 
1573  /// Compute the origin address for a given function argument.
1574  Value *getOriginPtrForArgument(Value *A, IRBuilder<> &IRB,
1575  int ArgOffset) {
1576  if (!MS.TrackOrigins)
1577  return nullptr;
1578  Value *Base = IRB.CreatePointerCast(MS.ParamOriginTLS, MS.IntptrTy);
1579  if (ArgOffset)
1580  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
1581  return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
1582  "_msarg_o");
1583  }
1584 
1585  /// Compute the shadow address for a retval.
1586  Value *getShadowPtrForRetval(Value *A, IRBuilder<> &IRB) {
1587  return IRB.CreatePointerCast(MS.RetvalTLS,
1588  PointerType::get(getShadowTy(A), 0),
1589  "_msret");
1590  }
1591 
1592  /// Compute the origin address for a retval.
1593  Value *getOriginPtrForRetval(IRBuilder<> &IRB) {
1594  // We keep a single origin for the entire retval. Might be too optimistic.
1595  return MS.RetvalOriginTLS;
1596  }
1597 
1598  /// Set SV to be the shadow value for V.
1599  void setShadow(Value *V, Value *SV) {
1600  assert(!ShadowMap.count(V) && "Values may only have one shadow");
1601  ShadowMap[V] = PropagateShadow ? SV : getCleanShadow(V);
1602  }
1603 
1604  /// Set Origin to be the origin value for V.
1605  void setOrigin(Value *V, Value *Origin) {
1606  if (!MS.TrackOrigins) return;
1607  assert(!OriginMap.count(V) && "Values may only have one origin");
1608  LLVM_DEBUG(dbgs() << "ORIGIN: " << *V << " ==> " << *Origin << "\n");
1609  OriginMap[V] = Origin;
1610  }
1611 
1612  Constant *getCleanShadow(Type *OrigTy) {
1613  Type *ShadowTy = getShadowTy(OrigTy);
1614  if (!ShadowTy)
1615  return nullptr;
1616  return Constant::getNullValue(ShadowTy);
1617  }
1618 
1619  /// Create a clean shadow value for a given value.
1620  ///
1621  /// Clean shadow (all zeroes) means all bits of the value are defined
1622  /// (initialized).
1623  Constant *getCleanShadow(Value *V) {
1624  return getCleanShadow(V->getType());
1625  }
1626 
1627  /// Create a dirty shadow of a given shadow type.
1628  Constant *getPoisonedShadow(Type *ShadowTy) {
1629  assert(ShadowTy);
1630  if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1631  return Constant::getAllOnesValue(ShadowTy);
1632  if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1633  SmallVector<Constant *, 4> Vals(AT->getNumElements(),
1634  getPoisonedShadow(AT->getElementType()));
1635  return ConstantArray::get(AT, Vals);
1636  }
1637  if (StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1639  for (unsigned i = 0, n = ST->getNumElements(); i < n; i++)
1640  Vals.push_back(getPoisonedShadow(ST->getElementType(i)));
1641  return ConstantStruct::get(ST, Vals);
1642  }
1643  llvm_unreachable("Unexpected shadow type");
1644  }
1645 
1646  /// Create a dirty shadow for a given value.
1647  Constant *getPoisonedShadow(Value *V) {
1648  Type *ShadowTy = getShadowTy(V);
1649  if (!ShadowTy)
1650  return nullptr;
1651  return getPoisonedShadow(ShadowTy);
1652  }
1653 
1654  /// Create a clean (zero) origin.
1655  Value *getCleanOrigin() {
1656  return Constant::getNullValue(MS.OriginTy);
1657  }
1658 
1659  /// Get the shadow value for a given Value.
1660  ///
1661  /// This function either returns the value set earlier with setShadow,
1662  /// or extracts if from ParamTLS (for function arguments).
1663  Value *getShadow(Value *V) {
1664  if (!PropagateShadow) return getCleanShadow(V);
1665  if (Instruction *I = dyn_cast<Instruction>(V)) {
1666  if (I->getMetadata("nosanitize"))
1667  return getCleanShadow(V);
1668  // For instructions the shadow is already stored in the map.
1669  Value *Shadow = ShadowMap[V];
1670  if (!Shadow) {
1671  LLVM_DEBUG(dbgs() << "No shadow: " << *V << "\n" << *(I->getParent()));
1672  (void)I;
1673  assert(Shadow && "No shadow for a value");
1674  }
1675  return Shadow;
1676  }
1677  if (UndefValue *U = dyn_cast<UndefValue>(V)) {
1678  Value *AllOnes = PoisonUndef ? getPoisonedShadow(V) : getCleanShadow(V);
1679  LLVM_DEBUG(dbgs() << "Undef: " << *U << " ==> " << *AllOnes << "\n");
1680  (void)U;
1681  return AllOnes;
1682  }
1683  if (Argument *A = dyn_cast<Argument>(V)) {
1684  // For arguments we compute the shadow on demand and store it in the map.
1685  Value **ShadowPtr = &ShadowMap[V];
1686  if (*ShadowPtr)
1687  return *ShadowPtr;
1688  Function *F = A->getParent();
1689  IRBuilder<> EntryIRB(FnPrologueEnd);
1690  unsigned ArgOffset = 0;
1691  const DataLayout &DL = F->getParent()->getDataLayout();
1692  for (auto &FArg : F->args()) {
1693  if (!FArg.getType()->isSized()) {
1694  LLVM_DEBUG(dbgs() << "Arg is not sized\n");
1695  continue;
1696  }
1697 
1698  bool FArgByVal = FArg.hasByValAttr();
1699  bool FArgNoUndef = FArg.hasAttribute(Attribute::NoUndef);
1700  bool FArgEagerCheck = ClEagerChecks && !FArgByVal && FArgNoUndef;
1701  unsigned Size =
1702  FArg.hasByValAttr()
1703  ? DL.getTypeAllocSize(FArg.getParamByValType())
1704  : DL.getTypeAllocSize(FArg.getType());
1705 
1706  if (A == &FArg) {
1707  bool Overflow = ArgOffset + Size > kParamTLSSize;
1708  if (FArgEagerCheck) {
1709  *ShadowPtr = getCleanShadow(V);
1710  setOrigin(A, getCleanOrigin());
1711  continue;
1712  } else if (FArgByVal) {
1713  Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1714  // ByVal pointer itself has clean shadow. We copy the actual
1715  // argument shadow to the underlying memory.
1716  // Figure out maximal valid memcpy alignment.
1717  const Align ArgAlign = DL.getValueOrABITypeAlignment(
1718  MaybeAlign(FArg.getParamAlignment()), FArg.getParamByValType());
1719  Value *CpShadowPtr =
1720  getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1721  /*isStore*/ true)
1722  .first;
1723  // TODO(glider): need to copy origins.
1724  if (Overflow) {
1725  // ParamTLS overflow.
1726  EntryIRB.CreateMemSet(
1727  CpShadowPtr, Constant::getNullValue(EntryIRB.getInt8Ty()),
1728  Size, ArgAlign);
1729  } else {
1730  const Align CopyAlign = std::min(ArgAlign, kShadowTLSAlignment);
1731  Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign, Base,
1732  CopyAlign, Size);
1733  LLVM_DEBUG(dbgs() << " ByValCpy: " << *Cpy << "\n");
1734  (void)Cpy;
1735  }
1736  *ShadowPtr = getCleanShadow(V);
1737  } else {
1738  // Shadow over TLS
1739  Value *Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1740  if (Overflow) {
1741  // ParamTLS overflow.
1742  *ShadowPtr = getCleanShadow(V);
1743  } else {
1744  *ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg), Base,
1746  }
1747  }
1748  LLVM_DEBUG(dbgs()
1749  << " ARG: " << FArg << " ==> " << **ShadowPtr << "\n");
1750  if (MS.TrackOrigins && !Overflow) {
1751  Value *OriginPtr =
1752  getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1753  setOrigin(A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
1754  } else {
1755  setOrigin(A, getCleanOrigin());
1756  }
1757 
1758  break;
1759  }
1760 
1761  if (!FArgEagerCheck)
1762  ArgOffset += alignTo(Size, kShadowTLSAlignment);
1763  }
1764  assert(*ShadowPtr && "Could not find shadow for an argument");
1765  return *ShadowPtr;
1766  }
1767  // For everything else the shadow is zero.
1768  return getCleanShadow(V);
1769  }
1770 
1771  /// Get the shadow for i-th argument of the instruction I.
1772  Value *getShadow(Instruction *I, int i) {
1773  return getShadow(I->getOperand(i));
1774  }
1775 
1776  /// Get the origin for a value.
1777  Value *getOrigin(Value *V) {
1778  if (!MS.TrackOrigins) return nullptr;
1779  if (!PropagateShadow) return getCleanOrigin();
1780  if (isa<Constant>(V)) return getCleanOrigin();
1781  assert((isa<Instruction>(V) || isa<Argument>(V)) &&
1782  "Unexpected value type in getOrigin()");
1783  if (Instruction *I = dyn_cast<Instruction>(V)) {
1784  if (I->getMetadata("nosanitize"))
1785  return getCleanOrigin();
1786  }
1787  Value *Origin = OriginMap[V];
1788  assert(Origin && "Missing origin");
1789  return Origin;
1790  }
1791 
1792  /// Get the origin for i-th argument of the instruction I.
1793  Value *getOrigin(Instruction *I, int i) {
1794  return getOrigin(I->getOperand(i));
1795  }
1796 
1797  /// Remember the place where a shadow check should be inserted.
1798  ///
1799  /// This location will be later instrumented with a check that will print a
1800  /// UMR warning in runtime if the shadow value is not 0.
1801  void insertShadowCheck(Value *Shadow, Value *Origin, Instruction *OrigIns) {
1802  assert(Shadow);
1803  if (!InsertChecks) return;
1804 #ifndef NDEBUG
1805  Type *ShadowTy = Shadow->getType();
1806  assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
1807  isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
1808  "Can only insert checks for integer, vector, and aggregate shadow "
1809  "types");
1810 #endif
1811  InstrumentationList.push_back(
1812  ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
1813  }
1814 
1815  /// Remember the place where a shadow check should be inserted.
1816  ///
1817  /// This location will be later instrumented with a check that will print a
1818  /// UMR warning in runtime if the value is not fully defined.
1819  void insertShadowCheck(Value *Val, Instruction *OrigIns) {
1820  assert(Val);
1821  Value *Shadow, *Origin;
1822  if (ClCheckConstantShadow) {
1823  Shadow = getShadow(Val);
1824  if (!Shadow) return;
1825  Origin = getOrigin(Val);
1826  } else {
1827  Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
1828  if (!Shadow) return;
1829  Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
1830  }
1831  insertShadowCheck(Shadow, Origin, OrigIns);
1832  }
1833 
1835  switch (a) {
1841  return AtomicOrdering::Release;
1847  }
1848  llvm_unreachable("Unknown ordering");
1849  }
1850 
1851  Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
1852  constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
1853  uint32_t OrderingTable[NumOrderings] = {};
1854 
1855  OrderingTable[(int)AtomicOrderingCABI::relaxed] =
1856  OrderingTable[(int)AtomicOrderingCABI::release] =
1858  OrderingTable[(int)AtomicOrderingCABI::consume] =
1859  OrderingTable[(int)AtomicOrderingCABI::acquire] =
1860  OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
1862  OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
1864 
1865  return ConstantDataVector::get(IRB.getContext(),
1866  makeArrayRef(OrderingTable, NumOrderings));
1867  }
1868 
1870  switch (a) {
1876  return AtomicOrdering::Acquire;
1882  }
1883  llvm_unreachable("Unknown ordering");
1884  }
1885 
1886  Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
1887  constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
1888  uint32_t OrderingTable[NumOrderings] = {};
1889 
1890  OrderingTable[(int)AtomicOrderingCABI::relaxed] =
1891  OrderingTable[(int)AtomicOrderingCABI::acquire] =
1892  OrderingTable[(int)AtomicOrderingCABI::consume] =
1894  OrderingTable[(int)AtomicOrderingCABI::release] =
1895  OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
1897  OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
1899 
1900  return ConstantDataVector::get(IRB.getContext(),
1901  makeArrayRef(OrderingTable, NumOrderings));
1902  }
1903 
1904  // ------------------- Visitors.
1906  void visit(Instruction &I) {
1907  if (I.getMetadata("nosanitize"))
1908  return;
1909  // Don't want to visit if we're in the prologue
1910  if (isInPrologue(I))
1911  return;
1913  }
1914 
1915  /// Instrument LoadInst
1916  ///
1917  /// Loads the corresponding shadow and (optionally) origin.
1918  /// Optionally, checks that the load address is fully defined.
1919  void visitLoadInst(LoadInst &I) {
1920  assert(I.getType()->isSized() && "Load type must have size");
1921  assert(!I.getMetadata("nosanitize"));
1922  IRBuilder<> IRB(I.getNextNode());
1923  Type *ShadowTy = getShadowTy(&I);
1924  Value *Addr = I.getPointerOperand();
1925  Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
1926  const Align Alignment = assumeAligned(I.getAlignment());
1927  if (PropagateShadow) {
1928  std::tie(ShadowPtr, OriginPtr) =
1929  getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
1930  setShadow(&I,
1931  IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
1932  } else {
1933  setShadow(&I, getCleanShadow(&I));
1934  }
1935 
1937  insertShadowCheck(I.getPointerOperand(), &I);
1938 
1939  if (I.isAtomic())
1940  I.setOrdering(addAcquireOrdering(I.getOrdering()));
1941 
1942  if (MS.TrackOrigins) {
1943  if (PropagateShadow) {
1944  const Align OriginAlignment = std::max(kMinOriginAlignment, Alignment);
1945  setOrigin(
1946  &I, IRB.CreateAlignedLoad(MS.OriginTy, OriginPtr, OriginAlignment));
1947  } else {
1948  setOrigin(&I, getCleanOrigin());
1949  }
1950  }
1951  }
1952 
1953  /// Instrument StoreInst
1954  ///
1955  /// Stores the corresponding shadow and (optionally) origin.
1956  /// Optionally, checks that the store address is fully defined.
1957  void visitStoreInst(StoreInst &I) {
1958  StoreList.push_back(&I);
1960  insertShadowCheck(I.getPointerOperand(), &I);
1961  }
1962 
1963  void handleCASOrRMW(Instruction &I) {
1964  assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
1965 
1966  IRBuilder<> IRB(&I);
1967  Value *Addr = I.getOperand(0);
1968  Value *Val = I.getOperand(1);
1969  Value *ShadowPtr = getShadowOriginPtr(Addr, IRB, Val->getType(), Align(1),
1970  /*isStore*/ true)
1971  .first;
1972 
1974  insertShadowCheck(Addr, &I);
1975 
1976  // Only test the conditional argument of cmpxchg instruction.
1977  // The other argument can potentially be uninitialized, but we can not
1978  // detect this situation reliably without possible false positives.
1979  if (isa<AtomicCmpXchgInst>(I))
1980  insertShadowCheck(Val, &I);
1981 
1982  IRB.CreateStore(getCleanShadow(Val), ShadowPtr);
1983 
1984  setShadow(&I, getCleanShadow(&I));
1985  setOrigin(&I, getCleanOrigin());
1986  }
1987 
1988  void visitAtomicRMWInst(AtomicRMWInst &I) {
1989  handleCASOrRMW(I);
1990  I.setOrdering(addReleaseOrdering(I.getOrdering()));
1991  }
1992 
1993  void visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
1994  handleCASOrRMW(I);
1995  I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
1996  }
1997 
1998  // Vector manipulation.
1999  void visitExtractElementInst(ExtractElementInst &I) {
2000  insertShadowCheck(I.getOperand(1), &I);
2001  IRBuilder<> IRB(&I);
2002  setShadow(&I, IRB.CreateExtractElement(getShadow(&I, 0), I.getOperand(1),
2003  "_msprop"));
2004  setOrigin(&I, getOrigin(&I, 0));
2005  }
2006 
2007  void visitInsertElementInst(InsertElementInst &I) {
2008  insertShadowCheck(I.getOperand(2), &I);
2009  IRBuilder<> IRB(&I);
2010  setShadow(&I, IRB.CreateInsertElement(getShadow(&I, 0), getShadow(&I, 1),
2011  I.getOperand(2), "_msprop"));
2012  setOriginForNaryOp(I);
2013  }
2014 
2015  void visitShuffleVectorInst(ShuffleVectorInst &I) {
2016  IRBuilder<> IRB(&I);
2017  setShadow(&I, IRB.CreateShuffleVector(getShadow(&I, 0), getShadow(&I, 1),
2018  I.getShuffleMask(), "_msprop"));
2019  setOriginForNaryOp(I);
2020  }
2021 
2022  // Casts.
2023  void visitSExtInst(SExtInst &I) {
2024  IRBuilder<> IRB(&I);
2025  setShadow(&I, IRB.CreateSExt(getShadow(&I, 0), I.getType(), "_msprop"));
2026  setOrigin(&I, getOrigin(&I, 0));
2027  }
2028 
2029  void visitZExtInst(ZExtInst &I) {
2030  IRBuilder<> IRB(&I);
2031  setShadow(&I, IRB.CreateZExt(getShadow(&I, 0), I.getType(), "_msprop"));
2032  setOrigin(&I, getOrigin(&I, 0));
2033  }
2034 
2035  void visitTruncInst(TruncInst &I) {
2036  IRBuilder<> IRB(&I);
2037  setShadow(&I, IRB.CreateTrunc(getShadow(&I, 0), I.getType(), "_msprop"));
2038  setOrigin(&I, getOrigin(&I, 0));
2039  }
2040 
2041  void visitBitCastInst(BitCastInst &I) {
2042  // Special case: if this is the bitcast (there is exactly 1 allowed) between
2043  // a musttail call and a ret, don't instrument. New instructions are not
2044  // allowed after a musttail call.
2045  if (auto *CI = dyn_cast<CallInst>(I.getOperand(0)))
2046  if (CI->isMustTailCall())
2047  return;
2048  IRBuilder<> IRB(&I);
2049  setShadow(&I, IRB.CreateBitCast(getShadow(&I, 0), getShadowTy(&I)));
2050  setOrigin(&I, getOrigin(&I, 0));
2051  }
2052 
2053  void visitPtrToIntInst(PtrToIntInst &I) {
2054  IRBuilder<> IRB(&I);
2055  setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2056  "_msprop_ptrtoint"));
2057  setOrigin(&I, getOrigin(&I, 0));
2058  }
2059 
2060  void visitIntToPtrInst(IntToPtrInst &I) {
2061  IRBuilder<> IRB(&I);
2062  setShadow(&I, IRB.CreateIntCast(getShadow(&I, 0), getShadowTy(&I), false,
2063  "_msprop_inttoptr"));
2064  setOrigin(&I, getOrigin(&I, 0));
2065  }
2066 
2067  void visitFPToSIInst(CastInst& I) { handleShadowOr(I); }
2068  void visitFPToUIInst(CastInst& I) { handleShadowOr(I); }
2069  void visitSIToFPInst(CastInst& I) { handleShadowOr(I); }
2070  void visitUIToFPInst(CastInst& I) { handleShadowOr(I); }
2071  void visitFPExtInst(CastInst& I) { handleShadowOr(I); }
2072  void visitFPTruncInst(CastInst& I) { handleShadowOr(I); }
2073 
2074  /// Propagate shadow for bitwise AND.
2075  ///
2076  /// This code is exact, i.e. if, for example, a bit in the left argument
2077  /// is defined and 0, then neither the value not definedness of the
2078  /// corresponding bit in B don't affect the resulting shadow.
2079  void visitAnd(BinaryOperator &I) {
2080  IRBuilder<> IRB(&I);
2081  // "And" of 0 and a poisoned value results in unpoisoned value.
2082  // 1&1 => 1; 0&1 => 0; p&1 => p;
2083  // 1&0 => 0; 0&0 => 0; p&0 => 0;
2084  // 1&p => p; 0&p => 0; p&p => p;
2085  // S = (S1 & S2) | (V1 & S2) | (S1 & V2)
2086  Value *S1 = getShadow(&I, 0);
2087  Value *S2 = getShadow(&I, 1);
2088  Value *V1 = I.getOperand(0);
2089  Value *V2 = I.getOperand(1);
2090  if (V1->getType() != S1->getType()) {
2091  V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2092  V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2093  }
2094  Value *S1S2 = IRB.CreateAnd(S1, S2);
2095  Value *V1S2 = IRB.CreateAnd(V1, S2);
2096  Value *S1V2 = IRB.CreateAnd(S1, V2);
2097  setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2098  setOriginForNaryOp(I);
2099  }
2100 
2101  void visitOr(BinaryOperator &I) {
2102  IRBuilder<> IRB(&I);
2103  // "Or" of 1 and a poisoned value results in unpoisoned value.
2104  // 1|1 => 1; 0|1 => 1; p|1 => 1;
2105  // 1|0 => 1; 0|0 => 0; p|0 => p;
2106  // 1|p => 1; 0|p => p; p|p => p;
2107  // S = (S1 & S2) | (~V1 & S2) | (S1 & ~V2)
2108  Value *S1 = getShadow(&I, 0);
2109  Value *S2 = getShadow(&I, 1);
2110  Value *V1 = IRB.CreateNot(I.getOperand(0));
2111  Value *V2 = IRB.CreateNot(I.getOperand(1));
2112  if (V1->getType() != S1->getType()) {
2113  V1 = IRB.CreateIntCast(V1, S1->getType(), false);
2114  V2 = IRB.CreateIntCast(V2, S2->getType(), false);
2115  }
2116  Value *S1S2 = IRB.CreateAnd(S1, S2);
2117  Value *V1S2 = IRB.CreateAnd(V1, S2);
2118  Value *S1V2 = IRB.CreateAnd(S1, V2);
2119  setShadow(&I, IRB.CreateOr({S1S2, V1S2, S1V2}));
2120  setOriginForNaryOp(I);
2121  }
2122 
2123  /// Default propagation of shadow and/or origin.
2124  ///
2125  /// This class implements the general case of shadow propagation, used in all
2126  /// cases where we don't know and/or don't care about what the operation
2127  /// actually does. It converts all input shadow values to a common type
2128  /// (extending or truncating as necessary), and bitwise OR's them.
2129  ///
2130  /// This is much cheaper than inserting checks (i.e. requiring inputs to be
2131  /// fully initialized), and less prone to false positives.
2132  ///
2133  /// This class also implements the general case of origin propagation. For a
2134  /// Nary operation, result origin is set to the origin of an argument that is
2135  /// not entirely initialized. If there is more than one such arguments, the
2136  /// rightmost of them is picked. It does not matter which one is picked if all
2137  /// arguments are initialized.
2138  template <bool CombineShadow>
2139  class Combiner {
2140  Value *Shadow = nullptr;
2141  Value *Origin = nullptr;
2142  IRBuilder<> &IRB;
2143  MemorySanitizerVisitor *MSV;
2144 
2145  public:
2146  Combiner(MemorySanitizerVisitor *MSV, IRBuilder<> &IRB)
2147  : IRB(IRB), MSV(MSV) {}
2148 
2149  /// Add a pair of shadow and origin values to the mix.
2150  Combiner &Add(Value *OpShadow, Value *OpOrigin) {
2151  if (CombineShadow) {
2152  assert(OpShadow);
2153  if (!Shadow)
2154  Shadow = OpShadow;
2155  else {
2156  OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2157  Shadow = IRB.CreateOr(Shadow, OpShadow, "_msprop");
2158  }
2159  }
2160 
2161  if (MSV->MS.TrackOrigins) {
2162  assert(OpOrigin);
2163  if (!Origin) {
2164  Origin = OpOrigin;
2165  } else {
2166  Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2167  // No point in adding something that might result in 0 origin value.
2168  if (!ConstOrigin || !ConstOrigin->isNullValue()) {
2169  Value *FlatShadow = MSV->convertShadowToScalar(OpShadow, IRB);
2170  Value *Cond =
2171  IRB.CreateICmpNE(FlatShadow, MSV->getCleanShadow(FlatShadow));
2172  Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2173  }
2174  }
2175  }
2176  return *this;
2177  }
2178 
2179  /// Add an application value to the mix.
2180  Combiner &Add(Value *V) {
2181  Value *OpShadow = MSV->getShadow(V);
2182  Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) : nullptr;
2183  return Add(OpShadow, OpOrigin);
2184  }
2185 
2186  /// Set the current combined values as the given instruction's shadow
2187  /// and origin.
2188  void Done(Instruction *I) {
2189  if (CombineShadow) {
2190  assert(Shadow);
2191  Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(I));
2192  MSV->setShadow(I, Shadow);
2193  }
2194  if (MSV->MS.TrackOrigins) {
2195  assert(Origin);
2196  MSV->setOrigin(I, Origin);
2197  }
2198  }
2199  };
2200 
2201  using ShadowAndOriginCombiner = Combiner<true>;
2202  using OriginCombiner = Combiner<false>;
2203 
2204  /// Propagate origin for arbitrary operation.
2205  void setOriginForNaryOp(Instruction &I) {
2206  if (!MS.TrackOrigins) return;
2207  IRBuilder<> IRB(&I);
2208  OriginCombiner OC(this, IRB);
2209  for (Use &Op : I.operands())
2210  OC.Add(Op.get());
2211  OC.Done(&I);
2212  }
2213 
2214  size_t VectorOrPrimitiveTypeSizeInBits(Type *Ty) {
2215  assert(!(Ty->isVectorTy() && Ty->getScalarType()->isPointerTy()) &&
2216  "Vector of pointers is not a valid shadow type");
2217  return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2218  Ty->getScalarSizeInBits()
2219  : Ty->getPrimitiveSizeInBits();
2220  }
2221 
2222  /// Cast between two shadow types, extending or truncating as
2223  /// necessary.
2224  Value *CreateShadowCast(IRBuilder<> &IRB, Value *V, Type *dstTy,
2225  bool Signed = false) {
2226  Type *srcTy = V->getType();
2227  size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2228  size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2229  if (srcSizeInBits > 1 && dstSizeInBits == 1)
2230  return IRB.CreateICmpNE(V, getCleanShadow(V));
2231 
2232  if (dstTy->isIntegerTy() && srcTy->isIntegerTy())
2233  return IRB.CreateIntCast(V, dstTy, Signed);
2234  if (dstTy->isVectorTy() && srcTy->isVectorTy() &&
2235  cast<FixedVectorType>(dstTy)->getNumElements() ==
2236  cast<FixedVectorType>(srcTy)->getNumElements())
2237  return IRB.CreateIntCast(V, dstTy, Signed);
2238  Value *V1 = IRB.CreateBitCast(V, Type::getIntNTy(*MS.C, srcSizeInBits));
2239  Value *V2 =
2240  IRB.CreateIntCast(V1, Type::getIntNTy(*MS.C, dstSizeInBits), Signed);
2241  return IRB.CreateBitCast(V2, dstTy);
2242  // TODO: handle struct types.
2243  }
2244 
2245  /// Cast an application value to the type of its own shadow.
2246  Value *CreateAppToShadowCast(IRBuilder<> &IRB, Value *V) {
2247  Type *ShadowTy = getShadowTy(V);
2248  if (V->getType() == ShadowTy)
2249  return V;
2250  if (V->getType()->isPtrOrPtrVectorTy())
2251  return IRB.CreatePtrToInt(V, ShadowTy);
2252  else
2253  return IRB.CreateBitCast(V, ShadowTy);
2254  }
2255 
2256  /// Propagate shadow for arbitrary operation.
2257  void handleShadowOr(Instruction &I) {
2258  IRBuilder<> IRB(&I);
2259  ShadowAndOriginCombiner SC(this, IRB);
2260  for (Use &Op : I.operands())
2261  SC.Add(Op.get());
2262  SC.Done(&I);
2263  }
2264 
2265  void visitFNeg(UnaryOperator &I) { handleShadowOr(I); }
2266 
2267  // Handle multiplication by constant.
2268  //
2269  // Handle a special case of multiplication by constant that may have one or
2270  // more zeros in the lower bits. This makes corresponding number of lower bits
2271  // of the result zero as well. We model it by shifting the other operand
2272  // shadow left by the required number of bits. Effectively, we transform
2273  // (X * (A * 2**B)) to ((X << B) * A) and instrument (X << B) as (Sx << B).
2274  // We use multiplication by 2**N instead of shift to cover the case of
2275  // multiplication by 0, which may occur in some elements of a vector operand.
2276  void handleMulByConstant(BinaryOperator &I, Constant *ConstArg,
2277  Value *OtherArg) {
2278  Constant *ShadowMul;
2279  Type *Ty = ConstArg->getType();
2280  if (auto *VTy = dyn_cast<VectorType>(Ty)) {
2281  unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2282  Type *EltTy = VTy->getElementType();
2283  SmallVector<Constant *, 16> Elements;
2284  for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
2285  if (ConstantInt *Elt =
2286  dyn_cast<ConstantInt>(ConstArg->getAggregateElement(Idx))) {
2287  const APInt &V = Elt->getValue();
2288  APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2289  Elements.push_back(ConstantInt::get(EltTy, V2));
2290  } else {
2291  Elements.push_back(ConstantInt::get(EltTy, 1));
2292  }
2293  }
2294  ShadowMul = ConstantVector::get(Elements);
2295  } else {
2296  if (ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2297  const APInt &V = Elt->getValue();
2298  APInt V2 = APInt(V.getBitWidth(), 1) << V.countTrailingZeros();
2299  ShadowMul = ConstantInt::get(Ty, V2);
2300  } else {
2301  ShadowMul = ConstantInt::get(Ty, 1);
2302  }
2303  }
2304 
2305  IRBuilder<> IRB(&I);
2306  setShadow(&I,
2307  IRB.CreateMul(getShadow(OtherArg), ShadowMul, "msprop_mul_cst"));
2308  setOrigin(&I, getOrigin(OtherArg));
2309  }
2310 
2311  void visitMul(BinaryOperator &I) {
2312  Constant *constOp0 = dyn_cast<Constant>(I.getOperand(0));
2313  Constant *constOp1 = dyn_cast<Constant>(I.getOperand(1));
2314  if (constOp0 && !constOp1)
2315  handleMulByConstant(I, constOp0, I.getOperand(1));
2316  else if (constOp1 && !constOp0)
2317  handleMulByConstant(I, constOp1, I.getOperand(0));
2318  else
2319  handleShadowOr(I);
2320  }
2321 
2322  void visitFAdd(BinaryOperator &I) { handleShadowOr(I); }
2323  void visitFSub(BinaryOperator &I) { handleShadowOr(I); }
2324  void visitFMul(BinaryOperator &I) { handleShadowOr(I); }
2325  void visitAdd(BinaryOperator &I) { handleShadowOr(I); }
2326  void visitSub(BinaryOperator &I) { handleShadowOr(I); }
2327  void visitXor(BinaryOperator &I) { handleShadowOr(I); }
2328 
2329  void handleIntegerDiv(Instruction &I) {
2330  IRBuilder<> IRB(&I);
2331  // Strict on the second argument.
2332  insertShadowCheck(I.getOperand(1), &I);
2333  setShadow(&I, getShadow(&I, 0));
2334  setOrigin(&I, getOrigin(&I, 0));
2335  }
2336 
2337  void visitUDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2338  void visitSDiv(BinaryOperator &I) { handleIntegerDiv(I); }
2339  void visitURem(BinaryOperator &I) { handleIntegerDiv(I); }
2340  void visitSRem(BinaryOperator &I) { handleIntegerDiv(I); }
2341 
2342  // Floating point division is side-effect free. We can not require that the
2343  // divisor is fully initialized and must propagate shadow. See PR37523.
2344  void visitFDiv(BinaryOperator &I) { handleShadowOr(I); }
2345  void visitFRem(BinaryOperator &I) { handleShadowOr(I); }
2346 
2347  /// Instrument == and != comparisons.
2348  ///
2349  /// Sometimes the comparison result is known even if some of the bits of the
2350  /// arguments are not.
2351  void handleEqualityComparison(ICmpInst &I) {
2352  IRBuilder<> IRB(&I);
2353  Value *A = I.getOperand(0);
2354  Value *B = I.getOperand(1);
2355  Value *Sa = getShadow(A);
2356  Value *Sb = getShadow(B);
2357 
2358  // Get rid of pointers and vectors of pointers.
2359  // For ints (and vectors of ints), types of A and Sa match,
2360  // and this is a no-op.
2361  A = IRB.CreatePointerCast(A, Sa->getType());
2362  B = IRB.CreatePointerCast(B, Sb->getType());
2363 
2364  // A == B <==> (C = A^B) == 0
2365  // A != B <==> (C = A^B) != 0
2366  // Sc = Sa | Sb
2367  Value *C = IRB.CreateXor(A, B);
2368  Value *Sc = IRB.CreateOr(Sa, Sb);
2369  // Now dealing with i = (C == 0) comparison (or C != 0, does not matter now)
2370  // Result is defined if one of the following is true
2371  // * there is a defined 1 bit in C
2372  // * C is fully defined
2373  // Si = !(C & ~Sc) && Sc
2374  Value *Zero = Constant::getNullValue(Sc->getType());
2375  Value *MinusOne = Constant::getAllOnesValue(Sc->getType());
2376  Value *Si =
2377  IRB.CreateAnd(IRB.CreateICmpNE(Sc, Zero),
2378  IRB.CreateICmpEQ(
2379  IRB.CreateAnd(IRB.CreateXor(Sc, MinusOne), C), Zero));
2380  Si->setName("_msprop_icmp");
2381  setShadow(&I, Si);
2382  setOriginForNaryOp(I);
2383  }
2384 
2385  /// Build the lowest possible value of V, taking into account V's
2386  /// uninitialized bits.
2387  Value *getLowestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2388  bool isSigned) {
2389  if (isSigned) {
2390  // Split shadow into sign bit and other bits.
2391  Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2392  Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2393  // Maximise the undefined shadow bit, minimize other undefined bits.
2394  return
2395  IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaOtherBits)), SaSignBit);
2396  } else {
2397  // Minimize undefined bits.
2398  return IRB.CreateAnd(A, IRB.CreateNot(Sa));
2399  }
2400  }
2401 
2402  /// Build the highest possible value of V, taking into account V's
2403  /// uninitialized bits.
2404  Value *getHighestPossibleValue(IRBuilder<> &IRB, Value *A, Value *Sa,
2405  bool isSigned) {
2406  if (isSigned) {
2407  // Split shadow into sign bit and other bits.
2408  Value *SaOtherBits = IRB.CreateLShr(IRB.CreateShl(Sa, 1), 1);
2409  Value *SaSignBit = IRB.CreateXor(Sa, SaOtherBits);
2410  // Minimise the undefined shadow bit, maximise other undefined bits.
2411  return
2412  IRB.CreateOr(IRB.CreateAnd(A, IRB.CreateNot(SaSignBit)), SaOtherBits);
2413  } else {
2414  // Maximize undefined bits.
2415  return IRB.CreateOr(A, Sa);
2416  }
2417  }
2418 
2419  /// Instrument relational comparisons.
2420  ///
2421  /// This function does exact shadow propagation for all relational
2422  /// comparisons of integers, pointers and vectors of those.
2423  /// FIXME: output seems suboptimal when one of the operands is a constant
2424  void handleRelationalComparisonExact(ICmpInst &I) {
2425  IRBuilder<> IRB(&I);
2426  Value *A = I.getOperand(0);
2427  Value *B = I.getOperand(1);
2428  Value *Sa = getShadow(A);
2429  Value *Sb = getShadow(B);
2430 
2431  // Get rid of pointers and vectors of pointers.
2432  // For ints (and vectors of ints), types of A and Sa match,
2433  // and this is a no-op.
2434  A = IRB.CreatePointerCast(A, Sa->getType());
2435  B = IRB.CreatePointerCast(B, Sb->getType());
2436 
2437  // Let [a0, a1] be the interval of possible values of A, taking into account
2438  // its undefined bits. Let [b0, b1] be the interval of possible values of B.
2439  // Then (A cmp B) is defined iff (a0 cmp b1) == (a1 cmp b0).
2440  bool IsSigned = I.isSigned();
2441  Value *S1 = IRB.CreateICmp(I.getPredicate(),
2442  getLowestPossibleValue(IRB, A, Sa, IsSigned),
2443  getHighestPossibleValue(IRB, B, Sb, IsSigned));
2444  Value *S2 = IRB.CreateICmp(I.getPredicate(),
2445  getHighestPossibleValue(IRB, A, Sa, IsSigned),
2446  getLowestPossibleValue(IRB, B, Sb, IsSigned));
2447  Value *Si = IRB.CreateXor(S1, S2);
2448  setShadow(&I, Si);
2449  setOriginForNaryOp(I);
2450  }
2451 
2452  /// Instrument signed relational comparisons.
2453  ///
2454  /// Handle sign bit tests: x<0, x>=0, x<=-1, x>-1 by propagating the highest
2455  /// bit of the shadow. Everything else is delegated to handleShadowOr().
2456  void handleSignedRelationalComparison(ICmpInst &I) {
2457  Constant *constOp;
2458  Value *op = nullptr;
2459  CmpInst::Predicate pre;
2460  if ((constOp = dyn_cast<Constant>(I.getOperand(1)))) {
2461  op = I.getOperand(0);
2462  pre = I.getPredicate();
2463  } else if ((constOp = dyn_cast<Constant>(I.getOperand(0)))) {
2464  op = I.getOperand(1);
2465  pre = I.getSwappedPredicate();
2466  } else {
2467  handleShadowOr(I);
2468  return;
2469  }
2470 
2471  if ((constOp->isNullValue() &&
2472  (pre == CmpInst::ICMP_SLT || pre == CmpInst::ICMP_SGE)) ||
2473  (constOp->isAllOnesValue() &&
2474  (pre == CmpInst::ICMP_SGT || pre == CmpInst::ICMP_SLE))) {
2475  IRBuilder<> IRB(&I);
2476  Value *Shadow = IRB.CreateICmpSLT(getShadow(op), getCleanShadow(op),
2477  "_msprop_icmp_s");
2478  setShadow(&I, Shadow);
2479  setOrigin(&I, getOrigin(op));
2480  } else {
2481  handleShadowOr(I);
2482  }
2483  }
2484 
2485  void visitICmpInst(ICmpInst &I) {
2486  if (!ClHandleICmp) {
2487  handleShadowOr(I);
2488  return;
2489  }
2490  if (I.isEquality()) {
2491  handleEqualityComparison(I);
2492  return;
2493  }
2494 
2495  assert(I.isRelational());
2496  if (ClHandleICmpExact) {
2497  handleRelationalComparisonExact(I);
2498  return;
2499  }
2500  if (I.isSigned()) {
2501  handleSignedRelationalComparison(I);
2502  return;
2503  }
2504 
2505  assert(I.isUnsigned());
2506  if ((isa<Constant>(I.getOperand(0)) || isa<Constant>(I.getOperand(1)))) {
2507  handleRelationalComparisonExact(I);
2508  return;
2509  }
2510 
2511  handleShadowOr(I);
2512  }
2513 
2514  void visitFCmpInst(FCmpInst &I) {
2515  handleShadowOr(I);
2516  }
2517 
2518  void handleShift(BinaryOperator &I) {
2519  IRBuilder<> IRB(&I);
2520  // If any of the S2 bits are poisoned, the whole thing is poisoned.
2521  // Otherwise perform the same shift on S1.
2522  Value *S1 = getShadow(&I, 0);
2523  Value *S2 = getShadow(&I, 1);
2524  Value *S2Conv = IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)),
2525  S2->getType());
2526  Value *V2 = I.getOperand(1);
2527  Value *Shift = IRB.CreateBinOp(I.getOpcode(), S1, V2);
2528  setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2529  setOriginForNaryOp(I);
2530  }
2531 
2532  void visitShl(BinaryOperator &I) { handleShift(I); }
2533  void visitAShr(BinaryOperator &I) { handleShift(I); }
2534  void visitLShr(BinaryOperator &I) { handleShift(I); }
2535 
2536  void handleFunnelShift(IntrinsicInst &I) {
2537  IRBuilder<> IRB(&I);
2538  // If any of the S2 bits are poisoned, the whole thing is poisoned.
2539  // Otherwise perform the same shift on S0 and S1.
2540  Value *S0 = getShadow(&I, 0);
2541  Value *S1 = getShadow(&I, 1);
2542  Value *S2 = getShadow(&I, 2);
2543  Value *S2Conv =
2544  IRB.CreateSExt(IRB.CreateICmpNE(S2, getCleanShadow(S2)), S2->getType());
2545  Value *V2 = I.getOperand(2);
2547  I.getModule(), I.getIntrinsicID(), S2Conv->getType());
2548  Value *Shift = IRB.CreateCall(Intrin, {S0, S1, V2});
2549  setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2550  setOriginForNaryOp(I);
2551  }
2552 
2553  /// Instrument llvm.memmove
2554  ///
2555  /// At this point we don't know if llvm.memmove will be inlined or not.
2556  /// If we don't instrument it and it gets inlined,
2557  /// our interceptor will not kick in and we will lose the memmove.
2558  /// If we instrument the call here, but it does not get inlined,
2559  /// we will memove the shadow twice: which is bad in case
2560  /// of overlapping regions. So, we simply lower the intrinsic to a call.
2561  ///
2562  /// Similar situation exists for memcpy and memset.
2563  void visitMemMoveInst(MemMoveInst &I) {
2564  IRBuilder<> IRB(&I);
2565  IRB.CreateCall(
2566  MS.MemmoveFn,
2567  {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2568  IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2569  IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2570  I.eraseFromParent();
2571  }
2572 
2573  // Similar to memmove: avoid copying shadow twice.
2574  // This is somewhat unfortunate as it may slowdown small constant memcpys.
2575  // FIXME: consider doing manual inline for small constant sizes and proper
2576  // alignment.
2577  void visitMemCpyInst(MemCpyInst &I) {
2578  IRBuilder<> IRB(&I);
2579  IRB.CreateCall(
2580  MS.MemcpyFn,
2581  {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2582  IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2583  IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2584  I.eraseFromParent();
2585  }
2586 
2587  // Same as memcpy.
2588  void visitMemSetInst(MemSetInst &I) {
2589  IRBuilder<> IRB(&I);
2590  IRB.CreateCall(
2591  MS.MemsetFn,
2592  {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2593  IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2594  IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2595  I.eraseFromParent();
2596  }
2597 
2598  void visitVAStartInst(VAStartInst &I) {
2599  VAHelper->visitVAStartInst(I);
2600  }
2601 
2602  void visitVACopyInst(VACopyInst &I) {
2603  VAHelper->visitVACopyInst(I);
2604  }
2605 
2606  /// Handle vector store-like intrinsics.
2607  ///
2608  /// Instrument intrinsics that look like a simple SIMD store: writes memory,
2609  /// has 1 pointer argument and 1 vector argument, returns void.
2610  bool handleVectorStoreIntrinsic(IntrinsicInst &I) {
2611  IRBuilder<> IRB(&I);
2612  Value* Addr = I.getArgOperand(0);
2613  Value *Shadow = getShadow(&I, 1);
2614  Value *ShadowPtr, *OriginPtr;
2615 
2616  // We don't know the pointer alignment (could be unaligned SSE store!).
2617  // Have to assume to worst case.
2618  std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2619  Addr, IRB, Shadow->getType(), Align(1), /*isStore*/ true);
2620  IRB.CreateAlignedStore(Shadow, ShadowPtr, Align(1));
2621 
2623  insertShadowCheck(Addr, &I);
2624 
2625  // FIXME: factor out common code from materializeStores
2626  if (MS.TrackOrigins) IRB.CreateStore(getOrigin(&I, 1), OriginPtr);
2627  return true;
2628  }
2629 
2630  /// Handle vector load-like intrinsics.
2631  ///
2632  /// Instrument intrinsics that look like a simple SIMD load: reads memory,
2633  /// has 1 pointer argument, returns a vector.
2634  bool handleVectorLoadIntrinsic(IntrinsicInst &I) {
2635  IRBuilder<> IRB(&I);
2636  Value *Addr = I.getArgOperand(0);
2637 
2638  Type *ShadowTy = getShadowTy(&I);
2639  Value *ShadowPtr = nullptr, *OriginPtr = nullptr;
2640  if (PropagateShadow) {
2641  // We don't know the pointer alignment (could be unaligned SSE load!).
2642  // Have to assume to worst case.
2643  const Align Alignment = Align(1);
2644  std::tie(ShadowPtr, OriginPtr) =
2645  getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
2646  setShadow(&I,
2647  IRB.CreateAlignedLoad(ShadowTy, ShadowPtr, Alignment, "_msld"));
2648  } else {
2649  setShadow(&I, getCleanShadow(&I));
2650  }
2651 
2653  insertShadowCheck(Addr, &I);
2654 
2655  if (MS.TrackOrigins) {
2656  if (PropagateShadow)
2657  setOrigin(&I, IRB.CreateLoad(MS.OriginTy, OriginPtr));
2658  else
2659  setOrigin(&I, getCleanOrigin());
2660  }
2661  return true;
2662  }
2663 
2664  /// Handle (SIMD arithmetic)-like intrinsics.
2665  ///
2666  /// Instrument intrinsics with any number of arguments of the same type,
2667  /// equal to the return type. The type should be simple (no aggregates or
2668  /// pointers; vectors are fine).
2669  /// Caller guarantees that this intrinsic does not access memory.
2670  bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I) {
2671  Type *RetTy = I.getType();
2672  if (!(RetTy->isIntOrIntVectorTy() ||
2673  RetTy->isFPOrFPVectorTy() ||
2674  RetTy->isX86_MMXTy()))
2675  return false;
2676 
2677  unsigned NumArgOperands = I.getNumArgOperands();
2678  for (unsigned i = 0; i < NumArgOperands; ++i) {
2679  Type *Ty = I.getArgOperand(i)->getType();
2680  if (Ty != RetTy)
2681  return false;
2682  }
2683 
2684  IRBuilder<> IRB(&I);
2685  ShadowAndOriginCombiner SC(this, IRB);
2686  for (unsigned i = 0; i < NumArgOperands; ++i)
2687  SC.Add(I.getArgOperand(i));
2688  SC.Done(&I);
2689 
2690  return true;
2691  }
2692 
2693  /// Heuristically instrument unknown intrinsics.
2694  ///
2695  /// The main purpose of this code is to do something reasonable with all
2696  /// random intrinsics we might encounter, most importantly - SIMD intrinsics.
2697  /// We recognize several classes of intrinsics by their argument types and
2698  /// ModRefBehaviour and apply special instrumentation when we are reasonably
2699  /// sure that we know what the intrinsic does.
2700  ///
2701  /// We special-case intrinsics where this approach fails. See llvm.bswap
2702  /// handling as an example of that.
2703  bool handleUnknownIntrinsic(IntrinsicInst &I) {
2704  unsigned NumArgOperands = I.getNumArgOperands();
2705  if (NumArgOperands == 0)
2706  return false;
2707 
2708  if (NumArgOperands == 2 &&
2709  I.getArgOperand(0)->getType()->isPointerTy() &&
2710  I.getArgOperand(1)->getType()->isVectorTy() &&
2711  I.getType()->isVoidTy() &&
2712  !I.onlyReadsMemory()) {
2713  // This looks like a vector store.
2714  return handleVectorStoreIntrinsic(I);
2715  }
2716 
2717  if (NumArgOperands == 1 &&
2718  I.getArgOperand(0)->getType()->isPointerTy() &&
2719  I.getType()->isVectorTy() &&
2720  I.onlyReadsMemory()) {
2721  // This looks like a vector load.
2722  return handleVectorLoadIntrinsic(I);
2723  }
2724 
2725  if (I.doesNotAccessMemory())
2726  if (maybeHandleSimpleNomemIntrinsic(I))
2727  return true;
2728 
2729  // FIXME: detect and handle SSE maskstore/maskload
2730  return false;
2731  }
2732 
2733  void handleInvariantGroup(IntrinsicInst &I) {
2734  setShadow(&I, getShadow(&I, 0));
2735  setOrigin(&I, getOrigin(&I, 0));
2736  }
2737 
2738  void handleLifetimeStart(IntrinsicInst &I) {
2739  if (!PoisonStack)
2740  return;
2741  AllocaInst *AI = llvm::findAllocaForValue(I.getArgOperand(1));
2742  if (!AI)
2743  InstrumentLifetimeStart = false;
2744  LifetimeStartList.push_back(std::make_pair(&I, AI));
2745  }
2746 
2747  void handleBswap(IntrinsicInst &I) {
2748  IRBuilder<> IRB(&I);
2749  Value *Op = I.getArgOperand(0);
2750  Type *OpType = Op->getType();
2751  Function *BswapFunc = Intrinsic::getDeclaration(
2752  F.getParent(), Intrinsic::bswap, makeArrayRef(&OpType, 1));
2753  setShadow(&I, IRB.CreateCall(BswapFunc, getShadow(Op)));
2754  setOrigin(&I, getOrigin(Op));
2755  }
2756 
2757  // Instrument vector convert intrinsic.
2758  //
2759  // This function instruments intrinsics like cvtsi2ss:
2760  // %Out = int_xxx_cvtyyy(%ConvertOp)
2761  // or
2762  // %Out = int_xxx_cvtyyy(%CopyOp, %ConvertOp)
2763  // Intrinsic converts \p NumUsedElements elements of \p ConvertOp to the same
2764  // number \p Out elements, and (if has 2 arguments) copies the rest of the
2765  // elements from \p CopyOp.
2766  // In most cases conversion involves floating-point value which may trigger a
2767  // hardware exception when not fully initialized. For this reason we require
2768  // \p ConvertOp[0:NumUsedElements] to be fully initialized and trap otherwise.
2769  // We copy the shadow of \p CopyOp[NumUsedElements:] to \p
2770  // Out[NumUsedElements:]. This means that intrinsics without \p CopyOp always
2771  // return a fully initialized value.
2772  void handleVectorConvertIntrinsic(IntrinsicInst &I, int NumUsedElements,
2773  bool HasRoundingMode = false) {
2774  IRBuilder<> IRB(&I);
2775  Value *CopyOp, *ConvertOp;
2776 
2777  assert((!HasRoundingMode ||
2778  isa<ConstantInt>(I.getArgOperand(I.getNumArgOperands() - 1))) &&
2779  "Invalid rounding mode");
2780 
2781  switch (I.getNumArgOperands() - HasRoundingMode) {
2782  case 2:
2783  CopyOp = I.getArgOperand(0);
2784  ConvertOp = I.getArgOperand(1);
2785  break;
2786  case 1:
2787  ConvertOp = I.getArgOperand(0);
2788  CopyOp = nullptr;
2789  break;
2790  default:
2791  llvm_unreachable("Cvt intrinsic with unsupported number of arguments.");
2792  }
2793 
2794  // The first *NumUsedElements* elements of ConvertOp are converted to the
2795  // same number of output elements. The rest of the output is copied from
2796  // CopyOp, or (if not available) filled with zeroes.
2797  // Combine shadow for elements of ConvertOp that are used in this operation,
2798  // and insert a check.
2799  // FIXME: consider propagating shadow of ConvertOp, at least in the case of
2800  // int->any conversion.
2801  Value *ConvertShadow = getShadow(ConvertOp);
2802  Value *AggShadow = nullptr;
2803  if (ConvertOp->getType()->isVectorTy()) {
2804  AggShadow = IRB.CreateExtractElement(
2805  ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
2806  for (int i = 1; i < NumUsedElements; ++i) {
2807  Value *MoreShadow = IRB.CreateExtractElement(
2808  ConvertShadow, ConstantInt::get(IRB.getInt32Ty(), i));
2809  AggShadow = IRB.CreateOr(AggShadow, MoreShadow);
2810  }
2811  } else {
2812  AggShadow = ConvertShadow;
2813  }
2814  assert(AggShadow->getType()->isIntegerTy());
2815  insertShadowCheck(AggShadow, getOrigin(ConvertOp), &I);
2816 
2817  // Build result shadow by zero-filling parts of CopyOp shadow that come from
2818  // ConvertOp.
2819  if (CopyOp) {
2820  assert(CopyOp->getType() == I.getType());
2821  assert(CopyOp->getType()->isVectorTy());
2822  Value *ResultShadow = getShadow(CopyOp);
2823  Type *EltTy = cast<VectorType>(ResultShadow->getType())->getElementType();
2824  for (int i = 0; i < NumUsedElements; ++i) {
2825  ResultShadow = IRB.CreateInsertElement(
2826  ResultShadow, ConstantInt::getNullValue(EltTy),
2827  ConstantInt::get(IRB.getInt32Ty(), i));
2828  }
2829  setShadow(&I, ResultShadow);
2830  setOrigin(&I, getOrigin(CopyOp));
2831  } else {
2832  setShadow(&I, getCleanShadow(&I));
2833  setOrigin(&I, getCleanOrigin());
2834  }
2835  }
2836 
2837  // Given a scalar or vector, extract lower 64 bits (or less), and return all
2838  // zeroes if it is zero, and all ones otherwise.
2839  Value *Lower64ShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2840  if (S->getType()->isVectorTy())
2841  S = CreateShadowCast(IRB, S, IRB.getInt64Ty(), /* Signed */ true);
2842  assert(S->getType()->getPrimitiveSizeInBits() <= 64);
2843  Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2844  return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2845  }
2846 
2847  // Given a vector, extract its first element, and return all
2848  // zeroes if it is zero, and all ones otherwise.
2849  Value *LowerElementShadowExtend(IRBuilder<> &IRB, Value *S, Type *T) {
2850  Value *S1 = IRB.CreateExtractElement(S, (uint64_t)0);
2851  Value *S2 = IRB.CreateICmpNE(S1, getCleanShadow(S1));
2852  return CreateShadowCast(IRB, S2, T, /* Signed */ true);
2853  }
2854 
2855  Value *VariableShadowExtend(IRBuilder<> &IRB, Value *S) {
2856  Type *T = S->getType();
2857  assert(T->isVectorTy());
2858  Value *S2 = IRB.CreateICmpNE(S, getCleanShadow(S));
2859  return IRB.CreateSExt(S2, T);
2860  }
2861 
2862  // Instrument vector shift intrinsic.
2863  //
2864  // This function instruments intrinsics like int_x86_avx2_psll_w.
2865  // Intrinsic shifts %In by %ShiftSize bits.
2866  // %ShiftSize may be a vector. In that case the lower 64 bits determine shift
2867  // size, and the rest is ignored. Behavior is defined even if shift size is
2868  // greater than register (or field) width.
2869  void handleVectorShiftIntrinsic(IntrinsicInst &I, bool Variable) {
2870  assert(I.getNumArgOperands() == 2);
2871  IRBuilder<> IRB(&I);
2872  // If any of the S2 bits are poisoned, the whole thing is poisoned.
2873  // Otherwise perform the same shift on S1.
2874  Value *S1 = getShadow(&I, 0);
2875  Value *S2 = getShadow(&I, 1);
2876  Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
2877  : Lower64ShadowExtend(IRB, S2, getShadowTy(&I));
2878  Value *V1 = I.getOperand(0);
2879  Value *V2 = I.getOperand(1);
2880  Value *Shift = IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
2881  {IRB.CreateBitCast(S1, V1->getType()), V2});
2882  Shift = IRB.CreateBitCast(Shift, getShadowTy(&I));
2883  setShadow(&I, IRB.CreateOr(Shift, S2Conv));
2884  setOriginForNaryOp(I);
2885  }
2886 
2887  // Get an X86_MMX-sized vector type.
2888  Type *getMMXVectorTy(unsigned EltSizeInBits) {
2889  const unsigned X86_MMXSizeInBits = 64;
2890  assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
2891  "Illegal MMX vector element size");
2892  return FixedVectorType::get(IntegerType::get(*MS.C, EltSizeInBits),
2893  X86_MMXSizeInBits / EltSizeInBits);
2894  }
2895 
2896  // Returns a signed counterpart for an (un)signed-saturate-and-pack
2897  // intrinsic.
2898  Intrinsic::ID getSignedPackIntrinsic(Intrinsic::ID id) {
2899  switch (id) {
2900  case Intrinsic::x86_sse2_packsswb_128:
2901  case Intrinsic::x86_sse2_packuswb_128:
2902  return Intrinsic::x86_sse2_packsswb_128;
2903 
2904  case Intrinsic::x86_sse2_packssdw_128:
2905  case Intrinsic::x86_sse41_packusdw:
2906  return Intrinsic::x86_sse2_packssdw_128;
2907 
2908  case Intrinsic::x86_avx2_packsswb:
2909  case Intrinsic::x86_avx2_packuswb:
2910  return Intrinsic::x86_avx2_packsswb;
2911 
2912  case Intrinsic::x86_avx2_packssdw:
2913  case Intrinsic::x86_avx2_packusdw:
2914  return Intrinsic::x86_avx2_packssdw;
2915 
2916  case Intrinsic::x86_mmx_packsswb:
2917  case Intrinsic::x86_mmx_packuswb:
2918  return Intrinsic::x86_mmx_packsswb;
2919 
2920  case Intrinsic::x86_mmx_packssdw:
2921  return Intrinsic::x86_mmx_packssdw;
2922  default:
2923  llvm_unreachable("unexpected intrinsic id");
2924  }
2925  }
2926 
2927  // Instrument vector pack intrinsic.
2928  //
2929  // This function instruments intrinsics like x86_mmx_packsswb, that
2930  // packs elements of 2 input vectors into half as many bits with saturation.
2931  // Shadow is propagated with the signed variant of the same intrinsic applied
2932  // to sext(Sa != zeroinitializer), sext(Sb != zeroinitializer).
2933  // EltSizeInBits is used only for x86mmx arguments.
2934  void handleVectorPackIntrinsic(IntrinsicInst &I, unsigned EltSizeInBits = 0) {
2935  assert(I.getNumArgOperands() == 2);
2936  bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2937  IRBuilder<> IRB(&I);
2938  Value *S1 = getShadow(&I, 0);
2939  Value *S2 = getShadow(&I, 1);
2940  assert(isX86_MMX || S1->getType()->isVectorTy());
2941 
2942  // SExt and ICmpNE below must apply to individual elements of input vectors.
2943  // In case of x86mmx arguments, cast them to appropriate vector types and
2944  // back.
2945  Type *T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->getType();
2946  if (isX86_MMX) {
2947  S1 = IRB.CreateBitCast(S1, T);
2948  S2 = IRB.CreateBitCast(S2, T);
2949  }
2950  Value *S1_ext = IRB.CreateSExt(
2951  IRB.CreateICmpNE(S1, Constant::getNullValue(T)), T);
2952  Value *S2_ext = IRB.CreateSExt(
2953  IRB.CreateICmpNE(S2, Constant::getNullValue(T)), T);
2954  if (isX86_MMX) {
2955  Type *X86_MMXTy = Type::getX86_MMXTy(*MS.C);
2956  S1_ext = IRB.CreateBitCast(S1_ext, X86_MMXTy);
2957  S2_ext = IRB.CreateBitCast(S2_ext, X86_MMXTy);
2958  }
2959 
2960  Function *ShadowFn = Intrinsic::getDeclaration(
2961  F.getParent(), getSignedPackIntrinsic(I.getIntrinsicID()));
2962 
2963  Value *S =
2964  IRB.CreateCall(ShadowFn, {S1_ext, S2_ext}, "_msprop_vector_pack");
2965  if (isX86_MMX) S = IRB.CreateBitCast(S, getShadowTy(&I));
2966  setShadow(&I, S);
2967  setOriginForNaryOp(I);
2968  }
2969 
2970  // Instrument sum-of-absolute-differences intrinsic.
2971  void handleVectorSadIntrinsic(IntrinsicInst &I) {
2972  const unsigned SignificantBitsPerResultElement = 16;
2973  bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2974  Type *ResTy = isX86_MMX ? IntegerType::get(*MS.C, 64) : I.getType();
2975  unsigned ZeroBitsPerResultElement =
2976  ResTy->getScalarSizeInBits() - SignificantBitsPerResultElement;
2977 
2978  IRBuilder<> IRB(&I);
2979  Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2980  S = IRB.CreateBitCast(S, ResTy);
2981  S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2982  ResTy);
2983  S = IRB.CreateLShr(S, ZeroBitsPerResultElement);
2984  S = IRB.CreateBitCast(S, getShadowTy(&I));
2985  setShadow(&I, S);
2986  setOriginForNaryOp(I);
2987  }
2988 
2989  // Instrument multiply-add intrinsic.
2990  void handleVectorPmaddIntrinsic(IntrinsicInst &I,
2991  unsigned EltSizeInBits = 0) {
2992  bool isX86_MMX = I.getOperand(0)->getType()->isX86_MMXTy();
2993  Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) : I.getType();
2994  IRBuilder<> IRB(&I);
2995  Value *S = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
2996  S = IRB.CreateBitCast(S, ResTy);
2997  S = IRB.CreateSExt(IRB.CreateICmpNE(S, Constant::getNullValue(ResTy)),
2998  ResTy);
2999  S = IRB.CreateBitCast(S, getShadowTy(&I));
3000  setShadow(&I, S);
3001  setOriginForNaryOp(I);
3002  }
3003 
3004  // Instrument compare-packed intrinsic.
3005  // Basically, an or followed by sext(icmp ne 0) to end up with all-zeros or
3006  // all-ones shadow.
3007  void handleVectorComparePackedIntrinsic(IntrinsicInst &I) {
3008  IRBuilder<> IRB(&I);
3009  Type *ResTy = getShadowTy(&I);
3010  Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
3011  Value *S = IRB.CreateSExt(
3012  IRB.CreateICmpNE(S0, Constant::getNullValue(ResTy)), ResTy);
3013  setShadow(&I, S);
3014  setOriginForNaryOp(I);
3015  }
3016 
3017  // Instrument compare-scalar intrinsic.
3018  // This handles both cmp* intrinsics which return the result in the first
3019  // element of a vector, and comi* which return the result as i32.
3020  void handleVectorCompareScalarIntrinsic(IntrinsicInst &I) {
3021  IRBuilder<> IRB(&I);
3022  Value *S0 = IRB.CreateOr(getShadow(&I, 0), getShadow(&I, 1));
3023  Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&I));
3024  setShadow(&I, S);
3025  setOriginForNaryOp(I);
3026  }
3027 
3028  // Instrument generic vector reduction intrinsics
3029  // by ORing together all their fields.
3030  void handleVectorReduceIntrinsic(IntrinsicInst &I) {
3031  IRBuilder<> IRB(&I);
3032  Value *S = IRB.CreateOrReduce(getShadow(&I, 0));
3033  setShadow(&I, S);
3034  setOrigin(&I, getOrigin(&I, 0));
3035  }
3036 
3037  // Instrument vector.reduce.or intrinsic.
3038  // Valid (non-poisoned) set bits in the operand pull low the
3039  // corresponding shadow bits.
3040  void handleVectorReduceOrIntrinsic(IntrinsicInst &I) {
3041  IRBuilder<> IRB(&I);
3042  Value *OperandShadow = getShadow(&I, 0);
3043  Value *OperandUnsetBits = IRB.CreateNot(I.getOperand(0));
3044  Value *OperandUnsetOrPoison = IRB.CreateOr(OperandUnsetBits, OperandShadow);
3045  // Bit N is clean if any field's bit N is 1 and unpoison
3046  Value *OutShadowMask = IRB.CreateAndReduce(OperandUnsetOrPoison);
3047  // Otherwise, it is clean if every field's bit N is unpoison
3048  Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3049  Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3050 
3051  setShadow(&I, S);
3052  setOrigin(&I, getOrigin(&I, 0));
3053  }
3054 
3055  // Instrument vector.reduce.and intrinsic.
3056  // Valid (non-poisoned) unset bits in the operand pull down the
3057  // corresponding shadow bits.
3058  void handleVectorReduceAndIntrinsic(IntrinsicInst &I) {
3059  IRBuilder<> IRB(&I);
3060  Value *OperandShadow = getShadow(&I, 0);
3061  Value *OperandSetOrPoison = IRB.CreateOr(I.getOperand(0), OperandShadow);
3062  // Bit N is clean if any field's bit N is 0 and unpoison
3063  Value *OutShadowMask = IRB.CreateAndReduce(OperandSetOrPoison);
3064  // Otherwise, it is clean if every field's bit N is unpoison
3065  Value *OrShadow = IRB.CreateOrReduce(OperandShadow);
3066  Value *S = IRB.CreateAnd(OutShadowMask, OrShadow);
3067 
3068  setShadow(&I, S);
3069  setOrigin(&I, getOrigin(&I, 0));
3070  }
3071 
3072  void handleStmxcsr(IntrinsicInst &I) {
3073  IRBuilder<> IRB(&I);
3074  Value* Addr = I.getArgOperand(0);
3075  Type *Ty = IRB.getInt32Ty();
3076  Value *ShadowPtr =
3077  getShadowOriginPtr(Addr, IRB, Ty, Align(1), /*isStore*/ true).first;
3078 
3079  IRB.CreateStore(getCleanShadow(Ty),
3080  IRB.CreatePointerCast(ShadowPtr, Ty->getPointerTo()));
3081 
3083  insertShadowCheck(Addr, &I);
3084  }
3085 
3086  void handleLdmxcsr(IntrinsicInst &I) {
3087  if (!InsertChecks) return;
3088 
3089  IRBuilder<> IRB(&I);
3090  Value *Addr = I.getArgOperand(0);
3091  Type *Ty = IRB.getInt32Ty();
3092  const Align Alignment = Align(1);
3093  Value *ShadowPtr, *OriginPtr;
3094  std::tie(ShadowPtr, OriginPtr) =
3095  getShadowOriginPtr(Addr, IRB, Ty, Alignment, /*isStore*/ false);
3096 
3098  insertShadowCheck(Addr, &I);
3099 
3100  Value *Shadow = IRB.CreateAlignedLoad(Ty, ShadowPtr, Alignment, "_ldmxcsr");
3101  Value *Origin = MS.TrackOrigins ? IRB.CreateLoad(MS.OriginTy, OriginPtr)
3102  : getCleanOrigin();
3103  insertShadowCheck(Shadow, Origin, &I);
3104  }
3105 
3106  void handleMaskedStore(IntrinsicInst &I) {
3107  IRBuilder<> IRB(&I);
3108  Value *V = I.getArgOperand(0);
3109  Value *Addr = I.getArgOperand(1);
3110  const Align Alignment(
3111  cast<ConstantInt>(I.getArgOperand(2))->getZExtValue());
3112  Value *Mask = I.getArgOperand(3);
3113  Value *Shadow = getShadow(V);
3114 
3115  Value *ShadowPtr;
3116  Value *OriginPtr;
3117  std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3118  Addr, IRB, Shadow->getType(), Alignment, /*isStore*/ true);
3119 
3120  if (ClCheckAccessAddress) {
3121  insertShadowCheck(Addr, &I);
3122  // Uninitialized mask is kind of like uninitialized address, but not as
3123  // scary.
3124  insertShadowCheck(Mask, &I);
3125  }
3126 
3127  IRB.CreateMaskedStore(Shadow, ShadowPtr, Alignment, Mask);
3128 
3129  if (MS.TrackOrigins) {
3130  auto &DL = F.getParent()->getDataLayout();
3131  paintOrigin(IRB, getOrigin(V), OriginPtr,
3132  DL.getTypeStoreSize(Shadow->getType()),
3133  std::max(Alignment, kMinOriginAlignment));
3134  }
3135  }
3136 
3137  bool handleMaskedLoad(IntrinsicInst &I) {
3138  IRBuilder<> IRB(&I);
3139  Value *Addr = I.getArgOperand(0);
3140  const Align Alignment(
3141  cast<ConstantInt>(I.getArgOperand(1))->getZExtValue());
3142  Value *Mask = I.getArgOperand(2);
3143  Value *PassThru = I.getArgOperand(3);
3144 
3145  Type *ShadowTy = getShadowTy(&I);
3146  Value *ShadowPtr, *OriginPtr;
3147  if (PropagateShadow) {
3148  std::tie(ShadowPtr, OriginPtr) =
3149  getShadowOriginPtr(Addr, IRB, ShadowTy, Alignment, /*isStore*/ false);
3150  setShadow(&I, IRB.CreateMaskedLoad(ShadowTy, ShadowPtr, Alignment, Mask,
3151  getShadow(PassThru), "_msmaskedld"));
3152  } else {
3153  setShadow(&I, getCleanShadow(&I));
3154  }
3155 
3156  if (ClCheckAccessAddress) {
3157  insertShadowCheck(Addr, &I);
3158  insertShadowCheck(Mask, &I);
3159  }
3160 
3161  if (MS.TrackOrigins) {
3162  if (PropagateShadow) {
3163  // Choose between PassThru's and the loaded value's origins.
3164  Value *MaskedPassThruShadow = IRB.CreateAnd(
3165  getShadow(PassThru), IRB.CreateSExt(IRB.CreateNeg(Mask), ShadowTy));
3166 
3167  Value *Acc = IRB.CreateExtractElement(
3168  MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), 0));
3169  for (int i = 1, N = cast<FixedVectorType>(PassThru->getType())
3170  ->getNumElements();
3171  i < N; ++i) {
3172  Value *More = IRB.CreateExtractElement(
3173  MaskedPassThruShadow, ConstantInt::get(IRB.getInt32Ty(), i));
3174  Acc = IRB.CreateOr(Acc, More);
3175  }
3176 
3177  Value *Origin = IRB.CreateSelect(
3178  IRB.CreateICmpNE(Acc, Constant::getNullValue(Acc->getType())),
3179  getOrigin(PassThru), IRB.CreateLoad(MS.OriginTy, OriginPtr));
3180 
3181  setOrigin(&I, Origin);
3182  } else {
3183  setOrigin(&I, getCleanOrigin());
3184  }
3185  }
3186  return true;
3187  }
3188 
3189  // Instrument BMI / BMI2 intrinsics.
3190  // All of these intrinsics are Z = I(X, Y)
3191  // where the types of all operands and the result match, and are either i32 or i64.
3192  // The following instrumentation happens to work for all of them:
3193  // Sz = I(Sx, Y) | (sext (Sy != 0))
3194  void handleBmiIntrinsic(IntrinsicInst &I) {
3195  IRBuilder<> IRB(&I);
3196  Type *ShadowTy = getShadowTy(&I);
3197 
3198  // If any bit of the mask operand is poisoned, then the whole thing is.
3199  Value *SMask = getShadow(&I, 1);
3200  SMask = IRB.CreateSExt(IRB.CreateICmpNE(SMask, getCleanShadow(ShadowTy)),
3201  ShadowTy);
3202  // Apply the same intrinsic to the shadow of the first operand.
3203  Value *S = IRB.CreateCall(I.getCalledFunction(),
3204  {getShadow(&I, 0), I.getOperand(1)});
3205  S = IRB.CreateOr(SMask, S);
3206  setShadow(&I, S);
3207  setOriginForNaryOp(I);
3208  }
3209 
3210  SmallVector<int, 8> getPclmulMask(unsigned Width, bool OddElements) {
3212  for (unsigned X = OddElements ? 1 : 0; X < Width; X += 2) {
3213  Mask.append(2, X);
3214  }
3215  return Mask;
3216  }
3217 
3218  // Instrument pclmul intrinsics.
3219  // These intrinsics operate either on odd or on even elements of the input
3220  // vectors, depending on the constant in the 3rd argument, ignoring the rest.
3221  // Replace the unused elements with copies of the used ones, ex:
3222  // (0, 1, 2, 3) -> (0, 0, 2, 2) (even case)
3223  // or
3224  // (0, 1, 2, 3) -> (1, 1, 3, 3) (odd case)
3225  // and then apply the usual shadow combining logic.
3226  void handlePclmulIntrinsic(IntrinsicInst &I) {
3227  IRBuilder<> IRB(&I);
3228  unsigned Width =
3229  cast<FixedVectorType>(I.getArgOperand(0)->getType())->getNumElements();
3230  assert(isa<ConstantInt>(I.getArgOperand(2)) &&
3231  "pclmul 3rd operand must be a constant");
3232  unsigned Imm = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3233  Value *Shuf0 = IRB.CreateShuffleVector(getShadow(&I, 0),
3234  getPclmulMask(Width, Imm & 0x01));
3235  Value *Shuf1 = IRB.CreateShuffleVector(getShadow(&I, 1),
3236  getPclmulMask(Width, Imm & 0x10));
3237  ShadowAndOriginCombiner SOC(this, IRB);
3238  SOC.Add(Shuf0, getOrigin(&I, 0));
3239  SOC.Add(Shuf1, getOrigin(&I, 1));
3240  SOC.Done(&I);
3241  }
3242 
3243  // Instrument _mm_*_sd intrinsics
3244  void handleUnarySdIntrinsic(IntrinsicInst &I) {
3245  IRBuilder<> IRB(&I);
3246  Value *First = getShadow(&I, 0);
3247  Value *Second = getShadow(&I, 1);
3248  // High word of first operand, low word of second
3249  Value *Shadow =
3250  IRB.CreateShuffleVector(First, Second, llvm::makeArrayRef<int>({2, 1}));
3251 
3252  setShadow(&I, Shadow);
3253  setOriginForNaryOp(I);
3254  }
3255 
3256  void handleBinarySdIntrinsic(IntrinsicInst &I) {
3257  IRBuilder<> IRB(&I);
3258  Value *First = getShadow(&I, 0);
3259  Value *Second = getShadow(&I, 1);
3260  Value *OrShadow = IRB.CreateOr(First, Second);
3261  // High word of first operand, low word of both OR'd together
3262  Value *Shadow = IRB.CreateShuffleVector(First, OrShadow,
3263  llvm::makeArrayRef<int>({2, 1}));
3264 
3265  setShadow(&I, Shadow);
3266  setOriginForNaryOp(I);
3267  }
3268 
3269  // Instrument abs intrinsic.
3270  // handleUnknownIntrinsic can't handle it because of the last
3271  // is_int_min_poison argument which does not match the result type.
3272  void handleAbsIntrinsic(IntrinsicInst &I) {
3273  assert(I.getType()->isIntOrIntVectorTy());
3274  assert(I.getArgOperand(0)->getType() == I.getType());
3275 
3276  // FIXME: Handle is_int_min_poison.
3277  IRBuilder<> IRB(&I);
3278  setShadow(&I, getShadow(&I, 0));
3279  setOrigin(&I, getOrigin(&I, 0));
3280  }
3281 
3282  void visitIntrinsicInst(IntrinsicInst &I) {
3283  switch (I.getIntrinsicID()) {
3284  case Intrinsic::abs:
3285  handleAbsIntrinsic(I);
3286  break;
3287  case Intrinsic::lifetime_start:
3288  handleLifetimeStart(I);
3289  break;
3290  case Intrinsic::launder_invariant_group:
3291  case Intrinsic::strip_invariant_group:
3292  handleInvariantGroup(I);
3293  break;
3294  case Intrinsic::bswap:
3295  handleBswap(I);
3296  break;
3297  case Intrinsic::masked_store:
3298  handleMaskedStore(I);
3299  break;
3300  case Intrinsic::masked_load:
3301  handleMaskedLoad(I);
3302  break;
3303  case Intrinsic::vector_reduce_and:
3304  handleVectorReduceAndIntrinsic(I);
3305  break;
3306  case Intrinsic::vector_reduce_or:
3307  handleVectorReduceOrIntrinsic(I);
3308  break;
3309  case Intrinsic::vector_reduce_add:
3310  case Intrinsic::vector_reduce_xor:
3311  case Intrinsic::vector_reduce_mul:
3312  handleVectorReduceIntrinsic(I);
3313  break;
3314  case Intrinsic::x86_sse_stmxcsr:
3315  handleStmxcsr(I);
3316  break;
3317  case Intrinsic::x86_sse_ldmxcsr:
3318  handleLdmxcsr(I);
3319  break;
3320  case Intrinsic::x86_avx512_vcvtsd2usi64:
3321  case Intrinsic::x86_avx512_vcvtsd2usi32:
3322  case Intrinsic::x86_avx512_vcvtss2usi64:
3323  case Intrinsic::x86_avx512_vcvtss2usi32:
3324  case Intrinsic::x86_avx512_cvttss2usi64:
3325  case Intrinsic::x86_avx512_cvttss2usi:
3326  case Intrinsic::x86_avx512_cvttsd2usi64:
3327  case Intrinsic::x86_avx512_cvttsd2usi:
3328  case Intrinsic::x86_avx512_cvtusi2ss:
3329  case Intrinsic::x86_avx512_cvtusi642sd:
3330  case Intrinsic::x86_avx512_cvtusi642ss:
3331  handleVectorConvertIntrinsic(I, 1, true);
3332  break;
3333  case Intrinsic::x86_sse2_cvtsd2si64:
3334  case Intrinsic::x86_sse2_cvtsd2si:
3335  case Intrinsic::x86_sse2_cvtsd2ss:
3336  case Intrinsic::x86_sse2_cvttsd2si64:
3337  case Intrinsic::x86_sse2_cvttsd2si:
3338  case Intrinsic::x86_sse_cvtss2si64:
3339  case Intrinsic::x86_sse_cvtss2si:
3340  case Intrinsic::x86_sse_cvttss2si64:
3341  case Intrinsic::x86_sse_cvttss2si:
3342  handleVectorConvertIntrinsic(I, 1);
3343  break;
3344  case Intrinsic::x86_sse_cvtps2pi:
3345  case Intrinsic::x86_sse_cvttps2pi:
3346  handleVectorConvertIntrinsic(I, 2);
3347  break;
3348 
3349  case Intrinsic::x86_avx512_psll_w_512:
3350  case Intrinsic::x86_avx512_psll_d_512:
3351  case Intrinsic::x86_avx512_psll_q_512:
3352  case Intrinsic::x86_avx512_pslli_w_512:
3353  case Intrinsic::x86_avx512_pslli_d_512:
3354  case Intrinsic::x86_avx512_pslli_q_512:
3355  case Intrinsic::x86_avx512_psrl_w_512:
3356  case Intrinsic::x86_avx512_psrl_d_512:
3357  case Intrinsic::x86_avx512_psrl_q_512:
3358  case Intrinsic::x86_avx512_psra_w_512:
3359  case Intrinsic::x86_avx512_psra_d_512:
3360  case Intrinsic::x86_avx512_psra_q_512:
3361  case Intrinsic::x86_avx512_psrli_w_512:
3362  case Intrinsic::x86_avx512_psrli_d_512:
3363  case Intrinsic::x86_avx512_psrli_q_512:
3364  case Intrinsic::x86_avx512_psrai_w_512:
3365  case Intrinsic::x86_avx512_psrai_d_512:
3366  case Intrinsic::x86_avx512_psrai_q_512:
3367  case Intrinsic::x86_avx512_psra_q_256:
3368  case Intrinsic::x86_avx512_psra_q_128:
3369  case Intrinsic::x86_avx512_psrai_q_256:
3370  case Intrinsic::x86_avx512_psrai_q_128:
3371  case Intrinsic::x86_avx2_psll_w:
3372  case Intrinsic::x86_avx2_psll_d:
3373  case Intrinsic::x86_avx2_psll_q:
3374  case Intrinsic::x86_avx2_pslli_w:
3375  case Intrinsic::x86_avx2_pslli_d:
3376  case Intrinsic::x86_avx2_pslli_q:
3377  case Intrinsic::x86_avx2_psrl_w:
3378  case Intrinsic::x86_avx2_psrl_d:
3379  case Intrinsic::x86_avx2_psrl_q:
3380  case Intrinsic::x86_avx2_psra_w:
3381  case Intrinsic::x86_avx2_psra_d:
3382  case Intrinsic::x86_avx2_psrli_w:
3383  case Intrinsic::x86_avx2_psrli_d:
3384  case Intrinsic::x86_avx2_psrli_q:
3385  case Intrinsic::x86_avx2_psrai_w:
3386  case Intrinsic::x86_avx2_psrai_d:
3387  case Intrinsic::x86_sse2_psll_w:
3388  case Intrinsic::x86_sse2_psll_d:
3389  case Intrinsic::x86_sse2_psll_q:
3390  case Intrinsic::x86_sse2_pslli_w:
3391  case Intrinsic::x86_sse2_pslli_d:
3392  case Intrinsic::x86_sse2_pslli_q:
3393  case Intrinsic::x86_sse2_psrl_w:
3394  case Intrinsic::x86_sse2_psrl_d:
3395  case Intrinsic::x86_sse2_psrl_q:
3396  case Intrinsic::x86_sse2_psra_w:
3397  case Intrinsic::x86_sse2_psra_d:
3398  case Intrinsic::x86_sse2_psrli_w:
3399  case Intrinsic::x86_sse2_psrli_d:
3400  case Intrinsic::x86_sse2_psrli_q:
3401  case Intrinsic::x86_sse2_psrai_w:
3402  case Intrinsic::x86_sse2_psrai_d:
3403  case Intrinsic::x86_mmx_psll_w:
3404  case Intrinsic::x86_mmx_psll_d:
3405  case Intrinsic::x86_mmx_psll_q:
3406  case Intrinsic::x86_mmx_pslli_w:
3407  case Intrinsic::x86_mmx_pslli_d:
3408  case Intrinsic::x86_mmx_pslli_q:
3409  case Intrinsic::x86_mmx_psrl_w:
3410  case Intrinsic::x86_mmx_psrl_d:
3411  case Intrinsic::x86_mmx_psrl_q:
3412  case Intrinsic::x86_mmx_psra_w:
3413  case Intrinsic::x86_mmx_psra_d:
3414  case Intrinsic::x86_mmx_psrli_w:
3415  case Intrinsic::x86_mmx_psrli_d:
3416  case Intrinsic::x86_mmx_psrli_q:
3417  case Intrinsic::x86_mmx_psrai_w:
3418  case Intrinsic::x86_mmx_psrai_d:
3419  handleVectorShiftIntrinsic(I, /* Variable */ false);
3420  break;
3421  case Intrinsic::x86_avx2_psllv_d:
3422  case Intrinsic::x86_avx2_psllv_d_256:
3423  case Intrinsic::x86_avx512_psllv_d_512:
3424  case Intrinsic::x86_avx2_psllv_q:
3425  case Intrinsic::x86_avx2_psllv_q_256:
3426  case Intrinsic::x86_avx512_psllv_q_512:
3427  case Intrinsic::x86_avx2_psrlv_d:
3428  case Intrinsic::x86_avx2_psrlv_d_256:
3429  case Intrinsic::x86_avx512_psrlv_d_512:
3430  case Intrinsic::x86_avx2_psrlv_q:
3431  case Intrinsic::x86_avx2_psrlv_q_256:
3432  case Intrinsic::x86_avx512_psrlv_q_512:
3433  case Intrinsic::x86_avx2_psrav_d:
3434  case Intrinsic::x86_avx2_psrav_d_256:
3435  case Intrinsic::x86_avx512_psrav_d_512:
3436  case Intrinsic::x86_avx512_psrav_q_128:
3437  case Intrinsic::x86_avx512_psrav_q_256:
3438  case Intrinsic::x86_avx512_psrav_q_512:
3439  handleVectorShiftIntrinsic(I, /* Variable */ true);
3440  break;
3441 
3442  case Intrinsic::x86_sse2_packsswb_128:
3443  case Intrinsic::x86_sse2_packssdw_128:
3444  case Intrinsic::x86_sse2_packuswb_128:
3445  case Intrinsic::x86_sse41_packusdw:
3446  case Intrinsic::x86_avx2_packsswb:
3447  case Intrinsic::x86_avx2_packssdw:
3448  case Intrinsic::x86_avx2_packuswb:
3449  case Intrinsic::x86_avx2_packusdw:
3450  handleVectorPackIntrinsic(I);
3451  break;
3452 
3453  case Intrinsic::x86_mmx_packsswb:
3454  case Intrinsic::x86_mmx_packuswb:
3455  handleVectorPackIntrinsic(I, 16);
3456  break;
3457 
3458  case Intrinsic::x86_mmx_packssdw:
3459  handleVectorPackIntrinsic(I, 32);
3460  break;
3461 
3462  case Intrinsic::x86_mmx_psad_bw:
3463  case Intrinsic::x86_sse2_psad_bw:
3464  case Intrinsic::x86_avx2_psad_bw:
3465  handleVectorSadIntrinsic(I);
3466  break;
3467 
3468  case Intrinsic::x86_sse2_pmadd_wd:
3469  case Intrinsic::x86_avx2_pmadd_wd:
3470  case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3471  case Intrinsic::x86_avx2_pmadd_ub_sw:
3472  handleVectorPmaddIntrinsic(I);
3473  break;
3474 
3475  case Intrinsic::x86_ssse3_pmadd_ub_sw:
3476  handleVectorPmaddIntrinsic(I, 8);
3477  break;
3478 
3479  case Intrinsic::x86_mmx_pmadd_wd:
3480  handleVectorPmaddIntrinsic(I, 16);
3481  break;
3482 
3483  case Intrinsic::x86_sse_cmp_ss:
3484  case Intrinsic::x86_sse2_cmp_sd:
3485  case Intrinsic::x86_sse_comieq_ss:
3486  case Intrinsic::x86_sse_comilt_ss:
3487  case Intrinsic::x86_sse_comile_ss:
3488  case Intrinsic::x86_sse_comigt_ss:
3489  case Intrinsic::x86_sse_comige_ss:
3490  case Intrinsic::x86_sse_comineq_ss:
3491  case Intrinsic::x86_sse_ucomieq_ss:
3492  case Intrinsic::x86_sse_ucomilt_ss:
3493  case Intrinsic::x86_sse_ucomile_ss:
3494  case Intrinsic::x86_sse_ucomigt_ss:
3495  case Intrinsic::x86_sse_ucomige_ss:
3496  case Intrinsic::x86_sse_ucomineq_ss:
3497  case Intrinsic::x86_sse2_comieq_sd:
3498  case Intrinsic::x86_sse2_comilt_sd:
3499  case Intrinsic::x86_sse2_comile_sd:
3500  case Intrinsic::x86_sse2_comigt_sd:
3501  case Intrinsic::x86_sse2_comige_sd:
3502  case Intrinsic::x86_sse2_comineq_sd:
3503  case Intrinsic::x86_sse2_ucomieq_sd:
3504  case Intrinsic::x86_sse2_ucomilt_sd:
3505  case Intrinsic::x86_sse2_ucomile_sd:
3506  case Intrinsic::x86_sse2_ucomigt_sd:
3507  case Intrinsic::x86_sse2_ucomige_sd:
3508  case Intrinsic::x86_sse2_ucomineq_sd:
3509  handleVectorCompareScalarIntrinsic(I);
3510  break;
3511 
3512  case Intrinsic::x86_sse_cmp_ps:
3513  case Intrinsic::x86_sse2_cmp_pd:
3514  // FIXME: For x86_avx_cmp_pd_256 and x86_avx_cmp_ps_256 this function
3515  // generates reasonably looking IR that fails in the backend with "Do not
3516  // know how to split the result of this operator!".
3517  handleVectorComparePackedIntrinsic(I);
3518  break;
3519 
3520  case Intrinsic::x86_bmi_bextr_32:
3521  case Intrinsic::x86_bmi_bextr_64:
3522  case Intrinsic::x86_bmi_bzhi_32:
3523  case Intrinsic::x86_bmi_bzhi_64:
3524  case Intrinsic::x86_bmi_pdep_32:
3525  case Intrinsic::x86_bmi_pdep_64:
3526  case Intrinsic::x86_bmi_pext_32:
3527  case Intrinsic::x86_bmi_pext_64:
3528  handleBmiIntrinsic(I);
3529  break;
3530 
3531  case Intrinsic::x86_pclmulqdq:
3532  case Intrinsic::x86_pclmulqdq_256:
3533  case Intrinsic::x86_pclmulqdq_512:
3534  handlePclmulIntrinsic(I);
3535  break;
3536 
3537  case Intrinsic::x86_sse41_round_sd:
3538  handleUnarySdIntrinsic(I);
3539  break;
3540  case Intrinsic::x86_sse2_max_sd:
3541  case Intrinsic::x86_sse2_min_sd:
3542  handleBinarySdIntrinsic(I);
3543  break;
3544 
3545  case Intrinsic::fshl:
3546  case Intrinsic::fshr:
3547  handleFunnelShift(I);
3548  break;
3549 
3550  case Intrinsic::is_constant:
3551  // The result of llvm.is.constant() is always defined.
3552  setShadow(&I, getCleanShadow(&I));
3553  setOrigin(&I, getCleanOrigin());
3554  break;
3555 
3556  default:
3557  if (!handleUnknownIntrinsic(I))
3558  visitInstruction(I);
3559  break;
3560  }
3561  }
3562 
3563  void visitLibAtomicLoad(CallBase &CB) {
3564  // Since we use getNextNode here, we can't have CB terminate the BB.
3565  assert(isa<CallInst>(CB));
3566 
3567  IRBuilder<> IRB(&CB);
3568  Value *Size = CB.getArgOperand(0);
3569  Value *SrcPtr = CB.getArgOperand(1);
3570  Value *DstPtr = CB.getArgOperand(2);
3571  Value *Ordering = CB.getArgOperand(3);
3572  // Convert the call to have at least Acquire ordering to make sure
3573  // the shadow operations aren't reordered before it.
3574  Value *NewOrdering =
3575  IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
3576  CB.setArgOperand(3, NewOrdering);
3577 
3578  IRBuilder<> NextIRB(CB.getNextNode());
3579  NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3580 
3581  Value *SrcShadowPtr, *SrcOriginPtr;
3582  std::tie(SrcShadowPtr, SrcOriginPtr) =
3583  getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
3584  /*isStore*/ false);
3585  Value *DstShadowPtr =
3586  getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(), Align(1),
3587  /*isStore*/ true)
3588  .first;
3589 
3590  NextIRB.CreateMemCpy(DstShadowPtr, Align(1), SrcShadowPtr, Align(1), Size);
3591  if (MS.TrackOrigins) {
3592  Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
3594  Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
3595  NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
3596  }
3597  }
3598 
3599  void visitLibAtomicStore(CallBase &CB) {
3600  IRBuilder<> IRB(&CB);
3601  Value *Size = CB.getArgOperand(0);
3602  Value *DstPtr = CB.getArgOperand(2);
3603  Value *Ordering = CB.getArgOperand(3);
3604  // Convert the call to have at least Release ordering to make sure
3605  // the shadow operations aren't reordered after it.
3606  Value *NewOrdering =
3607  IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
3608  CB.setArgOperand(3, NewOrdering);
3609 
3610  Value *DstShadowPtr =
3611  getShadowOriginPtr(DstPtr, IRB, IRB.getInt8Ty(), Align(1),
3612  /*isStore*/ true)
3613  .first;
3614 
3615  // Atomic store always paints clean shadow/origin. See file header.
3616  IRB.CreateMemSet(DstShadowPtr, getCleanShadow(IRB.getInt8Ty()), Size,
3617  Align(1));
3618  }
3619 
3620  void visitCallBase(CallBase &CB) {
3621  assert(!CB.getMetadata("nosanitize"));
3622  if (CB.isInlineAsm()) {
3623  // For inline asm (either a call to asm function, or callbr instruction),
3624  // do the usual thing: check argument shadow and mark all outputs as
3625  // clean. Note that any side effects of the inline asm that are not
3626  // immediately visible in its constraints are not handled.
3627  if (ClHandleAsmConservative && MS.CompileKernel)
3628  visitAsmInstruction(CB);
3629  else
3630  visitInstruction(CB);
3631  return;
3632  }
3633  LibFunc LF;
3634  if (TLI->getLibFunc(CB, LF)) {
3635  // libatomic.a functions need to have special handling because there isn't
3636  // a good way to intercept them or compile the library with
3637  // instrumentation.
3638  switch (LF) {
3639  case LibFunc_atomic_load:
3640  if (!isa<CallInst>(CB)) {
3641  llvm::errs() << "MSAN -- cannot instrument invoke of libatomic load."
3642  "Ignoring!\n";
3643  break;
3644  }
3645  visitLibAtomicLoad(CB);
3646  return;
3647  case LibFunc_atomic_store:
3648  visitLibAtomicStore(CB);
3649  return;
3650  default:
3651  break;
3652  }
3653  }
3654 
3655  if (auto *Call = dyn_cast<CallInst>(&CB)) {
3656  assert(!isa<IntrinsicInst>(Call) && "intrinsics are handled elsewhere");
3657 
3658  // We are going to insert code that relies on the fact that the callee
3659  // will become a non-readonly function after it is instrumented by us. To
3660  // prevent this code from being optimized out, mark that function
3661  // non-readonly in advance.
3662  AttrBuilder B;
3663  B.addAttribute(Attribute::ReadOnly)
3664  .addAttribute(Attribute::ReadNone)
3665  .addAttribute(Attribute::WriteOnly)
3666  .addAttribute(Attribute::ArgMemOnly)
3667  .addAttribute(Attribute::Speculatable);
3668 
3669  Call->removeFnAttrs(B);
3670  if (Function *Func = Call->getCalledFunction()) {
3671  Func->removeFnAttrs(B);
3672  }
3673 
3675  }
3676  IRBuilder<> IRB(&CB);
3677  bool MayCheckCall = ClEagerChecks;
3678  if (Function *Func = CB.getCalledFunction()) {
3679  // __sanitizer_unaligned_{load,store} functions may be called by users
3680  // and always expects shadows in the TLS. So don't check them.
3681  MayCheckCall &= !Func->getName().startswith("__sanitizer_unaligned_");
3682  }
3683 
3684  unsigned ArgOffset = 0;
3685  LLVM_DEBUG(dbgs() << " CallSite: " << CB << "\n");
3686  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
3687  ++ArgIt) {
3688  Value *A = *ArgIt;
3689  unsigned i = ArgIt - CB.arg_begin();
3690  if (!A->getType()->isSized()) {
3691  LLVM_DEBUG(dbgs() << "Arg " << i << " is not sized: " << CB << "\n");
3692  continue;
3693  }
3694  unsigned Size = 0;
3695  Value *Store = nullptr;
3696  // Compute the Shadow for arg even if it is ByVal, because
3697  // in that case getShadow() will copy the actual arg shadow to
3698  // __msan_param_tls.
3699  Value *ArgShadow = getShadow(A);
3700  Value *ArgShadowBase = getShadowPtrForArgument(A, IRB, ArgOffset);
3701  LLVM_DEBUG(dbgs() << " Arg#" << i << ": " << *A
3702  << " Shadow: " << *ArgShadow << "\n");
3703  bool ArgIsInitialized = false;
3704  const DataLayout &DL = F.getParent()->getDataLayout();
3705 
3706  bool ByVal = CB.paramHasAttr(i, Attribute::ByVal);
3707  bool NoUndef = CB.paramHasAttr(i, Attribute::NoUndef);
3708  bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
3709 
3710  if (EagerCheck) {
3711  insertShadowCheck(A, &CB);
3712  continue;
3713  }
3714  if (ByVal) {
3715  // ByVal requires some special handling as it's too big for a single
3716  // load
3717  assert(A->getType()->isPointerTy() &&
3718  "ByVal argument is not a pointer!");
3719  Size = DL.getTypeAllocSize(CB.getParamByValType(i));
3720  if (ArgOffset + Size > kParamTLSSize) break;
3721  const MaybeAlign ParamAlignment(CB.getParamAlign(i));
3722  MaybeAlign Alignment = llvm::None;
3723  if (ParamAlignment)
3724  Alignment = std::min(*ParamAlignment, kShadowTLSAlignment);
3725  Value *AShadowPtr =
3726  getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), Alignment,
3727  /*isStore*/ false)
3728  .first;
3729 
3730  Store = IRB.CreateMemCpy(ArgShadowBase, Alignment, AShadowPtr,
3731  Alignment, Size);
3732  // TODO(glider): need to copy origins.
3733  } else {
3734  // Any other parameters mean we need bit-grained tracking of uninit data
3735  Size = DL.getTypeAllocSize(A->getType());
3736  if (ArgOffset + Size > kParamTLSSize) break;
3737  Store = IRB.CreateAlignedStore(ArgShadow, ArgShadowBase,
3739  Constant *Cst = dyn_cast<Constant>(ArgShadow);
3740  if (Cst && Cst->isNullValue()) ArgIsInitialized = true;
3741  }
3742  if (MS.TrackOrigins && !ArgIsInitialized)
3743  IRB.CreateStore(getOrigin(A),
3744  getOriginPtrForArgument(A, IRB, ArgOffset));
3745  (void)Store;
3746  assert(Size != 0 && Store != nullptr);
3747  LLVM_DEBUG(dbgs() << " Param:" << *Store << "\n");
3748  ArgOffset += alignTo(Size, kShadowTLSAlignment);
3749  }
3750  LLVM_DEBUG(dbgs() << " done with call args\n");
3751 
3752  FunctionType *FT = CB.getFunctionType();
3753  if (FT->isVarArg()) {
3754  VAHelper->visitCallBase(CB, IRB);
3755  }
3756 
3757  // Now, get the shadow for the RetVal.
3758  if (!CB.getType()->isSized())
3759  return;
3760  // Don't emit the epilogue for musttail call returns.
3761  if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
3762  return;
3763 
3764  if (MayCheckCall && CB.hasRetAttr(Attribute::NoUndef)) {
3765  setShadow(&CB, getCleanShadow(&CB));
3766  setOrigin(&CB, getCleanOrigin());
3767  return;
3768  }
3769 
3770  IRBuilder<> IRBBefore(&CB);
3771  // Until we have full dynamic coverage, make sure the retval shadow is 0.
3772  Value *Base = getShadowPtrForRetval(&CB, IRBBefore);
3773  IRBBefore.CreateAlignedStore(getCleanShadow(&CB), Base,
3775  BasicBlock::iterator NextInsn;
3776  if (isa<CallInst>(CB)) {
3777  NextInsn = ++CB.getIterator();
3778  assert(NextInsn != CB.getParent()->end());
3779  } else {
3780  BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
3781  if (!NormalDest->getSinglePredecessor()) {
3782  // FIXME: this case is tricky, so we are just conservative here.
3783  // Perhaps we need to split the edge between this BB and NormalDest,
3784  // but a naive attempt to use SplitEdge leads to a crash.
3785  setShadow(&CB, getCleanShadow(&CB));
3786  setOrigin(&CB, getCleanOrigin());
3787  return;
3788  }
3789  // FIXME: NextInsn is likely in a basic block that has not been visited yet.
3790  // Anything inserted there will be instrumented by MSan later!
3791  NextInsn = NormalDest->getFirstInsertionPt();
3792  assert(NextInsn != NormalDest->end() &&
3793  "Could not find insertion point for retval shadow load");
3794  }
3795  IRBuilder<> IRBAfter(&*NextInsn);
3796  Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
3797  getShadowTy(&CB), getShadowPtrForRetval(&CB, IRBAfter),
3798  kShadowTLSAlignment, "_msret");
3799  setShadow(&CB, RetvalShadow);
3800  if (MS.TrackOrigins)
3801  setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
3802  getOriginPtrForRetval(IRBAfter)));
3803  }
3804 
3805  bool isAMustTailRetVal(Value *RetVal) {
3806  if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
3807  RetVal = I->getOperand(0);
3808  }
3809  if (auto *I = dyn_cast<CallInst>(RetVal)) {
3810  return I->isMustTailCall();
3811  }
3812  return false;
3813  }
3814 
3815  void visitReturnInst(ReturnInst &I) {
3816  IRBuilder<> IRB(&I);
3817  Value *RetVal = I.getReturnValue();
3818  if (!RetVal) return;
3819  // Don't emit the epilogue for musttail call returns.
3820  if (isAMustTailRetVal(RetVal)) return;
3821  Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
3822  bool HasNoUndef =
3823  F.hasRetAttribute(Attribute::NoUndef);
3824  bool StoreShadow = !(ClEagerChecks && HasNoUndef);
3825  // FIXME: Consider using SpecialCaseList to specify a list of functions that
3826  // must always return fully initialized values. For now, we hardcode "main".
3827  bool EagerCheck = (ClEagerChecks && HasNoUndef) || (F.getName() == "main");
3828 
3829  Value *Shadow = getShadow(RetVal);
3830  bool StoreOrigin = true;
3831  if (EagerCheck) {
3832  insertShadowCheck(RetVal, &I);
3833  Shadow = getCleanShadow(RetVal);
3834  StoreOrigin = false;
3835  }
3836 
3837  // The caller may still expect information passed over TLS if we pass our
3838  // check
3839  if (StoreShadow) {
3840  IRB.CreateAlignedStore(Shadow, ShadowPtr, kShadowTLSAlignment);
3841  if (MS.TrackOrigins && StoreOrigin)
3842  IRB.CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
3843  }
3844  }
3845 
3846  void visitPHINode(PHINode &I) {
3847  IRBuilder<> IRB(&I);
3848  if (!PropagateShadow) {
3849  setShadow(&I, getCleanShadow(&I));
3850  setOrigin(&I, getCleanOrigin());
3851  return;
3852  }
3853 
3854  ShadowPHINodes.push_back(&I);
3855  setShadow(&I, IRB.CreatePHI(getShadowTy(&I), I.getNumIncomingValues(),
3856  "_msphi_s"));
3857  if (MS.TrackOrigins)
3858  setOrigin(&I, IRB.CreatePHI(MS.OriginTy, I.getNumIncomingValues(),
3859  "_msphi_o"));
3860  }
3861 
3862  Value *getLocalVarDescription(AllocaInst &I) {
3863  SmallString<2048> StackDescriptionStorage;
3864  raw_svector_ostream StackDescription(StackDescriptionStorage);
3865  // We create a string with a description of the stack allocation and
3866  // pass it into __msan_set_alloca_origin.
3867  // It will be printed by the run-time if stack-originated UMR is found.
3868  // The first 4 bytes of the string are set to '----' and will be replaced
3869  // by __msan_va_arg_overflow_size_tls at the first call.
3870  StackDescription << "----" << I.getName() << "@" << F.getName();
3871  return createPrivateNonConstGlobalForString(*F.getParent(),
3872  StackDescription.str());
3873  }
3874 
3875  void poisonAllocaUserspace(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3876  if (PoisonStack && ClPoisonStackWithCall) {
3877  IRB.CreateCall(MS.MsanPoisonStackFn,
3878  {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3879  } else {
3880  Value *ShadowBase, *OriginBase;
3881  std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
3882  &I, IRB, IRB.getInt8Ty(), Align(1), /*isStore*/ true);
3883 
3884  Value *PoisonValue = IRB.getInt8(PoisonStack ? ClPoisonStackPattern : 0);
3885  IRB.CreateMemSet(ShadowBase, PoisonValue, Len,
3886  MaybeAlign(I.getAlignment()));
3887  }
3888 
3889  if (PoisonStack && MS.TrackOrigins) {
3890  Value *Descr = getLocalVarDescription(I);
3891  IRB.CreateCall(MS.MsanSetAllocaOrigin4Fn,
3892  {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3893  IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy()),
3894  IRB.CreatePointerCast(&F, MS.IntptrTy)});
3895  }
3896  }
3897 
3898  void poisonAllocaKmsan(AllocaInst &I, IRBuilder<> &IRB, Value *Len) {
3899  Value *Descr = getLocalVarDescription(I);
3900  if (PoisonStack) {
3901  IRB.CreateCall(MS.MsanPoisonAllocaFn,
3902  {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
3903  IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
3904  } else {
3905  IRB.CreateCall(MS.MsanUnpoisonAllocaFn,
3906  {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
3907  }
3908  }
3909 
3910  void instrumentAlloca(AllocaInst &I, Instruction *InsPoint = nullptr) {
3911  if (!InsPoint)
3912  InsPoint = &I;
3913  IRBuilder<> IRB(InsPoint->getNextNode());
3914  const DataLayout &DL = F.getParent()->getDataLayout();
3915  uint64_t TypeSize = DL.getTypeAllocSize(I.getAllocatedType());
3916  Value *Len = ConstantInt::get(MS.IntptrTy, TypeSize);
3917  if (I.isArrayAllocation())
3918  Len = IRB.CreateMul(Len, I.getArraySize());
3919 
3920  if (MS.CompileKernel)
3921  poisonAllocaKmsan(I, IRB, Len);
3922  else
3923  poisonAllocaUserspace(I, IRB, Len);
3924  }
3925 
3926  void visitAllocaInst(AllocaInst &I) {
3927  setShadow(&I, getCleanShadow(&I));
3928  setOrigin(&I, getCleanOrigin());
3929  // We'll get to this alloca later unless it's poisoned at the corresponding
3930  // llvm.lifetime.start.
3931  AllocaSet.insert(&I);
3932  }
3933 
3934  void visitSelectInst(SelectInst& I) {
3935  IRBuilder<> IRB(&I);
3936  // a = select b, c, d
3937  Value *B = I.getCondition();
3938  Value *C = I.getTrueValue();
3939  Value *D = I.getFalseValue();
3940  Value *Sb = getShadow(B);
3941  Value *Sc = getShadow(C);
3942  Value *Sd = getShadow(D);
3943 
3944  // Result shadow if condition shadow is 0.
3945  Value *Sa0 = IRB.CreateSelect(B, Sc, Sd);
3946  Value *Sa1;
3947  if (I.getType()->isAggregateType()) {
3948  // To avoid "sign extending" i1 to an arbitrary aggregate type, we just do
3949  // an extra "select". This results in much more compact IR.
3950  // Sa = select Sb, poisoned, (select b, Sc, Sd)
3951  Sa1 = getPoisonedShadow(getShadowTy(I.getType()));
3952  } else {
3953  // Sa = select Sb, [ (c^d) | Sc | Sd ], [ b ? Sc : Sd ]
3954  // If Sb (condition is poisoned), look for bits in c and d that are equal
3955  // and both unpoisoned.
3956  // If !Sb (condition is unpoisoned), simply pick one of Sc and Sd.
3957 
3958  // Cast arguments to shadow-compatible type.
3959  C = CreateAppToShadowCast(IRB, C);
3960  D = CreateAppToShadowCast(IRB, D);
3961 
3962  // Result shadow if condition shadow is 1.
3963  Sa1 = IRB.CreateOr({IRB.CreateXor(C, D), Sc, Sd});
3964  }
3965  Value *Sa = IRB.CreateSelect(Sb, Sa1, Sa0, "_msprop_select");
3966  setShadow(&I, Sa);
3967  if (MS.TrackOrigins) {
3968  // Origins are always i32, so any vector conditions must be flattened.
3969  // FIXME: consider tracking vector origins for app vectors?
3970  if (B->getType()->isVectorTy()) {
3971  Type *FlatTy = getShadowTyNoVec(B->getType());
3972  B = IRB.CreateICmpNE(IRB.CreateBitCast(B, FlatTy),
3973  ConstantInt::getNullValue(FlatTy));
3974  Sb = IRB.CreateICmpNE(IRB.CreateBitCast(Sb, FlatTy),
3975  ConstantInt::getNullValue(FlatTy));
3976  }
3977  // a = select b, c, d
3978  // Oa = Sb ? Ob : (b ? Oc : Od)
3979  setOrigin(
3980  &I, IRB.CreateSelect(Sb, getOrigin(I.getCondition()),
3981  IRB.CreateSelect(B, getOrigin(I.getTrueValue()),
3982  getOrigin(I.getFalseValue()))));
3983  }
3984  }
3985 
3986  void visitLandingPadInst(LandingPadInst &I) {
3987  // Do nothing.
3988  // See https://github.com/google/sanitizers/issues/504
3989  setShadow(&I, getCleanShadow(&I));
3990  setOrigin(&I, getCleanOrigin());
3991  }
3992 
3993  void visitCatchSwitchInst(CatchSwitchInst &I) {
3994  setShadow(&I, getCleanShadow(&I));
3995  setOrigin(&I, getCleanOrigin());
3996  }
3997 
3998  void visitFuncletPadInst(FuncletPadInst &I) {
3999  setShadow(&I, getCleanShadow(&I));
4000  setOrigin(&I, getCleanOrigin());
4001  }
4002 
4003  void visitGetElementPtrInst(GetElementPtrInst &I) {
4004  handleShadowOr(I);
4005  }
4006 
4007  void visitExtractValueInst(ExtractValueInst &I) {
4008  IRBuilder<> IRB(&I);
4009  Value *Agg = I.getAggregateOperand();
4010  LLVM_DEBUG(dbgs() << "ExtractValue: " << I << "\n");
4011  Value *AggShadow = getShadow(Agg);
4012  LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
4013  Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
4014  LLVM_DEBUG(dbgs() << " ResShadow: " << *ResShadow << "\n");
4015  setShadow(&I, ResShadow);
4016  setOriginForNaryOp(I);
4017  }
4018 
4019  void visitInsertValueInst(InsertValueInst &I) {
4020  IRBuilder<> IRB(&I);
4021  LLVM_DEBUG(dbgs() << "InsertValue: " << I << "\n");
4022  Value *AggShadow = getShadow(I.getAggregateOperand());
4023  Value *InsShadow = getShadow(I.getInsertedValueOperand());
4024  LLVM_DEBUG(dbgs() << " AggShadow: " << *AggShadow << "\n");
4025  LLVM_DEBUG(dbgs() << " InsShadow: " << *InsShadow << "\n");
4026  Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
4027  LLVM_DEBUG(dbgs() << " Res: " << *Res << "\n");
4028  setShadow(&I, Res);
4029  setOriginForNaryOp(I);
4030  }
4031 
4032  void dumpInst(Instruction &I) {
4033  if (CallInst *CI = dyn_cast<CallInst>(&I)) {
4034  errs() << "ZZZ call " << CI->getCalledFunction()->getName() << "\n";
4035  } else {
4036  errs() << "ZZZ " << I.getOpcodeName() << "\n";
4037  }
4038  errs() << "QQQ " << I << "\n";
4039  }
4040 
4041  void visitResumeInst(ResumeInst &I) {
4042  LLVM_DEBUG(dbgs() << "Resume: " << I << "\n");
4043  // Nothing to do here.
4044  }
4045 
4046  void visitCleanupReturnInst(CleanupReturnInst &CRI) {
4047  LLVM_DEBUG(dbgs() << "CleanupReturn: " << CRI << "\n");
4048  // Nothing to do here.
4049  }
4050 
4051  void visitCatchReturnInst(CatchReturnInst &CRI) {
4052  LLVM_DEBUG(dbgs() << "CatchReturn: " << CRI << "\n");
4053  // Nothing to do here.
4054  }
4055 
4056  void instrumentAsmArgument(Value *Operand, Instruction &I, IRBuilder<> &IRB,
4057  const DataLayout &DL, bool isOutput) {
4058  // For each assembly argument, we check its value for being initialized.
4059  // If the argument is a pointer, we assume it points to a single element
4060  // of the corresponding type (or to a 8-byte word, if the type is unsized).
4061  // Each such pointer is instrumented with a call to the runtime library.
4062  Type *OpType = Operand->getType();
4063  // Check the operand value itself.
4064  insertShadowCheck(Operand, &I);
4065  if (!OpType->isPointerTy() || !isOutput) {
4066  assert(!isOutput);
4067  return;
4068  }
4069  Type *ElType = OpType->getPointerElementType();
4070  if (!ElType->isSized())
4071  return;
4072  int Size = DL.getTypeStoreSize(ElType);
4073  Value *Ptr = IRB.CreatePointerCast(Operand, IRB.getInt8PtrTy());
4074  Value *SizeVal = ConstantInt::get(MS.IntptrTy, Size);
4075  IRB.CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
4076  }
4077 
4078  /// Get the number of output arguments returned by pointers.
4079  int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
4080  int NumRetOutputs = 0;
4081  int NumOutputs = 0;
4082  Type *RetTy = cast<Value>(CB)->getType();
4083  if (!RetTy->isVoidTy()) {
4084  // Register outputs are returned via the CallInst return value.
4085  auto *ST = dyn_cast<StructType>(RetTy);
4086  if (ST)
4087  NumRetOutputs = ST->getNumElements();
4088  else
4089  NumRetOutputs = 1;
4090  }
4092  for (const InlineAsm::ConstraintInfo &Info : Constraints) {
4093  switch (Info.Type) {
4094  case InlineAsm::isOutput:
4095  NumOutputs++;
4096  break;
4097  default:
4098  break;
4099  }
4100  }
4101  return NumOutputs - NumRetOutputs;
4102  }
4103 
4104  void visitAsmInstruction(Instruction &I) {
4105  // Conservative inline assembly handling: check for poisoned shadow of
4106  // asm() arguments, then unpoison the result and all the memory locations
4107  // pointed to by those arguments.
4108  // An inline asm() statement in C++ contains lists of input and output
4109  // arguments used by the assembly code. These are mapped to operands of the
4110  // CallInst as follows:
4111  // - nR register outputs ("=r) are returned by value in a single structure
4112  // (SSA value of the CallInst);
4113  // - nO other outputs ("=m" and others) are returned by pointer as first
4114  // nO operands of the CallInst;
4115  // - nI inputs ("r", "m" and others) are passed to CallInst as the
4116  // remaining nI operands.
4117  // The total number of asm() arguments in the source is nR+nO+nI, and the
4118  // corresponding CallInst has nO+nI+1 operands (the last operand is the
4119  // function to be called).
4120  const DataLayout &DL = F.getParent()->getDataLayout();
4121  CallBase *CB = cast<CallBase>(&I);
4122  IRBuilder<> IRB(&I);
4123  InlineAsm *IA = cast<InlineAsm>(CB->getCalledOperand());
4124  int OutputArgs = getNumOutputArgs(IA, CB);
4125  // The last operand of a CallInst is the function itself.
4126  int NumOperands = CB->getNumOperands() - 1;
4127 
4128  // Check input arguments. Doing so before unpoisoning output arguments, so
4129  // that we won't overwrite uninit values before checking them.
4130  for (int i = OutputArgs; i < NumOperands; i++) {
4131  Value *Operand = CB->getOperand(i);
4132  instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ false);
4133  }
4134  // Unpoison output arguments. This must happen before the actual InlineAsm
4135  // call, so that the shadow for memory published in the asm() statement
4136  // remains valid.
4137  for (int i = 0; i < OutputArgs; i++) {
4138  Value *Operand = CB->getOperand(i);
4139  instrumentAsmArgument(Operand, I, IRB, DL, /*isOutput*/ true);
4140  }
4141 
4142  setShadow(&I, getCleanShadow(&I));
4143  setOrigin(&I, getCleanOrigin());
4144  }
4145 
4146  void visitFreezeInst(FreezeInst &I) {
4147  // Freeze always returns a fully defined value.
4148  setShadow(&I, getCleanShadow(&I));
4149  setOrigin(&I, getCleanOrigin());
4150  }
4151 
4152  void visitInstruction(Instruction &I) {
4153  // Everything else: stop propagating and check for poisoned shadow.
4155  dumpInst(I);
4156  LLVM_DEBUG(dbgs() << "DEFAULT: " << I << "\n");
4157  for (size_t i = 0, n = I.getNumOperands(); i < n; i++) {
4158  Value *Operand = I.getOperand(i);
4159  if (Operand->getType()->isSized())
4160  insertShadowCheck(Operand, &I);
4161  }
4162  setShadow(&I, getCleanShadow(&I));
4163  setOrigin(&I, getCleanOrigin());
4164  }
4165 };
4166 
4167 /// AMD64-specific implementation of VarArgHelper.
4168 struct VarArgAMD64Helper : public VarArgHelper {
4169  // An unfortunate workaround for asymmetric lowering of va_arg stuff.
4170  // See a comment in visitCallBase for more details.
4171  static const unsigned AMD64GpEndOffset = 48; // AMD64 ABI Draft 0.99.6 p3.5.7
4172  static const unsigned AMD64FpEndOffsetSSE = 176;
4173  // If SSE is disabled, fp_offset in va_list is zero.
4174  static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4175 
4176  unsigned AMD64FpEndOffset;
4177  Function &F;
4178  MemorySanitizer &MS;
4179  MemorySanitizerVisitor &MSV;
4180  Value *VAArgTLSCopy = nullptr;
4181  Value *VAArgTLSOriginCopy = nullptr;
4182  Value *VAArgOverflowSize = nullptr;
4183 
4184  SmallVector<CallInst*, 16> VAStartInstrumentationList;
4185 
4186  enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4187 
4188  VarArgAMD64Helper(Function &F, MemorySanitizer &MS,
4189  MemorySanitizerVisitor &MSV)
4190  : F(F), MS(MS), MSV(MSV) {
4191  AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4192  for (const auto &Attr : F.getAttributes().getFnAttrs()) {
4193  if (Attr.isStringAttribute() &&
4194  (Attr.getKindAsString() == "target-features")) {
4195  if (Attr.getValueAsString().contains("-sse"))
4196  AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4197  break;
4198  }
4199  }
4200  }
4201 
4202  ArgKind classifyArgument(Value* arg) {
4203  // A very rough approximation of X86_64 argument classification rules.
4204  Type *T = arg->getType();
4205  if (T->isFPOrFPVectorTy() || T->isX86_MMXTy())
4206  return AK_FloatingPoint;
4207  if (T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4208  return AK_GeneralPurpose;
4209  if (T->isPointerTy())
4210  return AK_GeneralPurpose;
4211  return AK_Memory;
4212  }
4213 
4214  // For VarArg functions, store the argument shadow in an ABI-specific format
4215  // that corresponds to va_list layout.
4216  // We do this because Clang lowers va_arg in the frontend, and this pass
4217  // only sees the low level code that deals with va_list internals.
4218  // A much easier alternative (provided that Clang emits va_arg instructions)
4219  // would have been to associate each live instance of va_list with a copy of
4220  // MSanParamTLS, and extract shadow on va_arg() call in the argument list
4221  // order.
4222  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4223  unsigned GpOffset = 0;
4224  unsigned FpOffset = AMD64GpEndOffset;
4225  unsigned OverflowOffset = AMD64FpEndOffset;
4226  const DataLayout &DL = F.getParent()->getDataLayout();
4227  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4228  ++ArgIt) {
4229  Value *A = *ArgIt;
4230  unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4231  bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4232  bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4233  if (IsByVal) {
4234  // ByVal arguments always go to the overflow area.
4235  // Fixed arguments passed through the overflow area will be stepped
4236  // over by va_start, so don't count them towards the offset.
4237  if (IsFixed)
4238  continue;
4239  assert(A->getType()->isPointerTy());
4240  Type *RealTy = CB.getParamByValType(ArgNo);
4241  uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4242  Value *ShadowBase = getShadowPtrForVAArgument(
4243  RealTy, IRB, OverflowOffset, alignTo(ArgSize, 8));
4244  Value *OriginBase = nullptr;
4245  if (MS.TrackOrigins)
4246  OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
4247  OverflowOffset += alignTo(ArgSize, 8);
4248  if (!ShadowBase)
4249  continue;
4250  Value *ShadowPtr, *OriginPtr;
4251  std::tie(ShadowPtr, OriginPtr) =
4252  MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(), kShadowTLSAlignment,
4253  /*isStore*/ false);
4254 
4255  IRB.CreateMemCpy(ShadowBase, kShadowTLSAlignment, ShadowPtr,
4256  kShadowTLSAlignment, ArgSize);
4257  if (MS.TrackOrigins)
4258  IRB.CreateMemCpy(OriginBase, kShadowTLSAlignment, OriginPtr,
4259  kShadowTLSAlignment, ArgSize);
4260  } else {
4261  ArgKind AK = classifyArgument(A);
4262  if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4263  AK = AK_Memory;
4264  if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4265  AK = AK_Memory;
4266  Value *ShadowBase, *OriginBase = nullptr;
4267  switch (AK) {
4268  case AK_GeneralPurpose:
4269  ShadowBase =
4270  getShadowPtrForVAArgument(A->getType(), IRB, GpOffset, 8);
4271  if (MS.TrackOrigins)
4272  OriginBase =
4273  getOriginPtrForVAArgument(A->getType(), IRB, GpOffset);
4274  GpOffset += 8;
4275  break;
4276  case AK_FloatingPoint:
4277  ShadowBase =
4278  getShadowPtrForVAArgument(A->getType(), IRB, FpOffset, 16);
4279  if (MS.TrackOrigins)
4280  OriginBase =
4281  getOriginPtrForVAArgument(A->getType(), IRB, FpOffset);
4282  FpOffset += 16;
4283  break;
4284  case AK_Memory:
4285  if (IsFixed)
4286  continue;
4287  uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4288  ShadowBase =
4289  getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset, 8);
4290  if (MS.TrackOrigins)
4291  OriginBase =
4292  getOriginPtrForVAArgument(A->getType(), IRB, OverflowOffset);
4293  OverflowOffset += alignTo(ArgSize, 8);
4294  }
4295  // Take fixed arguments into account for GpOffset and FpOffset,
4296  // but don't actually store shadows for them.
4297  // TODO(glider): don't call get*PtrForVAArgument() for them.
4298  if (IsFixed)
4299  continue;
4300  if (!ShadowBase)
4301  continue;
4302  Value *Shadow = MSV.getShadow(A);
4303  IRB.CreateAlignedStore(Shadow, ShadowBase, kShadowTLSAlignment);
4304  if (MS.TrackOrigins) {
4305  Value *Origin = MSV.getOrigin(A);
4306  unsigned StoreSize = DL.getTypeStoreSize(Shadow->getType());
4307  MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4309  }
4310  }
4311  }
4312  Constant *OverflowSize =
4313  ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4314  IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4315  }
4316 
4317  /// Compute the shadow address for a given va_arg.
4318  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4319  unsigned ArgOffset, unsigned ArgSize) {
4320  // Make sure we don't overflow __msan_va_arg_tls.
4321  if (ArgOffset + ArgSize > kParamTLSSize)
4322  return nullptr;
4323  Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4324  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4325  return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4326  "_msarg_va_s");
4327  }
4328 
4329  /// Compute the origin address for a given va_arg.
4330  Value *getOriginPtrForVAArgument(Type *Ty, IRBuilder<> &IRB, int ArgOffset) {
4331  Value *Base = IRB.CreatePointerCast(MS.VAArgOriginTLS, MS.IntptrTy);
4332  // getOriginPtrForVAArgument() is always called after
4333  // getShadowPtrForVAArgument(), so __msan_va_arg_origin_tls can never
4334  // overflow.
4335  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4336  return IRB.CreateIntToPtr(Base, PointerType::get(MS.OriginTy, 0),
4337  "_msarg_va_o");
4338  }
4339 
4340  void unpoisonVAListTagForInst(IntrinsicInst &I) {
4341  IRBuilder<> IRB(&I);
4342  Value *VAListTag = I.getArgOperand(0);
4343  Value *ShadowPtr, *OriginPtr;
4344  const Align Alignment = Align(8);
4345  std::tie(ShadowPtr, OriginPtr) =
4346  MSV.getShadowOriginPtr(VAListTag, IRB, IRB.getInt8Ty(), Alignment,
4347  /*isStore*/ true);
4348 
4349  // Unpoison the whole __va_list_tag.
4350  // FIXME: magic ABI constants.
4351  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4352  /* size */ 24, Alignment, false);
4353  // We shouldn't need to zero out the origins, as they're only checked for
4354  // nonzero shadow.
4355  }
4356 
4357  void visitVAStartInst(VAStartInst &I) override {
4358  if (F.getCallingConv() == CallingConv::Win64)
4359  return;
4360  VAStartInstrumentationList.push_back(&I);
4361  unpoisonVAListTagForInst(I);
4362  }
4363 
4364  void visitVACopyInst(VACopyInst &I) override {
4365  if (F.getCallingConv() == CallingConv::Win64) return;
4366  unpoisonVAListTagForInst(I);
4367  }
4368 
4369  void finalizeInstrumentation() override {
4370  assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4371  "finalizeInstrumentation called twice");
4372  if (!VAStartInstrumentationList.empty()) {
4373  // If there is a va_start in this function, make a backup copy of
4374  // va_arg_tls somewhere in the function entry block.
4375  IRBuilder<> IRB(MSV.FnPrologueEnd);
4376  VAArgOverflowSize =
4377  IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4378  Value *CopySize =
4379  IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset),
4380  VAArgOverflowSize);
4381  VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4382  IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4383  if (MS.TrackOrigins) {
4384  VAArgTLSOriginCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4385  IRB.CreateMemCpy(VAArgTLSOriginCopy, Align(8), MS.VAArgOriginTLS,
4386  Align(8), CopySize);
4387  }
4388  }
4389 
4390  // Instrument va_start.
4391  // Copy va_list shadow from the backup copy of the TLS contents.
4392  for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4393  CallInst *OrigInst = VAStartInstrumentationList[i];
4394  IRBuilder<> IRB(OrigInst->getNextNode());
4395  Value *VAListTag = OrigInst->getArgOperand(0);
4396 
4397  Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4398  Value *RegSaveAreaPtrPtr = IRB.CreateIntToPtr(
4399  IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4400  ConstantInt::get(MS.IntptrTy, 16)),
4401  PointerType::get(RegSaveAreaPtrTy, 0));
4402  Value *RegSaveAreaPtr =
4403  IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4404  Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4405  const Align Alignment = Align(16);
4406  std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4407  MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4408  Alignment, /*isStore*/ true);
4409  IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4410  AMD64FpEndOffset);
4411  if (MS.TrackOrigins)
4412  IRB.CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
4413  Alignment, AMD64FpEndOffset);
4414  Type *OverflowArgAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4415  Value *OverflowArgAreaPtrPtr = IRB.CreateIntToPtr(
4416  IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4417  ConstantInt::get(MS.IntptrTy, 8)),
4418  PointerType::get(OverflowArgAreaPtrTy, 0));
4419  Value *OverflowArgAreaPtr =
4420  IRB.CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
4421  Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4422  std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
4423  MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.getInt8Ty(),
4424  Alignment, /*isStore*/ true);
4425  Value *SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSCopy,
4426  AMD64FpEndOffset);
4427  IRB.CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4428  VAArgOverflowSize);
4429  if (MS.TrackOrigins) {
4430  SrcPtr = IRB.CreateConstGEP1_32(IRB.getInt8Ty(), VAArgTLSOriginCopy,
4431  AMD64FpEndOffset);
4432  IRB.CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
4433  VAArgOverflowSize);
4434  }
4435  }
4436  }
4437 };
4438 
4439 /// MIPS64-specific implementation of VarArgHelper.
4440 struct VarArgMIPS64Helper : public VarArgHelper {
4441  Function &F;
4442  MemorySanitizer &MS;
4443  MemorySanitizerVisitor &MSV;
4444  Value *VAArgTLSCopy = nullptr;
4445  Value *VAArgSize = nullptr;
4446 
4447  SmallVector<CallInst*, 16> VAStartInstrumentationList;
4448 
4449  VarArgMIPS64Helper(Function &F, MemorySanitizer &MS,
4450  MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4451 
4452  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4453  unsigned VAArgOffset = 0;
4454  const DataLayout &DL = F.getParent()->getDataLayout();
4455  for (auto ArgIt = CB.arg_begin() + CB.getFunctionType()->getNumParams(),
4456  End = CB.arg_end();
4457  ArgIt != End; ++ArgIt) {
4458  Triple TargetTriple(F.getParent()->getTargetTriple());
4459  Value *A = *ArgIt;
4460  Value *Base;
4461  uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4462  if (TargetTriple.getArch() == Triple::mips64) {
4463  // Adjusting the shadow for argument with size < 8 to match the placement
4464  // of bits in big endian system
4465  if (ArgSize < 8)
4466  VAArgOffset += (8 - ArgSize);
4467  }
4468  Base = getShadowPtrForVAArgument(A->getType(), IRB, VAArgOffset, ArgSize);
4469  VAArgOffset += ArgSize;
4470  VAArgOffset = alignTo(VAArgOffset, 8);
4471  if (!Base)
4472  continue;
4473  IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4474  }
4475 
4476  Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(), VAArgOffset);
4477  // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4478  // a new class member i.e. it is the total size of all VarArgs.
4479  IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4480  }
4481 
4482  /// Compute the shadow address for a given va_arg.
4483  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4484  unsigned ArgOffset, unsigned ArgSize) {
4485  // Make sure we don't overflow __msan_va_arg_tls.
4486  if (ArgOffset + ArgSize > kParamTLSSize)
4487  return nullptr;
4488  Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4489  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4490  return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4491  "_msarg");
4492  }
4493 
4494  void visitVAStartInst(VAStartInst &I) override {
4495  IRBuilder<> IRB(&I);
4496  VAStartInstrumentationList.push_back(&I);
4497  Value *VAListTag = I.getArgOperand(0);
4498  Value *ShadowPtr, *OriginPtr;
4499  const Align Alignment = Align(8);
4500  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4501  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4502  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4503  /* size */ 8, Alignment, false);
4504  }
4505 
4506  void visitVACopyInst(VACopyInst &I) override {
4507  IRBuilder<> IRB(&I);
4508  VAStartInstrumentationList.push_back(&I);
4509  Value *VAListTag = I.getArgOperand(0);
4510  Value *ShadowPtr, *OriginPtr;
4511  const Align Alignment = Align(8);
4512  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4513  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4514  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4515  /* size */ 8, Alignment, false);
4516  }
4517 
4518  void finalizeInstrumentation() override {
4519  assert(!VAArgSize && !VAArgTLSCopy &&
4520  "finalizeInstrumentation called twice");
4521  IRBuilder<> IRB(MSV.FnPrologueEnd);
4522  VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4523  Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4524  VAArgSize);
4525 
4526  if (!VAStartInstrumentationList.empty()) {
4527  // If there is a va_start in this function, make a backup copy of
4528  // va_arg_tls somewhere in the function entry block.
4529  VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4530  IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4531  }
4532 
4533  // Instrument va_start.
4534  // Copy va_list shadow from the backup copy of the TLS contents.
4535  for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4536  CallInst *OrigInst = VAStartInstrumentationList[i];
4537  IRBuilder<> IRB(OrigInst->getNextNode());
4538  Value *VAListTag = OrigInst->getArgOperand(0);
4539  Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4540  Value *RegSaveAreaPtrPtr =
4541  IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4542  PointerType::get(RegSaveAreaPtrTy, 0));
4543  Value *RegSaveAreaPtr =
4544  IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4545  Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4546  const Align Alignment = Align(8);
4547  std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4548  MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4549  Alignment, /*isStore*/ true);
4550  IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4551  CopySize);
4552  }
4553  }
4554 };
4555 
4556 /// AArch64-specific implementation of VarArgHelper.
4557 struct VarArgAArch64Helper : public VarArgHelper {
4558  static const unsigned kAArch64GrArgSize = 64;
4559  static const unsigned kAArch64VrArgSize = 128;
4560 
4561  static const unsigned AArch64GrBegOffset = 0;
4562  static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
4563  // Make VR space aligned to 16 bytes.
4564  static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
4565  static const unsigned AArch64VrEndOffset = AArch64VrBegOffset
4566  + kAArch64VrArgSize;
4567  static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
4568 
4569  Function &F;
4570  MemorySanitizer &MS;
4571  MemorySanitizerVisitor &MSV;
4572  Value *VAArgTLSCopy = nullptr;
4573  Value *VAArgOverflowSize = nullptr;
4574 
4575  SmallVector<CallInst*, 16> VAStartInstrumentationList;
4576 
4577  enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4578 
4579  VarArgAArch64Helper(Function &F, MemorySanitizer &MS,
4580  MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4581 
4582  ArgKind classifyArgument(Value* arg) {
4583  Type *T = arg->getType();
4584  if (T->isFPOrFPVectorTy())
4585  return AK_FloatingPoint;
4586  if ((T->isIntegerTy() && T->getPrimitiveSizeInBits() <= 64)
4587  || (T->isPointerTy()))
4588  return AK_GeneralPurpose;
4589  return AK_Memory;
4590  }
4591 
4592  // The instrumentation stores the argument shadow in a non ABI-specific
4593  // format because it does not know which argument is named (since Clang,
4594  // like x86_64 case, lowers the va_args in the frontend and this pass only
4595  // sees the low level code that deals with va_list internals).
4596  // The first seven GR registers are saved in the first 56 bytes of the
4597  // va_arg tls arra, followers by the first 8 FP/SIMD registers, and then
4598  // the remaining arguments.
4599  // Using constant offset within the va_arg TLS array allows fast copy
4600  // in the finalize instrumentation.
4601  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4602  unsigned GrOffset = AArch64GrBegOffset;
4603  unsigned VrOffset = AArch64VrBegOffset;
4604  unsigned OverflowOffset = AArch64VAEndOffset;
4605 
4606  const DataLayout &DL = F.getParent()->getDataLayout();
4607  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4608  ++ArgIt) {
4609  Value *A = *ArgIt;
4610  unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4611  bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4612  ArgKind AK = classifyArgument(A);
4613  if (AK == AK_GeneralPurpose && GrOffset >= AArch64GrEndOffset)
4614  AK = AK_Memory;
4615  if (AK == AK_FloatingPoint && VrOffset >= AArch64VrEndOffset)
4616  AK = AK_Memory;
4617  Value *Base;
4618  switch (AK) {
4619  case AK_GeneralPurpose:
4620  Base = getShadowPtrForVAArgument(A->getType(), IRB, GrOffset, 8);
4621  GrOffset += 8;
4622  break;
4623  case AK_FloatingPoint:
4624  Base = getShadowPtrForVAArgument(A->getType(), IRB, VrOffset, 8);
4625  VrOffset += 16;
4626  break;
4627  case AK_Memory:
4628  // Don't count fixed arguments in the overflow area - va_start will
4629  // skip right over them.
4630  if (IsFixed)
4631  continue;
4632  uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4633  Base = getShadowPtrForVAArgument(A->getType(), IRB, OverflowOffset,
4634  alignTo(ArgSize, 8));
4635  OverflowOffset += alignTo(ArgSize, 8);
4636  break;
4637  }
4638  // Count Gp/Vr fixed arguments to their respective offsets, but don't
4639  // bother to actually store a shadow.
4640  if (IsFixed)
4641  continue;
4642  if (!Base)
4643  continue;
4644  IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4645  }
4646  Constant *OverflowSize =
4647  ConstantInt::get(IRB.getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
4648  IRB.CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4649  }
4650 
4651  /// Compute the shadow address for a given va_arg.
4652  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4653  unsigned ArgOffset, unsigned ArgSize) {
4654  // Make sure we don't overflow __msan_va_arg_tls.
4655  if (ArgOffset + ArgSize > kParamTLSSize)
4656  return nullptr;
4657  Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4658  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4659  return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4660  "_msarg");
4661  }
4662 
4663  void visitVAStartInst(VAStartInst &I) override {
4664  IRBuilder<> IRB(&I);
4665  VAStartInstrumentationList.push_back(&I);
4666  Value *VAListTag = I.getArgOperand(0);
4667  Value *ShadowPtr, *OriginPtr;
4668  const Align Alignment = Align(8);
4669  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4670  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4671  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4672  /* size */ 32, Alignment, false);
4673  }
4674 
4675  void visitVACopyInst(VACopyInst &I) override {
4676  IRBuilder<> IRB(&I);
4677  VAStartInstrumentationList.push_back(&I);
4678  Value *VAListTag = I.getArgOperand(0);
4679  Value *ShadowPtr, *OriginPtr;
4680  const Align Alignment = Align(8);
4681  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4682  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4683  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4684  /* size */ 32, Alignment, false);
4685  }
4686 
4687  // Retrieve a va_list field of 'void*' size.
4688  Value* getVAField64(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4689  Value *SaveAreaPtrPtr =
4690  IRB.CreateIntToPtr(
4691  IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4692  ConstantInt::get(MS.IntptrTy, offset)),
4693  Type::getInt64PtrTy(*MS.C));
4694  return IRB.CreateLoad(Type::getInt64Ty(*MS.C), SaveAreaPtrPtr);
4695  }
4696 
4697  // Retrieve a va_list field of 'int' size.
4698  Value* getVAField32(IRBuilder<> &IRB, Value *VAListTag, int offset) {
4699  Value *SaveAreaPtr =
4700  IRB.CreateIntToPtr(
4701  IRB.CreateAdd(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4702  ConstantInt::get(MS.IntptrTy, offset)),
4703  Type::getInt32PtrTy(*MS.C));
4704  Value *SaveArea32 = IRB.CreateLoad(IRB.getInt32Ty(), SaveAreaPtr);
4705  return IRB.CreateSExt(SaveArea32, MS.IntptrTy);
4706  }
4707 
4708  void finalizeInstrumentation() override {
4709  assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4710  "finalizeInstrumentation called twice");
4711  if (!VAStartInstrumentationList.empty()) {
4712  // If there is a va_start in this function, make a backup copy of
4713  // va_arg_tls somewhere in the function entry block.
4714  IRBuilder<> IRB(MSV.FnPrologueEnd);
4715  VAArgOverflowSize =
4716  IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4717  Value *CopySize =
4718  IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset),
4719  VAArgOverflowSize);
4720  VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4721  IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4722  }
4723 
4724  Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
4725  Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
4726 
4727  // Instrument va_start, copy va_list shadow from the backup copy of
4728  // the TLS contents.
4729  for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4730  CallInst *OrigInst = VAStartInstrumentationList[i];
4731  IRBuilder<> IRB(OrigInst->getNextNode());
4732 
4733  Value *VAListTag = OrigInst->getArgOperand(0);
4734 
4735  // The variadic ABI for AArch64 creates two areas to save the incoming
4736  // argument registers (one for 64-bit general register xn-x7 and another
4737  // for 128-bit FP/SIMD vn-v7).
4738  // We need then to propagate the shadow arguments on both regions
4739  // 'va::__gr_top + va::__gr_offs' and 'va::__vr_top + va::__vr_offs'.
4740  // The remaining arguments are saved on shadow for 'va::stack'.
4741  // One caveat is it requires only to propagate the non-named arguments,
4742  // however on the call site instrumentation 'all' the arguments are
4743  // saved. So to copy the shadow values from the va_arg TLS array
4744  // we need to adjust the offset for both GR and VR fields based on
4745  // the __{gr,vr}_offs value (since they are stores based on incoming
4746  // named arguments).
4747 
4748  // Read the stack pointer from the va_list.
4749  Value *StackSaveAreaPtr = getVAField64(IRB, VAListTag, 0);
4750 
4751  // Read both the __gr_top and __gr_off and add them up.
4752  Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
4753  Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
4754 
4755  Value *GrRegSaveAreaPtr = IRB.CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea);
4756 
4757  // Read both the __vr_top and __vr_off and add them up.
4758  Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
4759  Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
4760 
4761  Value *VrRegSaveAreaPtr = IRB.CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea);
4762 
4763  // It does not know how many named arguments is being used and, on the
4764  // callsite all the arguments were saved. Since __gr_off is defined as
4765  // '0 - ((8 - named_gr) * 8)', the idea is to just propagate the variadic
4766  // argument by ignoring the bytes of shadow from named arguments.
4767  Value *GrRegSaveAreaShadowPtrOff =
4768  IRB.CreateAdd(GrArgSize, GrOffSaveArea);
4769 
4770  Value *GrRegSaveAreaShadowPtr =
4771  MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4772  Align(8), /*isStore*/ true)
4773  .first;
4774 
4775  Value *GrSrcPtr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4776  GrRegSaveAreaShadowPtrOff);
4777  Value *GrCopySize = IRB.CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
4778 
4779  IRB.CreateMemCpy(GrRegSaveAreaShadowPtr, Align(8), GrSrcPtr, Align(8),
4780  GrCopySize);
4781 
4782  // Again, but for FP/SIMD values.
4783  Value *VrRegSaveAreaShadowPtrOff =
4784  IRB.CreateAdd(VrArgSize, VrOffSaveArea);
4785 
4786  Value *VrRegSaveAreaShadowPtr =
4787  MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4788  Align(8), /*isStore*/ true)
4789  .first;
4790 
4791  Value *VrSrcPtr = IRB.CreateInBoundsGEP(
4792  IRB.getInt8Ty(),
4793  IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4794  IRB.getInt32(AArch64VrBegOffset)),
4795  VrRegSaveAreaShadowPtrOff);
4796  Value *VrCopySize = IRB.CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
4797 
4798  IRB.CreateMemCpy(VrRegSaveAreaShadowPtr, Align(8), VrSrcPtr, Align(8),
4799  VrCopySize);
4800 
4801  // And finally for remaining arguments.
4802  Value *StackSaveAreaShadowPtr =
4803  MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.getInt8Ty(),
4804  Align(16), /*isStore*/ true)
4805  .first;
4806 
4807  Value *StackSrcPtr =
4808  IRB.CreateInBoundsGEP(IRB.getInt8Ty(), VAArgTLSCopy,
4809  IRB.getInt32(AArch64VAEndOffset));
4810 
4811  IRB.CreateMemCpy(StackSaveAreaShadowPtr, Align(16), StackSrcPtr,
4812  Align(16), VAArgOverflowSize);
4813  }
4814  }
4815 };
4816 
4817 /// PowerPC64-specific implementation of VarArgHelper.
4818 struct VarArgPowerPC64Helper : public VarArgHelper {
4819  Function &F;
4820  MemorySanitizer &MS;
4821  MemorySanitizerVisitor &MSV;
4822  Value *VAArgTLSCopy = nullptr;
4823  Value *VAArgSize = nullptr;
4824 
4825  SmallVector<CallInst*, 16> VAStartInstrumentationList;
4826 
4827  VarArgPowerPC64Helper(Function &F, MemorySanitizer &MS,
4828  MemorySanitizerVisitor &MSV) : F(F), MS(MS), MSV(MSV) {}
4829 
4830  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
4831  // For PowerPC, we need to deal with alignment of stack arguments -
4832  // they are mostly aligned to 8 bytes, but vectors and i128 arrays
4833  // are aligned to 16 bytes, byvals can be aligned to 8 or 16 bytes,
4834  // For that reason, we compute current offset from stack pointer (which is
4835  // always properly aligned), and offset for the first vararg, then subtract
4836  // them.
4837  unsigned VAArgBase;
4838  Triple TargetTriple(F.getParent()->getTargetTriple());
4839  // Parameter save area starts at 48 bytes from frame pointer for ABIv1,
4840  // and 32 bytes for ABIv2. This is usually determined by target
4841  // endianness, but in theory could be overridden by function attribute.
4842  if (TargetTriple.getArch() == Triple::ppc64)
4843  VAArgBase = 48;
4844  else
4845  VAArgBase = 32;
4846  unsigned VAArgOffset = VAArgBase;
4847  const DataLayout &DL = F.getParent()->getDataLayout();
4848  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
4849  ++ArgIt) {
4850  Value *A = *ArgIt;
4851  unsigned ArgNo = CB.getArgOperandNo(ArgIt);
4852  bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
4853  bool IsByVal = CB.paramHasAttr(ArgNo, Attribute::ByVal);
4854  if (IsByVal) {
4855  assert(A->getType()->isPointerTy());
4856  Type *RealTy = CB.getParamByValType(ArgNo);
4857  uint64_t ArgSize = DL.getTypeAllocSize(RealTy);
4858  MaybeAlign ArgAlign = CB.getParamAlign(ArgNo);
4859  if (!ArgAlign || *ArgAlign < Align(8))
4860  ArgAlign = Align(8);
4861  VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4862  if (!IsFixed) {
4863  Value *Base = getShadowPtrForVAArgument(
4864  RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
4865  if (Base) {
4866  Value *AShadowPtr, *AOriginPtr;
4867  std::tie(AShadowPtr, AOriginPtr) =
4868  MSV.getShadowOriginPtr(A, IRB, IRB.getInt8Ty(),
4869  kShadowTLSAlignment, /*isStore*/ false);
4870 
4871  IRB.CreateMemCpy(Base, kShadowTLSAlignment, AShadowPtr,
4872  kShadowTLSAlignment, ArgSize);
4873  }
4874  }
4875  VAArgOffset += alignTo(ArgSize, 8);
4876  } else {
4877  Value *Base;
4878  uint64_t ArgSize = DL.getTypeAllocSize(A->getType());
4879  uint64_t ArgAlign = 8;
4880  if (A->getType()->isArrayTy()) {
4881  // Arrays are aligned to element size, except for long double
4882  // arrays, which are aligned to 8 bytes.
4883  Type *ElementTy = A->getType()->getArrayElementType();
4884  if (!ElementTy->isPPC_FP128Ty())
4885  ArgAlign = DL.getTypeAllocSize(ElementTy);
4886  } else if (A->getType()->isVectorTy()) {
4887  // Vectors are naturally aligned.
4888  ArgAlign = DL.getTypeAllocSize(A->getType());
4889  }
4890  if (ArgAlign < 8)
4891  ArgAlign = 8;
4892  VAArgOffset = alignTo(VAArgOffset, ArgAlign);
4893  if (DL.isBigEndian()) {
4894  // Adjusting the shadow for argument with size < 8 to match the placement
4895  // of bits in big endian system
4896  if (ArgSize < 8)
4897  VAArgOffset += (8 - ArgSize);
4898  }
4899  if (!IsFixed) {
4900  Base = getShadowPtrForVAArgument(A->getType(), IRB,
4901  VAArgOffset - VAArgBase, ArgSize);
4902  if (Base)
4903  IRB.CreateAlignedStore(MSV.getShadow(A), Base, kShadowTLSAlignment);
4904  }
4905  VAArgOffset += ArgSize;
4906  VAArgOffset = alignTo(VAArgOffset, 8);
4907  }
4908  if (IsFixed)
4909  VAArgBase = VAArgOffset;
4910  }
4911 
4912  Constant *TotalVAArgSize = ConstantInt::get(IRB.getInt64Ty(),
4913  VAArgOffset - VAArgBase);
4914  // Here using VAArgOverflowSizeTLS as VAArgSizeTLS to avoid creation of
4915  // a new class member i.e. it is the total size of all VarArgs.
4916  IRB.CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4917  }
4918 
4919  /// Compute the shadow address for a given va_arg.
4920  Value *getShadowPtrForVAArgument(Type *Ty, IRBuilder<> &IRB,
4921  unsigned ArgOffset, unsigned ArgSize) {
4922  // Make sure we don't overflow __msan_va_arg_tls.
4923  if (ArgOffset + ArgSize > kParamTLSSize)
4924  return nullptr;
4925  Value *Base = IRB.CreatePointerCast(MS.VAArgTLS, MS.IntptrTy);
4926  Base = IRB.CreateAdd(Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4927  return IRB.CreateIntToPtr(Base, PointerType::get(MSV.getShadowTy(Ty), 0),
4928  "_msarg");
4929  }
4930 
4931  void visitVAStartInst(VAStartInst &I) override {
4932  IRBuilder<> IRB(&I);
4933  VAStartInstrumentationList.push_back(&I);
4934  Value *VAListTag = I.getArgOperand(0);
4935  Value *ShadowPtr, *OriginPtr;
4936  const Align Alignment = Align(8);
4937  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4938  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4939  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4940  /* size */ 8, Alignment, false);
4941  }
4942 
4943  void visitVACopyInst(VACopyInst &I) override {
4944  IRBuilder<> IRB(&I);
4945  Value *VAListTag = I.getArgOperand(0);
4946  Value *ShadowPtr, *OriginPtr;
4947  const Align Alignment = Align(8);
4948  std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4949  VAListTag, IRB, IRB.getInt8Ty(), Alignment, /*isStore*/ true);
4950  // Unpoison the whole __va_list_tag.
4951  // FIXME: magic ABI constants.
4952  IRB.CreateMemSet(ShadowPtr, Constant::getNullValue(IRB.getInt8Ty()),
4953  /* size */ 8, Alignment, false);
4954  }
4955 
4956  void finalizeInstrumentation() override {
4957  assert(!VAArgSize && !VAArgTLSCopy &&
4958  "finalizeInstrumentation called twice");
4959  IRBuilder<> IRB(MSV.FnPrologueEnd);
4960  VAArgSize = IRB.CreateLoad(IRB.getInt64Ty(), MS.VAArgOverflowSizeTLS);
4961  Value *CopySize = IRB.CreateAdd(ConstantInt::get(MS.IntptrTy, 0),
4962  VAArgSize);
4963 
4964  if (!VAStartInstrumentationList.empty()) {
4965  // If there is a va_start in this function, make a backup copy of
4966  // va_arg_tls somewhere in the function entry block.
4967  VAArgTLSCopy = IRB.CreateAlloca(Type::getInt8Ty(*MS.C), CopySize);
4968  IRB.CreateMemCpy(VAArgTLSCopy, Align(8), MS.VAArgTLS, Align(8), CopySize);
4969  }
4970 
4971  // Instrument va_start.
4972  // Copy va_list shadow from the backup copy of the TLS contents.
4973  for (size_t i = 0, n = VAStartInstrumentationList.size(); i < n; i++) {
4974  CallInst *OrigInst = VAStartInstrumentationList[i];
4975  IRBuilder<> IRB(OrigInst->getNextNode());
4976  Value *VAListTag = OrigInst->getArgOperand(0);
4977  Type *RegSaveAreaPtrTy = Type::getInt64PtrTy(*MS.C);
4978  Value *RegSaveAreaPtrPtr =
4979  IRB.CreateIntToPtr(IRB.CreatePtrToInt(VAListTag, MS.IntptrTy),
4980  PointerType::get(RegSaveAreaPtrTy, 0));
4981  Value *RegSaveAreaPtr =
4982  IRB.CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4983  Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4984  const Align Alignment = Align(8);
4985  std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4986  MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.getInt8Ty(),
4987  Alignment, /*isStore*/ true);
4988  IRB.CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4989  CopySize);
4990  }
4991  }
4992 };
4993 
4994 /// SystemZ-specific implementation of VarArgHelper.
4995 struct VarArgSystemZHelper : public VarArgHelper {
4996  static const unsigned SystemZGpOffset = 16;
4997  static const unsigned SystemZGpEndOffset = 56;
4998  static const unsigned SystemZFpOffset = 128;
4999  static const unsigned SystemZFpEndOffset = 160;
5000  static const unsigned SystemZMaxVrArgs = 8;
5001  static const unsigned SystemZRegSaveAreaSize = 160;
5002  static const unsigned SystemZOverflowOffset = 160;
5003  static const unsigned SystemZVAListTagSize = 32;
5004  static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5005  static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5006 
5007  Function &F;
5008  MemorySanitizer &MS;
5009  MemorySanitizerVisitor &MSV;
5010  Value *VAArgTLSCopy = nullptr;
5011  Value *VAArgTLSOriginCopy = nullptr;
5012  Value *VAArgOverflowSize = nullptr;
5013 
5014  SmallVector<CallInst *, 16> VAStartInstrumentationList;
5015 
5016  enum class ArgKind {
5017  GeneralPurpose,
5018  FloatingPoint,
5019  Vector,
5020  Memory,
5021  Indirect,
5022  };
5023 
5024  enum class ShadowExtension { None, Zero, Sign };
5025 
5026  VarArgSystemZHelper(Function &F, MemorySanitizer &MS,
5027  MemorySanitizerVisitor &MSV)
5028  : F(F), MS(MS), MSV(MSV) {}
5029 
5030  ArgKind classifyArgument(Type *T, bool IsSoftFloatABI) {
5031  // T is a SystemZABIInfo::classifyArgumentType() output, and there are
5032  // only a few possibilities of what it can be. In particular, enums, single
5033  // element structs and large types have already been taken care of.
5034 
5035  // Some i128 and fp128 arguments are converted to pointers only in the
5036  // back end.
5037  if (T->isIntegerTy(128) || T->isFP128Ty())
5038  return ArgKind::Indirect;
5039  if (T->isFloatingPointTy())
5040  return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5041  if (T->isIntegerTy() || T->isPointerTy())
5042  return ArgKind::GeneralPurpose;
5043  if (T->isVectorTy())
5044  return ArgKind::Vector;
5045  return ArgKind::Memory;
5046  }
5047 
5048  ShadowExtension getShadowExtension(const CallBase &CB, unsigned ArgNo) {
5049  // ABI says: "One of the simple integer types no more than 64 bits wide.
5050  // ... If such an argument is shorter than 64 bits, replace it by a full
5051  // 64-bit integer representing the same number, using sign or zero
5052  // extension". Shadow for an integer argument has the same type as the
5053  // argument itself, so it can be sign or zero extended as well.
5054  bool ZExt = CB.paramHasAttr(ArgNo, Attribute::ZExt);
5055  bool SExt = CB.paramHasAttr(ArgNo, Attribute::SExt);
5056  if (ZExt) {
5057  assert(!SExt);
5058  return ShadowExtension::Zero;
5059  }
5060  if (SExt) {
5061  assert(!ZExt);
5062  return ShadowExtension::Sign;
5063  }
5064  return ShadowExtension::None;
5065  }
5066 
5067  void visitCallBase(CallBase &CB, IRBuilder<> &IRB) override {
5068  bool IsSoftFloatABI = CB.getCalledFunction()
5069  ->getFnAttribute("use-soft-float")
5070  .getValueAsBool();
5071  unsigned GpOffset = SystemZGpOffset;
5072  unsigned FpOffset = SystemZFpOffset;
5073  unsigned VrIndex = 0;
5074  unsigned OverflowOffset = SystemZOverflowOffset;
5075  const DataLayout &DL = F.getParent()->getDataLayout();
5076  for (auto ArgIt = CB.arg_begin(), End = CB.arg_end(); ArgIt != End;
5077  ++ArgIt) {
5078  Value *A = *ArgIt;
5079  unsigned ArgNo = CB.getArgOperandNo(ArgIt);
5080  bool IsFixed = ArgNo < CB.getFunctionType()->getNumParams();
5081  // SystemZABIInfo does not produce ByVal parameters.
5082  assert(!CB.paramHasAttr(ArgNo, Attribute::ByVal));
5083  Type *T = A->getType();
5084  ArgKind AK = classifyArgument(T, IsSoftFloatABI);
5085  if (AK == ArgKind::Indirect) {
5086  T = PointerType::get(T, 0);
5087  AK = ArgKind::GeneralPurpose;
5088  }
5089  if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5090  AK = ArgKind::Memory;
5091  if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5092  AK = ArgKind::Memory;
5093  if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5094  AK = ArgKind::Memory;
5095  Value *ShadowBase = nullptr;
5096  Value *OriginBase = nullptr;
5097  ShadowExtension SE = ShadowExtension::None;
5098  switch (AK) {
5099  case ArgKind::GeneralPurpose: {
5100  // Always keep track of GpOffset, but store shadow only for varargs.
5101  uint64_t ArgSize = 8;
5102  if (GpOffset + ArgSize <= kParamTLSSize) {
5103  if (!IsFixed) {
5104  SE = getShadowExtension(CB, ArgNo);
5105  uint64_t GapSize = 0;
5106  if (SE == ShadowExtension::None) {
5107  uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5108  assert(ArgAllocSize <= ArgSize);
5109  GapSize = ArgSize - ArgAllocSize;
5110  }
5111  ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5112  if (MS.TrackOrigins)
5113  OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5114  }
5115  GpOffset += ArgSize;
5116  } else {
5117  GpOffset = kParamTLSSize;
5118  }
5119  break;
5120  }
5121  case ArgKind::FloatingPoint: {
5122  // Always keep track of FpOffset, but store shadow only for varargs.
5123  uint64_t ArgSize = 8;
5124  if (FpOffset + ArgSize <= kParamTLSSize) {
5125  if (!IsFixed) {
5126  // PoP says: "A short floating-point datum requires only the
5127  // left-most 32 bit positions of a floating-point register".
5128  // Therefore, in contrast to AK_GeneralPurpose and AK_Memory,
5129  // don't extend shadow and don't mind the gap.
5130  ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5131  if (MS.TrackOrigins)
5132  OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5133  }
5134  FpOffset += ArgSize;
5135  } else {
5136  FpOffset = kParamTLSSize;
5137  }
5138  break;
5139  }
5140  case ArgKind::Vector: {
5141  // Keep track of VrIndex. No need to store shadow, since vector varargs
5142  // go through AK_Memory.
5143  assert(IsFixed);
5144  VrIndex++;
5145  break;
5146  }
5147  case ArgKind::Memory: {
5148  // Keep track of OverflowOffset and store shadow only for varargs.
5149  // Ignore fixed args, since we need to copy only the vararg portion of
5150  // the overflow area shadow.
5151  if (!IsFixed) {
5152  uint64_t ArgAllocSize = DL.getTypeAllocSize(T);
5153  uint64_t ArgSize = alignTo(ArgAllocSize, 8);
5154  if (OverflowOffset + ArgSize <= kParamTLSSize) {
5155  SE = getShadowExtension(CB, ArgNo);
5156  uint64_t GapSize =
5157  SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5158  ShadowBase =
5159  getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5160  if (MS.TrackOrigins)
5161  OriginBase =
5162  getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);