LLVM 20.0.0git
DataFlowSanitizer.cpp
Go to the documentation of this file.
1//===- DataFlowSanitizer.cpp - dynamic data flow analysis -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of DataFlowSanitizer, a generalised dynamic data flow
11/// analysis.
12///
13/// Unlike other Sanitizer tools, this tool is not designed to detect a specific
14/// class of bugs on its own. Instead, it provides a generic dynamic data flow
15/// analysis framework to be used by clients to help detect application-specific
16/// issues within their own code.
17///
18/// The analysis is based on automatic propagation of data flow labels (also
19/// known as taint labels) through a program as it performs computation.
20///
21/// Argument and return value labels are passed through TLS variables
22/// __dfsan_arg_tls and __dfsan_retval_tls.
23///
24/// Each byte of application memory is backed by a shadow memory byte. The
25/// shadow byte can represent up to 8 labels. On Linux/x86_64, memory is then
26/// laid out as follows:
27///
28/// +--------------------+ 0x800000000000 (top of memory)
29/// | application 3 |
30/// +--------------------+ 0x700000000000
31/// | invalid |
32/// +--------------------+ 0x610000000000
33/// | origin 1 |
34/// +--------------------+ 0x600000000000
35/// | application 2 |
36/// +--------------------+ 0x510000000000
37/// | shadow 1 |
38/// +--------------------+ 0x500000000000
39/// | invalid |
40/// +--------------------+ 0x400000000000
41/// | origin 3 |
42/// +--------------------+ 0x300000000000
43/// | shadow 3 |
44/// +--------------------+ 0x200000000000
45/// | origin 2 |
46/// +--------------------+ 0x110000000000
47/// | invalid |
48/// +--------------------+ 0x100000000000
49/// | shadow 2 |
50/// +--------------------+ 0x010000000000
51/// | application 1 |
52/// +--------------------+ 0x000000000000
53///
54/// MEM_TO_SHADOW(mem) = mem ^ 0x500000000000
55/// SHADOW_TO_ORIGIN(shadow) = shadow + 0x100000000000
56///
57/// For more information, please refer to the design document:
58/// http://clang.llvm.org/docs/DataFlowSanitizerDesign.html
59//
60//===----------------------------------------------------------------------===//
61
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DenseSet.h"
68#include "llvm/ADT/StringRef.h"
69#include "llvm/ADT/StringSet.h"
70#include "llvm/ADT/iterator.h"
75#include "llvm/IR/Argument.h"
77#include "llvm/IR/Attributes.h"
78#include "llvm/IR/BasicBlock.h"
79#include "llvm/IR/Constant.h"
80#include "llvm/IR/Constants.h"
81#include "llvm/IR/DataLayout.h"
83#include "llvm/IR/Dominators.h"
84#include "llvm/IR/Function.h"
85#include "llvm/IR/GlobalAlias.h"
86#include "llvm/IR/GlobalValue.h"
88#include "llvm/IR/IRBuilder.h"
89#include "llvm/IR/InstVisitor.h"
90#include "llvm/IR/InstrTypes.h"
91#include "llvm/IR/Instruction.h"
94#include "llvm/IR/MDBuilder.h"
95#include "llvm/IR/Module.h"
96#include "llvm/IR/PassManager.h"
97#include "llvm/IR/Type.h"
98#include "llvm/IR/User.h"
99#include "llvm/IR/Value.h"
101#include "llvm/Support/Casting.h"
110#include <algorithm>
111#include <cassert>
112#include <cstddef>
113#include <cstdint>
114#include <memory>
115#include <set>
116#include <string>
117#include <utility>
118#include <vector>
119
120using namespace llvm;
121
122// This must be consistent with ShadowWidthBits.
124
126
127// The size of TLS variables. These constants must be kept in sync with the ones
128// in dfsan.cpp.
129static const unsigned ArgTLSSize = 800;
130static const unsigned RetvalTLSSize = 800;
131
132// The -dfsan-preserve-alignment flag controls whether this pass assumes that
133// alignment requirements provided by the input IR are correct. For example,
134// if the input IR contains a load with alignment 8, this flag will cause
135// the shadow load to have alignment 16. This flag is disabled by default as
136// we have unfortunately encountered too much code (including Clang itself;
137// see PR14291) which performs misaligned access.
139 "dfsan-preserve-alignment",
140 cl::desc("respect alignment requirements provided by input IR"), cl::Hidden,
141 cl::init(false));
142
143// The ABI list files control how shadow parameters are passed. The pass treats
144// every function labelled "uninstrumented" in the ABI list file as conforming
145// to the "native" (i.e. unsanitized) ABI. Unless the ABI list contains
146// additional annotations for those functions, a call to one of those functions
147// will produce a warning message, as the labelling behaviour of the function is
148// unknown. The other supported annotations for uninstrumented functions are
149// "functional" and "discard", which are described below under
150// DataFlowSanitizer::WrapperKind.
151// Functions will often be labelled with both "uninstrumented" and one of
152// "functional" or "discard". This will leave the function unchanged by this
153// pass, and create a wrapper function that will call the original.
154//
155// Instrumented functions can also be annotated as "force_zero_labels", which
156// will make all shadow and return values set zero labels.
157// Functions should never be labelled with both "force_zero_labels" and
158// "uninstrumented" or any of the unistrumented wrapper kinds.
160 "dfsan-abilist",
161 cl::desc("File listing native ABI functions and how the pass treats them"),
162 cl::Hidden);
163
164// Controls whether the pass includes or ignores the labels of pointers in load
165// instructions.
167 "dfsan-combine-pointer-labels-on-load",
168 cl::desc("Combine the label of the pointer with the label of the data when "
169 "loading from memory."),
170 cl::Hidden, cl::init(true));
171
172// Controls whether the pass includes or ignores the labels of pointers in
173// stores instructions.
175 "dfsan-combine-pointer-labels-on-store",
176 cl::desc("Combine the label of the pointer with the label of the data when "
177 "storing in memory."),
178 cl::Hidden, cl::init(false));
179
180// Controls whether the pass propagates labels of offsets in GEP instructions.
182 "dfsan-combine-offset-labels-on-gep",
183 cl::desc(
184 "Combine the label of the offset with the label of the pointer when "
185 "doing pointer arithmetic."),
186 cl::Hidden, cl::init(true));
187
189 "dfsan-combine-taint-lookup-table",
190 cl::desc(
191 "When dfsan-combine-offset-labels-on-gep and/or "
192 "dfsan-combine-pointer-labels-on-load are false, this flag can "
193 "be used to re-enable combining offset and/or pointer taint when "
194 "loading specific constant global variables (i.e. lookup tables)."),
195 cl::Hidden);
196
198 "dfsan-debug-nonzero-labels",
199 cl::desc("Insert calls to __dfsan_nonzero_label on observing a parameter, "
200 "load or return with a nonzero label"),
201 cl::Hidden);
202
203// Experimental feature that inserts callbacks for certain data events.
204// Currently callbacks are only inserted for loads, stores, memory transfers
205// (i.e. memcpy and memmove), and comparisons.
206//
207// If this flag is set to true, the user must provide definitions for the
208// following callback functions:
209// void __dfsan_load_callback(dfsan_label Label, void* addr);
210// void __dfsan_store_callback(dfsan_label Label, void* addr);
211// void __dfsan_mem_transfer_callback(dfsan_label *Start, size_t Len);
212// void __dfsan_cmp_callback(dfsan_label CombinedLabel);
214 "dfsan-event-callbacks",
215 cl::desc("Insert calls to __dfsan_*_callback functions on data events."),
216 cl::Hidden, cl::init(false));
217
218// Experimental feature that inserts callbacks for conditionals, including:
219// conditional branch, switch, select.
220// This must be true for dfsan_set_conditional_callback() to have effect.
222 "dfsan-conditional-callbacks",
223 cl::desc("Insert calls to callback functions on conditionals."), cl::Hidden,
224 cl::init(false));
225
226// Experimental feature that inserts callbacks for data reaching a function,
227// either via function arguments and loads.
228// This must be true for dfsan_set_reaches_function_callback() to have effect.
230 "dfsan-reaches-function-callbacks",
231 cl::desc("Insert calls to callback functions on data reaching a function."),
232 cl::Hidden, cl::init(false));
233
234// Controls whether the pass tracks the control flow of select instructions.
236 "dfsan-track-select-control-flow",
237 cl::desc("Propagate labels from condition values of select instructions "
238 "to results."),
239 cl::Hidden, cl::init(true));
240
241// TODO: This default value follows MSan. DFSan may use a different value.
243 "dfsan-instrument-with-call-threshold",
244 cl::desc("If the function being instrumented requires more than "
245 "this number of origin stores, use callbacks instead of "
246 "inline checks (-1 means never use callbacks)."),
247 cl::Hidden, cl::init(3500));
248
249// Controls how to track origins.
250// * 0: do not track origins.
251// * 1: track origins at memory store operations.
252// * 2: track origins at memory load and store operations.
253// TODO: track callsites.
254static cl::opt<int> ClTrackOrigins("dfsan-track-origins",
255 cl::desc("Track origins of labels"),
256 cl::Hidden, cl::init(0));
257
259 "dfsan-ignore-personality-routine",
260 cl::desc("If a personality routine is marked uninstrumented from the ABI "
261 "list, do not create a wrapper for it."),
262 cl::Hidden, cl::init(false));
263
265 // Types of GlobalVariables are always pointer types.
266 Type *GType = G.getValueType();
267 // For now we support excluding struct types only.
268 if (StructType *SGType = dyn_cast<StructType>(GType)) {
269 if (!SGType->isLiteral())
270 return SGType->getName();
271 }
272 return "<unknown type>";
273}
274
275namespace {
276
277// Memory map parameters used in application-to-shadow address calculation.
278// Offset = (Addr & ~AndMask) ^ XorMask
279// Shadow = ShadowBase + Offset
280// Origin = (OriginBase + Offset) & ~3ULL
281struct MemoryMapParams {
282 uint64_t AndMask;
283 uint64_t XorMask;
284 uint64_t ShadowBase;
285 uint64_t OriginBase;
286};
287
288} // end anonymous namespace
289
290// NOLINTBEGIN(readability-identifier-naming)
291// aarch64 Linux
292const MemoryMapParams Linux_AArch64_MemoryMapParams = {
293 0, // AndMask (not used)
294 0x0B00000000000, // XorMask
295 0, // ShadowBase (not used)
296 0x0200000000000, // OriginBase
297};
298
299// x86_64 Linux
300const MemoryMapParams Linux_X86_64_MemoryMapParams = {
301 0, // AndMask (not used)
302 0x500000000000, // XorMask
303 0, // ShadowBase (not used)
304 0x100000000000, // OriginBase
305};
306// NOLINTEND(readability-identifier-naming)
307
308// loongarch64 Linux
309const MemoryMapParams Linux_LoongArch64_MemoryMapParams = {
310 0, // AndMask (not used)
311 0x500000000000, // XorMask
312 0, // ShadowBase (not used)
313 0x100000000000, // OriginBase
314};
315
316namespace {
317
318class DFSanABIList {
319 std::unique_ptr<SpecialCaseList> SCL;
320
321public:
322 DFSanABIList() = default;
323
324 void set(std::unique_ptr<SpecialCaseList> List) { SCL = std::move(List); }
325
326 /// Returns whether either this function or its source file are listed in the
327 /// given category.
328 bool isIn(const Function &F, StringRef Category) const {
329 return isIn(*F.getParent(), Category) ||
330 SCL->inSection("dataflow", "fun", F.getName(), Category);
331 }
332
333 /// Returns whether this global alias is listed in the given category.
334 ///
335 /// If GA aliases a function, the alias's name is matched as a function name
336 /// would be. Similarly, aliases of globals are matched like globals.
337 bool isIn(const GlobalAlias &GA, StringRef Category) const {
338 if (isIn(*GA.getParent(), Category))
339 return true;
340
341 if (isa<FunctionType>(GA.getValueType()))
342 return SCL->inSection("dataflow", "fun", GA.getName(), Category);
343
344 return SCL->inSection("dataflow", "global", GA.getName(), Category) ||
345 SCL->inSection("dataflow", "type", getGlobalTypeString(GA),
346 Category);
347 }
348
349 /// Returns whether this module is listed in the given category.
350 bool isIn(const Module &M, StringRef Category) const {
351 return SCL->inSection("dataflow", "src", M.getModuleIdentifier(), Category);
352 }
353};
354
355/// TransformedFunction is used to express the result of transforming one
356/// function type into another. This struct is immutable. It holds metadata
357/// useful for updating calls of the old function to the new type.
358struct TransformedFunction {
359 TransformedFunction(FunctionType *OriginalType, FunctionType *TransformedType,
360 const std::vector<unsigned> &ArgumentIndexMapping)
361 : OriginalType(OriginalType), TransformedType(TransformedType),
362 ArgumentIndexMapping(ArgumentIndexMapping) {}
363
364 // Disallow copies.
365 TransformedFunction(const TransformedFunction &) = delete;
366 TransformedFunction &operator=(const TransformedFunction &) = delete;
367
368 // Allow moves.
369 TransformedFunction(TransformedFunction &&) = default;
370 TransformedFunction &operator=(TransformedFunction &&) = default;
371
372 /// Type of the function before the transformation.
373 FunctionType *OriginalType;
374
375 /// Type of the function after the transformation.
377
378 /// Transforming a function may change the position of arguments. This
379 /// member records the mapping from each argument's old position to its new
380 /// position. Argument positions are zero-indexed. If the transformation
381 /// from F to F' made the first argument of F into the third argument of F',
382 /// then ArgumentIndexMapping[0] will equal 2.
383 std::vector<unsigned> ArgumentIndexMapping;
384};
385
386/// Given function attributes from a call site for the original function,
387/// return function attributes appropriate for a call to the transformed
388/// function.
390transformFunctionAttributes(const TransformedFunction &TransformedFunction,
391 LLVMContext &Ctx, AttributeList CallSiteAttrs) {
392
393 // Construct a vector of AttributeSet for each function argument.
394 std::vector<llvm::AttributeSet> ArgumentAttributes(
395 TransformedFunction.TransformedType->getNumParams());
396
397 // Copy attributes from the parameter of the original function to the
398 // transformed version. 'ArgumentIndexMapping' holds the mapping from
399 // old argument position to new.
400 for (unsigned I = 0, IE = TransformedFunction.ArgumentIndexMapping.size();
401 I < IE; ++I) {
402 unsigned TransformedIndex = TransformedFunction.ArgumentIndexMapping[I];
403 ArgumentAttributes[TransformedIndex] = CallSiteAttrs.getParamAttrs(I);
404 }
405
406 // Copy annotations on varargs arguments.
407 for (unsigned I = TransformedFunction.OriginalType->getNumParams(),
408 IE = CallSiteAttrs.getNumAttrSets();
409 I < IE; ++I) {
410 ArgumentAttributes.push_back(CallSiteAttrs.getParamAttrs(I));
411 }
412
413 return AttributeList::get(Ctx, CallSiteAttrs.getFnAttrs(),
414 CallSiteAttrs.getRetAttrs(),
415 llvm::ArrayRef(ArgumentAttributes));
416}
417
418class DataFlowSanitizer {
419 friend struct DFSanFunction;
420 friend class DFSanVisitor;
421
422 enum { ShadowWidthBits = 8, ShadowWidthBytes = ShadowWidthBits / 8 };
423
424 enum { OriginWidthBits = 32, OriginWidthBytes = OriginWidthBits / 8 };
425
426 /// How should calls to uninstrumented functions be handled?
427 enum WrapperKind {
428 /// This function is present in an uninstrumented form but we don't know
429 /// how it should be handled. Print a warning and call the function anyway.
430 /// Don't label the return value.
431 WK_Warning,
432
433 /// This function does not write to (user-accessible) memory, and its return
434 /// value is unlabelled.
435 WK_Discard,
436
437 /// This function does not write to (user-accessible) memory, and the label
438 /// of its return value is the union of the label of its arguments.
439 WK_Functional,
440
441 /// Instead of calling the function, a custom wrapper __dfsw_F is called,
442 /// where F is the name of the function. This function may wrap the
443 /// original function or provide its own implementation. WK_Custom uses an
444 /// extra pointer argument to return the shadow. This allows the wrapped
445 /// form of the function type to be expressed in C.
446 WK_Custom
447 };
448
449 Module *Mod;
450 LLVMContext *Ctx;
451 Type *Int8Ptr;
452 IntegerType *OriginTy;
453 PointerType *OriginPtrTy;
454 ConstantInt *ZeroOrigin;
455 /// The shadow type for all primitive types and vector types.
456 IntegerType *PrimitiveShadowTy;
457 PointerType *PrimitiveShadowPtrTy;
458 IntegerType *IntptrTy;
459 ConstantInt *ZeroPrimitiveShadow;
460 Constant *ArgTLS;
461 ArrayType *ArgOriginTLSTy;
462 Constant *ArgOriginTLS;
463 Constant *RetvalTLS;
464 Constant *RetvalOriginTLS;
465 FunctionType *DFSanUnionLoadFnTy;
466 FunctionType *DFSanLoadLabelAndOriginFnTy;
467 FunctionType *DFSanUnimplementedFnTy;
468 FunctionType *DFSanWrapperExternWeakNullFnTy;
469 FunctionType *DFSanSetLabelFnTy;
470 FunctionType *DFSanNonzeroLabelFnTy;
471 FunctionType *DFSanVarargWrapperFnTy;
472 FunctionType *DFSanConditionalCallbackFnTy;
473 FunctionType *DFSanConditionalCallbackOriginFnTy;
474 FunctionType *DFSanReachesFunctionCallbackFnTy;
475 FunctionType *DFSanReachesFunctionCallbackOriginFnTy;
476 FunctionType *DFSanCmpCallbackFnTy;
477 FunctionType *DFSanLoadStoreCallbackFnTy;
478 FunctionType *DFSanMemTransferCallbackFnTy;
479 FunctionType *DFSanChainOriginFnTy;
480 FunctionType *DFSanChainOriginIfTaintedFnTy;
481 FunctionType *DFSanMemOriginTransferFnTy;
482 FunctionType *DFSanMemShadowOriginTransferFnTy;
483 FunctionType *DFSanMemShadowOriginConditionalExchangeFnTy;
484 FunctionType *DFSanMaybeStoreOriginFnTy;
485 FunctionCallee DFSanUnionLoadFn;
486 FunctionCallee DFSanLoadLabelAndOriginFn;
487 FunctionCallee DFSanUnimplementedFn;
488 FunctionCallee DFSanWrapperExternWeakNullFn;
489 FunctionCallee DFSanSetLabelFn;
490 FunctionCallee DFSanNonzeroLabelFn;
491 FunctionCallee DFSanVarargWrapperFn;
492 FunctionCallee DFSanLoadCallbackFn;
493 FunctionCallee DFSanStoreCallbackFn;
494 FunctionCallee DFSanMemTransferCallbackFn;
495 FunctionCallee DFSanConditionalCallbackFn;
496 FunctionCallee DFSanConditionalCallbackOriginFn;
497 FunctionCallee DFSanReachesFunctionCallbackFn;
498 FunctionCallee DFSanReachesFunctionCallbackOriginFn;
499 FunctionCallee DFSanCmpCallbackFn;
500 FunctionCallee DFSanChainOriginFn;
501 FunctionCallee DFSanChainOriginIfTaintedFn;
502 FunctionCallee DFSanMemOriginTransferFn;
503 FunctionCallee DFSanMemShadowOriginTransferFn;
504 FunctionCallee DFSanMemShadowOriginConditionalExchangeFn;
505 FunctionCallee DFSanMaybeStoreOriginFn;
506 SmallPtrSet<Value *, 16> DFSanRuntimeFunctions;
507 MDNode *ColdCallWeights;
508 MDNode *OriginStoreWeights;
509 DFSanABIList ABIList;
510 DenseMap<Value *, Function *> UnwrappedFnMap;
511 AttributeMask ReadOnlyNoneAttrs;
512 StringSet<> CombineTaintLookupTableNames;
513
514 /// Memory map parameters used in calculation mapping application addresses
515 /// to shadow addresses and origin addresses.
516 const MemoryMapParams *MapParams;
517
518 Value *getShadowOffset(Value *Addr, IRBuilder<> &IRB);
519 Value *getShadowAddress(Value *Addr, BasicBlock::iterator Pos);
520 Value *getShadowAddress(Value *Addr, BasicBlock::iterator Pos,
521 Value *ShadowOffset);
522 std::pair<Value *, Value *> getShadowOriginAddress(Value *Addr,
523 Align InstAlignment,
525 bool isInstrumented(const Function *F);
526 bool isInstrumented(const GlobalAlias *GA);
527 bool isForceZeroLabels(const Function *F);
528 TransformedFunction getCustomFunctionType(FunctionType *T);
529 WrapperKind getWrapperKind(Function *F);
530 void addGlobalNameSuffix(GlobalValue *GV);
531 void buildExternWeakCheckIfNeeded(IRBuilder<> &IRB, Function *F);
532 Function *buildWrapperFunction(Function *F, StringRef NewFName,
534 FunctionType *NewFT);
535 void initializeCallbackFunctions(Module &M);
536 void initializeRuntimeFunctions(Module &M);
537 bool initializeModule(Module &M);
538
539 /// Advances \p OriginAddr to point to the next 32-bit origin and then loads
540 /// from it. Returns the origin's loaded value.
541 Value *loadNextOrigin(BasicBlock::iterator Pos, Align OriginAlign,
542 Value **OriginAddr);
543
544 /// Returns whether the given load byte size is amenable to inlined
545 /// optimization patterns.
546 bool hasLoadSizeForFastPath(uint64_t Size);
547
548 /// Returns whether the pass tracks origins. Supports only TLS ABI mode.
549 bool shouldTrackOrigins();
550
551 /// Returns a zero constant with the shadow type of OrigTy.
552 ///
553 /// getZeroShadow({T1,T2,...}) = {getZeroShadow(T1),getZeroShadow(T2,...}
554 /// getZeroShadow([n x T]) = [n x getZeroShadow(T)]
555 /// getZeroShadow(other type) = i16(0)
556 Constant *getZeroShadow(Type *OrigTy);
557 /// Returns a zero constant with the shadow type of V's type.
558 Constant *getZeroShadow(Value *V);
559
560 /// Checks if V is a zero shadow.
561 bool isZeroShadow(Value *V);
562
563 /// Returns the shadow type of OrigTy.
564 ///
565 /// getShadowTy({T1,T2,...}) = {getShadowTy(T1),getShadowTy(T2),...}
566 /// getShadowTy([n x T]) = [n x getShadowTy(T)]
567 /// getShadowTy(other type) = i16
568 Type *getShadowTy(Type *OrigTy);
569 /// Returns the shadow type of V's type.
570 Type *getShadowTy(Value *V);
571
572 const uint64_t NumOfElementsInArgOrgTLS = ArgTLSSize / OriginWidthBytes;
573
574public:
575 DataFlowSanitizer(const std::vector<std::string> &ABIListFiles);
576
577 bool runImpl(Module &M,
579};
580
581struct DFSanFunction {
582 DataFlowSanitizer &DFS;
583 Function *F;
584 DominatorTree DT;
585 bool IsNativeABI;
586 bool IsForceZeroLabels;
588 AllocaInst *LabelReturnAlloca = nullptr;
589 AllocaInst *OriginReturnAlloca = nullptr;
590 DenseMap<Value *, Value *> ValShadowMap;
591 DenseMap<Value *, Value *> ValOriginMap;
594
595 struct PHIFixupElement {
596 PHINode *Phi;
597 PHINode *ShadowPhi;
598 PHINode *OriginPhi;
599 };
600 std::vector<PHIFixupElement> PHIFixups;
601
602 DenseSet<Instruction *> SkipInsts;
603 std::vector<Value *> NonZeroChecks;
604
605 struct CachedShadow {
606 BasicBlock *Block; // The block where Shadow is defined.
607 Value *Shadow;
608 };
609 /// Maps a value to its latest shadow value in terms of domination tree.
610 DenseMap<std::pair<Value *, Value *>, CachedShadow> CachedShadows;
611 /// Maps a value to its latest collapsed shadow value it was converted to in
612 /// terms of domination tree. When ClDebugNonzeroLabels is on, this cache is
613 /// used at a post process where CFG blocks are split. So it does not cache
614 /// BasicBlock like CachedShadows, but uses domination between values.
615 DenseMap<Value *, Value *> CachedCollapsedShadows;
617
618 DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI,
619 bool IsForceZeroLabels, TargetLibraryInfo &TLI)
620 : DFS(DFS), F(F), IsNativeABI(IsNativeABI),
621 IsForceZeroLabels(IsForceZeroLabels), TLI(TLI) {
622 DT.recalculate(*F);
623 }
624
625 /// Computes the shadow address for a given function argument.
626 ///
627 /// Shadow = ArgTLS+ArgOffset.
628 Value *getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB);
629
630 /// Computes the shadow address for a return value.
631 Value *getRetvalTLS(Type *T, IRBuilder<> &IRB);
632
633 /// Computes the origin address for a given function argument.
634 ///
635 /// Origin = ArgOriginTLS[ArgNo].
636 Value *getArgOriginTLS(unsigned ArgNo, IRBuilder<> &IRB);
637
638 /// Computes the origin address for a return value.
639 Value *getRetvalOriginTLS();
640
641 Value *getOrigin(Value *V);
642 void setOrigin(Instruction *I, Value *Origin);
643 /// Generates IR to compute the origin of the last operand with a taint label.
644 Value *combineOperandOrigins(Instruction *Inst);
645 /// Before the instruction Pos, generates IR to compute the last origin with a
646 /// taint label. Labels and origins are from vectors Shadows and Origins
647 /// correspondingly. The generated IR is like
648 /// Sn-1 != Zero ? On-1: ... S2 != Zero ? O2: S1 != Zero ? O1: O0
649 /// When Zero is nullptr, it uses ZeroPrimitiveShadow. Otherwise it can be
650 /// zeros with other bitwidths.
651 Value *combineOrigins(const std::vector<Value *> &Shadows,
652 const std::vector<Value *> &Origins,
653 BasicBlock::iterator Pos, ConstantInt *Zero = nullptr);
654
655 Value *getShadow(Value *V);
656 void setShadow(Instruction *I, Value *Shadow);
657 /// Generates IR to compute the union of the two given shadows, inserting it
658 /// before Pos. The combined value is with primitive type.
659 Value *combineShadows(Value *V1, Value *V2, BasicBlock::iterator Pos);
660 /// Combines the shadow values of V1 and V2, then converts the combined value
661 /// with primitive type into a shadow value with the original type T.
662 Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
664 Value *combineOperandShadows(Instruction *Inst);
665
666 /// Generates IR to load shadow and origin corresponding to bytes [\p
667 /// Addr, \p Addr + \p Size), where addr has alignment \p
668 /// InstAlignment, and take the union of each of those shadows. The returned
669 /// shadow always has primitive type.
670 ///
671 /// When tracking loads is enabled, the returned origin is a chain at the
672 /// current stack if the returned shadow is tainted.
673 std::pair<Value *, Value *> loadShadowOrigin(Value *Addr, uint64_t Size,
674 Align InstAlignment,
676
677 void storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
678 Align InstAlignment, Value *PrimitiveShadow,
679 Value *Origin, BasicBlock::iterator Pos);
680 /// Applies PrimitiveShadow to all primitive subtypes of T, returning
681 /// the expanded shadow value.
682 ///
683 /// EFP({T1,T2, ...}, PS) = {EFP(T1,PS),EFP(T2,PS),...}
684 /// EFP([n x T], PS) = [n x EFP(T,PS)]
685 /// EFP(other types, PS) = PS
686 Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
688 /// Collapses Shadow into a single primitive shadow value, unioning all
689 /// primitive shadow values in the process. Returns the final primitive
690 /// shadow value.
691 ///
692 /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...)
693 /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...)
694 /// CTP(other types, PS) = PS
695 Value *collapseToPrimitiveShadow(Value *Shadow, BasicBlock::iterator Pos);
696
697 void storeZeroPrimitiveShadow(Value *Addr, uint64_t Size, Align ShadowAlign,
699
700 Align getShadowAlign(Align InstAlignment);
701
702 // If ClConditionalCallbacks is enabled, insert a callback after a given
703 // branch instruction using the given conditional expression.
704 void addConditionalCallbacksIfEnabled(Instruction &I, Value *Condition);
705
706 // If ClReachesFunctionCallbacks is enabled, insert a callback for each
707 // argument and load instruction.
708 void addReachesFunctionCallbacksIfEnabled(IRBuilder<> &IRB, Instruction &I,
709 Value *Data);
710
711 bool isLookupTableConstant(Value *P);
712
713private:
714 /// Collapses the shadow with aggregate type into a single primitive shadow
715 /// value.
716 template <class AggregateType>
717 Value *collapseAggregateShadow(AggregateType *AT, Value *Shadow,
718 IRBuilder<> &IRB);
719
720 Value *collapseToPrimitiveShadow(Value *Shadow, IRBuilder<> &IRB);
721
722 /// Returns the shadow value of an argument A.
723 Value *getShadowForTLSArgument(Argument *A);
724
725 /// The fast path of loading shadows.
726 std::pair<Value *, Value *>
727 loadShadowFast(Value *ShadowAddr, Value *OriginAddr, uint64_t Size,
728 Align ShadowAlign, Align OriginAlign, Value *FirstOrigin,
730
731 Align getOriginAlign(Align InstAlignment);
732
733 /// Because 4 contiguous bytes share one 4-byte origin, the most accurate load
734 /// is __dfsan_load_label_and_origin. This function returns the union of all
735 /// labels and the origin of the first taint label. However this is an
736 /// additional call with many instructions. To ensure common cases are fast,
737 /// checks if it is possible to load labels and origins without using the
738 /// callback function.
739 ///
740 /// When enabling tracking load instructions, we always use
741 /// __dfsan_load_label_and_origin to reduce code size.
742 bool useCallbackLoadLabelAndOrigin(uint64_t Size, Align InstAlignment);
743
744 /// Returns a chain at the current stack with previous origin V.
745 Value *updateOrigin(Value *V, IRBuilder<> &IRB);
746
747 /// Returns a chain at the current stack with previous origin V if Shadow is
748 /// tainted.
749 Value *updateOriginIfTainted(Value *Shadow, Value *Origin, IRBuilder<> &IRB);
750
751 /// Creates an Intptr = Origin | Origin << 32 if Intptr's size is 64. Returns
752 /// Origin otherwise.
753 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin);
754
755 /// Stores Origin into the address range [StoreOriginAddr, StoreOriginAddr +
756 /// Size).
757 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *StoreOriginAddr,
758 uint64_t StoreOriginSize, Align Alignment);
759
760 /// Stores Origin in terms of its Shadow value.
761 /// * Do not write origins for zero shadows because we do not trace origins
762 /// for untainted sinks.
763 /// * Use __dfsan_maybe_store_origin if there are too many origin store
764 /// instrumentations.
765 void storeOrigin(BasicBlock::iterator Pos, Value *Addr, uint64_t Size,
766 Value *Shadow, Value *Origin, Value *StoreOriginAddr,
767 Align InstAlignment);
768
769 /// Convert a scalar value to an i1 by comparing with 0.
770 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &Name = "");
771
772 bool shouldInstrumentWithCall();
773
774 /// Generates IR to load shadow and origin corresponding to bytes [\p
775 /// Addr, \p Addr + \p Size), where addr has alignment \p
776 /// InstAlignment, and take the union of each of those shadows. The returned
777 /// shadow always has primitive type.
778 std::pair<Value *, Value *>
779 loadShadowOriginSansLoadTracking(Value *Addr, uint64_t Size,
780 Align InstAlignment,
782 int NumOriginStores = 0;
783};
784
785class DFSanVisitor : public InstVisitor<DFSanVisitor> {
786public:
787 DFSanFunction &DFSF;
788
789 DFSanVisitor(DFSanFunction &DFSF) : DFSF(DFSF) {}
790
791 const DataLayout &getDataLayout() const {
792 return DFSF.F->getDataLayout();
793 }
794
795 // Combines shadow values and origins for all of I's operands.
796 void visitInstOperands(Instruction &I);
797
800 void visitBitCastInst(BitCastInst &BCI);
801 void visitCastInst(CastInst &CI);
802 void visitCmpInst(CmpInst &CI);
805 void visitLoadInst(LoadInst &LI);
806 void visitStoreInst(StoreInst &SI);
809 void visitReturnInst(ReturnInst &RI);
810 void visitLibAtomicLoad(CallBase &CB);
811 void visitLibAtomicStore(CallBase &CB);
812 void visitLibAtomicExchange(CallBase &CB);
813 void visitLibAtomicCompareExchange(CallBase &CB);
814 void visitCallBase(CallBase &CB);
815 void visitPHINode(PHINode &PN);
825 void visitBranchInst(BranchInst &BR);
826 void visitSwitchInst(SwitchInst &SW);
827
828private:
829 void visitCASOrRMW(Align InstAlignment, Instruction &I);
830
831 // Returns false when this is an invoke of a custom function.
832 bool visitWrappedCallBase(Function &F, CallBase &CB);
833
834 // Combines origins for all of I's operands.
835 void visitInstOperandOrigins(Instruction &I);
836
837 void addShadowArguments(Function &F, CallBase &CB, std::vector<Value *> &Args,
838 IRBuilder<> &IRB);
839
840 void addOriginArguments(Function &F, CallBase &CB, std::vector<Value *> &Args,
841 IRBuilder<> &IRB);
842
843 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB);
844 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB);
845};
846
847bool LibAtomicFunction(const Function &F) {
848 // This is a bit of a hack because TargetLibraryInfo is a function pass.
849 // The DFSan pass would need to be refactored to be function pass oriented
850 // (like MSan is) in order to fit together nicely with TargetLibraryInfo.
851 // We need this check to prevent them from being instrumented, or wrapped.
852 // Match on name and number of arguments.
853 if (!F.hasName() || F.isVarArg())
854 return false;
855 switch (F.arg_size()) {
856 case 4:
857 return F.getName() == "__atomic_load" || F.getName() == "__atomic_store";
858 case 5:
859 return F.getName() == "__atomic_exchange";
860 case 6:
861 return F.getName() == "__atomic_compare_exchange";
862 default:
863 return false;
864 }
865}
866
867} // end anonymous namespace
868
869DataFlowSanitizer::DataFlowSanitizer(
870 const std::vector<std::string> &ABIListFiles) {
871 std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
872 llvm::append_range(AllABIListFiles, ClABIListFiles);
873 // FIXME: should we propagate vfs::FileSystem to this constructor?
874 ABIList.set(
876
878 CombineTaintLookupTableNames.insert(v);
879}
880
881TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
882 SmallVector<Type *, 4> ArgTypes;
883
884 // Some parameters of the custom function being constructed are
885 // parameters of T. Record the mapping from parameters of T to
886 // parameters of the custom function, so that parameter attributes
887 // at call sites can be updated.
888 std::vector<unsigned> ArgumentIndexMapping;
889 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I) {
890 Type *ParamType = T->getParamType(I);
891 ArgumentIndexMapping.push_back(ArgTypes.size());
892 ArgTypes.push_back(ParamType);
893 }
894 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I)
895 ArgTypes.push_back(PrimitiveShadowTy);
896 if (T->isVarArg())
897 ArgTypes.push_back(PrimitiveShadowPtrTy);
898 Type *RetType = T->getReturnType();
899 if (!RetType->isVoidTy())
900 ArgTypes.push_back(PrimitiveShadowPtrTy);
901
902 if (shouldTrackOrigins()) {
903 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I)
904 ArgTypes.push_back(OriginTy);
905 if (T->isVarArg())
906 ArgTypes.push_back(OriginPtrTy);
907 if (!RetType->isVoidTy())
908 ArgTypes.push_back(OriginPtrTy);
909 }
910
911 return TransformedFunction(
912 T, FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg()),
913 ArgumentIndexMapping);
914}
915
916bool DataFlowSanitizer::isZeroShadow(Value *V) {
917 Type *T = V->getType();
918 if (!isa<ArrayType>(T) && !isa<StructType>(T)) {
919 if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
920 return CI->isZero();
921 return false;
922 }
923
924 return isa<ConstantAggregateZero>(V);
925}
926
927bool DataFlowSanitizer::hasLoadSizeForFastPath(uint64_t Size) {
928 uint64_t ShadowSize = Size * ShadowWidthBytes;
929 return ShadowSize % 8 == 0 || ShadowSize == 4;
930}
931
932bool DataFlowSanitizer::shouldTrackOrigins() {
933 static const bool ShouldTrackOrigins = ClTrackOrigins;
934 return ShouldTrackOrigins;
935}
936
937Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
938 if (!isa<ArrayType>(OrigTy) && !isa<StructType>(OrigTy))
939 return ZeroPrimitiveShadow;
940 Type *ShadowTy = getShadowTy(OrigTy);
941 return ConstantAggregateZero::get(ShadowTy);
942}
943
944Constant *DataFlowSanitizer::getZeroShadow(Value *V) {
945 return getZeroShadow(V->getType());
946}
947
949 Value *Shadow, SmallVector<unsigned, 4> &Indices, Type *SubShadowTy,
950 Value *PrimitiveShadow, IRBuilder<> &IRB) {
951 if (!isa<ArrayType>(SubShadowTy) && !isa<StructType>(SubShadowTy))
952 return IRB.CreateInsertValue(Shadow, PrimitiveShadow, Indices);
953
954 if (ArrayType *AT = dyn_cast<ArrayType>(SubShadowTy)) {
955 for (unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) {
956 Indices.push_back(Idx);
958 Shadow, Indices, AT->getElementType(), PrimitiveShadow, IRB);
959 Indices.pop_back();
960 }
961 return Shadow;
962 }
963
964 if (StructType *ST = dyn_cast<StructType>(SubShadowTy)) {
965 for (unsigned Idx = 0; Idx < ST->getNumElements(); Idx++) {
966 Indices.push_back(Idx);
968 Shadow, Indices, ST->getElementType(Idx), PrimitiveShadow, IRB);
969 Indices.pop_back();
970 }
971 return Shadow;
972 }
973 llvm_unreachable("Unexpected shadow type");
974}
975
976bool DFSanFunction::shouldInstrumentWithCall() {
977 return ClInstrumentWithCallThreshold >= 0 &&
978 NumOriginStores >= ClInstrumentWithCallThreshold;
979}
980
981Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
983 Type *ShadowTy = DFS.getShadowTy(T);
984
985 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
986 return PrimitiveShadow;
987
988 if (DFS.isZeroShadow(PrimitiveShadow))
989 return DFS.getZeroShadow(ShadowTy);
990
991 IRBuilder<> IRB(Pos->getParent(), Pos);
993 Value *Shadow = UndefValue::get(ShadowTy);
994 Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, ShadowTy,
995 PrimitiveShadow, IRB);
996
997 // Caches the primitive shadow value that built the shadow value.
998 CachedCollapsedShadows[Shadow] = PrimitiveShadow;
999 return Shadow;
1000}
1001
1002template <class AggregateType>
1003Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT, Value *Shadow,
1004 IRBuilder<> &IRB) {
1005 if (!AT->getNumElements())
1006 return DFS.ZeroPrimitiveShadow;
1007
1008 Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
1009 Value *Aggregator = collapseToPrimitiveShadow(FirstItem, IRB);
1010
1011 for (unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) {
1012 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1013 Value *ShadowInner = collapseToPrimitiveShadow(ShadowItem, IRB);
1014 Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1015 }
1016 return Aggregator;
1017}
1018
1019Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
1020 IRBuilder<> &IRB) {
1021 Type *ShadowTy = Shadow->getType();
1022 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
1023 return Shadow;
1024 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy))
1025 return collapseAggregateShadow<>(AT, Shadow, IRB);
1026 if (StructType *ST = dyn_cast<StructType>(ShadowTy))
1027 return collapseAggregateShadow<>(ST, Shadow, IRB);
1028 llvm_unreachable("Unexpected shadow type");
1029}
1030
1031Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
1033 Type *ShadowTy = Shadow->getType();
1034 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
1035 return Shadow;
1036
1037 // Checks if the cached collapsed shadow value dominates Pos.
1038 Value *&CS = CachedCollapsedShadows[Shadow];
1039 if (CS && DT.dominates(CS, Pos))
1040 return CS;
1041
1042 IRBuilder<> IRB(Pos->getParent(), Pos);
1043 Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB);
1044 // Caches the converted primitive shadow value.
1045 CS = PrimitiveShadow;
1046 return PrimitiveShadow;
1047}
1048
1049void DFSanFunction::addConditionalCallbacksIfEnabled(Instruction &I,
1050 Value *Condition) {
1052 return;
1053 }
1054 IRBuilder<> IRB(&I);
1055 Value *CondShadow = getShadow(Condition);
1056 CallInst *CI;
1057 if (DFS.shouldTrackOrigins()) {
1058 Value *CondOrigin = getOrigin(Condition);
1059 CI = IRB.CreateCall(DFS.DFSanConditionalCallbackOriginFn,
1060 {CondShadow, CondOrigin});
1061 } else {
1062 CI = IRB.CreateCall(DFS.DFSanConditionalCallbackFn, {CondShadow});
1063 }
1064 CI->addParamAttr(0, Attribute::ZExt);
1065}
1066
1067void DFSanFunction::addReachesFunctionCallbacksIfEnabled(IRBuilder<> &IRB,
1068 Instruction &I,
1069 Value *Data) {
1071 return;
1072 }
1073 const DebugLoc &dbgloc = I.getDebugLoc();
1074 Value *DataShadow = collapseToPrimitiveShadow(getShadow(Data), IRB);
1075 ConstantInt *CILine;
1076 llvm::Value *FilePathPtr;
1077
1078 if (dbgloc.get() == nullptr) {
1079 CILine = llvm::ConstantInt::get(I.getContext(), llvm::APInt(32, 0));
1080 FilePathPtr = IRB.CreateGlobalStringPtr(
1081 I.getFunction()->getParent()->getSourceFileName());
1082 } else {
1083 CILine = llvm::ConstantInt::get(I.getContext(),
1084 llvm::APInt(32, dbgloc.getLine()));
1085 FilePathPtr =
1086 IRB.CreateGlobalStringPtr(dbgloc->getFilename());
1087 }
1088
1089 llvm::Value *FunctionNamePtr =
1090 IRB.CreateGlobalStringPtr(I.getFunction()->getName());
1091
1092 CallInst *CB;
1093 std::vector<Value *> args;
1094
1095 if (DFS.shouldTrackOrigins()) {
1096 Value *DataOrigin = getOrigin(Data);
1097 args = { DataShadow, DataOrigin, FilePathPtr, CILine, FunctionNamePtr };
1098 CB = IRB.CreateCall(DFS.DFSanReachesFunctionCallbackOriginFn, args);
1099 } else {
1100 args = { DataShadow, FilePathPtr, CILine, FunctionNamePtr };
1101 CB = IRB.CreateCall(DFS.DFSanReachesFunctionCallbackFn, args);
1102 }
1103 CB->addParamAttr(0, Attribute::ZExt);
1104 CB->setDebugLoc(dbgloc);
1105}
1106
1107Type *DataFlowSanitizer::getShadowTy(Type *OrigTy) {
1108 if (!OrigTy->isSized())
1109 return PrimitiveShadowTy;
1110 if (isa<IntegerType>(OrigTy))
1111 return PrimitiveShadowTy;
1112 if (isa<VectorType>(OrigTy))
1113 return PrimitiveShadowTy;
1114 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy))
1115 return ArrayType::get(getShadowTy(AT->getElementType()),
1116 AT->getNumElements());
1117 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1119 for (unsigned I = 0, N = ST->getNumElements(); I < N; ++I)
1120 Elements.push_back(getShadowTy(ST->getElementType(I)));
1121 return StructType::get(*Ctx, Elements);
1122 }
1123 return PrimitiveShadowTy;
1124}
1125
1126Type *DataFlowSanitizer::getShadowTy(Value *V) {
1127 return getShadowTy(V->getType());
1128}
1129
1130bool DataFlowSanitizer::initializeModule(Module &M) {
1131 Triple TargetTriple(M.getTargetTriple());
1132 const DataLayout &DL = M.getDataLayout();
1133
1134 if (TargetTriple.getOS() != Triple::Linux)
1135 report_fatal_error("unsupported operating system");
1136 switch (TargetTriple.getArch()) {
1137 case Triple::aarch64:
1138 MapParams = &Linux_AArch64_MemoryMapParams;
1139 break;
1140 case Triple::x86_64:
1141 MapParams = &Linux_X86_64_MemoryMapParams;
1142 break;
1145 break;
1146 default:
1147 report_fatal_error("unsupported architecture");
1148 }
1149
1150 Mod = &M;
1151 Ctx = &M.getContext();
1152 Int8Ptr = PointerType::getUnqual(*Ctx);
1153 OriginTy = IntegerType::get(*Ctx, OriginWidthBits);
1154 OriginPtrTy = PointerType::getUnqual(OriginTy);
1155 PrimitiveShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
1156 PrimitiveShadowPtrTy = PointerType::getUnqual(PrimitiveShadowTy);
1157 IntptrTy = DL.getIntPtrType(*Ctx);
1158 ZeroPrimitiveShadow = ConstantInt::getSigned(PrimitiveShadowTy, 0);
1159 ZeroOrigin = ConstantInt::getSigned(OriginTy, 0);
1160
1161 Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1162 DFSanUnionLoadFnTy = FunctionType::get(PrimitiveShadowTy, DFSanUnionLoadArgs,
1163 /*isVarArg=*/false);
1164 Type *DFSanLoadLabelAndOriginArgs[2] = {Int8Ptr, IntptrTy};
1165 DFSanLoadLabelAndOriginFnTy =
1166 FunctionType::get(IntegerType::get(*Ctx, 64), DFSanLoadLabelAndOriginArgs,
1167 /*isVarArg=*/false);
1168 DFSanUnimplementedFnTy = FunctionType::get(
1169 Type::getVoidTy(*Ctx), PointerType::getUnqual(*Ctx), /*isVarArg=*/false);
1170 Type *DFSanWrapperExternWeakNullArgs[2] = {Int8Ptr, Int8Ptr};
1171 DFSanWrapperExternWeakNullFnTy =
1172 FunctionType::get(Type::getVoidTy(*Ctx), DFSanWrapperExternWeakNullArgs,
1173 /*isVarArg=*/false);
1174 Type *DFSanSetLabelArgs[4] = {PrimitiveShadowTy, OriginTy,
1175 PointerType::getUnqual(*Ctx), IntptrTy};
1176 DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
1177 DFSanSetLabelArgs, /*isVarArg=*/false);
1178 DFSanNonzeroLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx), std::nullopt,
1179 /*isVarArg=*/false);
1180 DFSanVarargWrapperFnTy = FunctionType::get(
1181 Type::getVoidTy(*Ctx), PointerType::getUnqual(*Ctx), /*isVarArg=*/false);
1182 DFSanConditionalCallbackFnTy =
1183 FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
1184 /*isVarArg=*/false);
1185 Type *DFSanConditionalCallbackOriginArgs[2] = {PrimitiveShadowTy, OriginTy};
1186 DFSanConditionalCallbackOriginFnTy = FunctionType::get(
1187 Type::getVoidTy(*Ctx), DFSanConditionalCallbackOriginArgs,
1188 /*isVarArg=*/false);
1189 Type *DFSanReachesFunctionCallbackArgs[4] = {PrimitiveShadowTy, Int8Ptr,
1190 OriginTy, Int8Ptr};
1191 DFSanReachesFunctionCallbackFnTy =
1192 FunctionType::get(Type::getVoidTy(*Ctx), DFSanReachesFunctionCallbackArgs,
1193 /*isVarArg=*/false);
1194 Type *DFSanReachesFunctionCallbackOriginArgs[5] = {
1195 PrimitiveShadowTy, OriginTy, Int8Ptr, OriginTy, Int8Ptr};
1196 DFSanReachesFunctionCallbackOriginFnTy = FunctionType::get(
1197 Type::getVoidTy(*Ctx), DFSanReachesFunctionCallbackOriginArgs,
1198 /*isVarArg=*/false);
1199 DFSanCmpCallbackFnTy =
1200 FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
1201 /*isVarArg=*/false);
1202 DFSanChainOriginFnTy =
1203 FunctionType::get(OriginTy, OriginTy, /*isVarArg=*/false);
1204 Type *DFSanChainOriginIfTaintedArgs[2] = {PrimitiveShadowTy, OriginTy};
1205 DFSanChainOriginIfTaintedFnTy = FunctionType::get(
1206 OriginTy, DFSanChainOriginIfTaintedArgs, /*isVarArg=*/false);
1207 Type *DFSanMaybeStoreOriginArgs[4] = {IntegerType::get(*Ctx, ShadowWidthBits),
1208 Int8Ptr, IntptrTy, OriginTy};
1209 DFSanMaybeStoreOriginFnTy = FunctionType::get(
1210 Type::getVoidTy(*Ctx), DFSanMaybeStoreOriginArgs, /*isVarArg=*/false);
1211 Type *DFSanMemOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1212 DFSanMemOriginTransferFnTy = FunctionType::get(
1213 Type::getVoidTy(*Ctx), DFSanMemOriginTransferArgs, /*isVarArg=*/false);
1214 Type *DFSanMemShadowOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1215 DFSanMemShadowOriginTransferFnTy =
1216 FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemShadowOriginTransferArgs,
1217 /*isVarArg=*/false);
1218 Type *DFSanMemShadowOriginConditionalExchangeArgs[5] = {
1219 IntegerType::get(*Ctx, 8), Int8Ptr, Int8Ptr, Int8Ptr, IntptrTy};
1220 DFSanMemShadowOriginConditionalExchangeFnTy = FunctionType::get(
1221 Type::getVoidTy(*Ctx), DFSanMemShadowOriginConditionalExchangeArgs,
1222 /*isVarArg=*/false);
1223 Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr};
1224 DFSanLoadStoreCallbackFnTy =
1225 FunctionType::get(Type::getVoidTy(*Ctx), DFSanLoadStoreCallbackArgs,
1226 /*isVarArg=*/false);
1227 Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1228 DFSanMemTransferCallbackFnTy =
1229 FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemTransferCallbackArgs,
1230 /*isVarArg=*/false);
1231
1232 ColdCallWeights = MDBuilder(*Ctx).createUnlikelyBranchWeights();
1233 OriginStoreWeights = MDBuilder(*Ctx).createUnlikelyBranchWeights();
1234 return true;
1235}
1236
1237bool DataFlowSanitizer::isInstrumented(const Function *F) {
1238 return !ABIList.isIn(*F, "uninstrumented");
1239}
1240
1241bool DataFlowSanitizer::isInstrumented(const GlobalAlias *GA) {
1242 return !ABIList.isIn(*GA, "uninstrumented");
1243}
1244
1245bool DataFlowSanitizer::isForceZeroLabels(const Function *F) {
1246 return ABIList.isIn(*F, "force_zero_labels");
1247}
1248
1249DataFlowSanitizer::WrapperKind DataFlowSanitizer::getWrapperKind(Function *F) {
1250 if (ABIList.isIn(*F, "functional"))
1251 return WK_Functional;
1252 if (ABIList.isIn(*F, "discard"))
1253 return WK_Discard;
1254 if (ABIList.isIn(*F, "custom"))
1255 return WK_Custom;
1256
1257 return WK_Warning;
1258}
1259
1260void DataFlowSanitizer::addGlobalNameSuffix(GlobalValue *GV) {
1261 std::string GVName = std::string(GV->getName()), Suffix = ".dfsan";
1262 GV->setName(GVName + Suffix);
1263
1264 // Try to change the name of the function in module inline asm. We only do
1265 // this for specific asm directives, currently only ".symver", to try to avoid
1266 // corrupting asm which happens to contain the symbol name as a substring.
1267 // Note that the substitution for .symver assumes that the versioned symbol
1268 // also has an instrumented name.
1269 std::string Asm = GV->getParent()->getModuleInlineAsm();
1270 std::string SearchStr = ".symver " + GVName + ",";
1271 size_t Pos = Asm.find(SearchStr);
1272 if (Pos != std::string::npos) {
1273 Asm.replace(Pos, SearchStr.size(), ".symver " + GVName + Suffix + ",");
1274 Pos = Asm.find('@');
1275
1276 if (Pos == std::string::npos)
1277 report_fatal_error(Twine("unsupported .symver: ", Asm));
1278
1279 Asm.replace(Pos, 1, Suffix + "@");
1280 GV->getParent()->setModuleInlineAsm(Asm);
1281 }
1282}
1283
1284void DataFlowSanitizer::buildExternWeakCheckIfNeeded(IRBuilder<> &IRB,
1285 Function *F) {
1286 // If the function we are wrapping was ExternWeak, it may be null.
1287 // The original code before calling this wrapper may have checked for null,
1288 // but replacing with a known-to-not-be-null wrapper can break this check.
1289 // When replacing uses of the extern weak function with the wrapper we try
1290 // to avoid replacing uses in conditionals, but this is not perfect.
1291 // In the case where we fail, and accidentally optimize out a null check
1292 // for a extern weak function, add a check here to help identify the issue.
1293 if (GlobalValue::isExternalWeakLinkage(F->getLinkage())) {
1294 std::vector<Value *> Args;
1295 Args.push_back(F);
1296 Args.push_back(IRB.CreateGlobalStringPtr(F->getName()));
1297 IRB.CreateCall(DFSanWrapperExternWeakNullFn, Args);
1298 }
1299}
1300
1301Function *
1302DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
1304 FunctionType *NewFT) {
1305 FunctionType *FT = F->getFunctionType();
1306 Function *NewF = Function::Create(NewFT, NewFLink, F->getAddressSpace(),
1307 NewFName, F->getParent());
1308 NewF->copyAttributesFrom(F);
1309 NewF->removeRetAttrs(
1310 AttributeFuncs::typeIncompatible(NewFT->getReturnType()));
1311
1312 BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", NewF);
1313 if (F->isVarArg()) {
1314 NewF->removeFnAttr("split-stack");
1315 CallInst::Create(DFSanVarargWrapperFn,
1316 IRBuilder<>(BB).CreateGlobalStringPtr(F->getName()), "",
1317 BB);
1318 new UnreachableInst(*Ctx, BB);
1319 } else {
1320 auto ArgIt = pointer_iterator<Argument *>(NewF->arg_begin());
1321 std::vector<Value *> Args(ArgIt, ArgIt + FT->getNumParams());
1322
1323 CallInst *CI = CallInst::Create(F, Args, "", BB);
1324 if (FT->getReturnType()->isVoidTy())
1325 ReturnInst::Create(*Ctx, BB);
1326 else
1327 ReturnInst::Create(*Ctx, CI, BB);
1328 }
1329
1330 return NewF;
1331}
1332
1333// Initialize DataFlowSanitizer runtime functions and declare them in the module
1334void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
1335 LLVMContext &C = M.getContext();
1336 {
1338 AL = AL.addFnAttribute(C, Attribute::NoUnwind);
1339 AL = AL.addFnAttribute(
1341 AL = AL.addRetAttribute(C, Attribute::ZExt);
1342 DFSanUnionLoadFn =
1343 Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
1344 }
1345 {
1347 AL = AL.addFnAttribute(C, Attribute::NoUnwind);
1348 AL = AL.addFnAttribute(
1350 AL = AL.addRetAttribute(C, Attribute::ZExt);
1351 DFSanLoadLabelAndOriginFn = Mod->getOrInsertFunction(
1352 "__dfsan_load_label_and_origin", DFSanLoadLabelAndOriginFnTy, AL);
1353 }
1354 DFSanUnimplementedFn =
1355 Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy);
1356 DFSanWrapperExternWeakNullFn = Mod->getOrInsertFunction(
1357 "__dfsan_wrapper_extern_weak_null", DFSanWrapperExternWeakNullFnTy);
1358 {
1360 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1361 AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
1362 DFSanSetLabelFn =
1363 Mod->getOrInsertFunction("__dfsan_set_label", DFSanSetLabelFnTy, AL);
1364 }
1365 DFSanNonzeroLabelFn =
1366 Mod->getOrInsertFunction("__dfsan_nonzero_label", DFSanNonzeroLabelFnTy);
1367 DFSanVarargWrapperFn = Mod->getOrInsertFunction("__dfsan_vararg_wrapper",
1368 DFSanVarargWrapperFnTy);
1369 {
1371 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1372 AL = AL.addRetAttribute(M.getContext(), Attribute::ZExt);
1373 DFSanChainOriginFn = Mod->getOrInsertFunction("__dfsan_chain_origin",
1374 DFSanChainOriginFnTy, AL);
1375 }
1376 {
1378 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1379 AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
1380 AL = AL.addRetAttribute(M.getContext(), Attribute::ZExt);
1381 DFSanChainOriginIfTaintedFn = Mod->getOrInsertFunction(
1382 "__dfsan_chain_origin_if_tainted", DFSanChainOriginIfTaintedFnTy, AL);
1383 }
1384 DFSanMemOriginTransferFn = Mod->getOrInsertFunction(
1385 "__dfsan_mem_origin_transfer", DFSanMemOriginTransferFnTy);
1386
1387 DFSanMemShadowOriginTransferFn = Mod->getOrInsertFunction(
1388 "__dfsan_mem_shadow_origin_transfer", DFSanMemShadowOriginTransferFnTy);
1389
1390 DFSanMemShadowOriginConditionalExchangeFn =
1391 Mod->getOrInsertFunction("__dfsan_mem_shadow_origin_conditional_exchange",
1392 DFSanMemShadowOriginConditionalExchangeFnTy);
1393
1394 {
1396 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1397 AL = AL.addParamAttribute(M.getContext(), 3, Attribute::ZExt);
1398 DFSanMaybeStoreOriginFn = Mod->getOrInsertFunction(
1399 "__dfsan_maybe_store_origin", DFSanMaybeStoreOriginFnTy, AL);
1400 }
1401
1402 DFSanRuntimeFunctions.insert(
1403 DFSanUnionLoadFn.getCallee()->stripPointerCasts());
1404 DFSanRuntimeFunctions.insert(
1405 DFSanLoadLabelAndOriginFn.getCallee()->stripPointerCasts());
1406 DFSanRuntimeFunctions.insert(
1407 DFSanUnimplementedFn.getCallee()->stripPointerCasts());
1408 DFSanRuntimeFunctions.insert(
1409 DFSanWrapperExternWeakNullFn.getCallee()->stripPointerCasts());
1410 DFSanRuntimeFunctions.insert(
1411 DFSanSetLabelFn.getCallee()->stripPointerCasts());
1412 DFSanRuntimeFunctions.insert(
1413 DFSanNonzeroLabelFn.getCallee()->stripPointerCasts());
1414 DFSanRuntimeFunctions.insert(
1415 DFSanVarargWrapperFn.getCallee()->stripPointerCasts());
1416 DFSanRuntimeFunctions.insert(
1417 DFSanLoadCallbackFn.getCallee()->stripPointerCasts());
1418 DFSanRuntimeFunctions.insert(
1419 DFSanStoreCallbackFn.getCallee()->stripPointerCasts());
1420 DFSanRuntimeFunctions.insert(
1421 DFSanMemTransferCallbackFn.getCallee()->stripPointerCasts());
1422 DFSanRuntimeFunctions.insert(
1423 DFSanConditionalCallbackFn.getCallee()->stripPointerCasts());
1424 DFSanRuntimeFunctions.insert(
1425 DFSanConditionalCallbackOriginFn.getCallee()->stripPointerCasts());
1426 DFSanRuntimeFunctions.insert(
1427 DFSanReachesFunctionCallbackFn.getCallee()->stripPointerCasts());
1428 DFSanRuntimeFunctions.insert(
1429 DFSanReachesFunctionCallbackOriginFn.getCallee()->stripPointerCasts());
1430 DFSanRuntimeFunctions.insert(
1431 DFSanCmpCallbackFn.getCallee()->stripPointerCasts());
1432 DFSanRuntimeFunctions.insert(
1433 DFSanChainOriginFn.getCallee()->stripPointerCasts());
1434 DFSanRuntimeFunctions.insert(
1435 DFSanChainOriginIfTaintedFn.getCallee()->stripPointerCasts());
1436 DFSanRuntimeFunctions.insert(
1437 DFSanMemOriginTransferFn.getCallee()->stripPointerCasts());
1438 DFSanRuntimeFunctions.insert(
1439 DFSanMemShadowOriginTransferFn.getCallee()->stripPointerCasts());
1440 DFSanRuntimeFunctions.insert(
1441 DFSanMemShadowOriginConditionalExchangeFn.getCallee()
1442 ->stripPointerCasts());
1443 DFSanRuntimeFunctions.insert(
1444 DFSanMaybeStoreOriginFn.getCallee()->stripPointerCasts());
1445}
1446
1447// Initializes event callback functions and declare them in the module
1448void DataFlowSanitizer::initializeCallbackFunctions(Module &M) {
1449 {
1451 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1452 DFSanLoadCallbackFn = Mod->getOrInsertFunction(
1453 "__dfsan_load_callback", DFSanLoadStoreCallbackFnTy, AL);
1454 }
1455 {
1457 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1458 DFSanStoreCallbackFn = Mod->getOrInsertFunction(
1459 "__dfsan_store_callback", DFSanLoadStoreCallbackFnTy, AL);
1460 }
1461 DFSanMemTransferCallbackFn = Mod->getOrInsertFunction(
1462 "__dfsan_mem_transfer_callback", DFSanMemTransferCallbackFnTy);
1463 {
1465 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1466 DFSanCmpCallbackFn = Mod->getOrInsertFunction("__dfsan_cmp_callback",
1467 DFSanCmpCallbackFnTy, AL);
1468 }
1469 {
1471 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1472 DFSanConditionalCallbackFn = Mod->getOrInsertFunction(
1473 "__dfsan_conditional_callback", DFSanConditionalCallbackFnTy, AL);
1474 }
1475 {
1477 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1478 DFSanConditionalCallbackOriginFn =
1479 Mod->getOrInsertFunction("__dfsan_conditional_callback_origin",
1480 DFSanConditionalCallbackOriginFnTy, AL);
1481 }
1482 {
1484 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1485 DFSanReachesFunctionCallbackFn =
1486 Mod->getOrInsertFunction("__dfsan_reaches_function_callback",
1487 DFSanReachesFunctionCallbackFnTy, AL);
1488 }
1489 {
1491 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1492 DFSanReachesFunctionCallbackOriginFn =
1493 Mod->getOrInsertFunction("__dfsan_reaches_function_callback_origin",
1494 DFSanReachesFunctionCallbackOriginFnTy, AL);
1495 }
1496}
1497
1498bool DataFlowSanitizer::runImpl(
1500 initializeModule(M);
1501
1502 if (ABIList.isIn(M, "skip"))
1503 return false;
1504
1505 const unsigned InitialGlobalSize = M.global_size();
1506 const unsigned InitialModuleSize = M.size();
1507
1508 bool Changed = false;
1509
1510 auto GetOrInsertGlobal = [this, &Changed](StringRef Name,
1511 Type *Ty) -> Constant * {
1513 if (GlobalVariable *G = dyn_cast<GlobalVariable>(C)) {
1514 Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
1515 G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
1516 }
1517 return C;
1518 };
1519
1520 // These globals must be kept in sync with the ones in dfsan.cpp.
1521 ArgTLS =
1522 GetOrInsertGlobal("__dfsan_arg_tls",
1523 ArrayType::get(Type::getInt64Ty(*Ctx), ArgTLSSize / 8));
1524 RetvalTLS = GetOrInsertGlobal(
1525 "__dfsan_retval_tls",
1526 ArrayType::get(Type::getInt64Ty(*Ctx), RetvalTLSSize / 8));
1527 ArgOriginTLSTy = ArrayType::get(OriginTy, NumOfElementsInArgOrgTLS);
1528 ArgOriginTLS = GetOrInsertGlobal("__dfsan_arg_origin_tls", ArgOriginTLSTy);
1529 RetvalOriginTLS = GetOrInsertGlobal("__dfsan_retval_origin_tls", OriginTy);
1530
1531 (void)Mod->getOrInsertGlobal("__dfsan_track_origins", OriginTy, [&] {
1532 Changed = true;
1533 return new GlobalVariable(
1534 M, OriginTy, true, GlobalValue::WeakODRLinkage,
1535 ConstantInt::getSigned(OriginTy,
1536 shouldTrackOrigins() ? ClTrackOrigins : 0),
1537 "__dfsan_track_origins");
1538 });
1539
1540 initializeCallbackFunctions(M);
1541 initializeRuntimeFunctions(M);
1542
1543 std::vector<Function *> FnsToInstrument;
1544 SmallPtrSet<Function *, 2> FnsWithNativeABI;
1545 SmallPtrSet<Function *, 2> FnsWithForceZeroLabel;
1546 SmallPtrSet<Constant *, 1> PersonalityFns;
1547 for (Function &F : M)
1548 if (!F.isIntrinsic() && !DFSanRuntimeFunctions.contains(&F) &&
1549 !LibAtomicFunction(F) &&
1550 !F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation)) {
1551 FnsToInstrument.push_back(&F);
1552 if (F.hasPersonalityFn())
1553 PersonalityFns.insert(F.getPersonalityFn()->stripPointerCasts());
1554 }
1555
1557 for (auto *C : PersonalityFns) {
1558 assert(isa<Function>(C) && "Personality routine is not a function!");
1559 Function *F = cast<Function>(C);
1560 if (!isInstrumented(F))
1561 llvm::erase(FnsToInstrument, F);
1562 }
1563 }
1564
1565 // Give function aliases prefixes when necessary, and build wrappers where the
1566 // instrumentedness is inconsistent.
1567 for (GlobalAlias &GA : llvm::make_early_inc_range(M.aliases())) {
1568 // Don't stop on weak. We assume people aren't playing games with the
1569 // instrumentedness of overridden weak aliases.
1570 auto *F = dyn_cast<Function>(GA.getAliaseeObject());
1571 if (!F)
1572 continue;
1573
1574 bool GAInst = isInstrumented(&GA), FInst = isInstrumented(F);
1575 if (GAInst && FInst) {
1576 addGlobalNameSuffix(&GA);
1577 } else if (GAInst != FInst) {
1578 // Non-instrumented alias of an instrumented function, or vice versa.
1579 // Replace the alias with a native-ABI wrapper of the aliasee. The pass
1580 // below will take care of instrumenting it.
1581 Function *NewF =
1582 buildWrapperFunction(F, "", GA.getLinkage(), F->getFunctionType());
1583 GA.replaceAllUsesWith(NewF);
1584 NewF->takeName(&GA);
1585 GA.eraseFromParent();
1586 FnsToInstrument.push_back(NewF);
1587 }
1588 }
1589
1590 // TODO: This could be more precise.
1591 ReadOnlyNoneAttrs.addAttribute(Attribute::Memory);
1592
1593 // First, change the ABI of every function in the module. ABI-listed
1594 // functions keep their original ABI and get a wrapper function.
1595 for (std::vector<Function *>::iterator FI = FnsToInstrument.begin(),
1596 FE = FnsToInstrument.end();
1597 FI != FE; ++FI) {
1598 Function &F = **FI;
1599 FunctionType *FT = F.getFunctionType();
1600
1601 bool IsZeroArgsVoidRet = (FT->getNumParams() == 0 && !FT->isVarArg() &&
1602 FT->getReturnType()->isVoidTy());
1603
1604 if (isInstrumented(&F)) {
1605 if (isForceZeroLabels(&F))
1606 FnsWithForceZeroLabel.insert(&F);
1607
1608 // Instrumented functions get a '.dfsan' suffix. This allows us to more
1609 // easily identify cases of mismatching ABIs. This naming scheme is
1610 // mangling-compatible (see Itanium ABI), using a vendor-specific suffix.
1611 addGlobalNameSuffix(&F);
1612 } else if (!IsZeroArgsVoidRet || getWrapperKind(&F) == WK_Custom) {
1613 // Build a wrapper function for F. The wrapper simply calls F, and is
1614 // added to FnsToInstrument so that any instrumentation according to its
1615 // WrapperKind is done in the second pass below.
1616
1617 // If the function being wrapped has local linkage, then preserve the
1618 // function's linkage in the wrapper function.
1619 GlobalValue::LinkageTypes WrapperLinkage =
1620 F.hasLocalLinkage() ? F.getLinkage()
1622
1623 Function *NewF = buildWrapperFunction(
1624 &F,
1625 (shouldTrackOrigins() ? std::string("dfso$") : std::string("dfsw$")) +
1626 std::string(F.getName()),
1627 WrapperLinkage, FT);
1628 NewF->removeFnAttrs(ReadOnlyNoneAttrs);
1629
1630 // Extern weak functions can sometimes be null at execution time.
1631 // Code will sometimes check if an extern weak function is null.
1632 // This could look something like:
1633 // declare extern_weak i8 @my_func(i8)
1634 // br i1 icmp ne (i8 (i8)* @my_func, i8 (i8)* null), label %use_my_func,
1635 // label %avoid_my_func
1636 // The @"dfsw$my_func" wrapper is never null, so if we replace this use
1637 // in the comparison, the icmp will simplify to false and we have
1638 // accidentally optimized away a null check that is necessary.
1639 // This can lead to a crash when the null extern_weak my_func is called.
1640 //
1641 // To prevent (the most common pattern of) this problem,
1642 // do not replace uses in comparisons with the wrapper.
1643 // We definitely want to replace uses in call instructions.
1644 // Other uses (e.g. store the function address somewhere) might be
1645 // called or compared or both - this case may not be handled correctly.
1646 // We will default to replacing with wrapper in cases we are unsure.
1647 auto IsNotCmpUse = [](Use &U) -> bool {
1648 User *Usr = U.getUser();
1649 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
1650 // This is the most common case for icmp ne null
1651 if (CE->getOpcode() == Instruction::ICmp) {
1652 return false;
1653 }
1654 }
1655 if (Instruction *I = dyn_cast<Instruction>(Usr)) {
1656 if (I->getOpcode() == Instruction::ICmp) {
1657 return false;
1658 }
1659 }
1660 return true;
1661 };
1662 F.replaceUsesWithIf(NewF, IsNotCmpUse);
1663
1664 UnwrappedFnMap[NewF] = &F;
1665 *FI = NewF;
1666
1667 if (!F.isDeclaration()) {
1668 // This function is probably defining an interposition of an
1669 // uninstrumented function and hence needs to keep the original ABI.
1670 // But any functions it may call need to use the instrumented ABI, so
1671 // we instrument it in a mode which preserves the original ABI.
1672 FnsWithNativeABI.insert(&F);
1673
1674 // This code needs to rebuild the iterators, as they may be invalidated
1675 // by the push_back, taking care that the new range does not include
1676 // any functions added by this code.
1677 size_t N = FI - FnsToInstrument.begin(),
1678 Count = FE - FnsToInstrument.begin();
1679 FnsToInstrument.push_back(&F);
1680 FI = FnsToInstrument.begin() + N;
1681 FE = FnsToInstrument.begin() + Count;
1682 }
1683 // Hopefully, nobody will try to indirectly call a vararg
1684 // function... yet.
1685 } else if (FT->isVarArg()) {
1686 UnwrappedFnMap[&F] = &F;
1687 *FI = nullptr;
1688 }
1689 }
1690
1691 for (Function *F : FnsToInstrument) {
1692 if (!F || F->isDeclaration())
1693 continue;
1694
1696
1697 DFSanFunction DFSF(*this, F, FnsWithNativeABI.count(F),
1698 FnsWithForceZeroLabel.count(F), GetTLI(*F));
1699
1701 // Add callback for arguments reaching this function.
1702 for (auto &FArg : F->args()) {
1703 Instruction *Next = &F->getEntryBlock().front();
1704 Value *FArgShadow = DFSF.getShadow(&FArg);
1705 if (isZeroShadow(FArgShadow))
1706 continue;
1707 if (Instruction *FArgShadowInst = dyn_cast<Instruction>(FArgShadow)) {
1708 Next = FArgShadowInst->getNextNode();
1709 }
1710 if (shouldTrackOrigins()) {
1711 if (Instruction *Origin =
1712 dyn_cast<Instruction>(DFSF.getOrigin(&FArg))) {
1713 // Ensure IRB insertion point is after loads for shadow and origin.
1714 Instruction *OriginNext = Origin->getNextNode();
1715 if (Next->comesBefore(OriginNext)) {
1716 Next = OriginNext;
1717 }
1718 }
1719 }
1720 IRBuilder<> IRB(Next);
1721 DFSF.addReachesFunctionCallbacksIfEnabled(IRB, *Next, &FArg);
1722 }
1723 }
1724
1725 // DFSanVisitor may create new basic blocks, which confuses df_iterator.
1726 // Build a copy of the list before iterating over it.
1727 SmallVector<BasicBlock *, 4> BBList(depth_first(&F->getEntryBlock()));
1728
1729 for (BasicBlock *BB : BBList) {
1730 Instruction *Inst = &BB->front();
1731 while (true) {
1732 // DFSanVisitor may split the current basic block, changing the current
1733 // instruction's next pointer and moving the next instruction to the
1734 // tail block from which we should continue.
1735 Instruction *Next = Inst->getNextNode();
1736 // DFSanVisitor may delete Inst, so keep track of whether it was a
1737 // terminator.
1738 bool IsTerminator = Inst->isTerminator();
1739 if (!DFSF.SkipInsts.count(Inst))
1740 DFSanVisitor(DFSF).visit(Inst);
1741 if (IsTerminator)
1742 break;
1743 Inst = Next;
1744 }
1745 }
1746
1747 // We will not necessarily be able to compute the shadow for every phi node
1748 // until we have visited every block. Therefore, the code that handles phi
1749 // nodes adds them to the PHIFixups list so that they can be properly
1750 // handled here.
1751 for (DFSanFunction::PHIFixupElement &P : DFSF.PHIFixups) {
1752 for (unsigned Val = 0, N = P.Phi->getNumIncomingValues(); Val != N;
1753 ++Val) {
1754 P.ShadowPhi->setIncomingValue(
1755 Val, DFSF.getShadow(P.Phi->getIncomingValue(Val)));
1756 if (P.OriginPhi)
1757 P.OriginPhi->setIncomingValue(
1758 Val, DFSF.getOrigin(P.Phi->getIncomingValue(Val)));
1759 }
1760 }
1761
1762 // -dfsan-debug-nonzero-labels will split the CFG in all kinds of crazy
1763 // places (i.e. instructions in basic blocks we haven't even begun visiting
1764 // yet). To make our life easier, do this work in a pass after the main
1765 // instrumentation.
1767 for (Value *V : DFSF.NonZeroChecks) {
1769 if (Instruction *I = dyn_cast<Instruction>(V))
1770 Pos = std::next(I->getIterator());
1771 else
1772 Pos = DFSF.F->getEntryBlock().begin();
1773 while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos))
1774 Pos = std::next(Pos->getIterator());
1775 IRBuilder<> IRB(Pos->getParent(), Pos);
1776 Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos);
1777 Value *Ne =
1778 IRB.CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow);
1779 BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
1780 Ne, Pos, /*Unreachable=*/false, ColdCallWeights));
1781 IRBuilder<> ThenIRB(BI);
1782 ThenIRB.CreateCall(DFSF.DFS.DFSanNonzeroLabelFn, {});
1783 }
1784 }
1785 }
1786
1787 return Changed || !FnsToInstrument.empty() ||
1788 M.global_size() != InitialGlobalSize || M.size() != InitialModuleSize;
1789}
1790
1791Value *DFSanFunction::getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB) {
1792 Value *Base = IRB.CreatePointerCast(DFS.ArgTLS, DFS.IntptrTy);
1793 if (ArgOffset)
1794 Base = IRB.CreateAdd(Base, ConstantInt::get(DFS.IntptrTy, ArgOffset));
1795 return IRB.CreateIntToPtr(Base, PointerType::get(DFS.getShadowTy(T), 0),
1796 "_dfsarg");
1797}
1798
1799Value *DFSanFunction::getRetvalTLS(Type *T, IRBuilder<> &IRB) {
1800 return IRB.CreatePointerCast(
1801 DFS.RetvalTLS, PointerType::get(DFS.getShadowTy(T), 0), "_dfsret");
1802}
1803
1804Value *DFSanFunction::getRetvalOriginTLS() { return DFS.RetvalOriginTLS; }
1805
1806Value *DFSanFunction::getArgOriginTLS(unsigned ArgNo, IRBuilder<> &IRB) {
1807 return IRB.CreateConstInBoundsGEP2_64(DFS.ArgOriginTLSTy, DFS.ArgOriginTLS, 0,
1808 ArgNo, "_dfsarg_o");
1809}
1810
1811Value *DFSanFunction::getOrigin(Value *V) {
1812 assert(DFS.shouldTrackOrigins());
1813 if (!isa<Argument>(V) && !isa<Instruction>(V))
1814 return DFS.ZeroOrigin;
1815 Value *&Origin = ValOriginMap[V];
1816 if (!Origin) {
1817 if (Argument *A = dyn_cast<Argument>(V)) {
1818 if (IsNativeABI)
1819 return DFS.ZeroOrigin;
1820 if (A->getArgNo() < DFS.NumOfElementsInArgOrgTLS) {
1821 Instruction *ArgOriginTLSPos = &*F->getEntryBlock().begin();
1822 IRBuilder<> IRB(ArgOriginTLSPos);
1823 Value *ArgOriginPtr = getArgOriginTLS(A->getArgNo(), IRB);
1824 Origin = IRB.CreateLoad(DFS.OriginTy, ArgOriginPtr);
1825 } else {
1826 // Overflow
1827 Origin = DFS.ZeroOrigin;
1828 }
1829 } else {
1830 Origin = DFS.ZeroOrigin;
1831 }
1832 }
1833 return Origin;
1834}
1835
1836void DFSanFunction::setOrigin(Instruction *I, Value *Origin) {
1837 if (!DFS.shouldTrackOrigins())
1838 return;
1839 assert(!ValOriginMap.count(I));
1840 assert(Origin->getType() == DFS.OriginTy);
1841 ValOriginMap[I] = Origin;
1842}
1843
1844Value *DFSanFunction::getShadowForTLSArgument(Argument *A) {
1845 unsigned ArgOffset = 0;
1846 const DataLayout &DL = F->getDataLayout();
1847 for (auto &FArg : F->args()) {
1848 if (!FArg.getType()->isSized()) {
1849 if (A == &FArg)
1850 break;
1851 continue;
1852 }
1853
1854 unsigned Size = DL.getTypeAllocSize(DFS.getShadowTy(&FArg));
1855 if (A != &FArg) {
1856 ArgOffset += alignTo(Size, ShadowTLSAlignment);
1857 if (ArgOffset > ArgTLSSize)
1858 break; // ArgTLS overflows, uses a zero shadow.
1859 continue;
1860 }
1861
1862 if (ArgOffset + Size > ArgTLSSize)
1863 break; // ArgTLS overflows, uses a zero shadow.
1864
1865 Instruction *ArgTLSPos = &*F->getEntryBlock().begin();
1866 IRBuilder<> IRB(ArgTLSPos);
1867 Value *ArgShadowPtr = getArgTLS(FArg.getType(), ArgOffset, IRB);
1868 return IRB.CreateAlignedLoad(DFS.getShadowTy(&FArg), ArgShadowPtr,
1870 }
1871
1872 return DFS.getZeroShadow(A);
1873}
1874
1875Value *DFSanFunction::getShadow(Value *V) {
1876 if (!isa<Argument>(V) && !isa<Instruction>(V))
1877 return DFS.getZeroShadow(V);
1878 if (IsForceZeroLabels)
1879 return DFS.getZeroShadow(V);
1880 Value *&Shadow = ValShadowMap[V];
1881 if (!Shadow) {
1882 if (Argument *A = dyn_cast<Argument>(V)) {
1883 if (IsNativeABI)
1884 return DFS.getZeroShadow(V);
1885 Shadow = getShadowForTLSArgument(A);
1886 NonZeroChecks.push_back(Shadow);
1887 } else {
1888 Shadow = DFS.getZeroShadow(V);
1889 }
1890 }
1891 return Shadow;
1892}
1893
1894void DFSanFunction::setShadow(Instruction *I, Value *Shadow) {
1895 assert(!ValShadowMap.count(I));
1896 ValShadowMap[I] = Shadow;
1897}
1898
1899/// Compute the integer shadow offset that corresponds to a given
1900/// application address.
1901///
1902/// Offset = (Addr & ~AndMask) ^ XorMask
1903Value *DataFlowSanitizer::getShadowOffset(Value *Addr, IRBuilder<> &IRB) {
1904 assert(Addr != RetvalTLS && "Reinstrumenting?");
1905 Value *OffsetLong = IRB.CreatePointerCast(Addr, IntptrTy);
1906
1907 uint64_t AndMask = MapParams->AndMask;
1908 if (AndMask)
1909 OffsetLong =
1910 IRB.CreateAnd(OffsetLong, ConstantInt::get(IntptrTy, ~AndMask));
1911
1912 uint64_t XorMask = MapParams->XorMask;
1913 if (XorMask)
1914 OffsetLong = IRB.CreateXor(OffsetLong, ConstantInt::get(IntptrTy, XorMask));
1915 return OffsetLong;
1916}
1917
1918std::pair<Value *, Value *>
1919DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment,
1921 // Returns ((Addr & shadow_mask) + origin_base - shadow_base) & ~4UL
1922 IRBuilder<> IRB(Pos->getParent(), Pos);
1923 Value *ShadowOffset = getShadowOffset(Addr, IRB);
1924 Value *ShadowLong = ShadowOffset;
1925 uint64_t ShadowBase = MapParams->ShadowBase;
1926 if (ShadowBase != 0) {
1927 ShadowLong =
1928 IRB.CreateAdd(ShadowLong, ConstantInt::get(IntptrTy, ShadowBase));
1929 }
1930 IntegerType *ShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
1931 Value *ShadowPtr =
1932 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1933 Value *OriginPtr = nullptr;
1934 if (shouldTrackOrigins()) {
1935 Value *OriginLong = ShadowOffset;
1936 uint64_t OriginBase = MapParams->OriginBase;
1937 if (OriginBase != 0)
1938 OriginLong =
1939 IRB.CreateAdd(OriginLong, ConstantInt::get(IntptrTy, OriginBase));
1940 const Align Alignment = llvm::assumeAligned(InstAlignment.value());
1941 // When alignment is >= 4, Addr must be aligned to 4, otherwise it is UB.
1942 // So Mask is unnecessary.
1943 if (Alignment < MinOriginAlignment) {
1945 OriginLong = IRB.CreateAnd(OriginLong, ConstantInt::get(IntptrTy, ~Mask));
1946 }
1947 OriginPtr = IRB.CreateIntToPtr(OriginLong, OriginPtrTy);
1948 }
1949 return std::make_pair(ShadowPtr, OriginPtr);
1950}
1951
1952Value *DataFlowSanitizer::getShadowAddress(Value *Addr,
1954 Value *ShadowOffset) {
1955 IRBuilder<> IRB(Pos->getParent(), Pos);
1956 return IRB.CreateIntToPtr(ShadowOffset, PrimitiveShadowPtrTy);
1957}
1958
1959Value *DataFlowSanitizer::getShadowAddress(Value *Addr,
1961 IRBuilder<> IRB(Pos->getParent(), Pos);
1962 Value *ShadowOffset = getShadowOffset(Addr, IRB);
1963 return getShadowAddress(Addr, Pos, ShadowOffset);
1964}
1965
1966Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
1968 Value *PrimitiveValue = combineShadows(V1, V2, Pos);
1969 return expandFromPrimitiveShadow(T, PrimitiveValue, Pos);
1970}
1971
1972// Generates IR to compute the union of the two given shadows, inserting it
1973// before Pos. The combined value is with primitive type.
1974Value *DFSanFunction::combineShadows(Value *V1, Value *V2,
1976 if (DFS.isZeroShadow(V1))
1977 return collapseToPrimitiveShadow(V2, Pos);
1978 if (DFS.isZeroShadow(V2))
1979 return collapseToPrimitiveShadow(V1, Pos);
1980 if (V1 == V2)
1981 return collapseToPrimitiveShadow(V1, Pos);
1982
1983 auto V1Elems = ShadowElements.find(V1);
1984 auto V2Elems = ShadowElements.find(V2);
1985 if (V1Elems != ShadowElements.end() && V2Elems != ShadowElements.end()) {
1986 if (std::includes(V1Elems->second.begin(), V1Elems->second.end(),
1987 V2Elems->second.begin(), V2Elems->second.end())) {
1988 return collapseToPrimitiveShadow(V1, Pos);
1989 }
1990 if (std::includes(V2Elems->second.begin(), V2Elems->second.end(),
1991 V1Elems->second.begin(), V1Elems->second.end())) {
1992 return collapseToPrimitiveShadow(V2, Pos);
1993 }
1994 } else if (V1Elems != ShadowElements.end()) {
1995 if (V1Elems->second.count(V2))
1996 return collapseToPrimitiveShadow(V1, Pos);
1997 } else if (V2Elems != ShadowElements.end()) {
1998 if (V2Elems->second.count(V1))
1999 return collapseToPrimitiveShadow(V2, Pos);
2000 }
2001
2002 auto Key = std::make_pair(V1, V2);
2003 if (V1 > V2)
2004 std::swap(Key.first, Key.second);
2005 CachedShadow &CCS = CachedShadows[Key];
2006 if (CCS.Block && DT.dominates(CCS.Block, Pos->getParent()))
2007 return CCS.Shadow;
2008
2009 // Converts inputs shadows to shadows with primitive types.
2010 Value *PV1 = collapseToPrimitiveShadow(V1, Pos);
2011 Value *PV2 = collapseToPrimitiveShadow(V2, Pos);
2012
2013 IRBuilder<> IRB(Pos->getParent(), Pos);
2014 CCS.Block = Pos->getParent();
2015 CCS.Shadow = IRB.CreateOr(PV1, PV2);
2016
2017 std::set<Value *> UnionElems;
2018 if (V1Elems != ShadowElements.end()) {
2019 UnionElems = V1Elems->second;
2020 } else {
2021 UnionElems.insert(V1);
2022 }
2023 if (V2Elems != ShadowElements.end()) {
2024 UnionElems.insert(V2Elems->second.begin(), V2Elems->second.end());
2025 } else {
2026 UnionElems.insert(V2);
2027 }
2028 ShadowElements[CCS.Shadow] = std::move(UnionElems);
2029
2030 return CCS.Shadow;
2031}
2032
2033// A convenience function which folds the shadows of each of the operands
2034// of the provided instruction Inst, inserting the IR before Inst. Returns
2035// the computed union Value.
2036Value *DFSanFunction::combineOperandShadows(Instruction *Inst) {
2037 if (Inst->getNumOperands() == 0)
2038 return DFS.getZeroShadow(Inst);
2039
2040 Value *Shadow = getShadow(Inst->getOperand(0));
2041 for (unsigned I = 1, N = Inst->getNumOperands(); I < N; ++I)
2042 Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(I)),
2043 Inst->getIterator());
2044
2045 return expandFromPrimitiveShadow(Inst->getType(), Shadow,
2046 Inst->getIterator());
2047}
2048
2049void DFSanVisitor::visitInstOperands(Instruction &I) {
2050 Value *CombinedShadow = DFSF.combineOperandShadows(&I);
2051 DFSF.setShadow(&I, CombinedShadow);
2052 visitInstOperandOrigins(I);
2053}
2054
2055Value *DFSanFunction::combineOrigins(const std::vector<Value *> &Shadows,
2056 const std::vector<Value *> &Origins,
2058 ConstantInt *Zero) {
2059 assert(Shadows.size() == Origins.size());
2060 size_t Size = Origins.size();
2061 if (Size == 0)
2062 return DFS.ZeroOrigin;
2063 Value *Origin = nullptr;
2064 if (!Zero)
2065 Zero = DFS.ZeroPrimitiveShadow;
2066 for (size_t I = 0; I != Size; ++I) {
2067 Value *OpOrigin = Origins[I];
2068 Constant *ConstOpOrigin = dyn_cast<Constant>(OpOrigin);
2069 if (ConstOpOrigin && ConstOpOrigin->isNullValue())
2070 continue;
2071 if (!Origin) {
2072 Origin = OpOrigin;
2073 continue;
2074 }
2075 Value *OpShadow = Shadows[I];
2076 Value *PrimitiveShadow = collapseToPrimitiveShadow(OpShadow, Pos);
2077 IRBuilder<> IRB(Pos->getParent(), Pos);
2078 Value *Cond = IRB.CreateICmpNE(PrimitiveShadow, Zero);
2079 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2080 }
2081 return Origin ? Origin : DFS.ZeroOrigin;
2082}
2083
2084Value *DFSanFunction::combineOperandOrigins(Instruction *Inst) {
2085 size_t Size = Inst->getNumOperands();
2086 std::vector<Value *> Shadows(Size);
2087 std::vector<Value *> Origins(Size);
2088 for (unsigned I = 0; I != Size; ++I) {
2089 Shadows[I] = getShadow(Inst->getOperand(I));
2090 Origins[I] = getOrigin(Inst->getOperand(I));
2091 }
2092 return combineOrigins(Shadows, Origins, Inst->getIterator());
2093}
2094
2095void DFSanVisitor::visitInstOperandOrigins(Instruction &I) {
2096 if (!DFSF.DFS.shouldTrackOrigins())
2097 return;
2098 Value *CombinedOrigin = DFSF.combineOperandOrigins(&I);
2099 DFSF.setOrigin(&I, CombinedOrigin);
2100}
2101
2102Align DFSanFunction::getShadowAlign(Align InstAlignment) {
2103 const Align Alignment = ClPreserveAlignment ? InstAlignment : Align(1);
2104 return Align(Alignment.value() * DFS.ShadowWidthBytes);
2105}
2106
2107Align DFSanFunction::getOriginAlign(Align InstAlignment) {
2108 const Align Alignment = llvm::assumeAligned(InstAlignment.value());
2109 return Align(std::max(MinOriginAlignment, Alignment));
2110}
2111
2112bool DFSanFunction::isLookupTableConstant(Value *P) {
2113 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(P->stripPointerCasts()))
2114 if (GV->isConstant() && GV->hasName())
2115 return DFS.CombineTaintLookupTableNames.count(GV->getName());
2116
2117 return false;
2118}
2119
2120bool DFSanFunction::useCallbackLoadLabelAndOrigin(uint64_t Size,
2121 Align InstAlignment) {
2122 // When enabling tracking load instructions, we always use
2123 // __dfsan_load_label_and_origin to reduce code size.
2124 if (ClTrackOrigins == 2)
2125 return true;
2126
2127 assert(Size != 0);
2128 // * if Size == 1, it is sufficient to load its origin aligned at 4.
2129 // * if Size == 2, we assume most cases Addr % 2 == 0, so it is sufficient to
2130 // load its origin aligned at 4. If not, although origins may be lost, it
2131 // should not happen very often.
2132 // * if align >= 4, Addr must be aligned to 4, otherwise it is UB. When
2133 // Size % 4 == 0, it is more efficient to load origins without callbacks.
2134 // * Otherwise we use __dfsan_load_label_and_origin.
2135 // This should ensure that common cases run efficiently.
2136 if (Size <= 2)
2137 return false;
2138
2139 const Align Alignment = llvm::assumeAligned(InstAlignment.value());
2140 return Alignment < MinOriginAlignment || !DFS.hasLoadSizeForFastPath(Size);
2141}
2142
2143Value *DataFlowSanitizer::loadNextOrigin(BasicBlock::iterator Pos,
2144 Align OriginAlign,
2145 Value **OriginAddr) {
2146 IRBuilder<> IRB(Pos->getParent(), Pos);
2147 *OriginAddr =
2148 IRB.CreateGEP(OriginTy, *OriginAddr, ConstantInt::get(IntptrTy, 1));
2149 return IRB.CreateAlignedLoad(OriginTy, *OriginAddr, OriginAlign);
2150}
2151
2152std::pair<Value *, Value *> DFSanFunction::loadShadowFast(
2153 Value *ShadowAddr, Value *OriginAddr, uint64_t Size, Align ShadowAlign,
2154 Align OriginAlign, Value *FirstOrigin, BasicBlock::iterator Pos) {
2155 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2156 const uint64_t ShadowSize = Size * DFS.ShadowWidthBytes;
2157
2158 assert(Size >= 4 && "Not large enough load size for fast path!");
2159
2160 // Used for origin tracking.
2161 std::vector<Value *> Shadows;
2162 std::vector<Value *> Origins;
2163
2164 // Load instructions in LLVM can have arbitrary byte sizes (e.g., 3, 12, 20)
2165 // but this function is only used in a subset of cases that make it possible
2166 // to optimize the instrumentation.
2167 //
2168 // Specifically, when the shadow size in bytes (i.e., loaded bytes x shadow
2169 // per byte) is either:
2170 // - a multiple of 8 (common)
2171 // - equal to 4 (only for load32)
2172 //
2173 // For the second case, we can fit the wide shadow in a 32-bit integer. In all
2174 // other cases, we use a 64-bit integer to hold the wide shadow.
2175 Type *WideShadowTy =
2176 ShadowSize == 4 ? Type::getInt32Ty(*DFS.Ctx) : Type::getInt64Ty(*DFS.Ctx);
2177
2178 IRBuilder<> IRB(Pos->getParent(), Pos);
2179 Value *CombinedWideShadow =
2180 IRB.CreateAlignedLoad(WideShadowTy, ShadowAddr, ShadowAlign);
2181
2182 unsigned WideShadowBitWidth = WideShadowTy->getIntegerBitWidth();
2183 const uint64_t BytesPerWideShadow = WideShadowBitWidth / DFS.ShadowWidthBits;
2184
2185 auto AppendWideShadowAndOrigin = [&](Value *WideShadow, Value *Origin) {
2186 if (BytesPerWideShadow > 4) {
2187 assert(BytesPerWideShadow == 8);
2188 // The wide shadow relates to two origin pointers: one for the first four
2189 // application bytes, and one for the latest four. We use a left shift to
2190 // get just the shadow bytes that correspond to the first origin pointer,
2191 // and then the entire shadow for the second origin pointer (which will be
2192 // chosen by combineOrigins() iff the least-significant half of the wide
2193 // shadow was empty but the other half was not).
2194 Value *WideShadowLo = IRB.CreateShl(
2195 WideShadow, ConstantInt::get(WideShadowTy, WideShadowBitWidth / 2));
2196 Shadows.push_back(WideShadow);
2197 Origins.push_back(DFS.loadNextOrigin(Pos, OriginAlign, &OriginAddr));
2198
2199 Shadows.push_back(WideShadowLo);
2200 Origins.push_back(Origin);
2201 } else {
2202 Shadows.push_back(WideShadow);
2203 Origins.push_back(Origin);
2204 }
2205 };
2206
2207 if (ShouldTrackOrigins)
2208 AppendWideShadowAndOrigin(CombinedWideShadow, FirstOrigin);
2209
2210 // First OR all the WideShadows (i.e., 64bit or 32bit shadow chunks) linearly;
2211 // then OR individual shadows within the combined WideShadow by binary ORing.
2212 // This is fewer instructions than ORing shadows individually, since it
2213 // needs logN shift/or instructions (N being the bytes of the combined wide
2214 // shadow).
2215 for (uint64_t ByteOfs = BytesPerWideShadow; ByteOfs < Size;
2216 ByteOfs += BytesPerWideShadow) {
2217 ShadowAddr = IRB.CreateGEP(WideShadowTy, ShadowAddr,
2218 ConstantInt::get(DFS.IntptrTy, 1));
2219 Value *NextWideShadow =
2220 IRB.CreateAlignedLoad(WideShadowTy, ShadowAddr, ShadowAlign);
2221 CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow);
2222 if (ShouldTrackOrigins) {
2223 Value *NextOrigin = DFS.loadNextOrigin(Pos, OriginAlign, &OriginAddr);
2224 AppendWideShadowAndOrigin(NextWideShadow, NextOrigin);
2225 }
2226 }
2227 for (unsigned Width = WideShadowBitWidth / 2; Width >= DFS.ShadowWidthBits;
2228 Width >>= 1) {
2229 Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
2230 CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
2231 }
2232 return {IRB.CreateTrunc(CombinedWideShadow, DFS.PrimitiveShadowTy),
2233 ShouldTrackOrigins
2234 ? combineOrigins(Shadows, Origins, Pos,
2236 : DFS.ZeroOrigin};
2237}
2238
2239std::pair<Value *, Value *> DFSanFunction::loadShadowOriginSansLoadTracking(
2240 Value *Addr, uint64_t Size, Align InstAlignment, BasicBlock::iterator Pos) {
2241 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2242
2243 // Non-escaped loads.
2244 if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
2245 const auto SI = AllocaShadowMap.find(AI);
2246 if (SI != AllocaShadowMap.end()) {
2247 IRBuilder<> IRB(Pos->getParent(), Pos);
2248 Value *ShadowLI = IRB.CreateLoad(DFS.PrimitiveShadowTy, SI->second);
2249 const auto OI = AllocaOriginMap.find(AI);
2250 assert(!ShouldTrackOrigins || OI != AllocaOriginMap.end());
2251 return {ShadowLI, ShouldTrackOrigins
2252 ? IRB.CreateLoad(DFS.OriginTy, OI->second)
2253 : nullptr};
2254 }
2255 }
2256
2257 // Load from constant addresses.
2260 bool AllConstants = true;
2261 for (const Value *Obj : Objs) {
2262 if (isa<Function>(Obj) || isa<BlockAddress>(Obj))
2263 continue;
2264 if (isa<GlobalVariable>(Obj) && cast<GlobalVariable>(Obj)->isConstant())
2265 continue;
2266
2267 AllConstants = false;
2268 break;
2269 }
2270 if (AllConstants)
2271 return {DFS.ZeroPrimitiveShadow,
2272 ShouldTrackOrigins ? DFS.ZeroOrigin : nullptr};
2273
2274 if (Size == 0)
2275 return {DFS.ZeroPrimitiveShadow,
2276 ShouldTrackOrigins ? DFS.ZeroOrigin : nullptr};
2277
2278 // Use callback to load if this is not an optimizable case for origin
2279 // tracking.
2280 if (ShouldTrackOrigins &&
2281 useCallbackLoadLabelAndOrigin(Size, InstAlignment)) {
2282 IRBuilder<> IRB(Pos->getParent(), Pos);
2283 CallInst *Call =
2284 IRB.CreateCall(DFS.DFSanLoadLabelAndOriginFn,
2285 {Addr, ConstantInt::get(DFS.IntptrTy, Size)});
2286 Call->addRetAttr(Attribute::ZExt);
2287 return {IRB.CreateTrunc(IRB.CreateLShr(Call, DFS.OriginWidthBits),
2288 DFS.PrimitiveShadowTy),
2289 IRB.CreateTrunc(Call, DFS.OriginTy)};
2290 }
2291
2292 // Other cases that support loading shadows or origins in a fast way.
2293 Value *ShadowAddr, *OriginAddr;
2294 std::tie(ShadowAddr, OriginAddr) =
2295 DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
2296
2297 const Align ShadowAlign = getShadowAlign(InstAlignment);
2298 const Align OriginAlign = getOriginAlign(InstAlignment);
2299 Value *Origin = nullptr;
2300 if (ShouldTrackOrigins) {
2301 IRBuilder<> IRB(Pos->getParent(), Pos);
2302 Origin = IRB.CreateAlignedLoad(DFS.OriginTy, OriginAddr, OriginAlign);
2303 }
2304
2305 // When the byte size is small enough, we can load the shadow directly with
2306 // just a few instructions.
2307 switch (Size) {
2308 case 1: {
2309 LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos);
2310 LI->setAlignment(ShadowAlign);
2311 return {LI, Origin};
2312 }
2313 case 2: {
2314 IRBuilder<> IRB(Pos->getParent(), Pos);
2315 Value *ShadowAddr1 = IRB.CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr,
2316 ConstantInt::get(DFS.IntptrTy, 1));
2317 Value *Load =
2318 IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr, ShadowAlign);
2319 Value *Load1 =
2320 IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr1, ShadowAlign);
2321 return {combineShadows(Load, Load1, Pos), Origin};
2322 }
2323 }
2324 bool HasSizeForFastPath = DFS.hasLoadSizeForFastPath(Size);
2325
2326 if (HasSizeForFastPath)
2327 return loadShadowFast(ShadowAddr, OriginAddr, Size, ShadowAlign,
2328 OriginAlign, Origin, Pos);
2329
2330 IRBuilder<> IRB(Pos->getParent(), Pos);
2331 CallInst *FallbackCall = IRB.CreateCall(
2332 DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
2333 FallbackCall->addRetAttr(Attribute::ZExt);
2334 return {FallbackCall, Origin};
2335}
2336
2337std::pair<Value *, Value *>
2338DFSanFunction::loadShadowOrigin(Value *Addr, uint64_t Size, Align InstAlignment,
2340 Value *PrimitiveShadow, *Origin;
2341 std::tie(PrimitiveShadow, Origin) =
2342 loadShadowOriginSansLoadTracking(Addr, Size, InstAlignment, Pos);
2343 if (DFS.shouldTrackOrigins()) {
2344 if (ClTrackOrigins == 2) {
2345 IRBuilder<> IRB(Pos->getParent(), Pos);
2346 auto *ConstantShadow = dyn_cast<Constant>(PrimitiveShadow);
2347 if (!ConstantShadow || !ConstantShadow->isZeroValue())
2348 Origin = updateOriginIfTainted(PrimitiveShadow, Origin, IRB);
2349 }
2350 }
2351 return {PrimitiveShadow, Origin};
2352}
2353
2355 switch (AO) {
2356 case AtomicOrdering::NotAtomic:
2357 return AtomicOrdering::NotAtomic;
2358 case AtomicOrdering::Unordered:
2359 case AtomicOrdering::Monotonic:
2360 case AtomicOrdering::Acquire:
2361 return AtomicOrdering::Acquire;
2362 case AtomicOrdering::Release:
2363 case AtomicOrdering::AcquireRelease:
2364 return AtomicOrdering::AcquireRelease;
2365 case AtomicOrdering::SequentiallyConsistent:
2366 return AtomicOrdering::SequentiallyConsistent;
2367 }
2368 llvm_unreachable("Unknown ordering");
2369}
2370
2372 if (!V->getType()->isPointerTy())
2373 return V;
2374
2375 // DFSan pass should be running on valid IR, but we'll
2376 // keep a seen set to ensure there are no issues.
2378 Visited.insert(V);
2379 do {
2380 if (auto *GEP = dyn_cast<GEPOperator>(V)) {
2381 V = GEP->getPointerOperand();
2382 } else if (Operator::getOpcode(V) == Instruction::BitCast) {
2383 V = cast<Operator>(V)->getOperand(0);
2384 if (!V->getType()->isPointerTy())
2385 return V;
2386 } else if (isa<GlobalAlias>(V)) {
2387 V = cast<GlobalAlias>(V)->getAliasee();
2388 }
2389 } while (Visited.insert(V).second);
2390
2391 return V;
2392}
2393
2394void DFSanVisitor::visitLoadInst(LoadInst &LI) {
2395 auto &DL = LI.getDataLayout();
2396 uint64_t Size = DL.getTypeStoreSize(LI.getType());
2397 if (Size == 0) {
2398 DFSF.setShadow(&LI, DFSF.DFS.getZeroShadow(&LI));
2399 DFSF.setOrigin(&LI, DFSF.DFS.ZeroOrigin);
2400 return;
2401 }
2402
2403 // When an application load is atomic, increase atomic ordering between
2404 // atomic application loads and stores to ensure happen-before order; load
2405 // shadow data after application data; store zero shadow data before
2406 // application data. This ensure shadow loads return either labels of the
2407 // initial application data or zeros.
2408 if (LI.isAtomic())
2410
2411 BasicBlock::iterator AfterLi = std::next(LI.getIterator());
2413 if (LI.isAtomic())
2414 Pos = std::next(Pos);
2415
2416 std::vector<Value *> Shadows;
2417 std::vector<Value *> Origins;
2418 Value *PrimitiveShadow, *Origin;
2419 std::tie(PrimitiveShadow, Origin) =
2420 DFSF.loadShadowOrigin(LI.getPointerOperand(), Size, LI.getAlign(), Pos);
2421 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2422 if (ShouldTrackOrigins) {
2423 Shadows.push_back(PrimitiveShadow);
2424 Origins.push_back(Origin);
2425 }
2427 DFSF.isLookupTableConstant(
2429 Value *PtrShadow = DFSF.getShadow(LI.getPointerOperand());
2430 PrimitiveShadow = DFSF.combineShadows(PrimitiveShadow, PtrShadow, Pos);
2431 if (ShouldTrackOrigins) {
2432 Shadows.push_back(PtrShadow);
2433 Origins.push_back(DFSF.getOrigin(LI.getPointerOperand()));
2434 }
2435 }
2436 if (!DFSF.DFS.isZeroShadow(PrimitiveShadow))
2437 DFSF.NonZeroChecks.push_back(PrimitiveShadow);
2438
2439 Value *Shadow =
2440 DFSF.expandFromPrimitiveShadow(LI.getType(), PrimitiveShadow, Pos);
2441 DFSF.setShadow(&LI, Shadow);
2442
2443 if (ShouldTrackOrigins) {
2444 DFSF.setOrigin(&LI, DFSF.combineOrigins(Shadows, Origins, Pos));
2445 }
2446
2447 if (ClEventCallbacks) {
2448 IRBuilder<> IRB(Pos->getParent(), Pos);
2450 CallInst *CI =
2451 IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr});
2452 CI->addParamAttr(0, Attribute::ZExt);
2453 }
2454
2455 IRBuilder<> IRB(AfterLi->getParent(), AfterLi);
2456 DFSF.addReachesFunctionCallbacksIfEnabled(IRB, LI, &LI);
2457}
2458
2459Value *DFSanFunction::updateOriginIfTainted(Value *Shadow, Value *Origin,
2460 IRBuilder<> &IRB) {
2461 assert(DFS.shouldTrackOrigins());
2462 return IRB.CreateCall(DFS.DFSanChainOriginIfTaintedFn, {Shadow, Origin});
2463}
2464
2465Value *DFSanFunction::updateOrigin(Value *V, IRBuilder<> &IRB) {
2466 if (!DFS.shouldTrackOrigins())
2467 return V;
2468 return IRB.CreateCall(DFS.DFSanChainOriginFn, V);
2469}
2470
2471Value *DFSanFunction::originToIntptr(IRBuilder<> &IRB, Value *Origin) {
2472 const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2473 const DataLayout &DL = F->getDataLayout();
2474 unsigned IntptrSize = DL.getTypeStoreSize(DFS.IntptrTy);
2475 if (IntptrSize == OriginSize)
2476 return Origin;
2477 assert(IntptrSize == OriginSize * 2);
2478 Origin = IRB.CreateIntCast(Origin, DFS.IntptrTy, /* isSigned */ false);
2479 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, OriginSize * 8));
2480}
2481
2482void DFSanFunction::paintOrigin(IRBuilder<> &IRB, Value *Origin,
2483 Value *StoreOriginAddr,
2484 uint64_t StoreOriginSize, Align Alignment) {
2485 const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2486 const DataLayout &DL = F->getDataLayout();
2487 const Align IntptrAlignment = DL.getABITypeAlign(DFS.IntptrTy);
2488 unsigned IntptrSize = DL.getTypeStoreSize(DFS.IntptrTy);
2489 assert(IntptrAlignment >= MinOriginAlignment);
2490 assert(IntptrSize >= OriginSize);
2491
2492 unsigned Ofs = 0;
2493 Align CurrentAlignment = Alignment;
2494 if (Alignment >= IntptrAlignment && IntptrSize > OriginSize) {
2495 Value *IntptrOrigin = originToIntptr(IRB, Origin);
2496 Value *IntptrStoreOriginPtr = IRB.CreatePointerCast(
2497 StoreOriginAddr, PointerType::get(DFS.IntptrTy, 0));
2498 for (unsigned I = 0; I < StoreOriginSize / IntptrSize; ++I) {
2499 Value *Ptr =
2500 I ? IRB.CreateConstGEP1_32(DFS.IntptrTy, IntptrStoreOriginPtr, I)
2501 : IntptrStoreOriginPtr;
2502 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
2503 Ofs += IntptrSize / OriginSize;
2504 CurrentAlignment = IntptrAlignment;
2505 }
2506 }
2507
2508 for (unsigned I = Ofs; I < (StoreOriginSize + OriginSize - 1) / OriginSize;
2509 ++I) {
2510 Value *GEP = I ? IRB.CreateConstGEP1_32(DFS.OriginTy, StoreOriginAddr, I)
2511 : StoreOriginAddr;
2512 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
2513 CurrentAlignment = MinOriginAlignment;
2514 }
2515}
2516
2517Value *DFSanFunction::convertToBool(Value *V, IRBuilder<> &IRB,
2518 const Twine &Name) {
2519 Type *VTy = V->getType();
2520 assert(VTy->isIntegerTy());
2521 if (VTy->getIntegerBitWidth() == 1)
2522 // Just converting a bool to a bool, so do nothing.
2523 return V;
2524 return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), Name);
2525}
2526
2527void DFSanFunction::storeOrigin(BasicBlock::iterator Pos, Value *Addr,
2528 uint64_t Size, Value *Shadow, Value *Origin,
2529 Value *StoreOriginAddr, Align InstAlignment) {
2530 // Do not write origins for zero shadows because we do not trace origins for
2531 // untainted sinks.
2532 const Align OriginAlignment = getOriginAlign(InstAlignment);
2533 Value *CollapsedShadow = collapseToPrimitiveShadow(Shadow, Pos);
2534 IRBuilder<> IRB(Pos->getParent(), Pos);
2535 if (auto *ConstantShadow = dyn_cast<Constant>(CollapsedShadow)) {
2536 if (!ConstantShadow->isZeroValue())
2537 paintOrigin(IRB, updateOrigin(Origin, IRB), StoreOriginAddr, Size,
2538 OriginAlignment);
2539 return;
2540 }
2541
2542 if (shouldInstrumentWithCall()) {
2543 IRB.CreateCall(
2544 DFS.DFSanMaybeStoreOriginFn,
2545 {CollapsedShadow, Addr, ConstantInt::get(DFS.IntptrTy, Size), Origin});
2546 } else {
2547 Value *Cmp = convertToBool(CollapsedShadow, IRB, "_dfscmp");
2548 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
2550 Cmp, &*IRB.GetInsertPoint(), false, DFS.OriginStoreWeights, &DTU);
2551 IRBuilder<> IRBNew(CheckTerm);
2552 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), StoreOriginAddr, Size,
2553 OriginAlignment);
2554 ++NumOriginStores;
2555 }
2556}
2557
2558void DFSanFunction::storeZeroPrimitiveShadow(Value *Addr, uint64_t Size,
2559 Align ShadowAlign,
2561 IRBuilder<> IRB(Pos->getParent(), Pos);
2562 IntegerType *ShadowTy =
2563 IntegerType::get(*DFS.Ctx, Size * DFS.ShadowWidthBits);
2564 Value *ExtZeroShadow = ConstantInt::get(ShadowTy, 0);
2565 Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
2566 IRB.CreateAlignedStore(ExtZeroShadow, ShadowAddr, ShadowAlign);
2567 // Do not write origins for 0 shadows because we do not trace origins for
2568 // untainted sinks.
2569}
2570
2571void DFSanFunction::storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
2572 Align InstAlignment,
2573 Value *PrimitiveShadow,
2574 Value *Origin,
2576 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins() && Origin;
2577
2578 if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
2579 const auto SI = AllocaShadowMap.find(AI);
2580 if (SI != AllocaShadowMap.end()) {
2581 IRBuilder<> IRB(Pos->getParent(), Pos);
2582 IRB.CreateStore(PrimitiveShadow, SI->second);
2583
2584 // Do not write origins for 0 shadows because we do not trace origins for
2585 // untainted sinks.
2586 if (ShouldTrackOrigins && !DFS.isZeroShadow(PrimitiveShadow)) {
2587 const auto OI = AllocaOriginMap.find(AI);
2588 assert(OI != AllocaOriginMap.end() && Origin);
2589 IRB.CreateStore(Origin, OI->second);
2590 }
2591 return;
2592 }
2593 }
2594
2595 const Align ShadowAlign = getShadowAlign(InstAlignment);
2596 if (DFS.isZeroShadow(PrimitiveShadow)) {
2597 storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, Pos);
2598 return;
2599 }
2600
2601 IRBuilder<> IRB(Pos->getParent(), Pos);
2602 Value *ShadowAddr, *OriginAddr;
2603 std::tie(ShadowAddr, OriginAddr) =
2604 DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
2605
2606 const unsigned ShadowVecSize = 8;
2607 assert(ShadowVecSize * DFS.ShadowWidthBits <= 128 &&
2608 "Shadow vector is too large!");
2609
2610 uint64_t Offset = 0;
2611 uint64_t LeftSize = Size;
2612 if (LeftSize >= ShadowVecSize) {
2613 auto *ShadowVecTy =
2614 FixedVectorType::get(DFS.PrimitiveShadowTy, ShadowVecSize);
2615 Value *ShadowVec = PoisonValue::get(ShadowVecTy);
2616 for (unsigned I = 0; I != ShadowVecSize; ++I) {
2617 ShadowVec = IRB.CreateInsertElement(
2618 ShadowVec, PrimitiveShadow,
2619 ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), I));
2620 }
2621 do {
2622 Value *CurShadowVecAddr =
2623 IRB.CreateConstGEP1_32(ShadowVecTy, ShadowAddr, Offset);
2624 IRB.CreateAlignedStore(ShadowVec, CurShadowVecAddr, ShadowAlign);
2625 LeftSize -= ShadowVecSize;
2626 ++Offset;
2627 } while (LeftSize >= ShadowVecSize);
2628 Offset *= ShadowVecSize;
2629 }
2630 while (LeftSize > 0) {
2631 Value *CurShadowAddr =
2632 IRB.CreateConstGEP1_32(DFS.PrimitiveShadowTy, ShadowAddr, Offset);
2633 IRB.CreateAlignedStore(PrimitiveShadow, CurShadowAddr, ShadowAlign);
2634 --LeftSize;
2635 ++Offset;
2636 }
2637
2638 if (ShouldTrackOrigins) {
2639 storeOrigin(Pos, Addr, Size, PrimitiveShadow, Origin, OriginAddr,
2640 InstAlignment);
2641 }
2642}
2643
2645 switch (AO) {
2646 case AtomicOrdering::NotAtomic:
2647 return AtomicOrdering::NotAtomic;
2648 case AtomicOrdering::Unordered:
2649 case AtomicOrdering::Monotonic:
2650 case AtomicOrdering::Release:
2651 return AtomicOrdering::Release;
2652 case AtomicOrdering::Acquire:
2653 case AtomicOrdering::AcquireRelease:
2654 return AtomicOrdering::AcquireRelease;
2655 case AtomicOrdering::SequentiallyConsistent:
2656 return AtomicOrdering::SequentiallyConsistent;
2657 }
2658 llvm_unreachable("Unknown ordering");
2659}
2660
2661void DFSanVisitor::visitStoreInst(StoreInst &SI) {
2662 auto &DL = SI.getDataLayout();
2663 Value *Val = SI.getValueOperand();
2664 uint64_t Size = DL.getTypeStoreSize(Val->getType());
2665 if (Size == 0)
2666 return;
2667
2668 // When an application store is atomic, increase atomic ordering between
2669 // atomic application loads and stores to ensure happen-before order; load
2670 // shadow data after application data; store zero shadow data before
2671 // application data. This ensure shadow loads return either labels of the
2672 // initial application data or zeros.
2673 if (SI.isAtomic())
2674 SI.setOrdering(addReleaseOrdering(SI.getOrdering()));
2675
2676 const bool ShouldTrackOrigins =
2677 DFSF.DFS.shouldTrackOrigins() && !SI.isAtomic();
2678 std::vector<Value *> Shadows;
2679 std::vector<Value *> Origins;
2680
2681 Value *Shadow =
2682 SI.isAtomic() ? DFSF.DFS.getZeroShadow(Val) : DFSF.getShadow(Val);
2683
2684 if (ShouldTrackOrigins) {
2685 Shadows.push_back(Shadow);
2686 Origins.push_back(DFSF.getOrigin(Val));
2687 }
2688
2689 Value *PrimitiveShadow;
2691 Value *PtrShadow = DFSF.getShadow(SI.getPointerOperand());
2692 if (ShouldTrackOrigins) {
2693 Shadows.push_back(PtrShadow);
2694 Origins.push_back(DFSF.getOrigin(SI.getPointerOperand()));
2695 }
2696 PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, SI.getIterator());
2697 } else {
2698 PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, SI.getIterator());
2699 }
2700 Value *Origin = nullptr;
2701 if (ShouldTrackOrigins)
2702 Origin = DFSF.combineOrigins(Shadows, Origins, SI.getIterator());
2703 DFSF.storePrimitiveShadowOrigin(SI.getPointerOperand(), Size, SI.getAlign(),
2704 PrimitiveShadow, Origin, SI.getIterator());
2705 if (ClEventCallbacks) {
2706 IRBuilder<> IRB(&SI);
2707 Value *Addr = SI.getPointerOperand();
2708 CallInst *CI =
2709 IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {PrimitiveShadow, Addr});
2710 CI->addParamAttr(0, Attribute::ZExt);
2711 }
2712}
2713
2714void DFSanVisitor::visitCASOrRMW(Align InstAlignment, Instruction &I) {
2715 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2716
2717 Value *Val = I.getOperand(1);
2718 const auto &DL = I.getDataLayout();
2719 uint64_t Size = DL.getTypeStoreSize(Val->getType());
2720 if (Size == 0)
2721 return;
2722
2723 // Conservatively set data at stored addresses and return with zero shadow to
2724 // prevent shadow data races.
2725 IRBuilder<> IRB(&I);
2726 Value *Addr = I.getOperand(0);
2727 const Align ShadowAlign = DFSF.getShadowAlign(InstAlignment);
2728 DFSF.storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, I.getIterator());
2729 DFSF.setShadow(&I, DFSF.DFS.getZeroShadow(&I));
2730 DFSF.setOrigin(&I, DFSF.DFS.ZeroOrigin);
2731}
2732
2733void DFSanVisitor::visitAtomicRMWInst(AtomicRMWInst &I) {
2734 visitCASOrRMW(I.getAlign(), I);
2735 // TODO: The ordering change follows MSan. It is possible not to change
2736 // ordering because we always set and use 0 shadows.
2737 I.setOrdering(addReleaseOrdering(I.getOrdering()));
2738}
2739
2740void DFSanVisitor::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2741 visitCASOrRMW(I.getAlign(), I);
2742 // TODO: The ordering change follows MSan. It is possible not to change
2743 // ordering because we always set and use 0 shadows.
2744 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
2745}
2746
2747void DFSanVisitor::visitUnaryOperator(UnaryOperator &UO) {
2748 visitInstOperands(UO);
2749}
2750
2751void DFSanVisitor::visitBinaryOperator(BinaryOperator &BO) {
2752 visitInstOperands(BO);
2753}
2754
2755void DFSanVisitor::visitBitCastInst(BitCastInst &BCI) {
2756 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2757 // a musttail call and a ret, don't instrument. New instructions are not
2758 // allowed after a musttail call.
2759 if (auto *CI = dyn_cast<CallInst>(BCI.getOperand(0)))
2760 if (CI->isMustTailCall())
2761 return;
2762 visitInstOperands(BCI);
2763}
2764
2765void DFSanVisitor::visitCastInst(CastInst &CI) { visitInstOperands(CI); }
2766
2767void DFSanVisitor::visitCmpInst(CmpInst &CI) {
2768 visitInstOperands(CI);
2769 if (ClEventCallbacks) {
2770 IRBuilder<> IRB(&CI);
2771 Value *CombinedShadow = DFSF.getShadow(&CI);
2772 CallInst *CallI =
2773 IRB.CreateCall(DFSF.DFS.DFSanCmpCallbackFn, CombinedShadow);
2774 CallI->addParamAttr(0, Attribute::ZExt);
2775 }
2776}
2777
2778void DFSanVisitor::visitLandingPadInst(LandingPadInst &LPI) {
2779 // We do not need to track data through LandingPadInst.
2780 //
2781 // For the C++ exceptions, if a value is thrown, this value will be stored
2782 // in a memory location provided by __cxa_allocate_exception(...) (on the
2783 // throw side) or __cxa_begin_catch(...) (on the catch side).
2784 // This memory will have a shadow, so with the loads and stores we will be
2785 // able to propagate labels on data thrown through exceptions, without any
2786 // special handling of the LandingPadInst.
2787 //
2788 // The second element in the pair result of the LandingPadInst is a
2789 // register value, but it is for a type ID and should never be tainted.
2790 DFSF.setShadow(&LPI, DFSF.DFS.getZeroShadow(&LPI));
2791 DFSF.setOrigin(&LPI, DFSF.DFS.ZeroOrigin);
2792}
2793
2794void DFSanVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
2796 DFSF.isLookupTableConstant(
2798 visitInstOperands(GEPI);
2799 return;
2800 }
2801
2802 // Only propagate shadow/origin of base pointer value but ignore those of
2803 // offset operands.
2804 Value *BasePointer = GEPI.getPointerOperand();
2805 DFSF.setShadow(&GEPI, DFSF.getShadow(BasePointer));
2806 if (DFSF.DFS.shouldTrackOrigins())
2807 DFSF.setOrigin(&GEPI, DFSF.getOrigin(BasePointer));
2808}
2809
2810void DFSanVisitor::visitExtractElementInst(ExtractElementInst &I) {
2811 visitInstOperands(I);
2812}
2813
2814void DFSanVisitor::visitInsertElementInst(InsertElementInst &I) {
2815 visitInstOperands(I);
2816}
2817
2818void DFSanVisitor::visitShuffleVectorInst(ShuffleVectorInst &I) {
2819 visitInstOperands(I);
2820}
2821
2822void DFSanVisitor::visitExtractValueInst(ExtractValueInst &I) {
2823 IRBuilder<> IRB(&I);
2824 Value *Agg = I.getAggregateOperand();
2825 Value *AggShadow = DFSF.getShadow(Agg);
2826 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
2827 DFSF.setShadow(&I, ResShadow);
2828 visitInstOperandOrigins(I);
2829}
2830
2831void DFSanVisitor::visitInsertValueInst(InsertValueInst &I) {
2832 IRBuilder<> IRB(&I);
2833 Value *AggShadow = DFSF.getShadow(I.getAggregateOperand());
2834 Value *InsShadow = DFSF.getShadow(I.getInsertedValueOperand());
2835 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
2836 DFSF.setShadow(&I, Res);
2837 visitInstOperandOrigins(I);
2838}
2839
2840void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
2841 bool AllLoadsStores = true;
2842 for (User *U : I.users()) {
2843 if (isa<LoadInst>(U))
2844 continue;
2845
2846 if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
2847 if (SI->getPointerOperand() == &I)
2848 continue;
2849 }
2850
2851 AllLoadsStores = false;
2852 break;
2853 }
2854 if (AllLoadsStores) {
2855 IRBuilder<> IRB(&I);
2856 DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.PrimitiveShadowTy);
2857 if (DFSF.DFS.shouldTrackOrigins()) {
2858 DFSF.AllocaOriginMap[&I] =
2859 IRB.CreateAlloca(DFSF.DFS.OriginTy, nullptr, "_dfsa");
2860 }
2861 }
2862 DFSF.setShadow(&I, DFSF.DFS.ZeroPrimitiveShadow);
2863 DFSF.setOrigin(&I, DFSF.DFS.ZeroOrigin);
2864}
2865
2866void DFSanVisitor::visitSelectInst(SelectInst &I) {
2867 Value *CondShadow = DFSF.getShadow(I.getCondition());
2868 Value *TrueShadow = DFSF.getShadow(I.getTrueValue());
2869 Value *FalseShadow = DFSF.getShadow(I.getFalseValue());
2870 Value *ShadowSel = nullptr;
2871 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2872 std::vector<Value *> Shadows;
2873 std::vector<Value *> Origins;
2874 Value *TrueOrigin =
2875 ShouldTrackOrigins ? DFSF.getOrigin(I.getTrueValue()) : nullptr;
2876 Value *FalseOrigin =
2877 ShouldTrackOrigins ? DFSF.getOrigin(I.getFalseValue()) : nullptr;
2878
2879 DFSF.addConditionalCallbacksIfEnabled(I, I.getCondition());
2880
2881 if (isa<VectorType>(I.getCondition()->getType())) {
2882 ShadowSel = DFSF.combineShadowsThenConvert(I.getType(), TrueShadow,
2883 FalseShadow, I.getIterator());
2884 if (ShouldTrackOrigins) {
2885 Shadows.push_back(TrueShadow);
2886 Shadows.push_back(FalseShadow);
2887 Origins.push_back(TrueOrigin);
2888 Origins.push_back(FalseOrigin);
2889 }
2890 } else {
2891 if (TrueShadow == FalseShadow) {
2892 ShadowSel = TrueShadow;
2893 if (ShouldTrackOrigins) {
2894 Shadows.push_back(TrueShadow);
2895 Origins.push_back(TrueOrigin);
2896 }
2897 } else {
2898 ShadowSel = SelectInst::Create(I.getCondition(), TrueShadow, FalseShadow,
2899 "", I.getIterator());
2900 if (ShouldTrackOrigins) {
2901 Shadows.push_back(ShadowSel);
2902 Origins.push_back(SelectInst::Create(I.getCondition(), TrueOrigin,
2903 FalseOrigin, "", I.getIterator()));
2904 }
2905 }
2906 }
2907 DFSF.setShadow(&I, ClTrackSelectControlFlow ? DFSF.combineShadowsThenConvert(
2908 I.getType(), CondShadow,
2909 ShadowSel, I.getIterator())
2910 : ShadowSel);
2911 if (ShouldTrackOrigins) {
2913 Shadows.push_back(CondShadow);
2914 Origins.push_back(DFSF.getOrigin(I.getCondition()));
2915 }
2916 DFSF.setOrigin(&I, DFSF.combineOrigins(Shadows, Origins, I.getIterator()));
2917 }
2918}
2919
2920void DFSanVisitor::visitMemSetInst(MemSetInst &I) {
2921 IRBuilder<> IRB(&I);
2922 Value *ValShadow = DFSF.getShadow(I.getValue());
2923 Value *ValOrigin = DFSF.DFS.shouldTrackOrigins()
2924 ? DFSF.getOrigin(I.getValue())
2925 : DFSF.DFS.ZeroOrigin;
2926 IRB.CreateCall(DFSF.DFS.DFSanSetLabelFn,
2927 {ValShadow, ValOrigin, I.getDest(),
2928 IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
2929}
2930
2931void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
2932 IRBuilder<> IRB(&I);
2933
2934 // CopyOrMoveOrigin transfers origins by refering to their shadows. So we
2935 // need to move origins before moving shadows.
2936 if (DFSF.DFS.shouldTrackOrigins()) {
2937 IRB.CreateCall(
2938 DFSF.DFS.DFSanMemOriginTransferFn,
2939 {I.getArgOperand(0), I.getArgOperand(1),
2940 IRB.CreateIntCast(I.getArgOperand(2), DFSF.DFS.IntptrTy, false)});
2941 }
2942
2943 Value *DestShadow = DFSF.DFS.getShadowAddress(I.getDest(), I.getIterator());
2944 Value *SrcShadow = DFSF.DFS.getShadowAddress(I.getSource(), I.getIterator());
2945 Value *LenShadow =
2946 IRB.CreateMul(I.getLength(), ConstantInt::get(I.getLength()->getType(),
2947 DFSF.DFS.ShadowWidthBytes));
2948 auto *MTI = cast<MemTransferInst>(
2949 IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
2950 {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
2951 MTI->setDestAlignment(DFSF.getShadowAlign(I.getDestAlign().valueOrOne()));
2952 MTI->setSourceAlignment(DFSF.getShadowAlign(I.getSourceAlign().valueOrOne()));
2953 if (ClEventCallbacks) {
2954 IRB.CreateCall(
2955 DFSF.DFS.DFSanMemTransferCallbackFn,
2956 {DestShadow, IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
2957 }
2958}
2959
2960void DFSanVisitor::visitBranchInst(BranchInst &BR) {
2961 if (!BR.isConditional())
2962 return;
2963
2964 DFSF.addConditionalCallbacksIfEnabled(BR, BR.getCondition());
2965}
2966
2967void DFSanVisitor::visitSwitchInst(SwitchInst &SW) {
2968 DFSF.addConditionalCallbacksIfEnabled(SW, SW.getCondition());
2969}
2970
2971static bool isAMustTailRetVal(Value *RetVal) {
2972 // Tail call may have a bitcast between return.
2973 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
2974 RetVal = I->getOperand(0);
2975 }
2976 if (auto *I = dyn_cast<CallInst>(RetVal)) {
2977 return I->isMustTailCall();
2978 }
2979 return false;
2980}
2981
2982void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
2983 if (!DFSF.IsNativeABI && RI.getReturnValue()) {
2984 // Don't emit the instrumentation for musttail call returns.
2986 return;
2987
2988 Value *S = DFSF.getShadow(RI.getReturnValue());
2989 IRBuilder<> IRB(&RI);
2990 Type *RT = DFSF.F->getFunctionType()->getReturnType();
2991 unsigned Size = getDataLayout().getTypeAllocSize(DFSF.DFS.getShadowTy(RT));
2992 if (Size <= RetvalTLSSize) {
2993 // If the size overflows, stores nothing. At callsite, oversized return
2994 // shadows are set to zero.
2995 IRB.CreateAlignedStore(S, DFSF.getRetvalTLS(RT, IRB), ShadowTLSAlignment);
2996 }
2997 if (DFSF.DFS.shouldTrackOrigins()) {
2998 Value *O = DFSF.getOrigin(RI.getReturnValue());
2999 IRB.CreateStore(O, DFSF.getRetvalOriginTLS());
3000 }
3001 }
3002}
3003
3004void DFSanVisitor::addShadowArguments(Function &F, CallBase &CB,
3005 std::vector<Value *> &Args,
3006 IRBuilder<> &IRB) {
3007 FunctionType *FT = F.getFunctionType();
3008
3009 auto *I = CB.arg_begin();
3010
3011 // Adds non-variable argument shadows.
3012 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N)
3013 Args.push_back(
3014 DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), CB.getIterator()));
3015
3016 // Adds variable argument shadows.
3017 if (FT->isVarArg()) {
3018 auto *LabelVATy = ArrayType::get(DFSF.DFS.PrimitiveShadowTy,
3019 CB.arg_size() - FT->getNumParams());
3020 auto *LabelVAAlloca =
3021 new AllocaInst(LabelVATy, getDataLayout().getAllocaAddrSpace(),
3022 "labelva", DFSF.F->getEntryBlock().begin());
3023
3024 for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) {
3025 auto *LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, N);
3026 IRB.CreateStore(
3027 DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), CB.getIterator()),
3028 LabelVAPtr);
3029 }
3030
3031 Args.push_back(IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, 0));
3032 }
3033
3034 // Adds the return value shadow.
3035 if (!FT->getReturnType()->isVoidTy()) {
3036 if (!DFSF.LabelReturnAlloca) {
3037 DFSF.LabelReturnAlloca = new AllocaInst(
3038 DFSF.DFS.PrimitiveShadowTy, getDataLayout().getAllocaAddrSpace(),
3039 "labelreturn", DFSF.F->getEntryBlock().begin());
3040 }
3041 Args.push_back(DFSF.LabelReturnAlloca);
3042 }
3043}
3044
3045void DFSanVisitor::addOriginArguments(Function &F, CallBase &CB,
3046 std::vector<Value *> &Args,
3047 IRBuilder<> &IRB) {
3048 FunctionType *FT = F.getFunctionType();
3049
3050 auto *I = CB.arg_begin();
3051
3052 // Add non-variable argument origins.
3053 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N)
3054 Args.push_back(DFSF.getOrigin(*I));
3055
3056 // Add variable argument origins.
3057 if (FT->isVarArg()) {
3058 auto *OriginVATy =
3059 ArrayType::get(DFSF.DFS.OriginTy, CB.arg_size() - FT->getNumParams());
3060 auto *OriginVAAlloca =
3061 new AllocaInst(OriginVATy, getDataLayout().getAllocaAddrSpace(),
3062 "originva", DFSF.F->getEntryBlock().begin());
3063
3064 for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) {
3065 auto *OriginVAPtr = IRB.CreateStructGEP(OriginVATy, OriginVAAlloca, N);
3066 IRB.CreateStore(DFSF.getOrigin(*I), OriginVAPtr);
3067 }
3068
3069 Args.push_back(IRB.CreateStructGEP(OriginVATy, OriginVAAlloca, 0));
3070 }
3071
3072 // Add the return value origin.
3073 if (!FT->getReturnType()->isVoidTy()) {
3074 if (!DFSF.OriginReturnAlloca) {
3075 DFSF.OriginReturnAlloca = new AllocaInst(
3076 DFSF.DFS.OriginTy, getDataLayout().getAllocaAddrSpace(),
3077 "originreturn", DFSF.F->getEntryBlock().begin());
3078 }
3079 Args.push_back(DFSF.OriginReturnAlloca);
3080 }
3081}
3082
3083bool DFSanVisitor::visitWrappedCallBase(Function &F, CallBase &CB) {
3084 IRBuilder<> IRB(&CB);
3085 switch (DFSF.DFS.getWrapperKind(&F)) {
3086 case DataFlowSanitizer::WK_Warning:
3087 CB.setCalledFunction(&F);
3088 IRB.CreateCall(DFSF.DFS.DFSanUnimplementedFn,
3089 IRB.CreateGlobalStringPtr(F.getName()));
3090 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3091 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3092 DFSF.setOrigin(&CB, DFSF.DFS.ZeroOrigin);
3093 return true;
3094 case DataFlowSanitizer::WK_Discard:
3095 CB.setCalledFunction(&F);
3096 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3097 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3098 DFSF.setOrigin(&CB, DFSF.DFS.ZeroOrigin);
3099 return true;
3100 case DataFlowSanitizer::WK_Functional:
3101 CB.setCalledFunction(&F);
3102 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3103 visitInstOperands(CB);
3104 return true;
3105 case DataFlowSanitizer::WK_Custom:
3106 // Don't try to handle invokes of custom functions, it's too complicated.
3107 // Instead, invoke the dfsw$ wrapper, which will in turn call the __dfsw_
3108 // wrapper.
3109 CallInst *CI = dyn_cast<CallInst>(&CB);
3110 if (!CI)
3111 return false;
3112
3113 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
3114 FunctionType *FT = F.getFunctionType();
3115 TransformedFunction CustomFn = DFSF.DFS.getCustomFunctionType(FT);
3116 std::string CustomFName = ShouldTrackOrigins ? "__dfso_" : "__dfsw_";
3117 CustomFName += F.getName();
3118 FunctionCallee CustomF = DFSF.DFS.Mod->getOrInsertFunction(
3119 CustomFName, CustomFn.TransformedType);
3120 if (Function *CustomFn = dyn_cast<Function>(CustomF.getCallee())) {
3121 CustomFn->copyAttributesFrom(&F);
3122
3123 // Custom functions returning non-void will write to the return label.
3124 if (!FT->getReturnType()->isVoidTy()) {
3125 CustomFn->removeFnAttrs(DFSF.DFS.ReadOnlyNoneAttrs);
3126 }
3127 }
3128
3129 std::vector<Value *> Args;
3130
3131 // Adds non-variable arguments.
3132 auto *I = CB.arg_begin();
3133 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N) {
3134 Args.push_back(*I);
3135 }
3136
3137 // Adds shadow arguments.
3138 const unsigned ShadowArgStart = Args.size();
3139 addShadowArguments(F, CB, Args, IRB);
3140
3141 // Adds origin arguments.
3142 const unsigned OriginArgStart = Args.size();
3143 if (ShouldTrackOrigins)
3144 addOriginArguments(F, CB, Args, IRB);
3145
3146 // Adds variable arguments.
3147 append_range(Args, drop_begin(CB.args(), FT->getNumParams()));
3148
3149 CallInst *CustomCI = IRB.CreateCall(CustomF, Args);
3150 CustomCI->setCallingConv(CI->getCallingConv());
3151 CustomCI->setAttributes(transformFunctionAttributes(
3152 CustomFn, CI->getContext(), CI->getAttributes()));
3153
3154 // Update the parameter attributes of the custom call instruction to
3155 // zero extend the shadow parameters. This is required for targets
3156 // which consider PrimitiveShadowTy an illegal type.
3157 for (unsigned N = 0; N < FT->getNumParams(); N++) {
3158 const unsigned ArgNo = ShadowArgStart + N;
3159 if (CustomCI->getArgOperand(ArgNo)->getType() ==
3160 DFSF.DFS.PrimitiveShadowTy)
3161 CustomCI->addParamAttr(ArgNo, Attribute::ZExt);
3162 if (ShouldTrackOrigins) {
3163 const unsigned OriginArgNo = OriginArgStart + N;
3164 if (CustomCI->getArgOperand(OriginArgNo)->getType() ==
3165 DFSF.DFS.OriginTy)
3166 CustomCI->addParamAttr(OriginArgNo, Attribute::ZExt);
3167 }
3168 }
3169
3170 // Loads the return value shadow and origin.
3171 if (!FT->getReturnType()->isVoidTy()) {
3172 LoadInst *LabelLoad =
3173 IRB.CreateLoad(DFSF.DFS.PrimitiveShadowTy, DFSF.LabelReturnAlloca);
3174 DFSF.setShadow(CustomCI,
3175 DFSF.expandFromPrimitiveShadow(
3176 FT->getReturnType(), LabelLoad, CB.getIterator()));
3177 if (ShouldTrackOrigins) {
3178 LoadInst *OriginLoad =
3179 IRB.CreateLoad(DFSF.DFS.OriginTy, DFSF.OriginReturnAlloca);
3180 DFSF.setOrigin(CustomCI, OriginLoad);
3181 }
3182 }
3183
3184 CI->replaceAllUsesWith(CustomCI);
3185 CI->eraseFromParent();
3186 return true;
3187 }
3188 return false;
3189}
3190
3191Value *DFSanVisitor::makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
3192 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
3193 uint32_t OrderingTable[NumOrderings] = {};
3194
3195 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
3196 OrderingTable[(int)AtomicOrderingCABI::acquire] =
3197 OrderingTable[(int)AtomicOrderingCABI::consume] =
3198 (int)AtomicOrderingCABI::acquire;
3199 OrderingTable[(int)AtomicOrderingCABI::release] =
3200 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
3201 (int)AtomicOrderingCABI::acq_rel;
3202 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
3203 (int)AtomicOrderingCABI::seq_cst;
3204
3205 return ConstantDataVector::get(IRB.getContext(), OrderingTable);
3206}
3207
3208void DFSanVisitor::visitLibAtomicLoad(CallBase &CB) {
3209 // Since we use getNextNode here, we can't have CB terminate the BB.
3210 assert(isa<CallInst>(CB));
3211
3212 IRBuilder<> IRB(&CB);
3213 Value *Size = CB.getArgOperand(0);
3214 Value *SrcPtr = CB.getArgOperand(1);
3215 Value *DstPtr = CB.getArgOperand(2);
3216 Value *Ordering = CB.getArgOperand(3);
3217 // Convert the call to have at least Acquire ordering to make sure
3218 // the shadow operations aren't reordered before it.
3219 Value *NewOrdering =
3220 IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
3221 CB.setArgOperand(3, NewOrdering);
3222
3223 IRBuilder<> NextIRB(CB.getNextNode());
3224 NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3225
3226 // TODO: Support ClCombinePointerLabelsOnLoad
3227 // TODO: Support ClEventCallbacks
3228
3229 NextIRB.CreateCall(
3230 DFSF.DFS.DFSanMemShadowOriginTransferFn,
3231 {DstPtr, SrcPtr, NextIRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3232}
3233
3234Value *DFSanVisitor::makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
3235 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
3236 uint32_t OrderingTable[NumOrderings] = {};
3237
3238 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
3239 OrderingTable[(int)AtomicOrderingCABI::release] =
3240 (int)AtomicOrderingCABI::release;
3241 OrderingTable[(int)AtomicOrderingCABI::consume] =
3242 OrderingTable[(int)AtomicOrderingCABI::acquire] =
3243 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
3244 (int)AtomicOrderingCABI::acq_rel;
3245 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
3246 (int)AtomicOrderingCABI::seq_cst;
3247
3248 return ConstantDataVector::get(IRB.getContext(), OrderingTable);
3249}
3250
3251void DFSanVisitor::visitLibAtomicStore(CallBase &CB) {
3252 IRBuilder<> IRB(&CB);
3253 Value *Size = CB.getArgOperand(0);
3254 Value *SrcPtr = CB.getArgOperand(1);
3255 Value *DstPtr = CB.getArgOperand(2);
3256 Value *Ordering = CB.getArgOperand(3);
3257 // Convert the call to have at least Release ordering to make sure
3258 // the shadow operations aren't reordered after it.
3259 Value *NewOrdering =
3260 IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
3261 CB.setArgOperand(3, NewOrdering);
3262
3263 // TODO: Support ClCombinePointerLabelsOnStore
3264 // TODO: Support ClEventCallbacks
3265
3266 IRB.CreateCall(
3267 DFSF.DFS.DFSanMemShadowOriginTransferFn,
3268 {DstPtr, SrcPtr, IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3269}
3270
3271void DFSanVisitor::visitLibAtomicExchange(CallBase &CB) {
3272 // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret, int
3273 // ordering)
3274 IRBuilder<> IRB(&CB);
3275 Value *Size = CB.getArgOperand(0);
3276 Value *TargetPtr = CB.getArgOperand(1);
3277 Value *SrcPtr = CB.getArgOperand(2);
3278 Value *DstPtr = CB.getArgOperand(3);
3279
3280 // This operation is not atomic for the shadow and origin memory.
3281 // This could result in DFSan false positives or false negatives.
3282 // For now we will assume these operations are rare, and
3283 // the additional complexity to address this is not warrented.
3284
3285 // Current Target to Dest
3286 IRB.CreateCall(
3287 DFSF.DFS.DFSanMemShadowOriginTransferFn,
3288 {DstPtr, TargetPtr, IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3289
3290 // Current Src to Target (overriding)
3291 IRB.CreateCall(
3292 DFSF.DFS.DFSanMemShadowOriginTransferFn,
3293 {TargetPtr, SrcPtr, IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3294}
3295
3296void DFSanVisitor::visitLibAtomicCompareExchange(CallBase &CB) {
3297 // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected, void
3298 // *desired, int success_order, int failure_order)
3299 Value *Size = CB.getArgOperand(0);
3300 Value *TargetPtr = CB.getArgOperand(1);
3301 Value *ExpectedPtr = CB.getArgOperand(2);
3302 Value *DesiredPtr = CB.getArgOperand(3);
3303
3304 // This operation is not atomic for the shadow and origin memory.
3305 // This could result in DFSan false positives or false negatives.
3306 // For now we will assume these operations are rare, and
3307 // the additional complexity to address this is not warrented.
3308
3309 IRBuilder<> NextIRB(CB.getNextNode());
3310 NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3311
3312 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3313
3314 // If original call returned true, copy Desired to Target.
3315 // If original call returned false, copy Target to Expected.
3316 NextIRB.CreateCall(DFSF.DFS.DFSanMemShadowOriginConditionalExchangeFn,
3317 {NextIRB.CreateIntCast(&CB, NextIRB.getInt8Ty(), false),
3318 TargetPtr, ExpectedPtr, DesiredPtr,
3319 NextIRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3320}
3321
3322void DFSanVisitor::visitCallBase(CallBase &CB) {
3324 if ((F && F->isIntrinsic()) || CB.isInlineAsm()) {
3325 visitInstOperands(CB);
3326 return;
3327 }
3328
3329 // Calls to this function are synthesized in wrappers, and we shouldn't
3330 // instrument them.
3331 if (F == DFSF.DFS.DFSanVarargWrapperFn.getCallee()->stripPointerCasts())
3332 return;
3333
3334 LibFunc LF;
3335 if (DFSF.TLI.getLibFunc(CB, LF)) {
3336 // libatomic.a functions need to have special handling because there isn't
3337 // a good way to intercept them or compile the library with
3338 // instrumentation.
3339 switch (LF) {
3340 case LibFunc_atomic_load:
3341 if (!isa<CallInst>(CB)) {
3342 llvm::errs() << "DFSAN -- cannot instrument invoke of libatomic load. "
3343 "Ignoring!\n";
3344 break;
3345 }
3346 visitLibAtomicLoad(CB);
3347 return;
3348 case LibFunc_atomic_store:
3349 visitLibAtomicStore(CB);
3350 return;
3351 default:
3352 break;
3353 }
3354 }
3355
3356 // TODO: These are not supported by TLI? They are not in the enum.
3357 if (F && F->hasName() && !F->isVarArg()) {
3358 if (F->getName() == "__atomic_exchange") {
3359 visitLibAtomicExchange(CB);
3360 return;
3361 }
3362 if (F->getName() == "__atomic_compare_exchange") {
3363 visitLibAtomicCompareExchange(CB);
3364 return;
3365 }
3366 }
3367
3369 DFSF.DFS.UnwrappedFnMap.find(CB.getCalledOperand());
3370 if (UnwrappedFnIt != DFSF.DFS.UnwrappedFnMap.end())
3371 if (visitWrappedCallBase(*UnwrappedFnIt->second, CB))
3372 return;
3373
3374 IRBuilder<> IRB(&CB);
3375
3376 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
3377 FunctionType *FT = CB.getFunctionType();
3378 const DataLayout &DL = getDataLayout();
3379
3380 // Stores argument shadows.
3381 unsigned ArgOffset = 0;
3382 for (unsigned I = 0, N = FT->getNumParams(); I != N; ++I) {
3383 if (ShouldTrackOrigins) {
3384 // Ignore overflowed origins
3385 Value *ArgShadow = DFSF.getShadow(CB.getArgOperand(I));
3386 if (I < DFSF.DFS.NumOfElementsInArgOrgTLS &&
3387 !DFSF.DFS.isZeroShadow(ArgShadow))
3388 IRB.CreateStore(DFSF.getOrigin(CB.getArgOperand(I)),
3389 DFSF.getArgOriginTLS(I, IRB));
3390 }
3391
3392 unsigned Size =
3393 DL.getTypeAllocSize(DFSF.DFS.getShadowTy(FT->getParamType(I)));
3394 // Stop storing if arguments' size overflows. Inside a function, arguments
3395 // after overflow have zero shadow values.
3396 if (ArgOffset + Size > ArgTLSSize)
3397 break;
3398 IRB.CreateAlignedStore(DFSF.getShadow(CB.getArgOperand(I)),
3399 DFSF.getArgTLS(FT->getParamType(I), ArgOffset, IRB),
3401 ArgOffset += alignTo(Size, ShadowTLSAlignment);
3402 }
3403
3404 Instruction *Next = nullptr;
3405 if (!CB.getType()->isVoidTy()) {
3406 if (InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
3407 if (II->getNormalDest()->getSinglePredecessor()) {
3408 Next = &II->getNormalDest()->front();
3409 } else {
3410 BasicBlock *NewBB =
3411 SplitEdge(II->getParent(), II->getNormalDest(), &DFSF.DT);
3412 Next = &NewBB->front();
3413 }
3414 } else {
3415 assert(CB.getIterator() != CB.getParent()->end());
3416 Next = CB.getNextNode();
3417 }
3418
3419 // Don't emit the epilogue for musttail call returns.
3420 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
3421 return;
3422
3423 // Loads the return value shadow.
3424 IRBuilder<> NextIRB(Next);
3425 unsigned Size = DL.getTypeAllocSize(DFSF.DFS.getShadowTy(&CB));
3426 if (Size > RetvalTLSSize) {
3427 // Set overflowed return shadow to be zero.
3428 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3429 } else {
3430 LoadInst *LI = NextIRB.CreateAlignedLoad(
3431 DFSF.DFS.getShadowTy(&CB), DFSF.getRetvalTLS(CB.getType(), NextIRB),
3432 ShadowTLSAlignment, "_dfsret");
3433 DFSF.SkipInsts.insert(LI);
3434 DFSF.setShadow(&CB, LI);
3435 DFSF.NonZeroChecks.push_back(LI);
3436 }
3437
3438 if (ShouldTrackOrigins) {
3439 LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.OriginTy,
3440 DFSF.getRetvalOriginTLS(), "_dfsret_o");
3441 DFSF.SkipInsts.insert(LI);
3442 DFSF.setOrigin(&CB, LI);
3443 }
3444
3445 DFSF.addReachesFunctionCallbacksIfEnabled(NextIRB, CB, &CB);
3446 }
3447}
3448
3449void DFSanVisitor::visitPHINode(PHINode &PN) {
3450 Type *ShadowTy = DFSF.DFS.getShadowTy(&PN);
3451 PHINode *ShadowPN = PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "",
3452 PN.getIterator());
3453
3454 // Give the shadow phi node valid predecessors to fool SplitEdge into working.
3455 Value *UndefShadow = UndefValue::get(ShadowTy);
3456 for (BasicBlock *BB : PN.blocks())
3457 ShadowPN->addIncoming(UndefShadow, BB);
3458
3459 DFSF.setShadow(&PN, ShadowPN);
3460
3461 PHINode *OriginPN = nullptr;
3462 if (DFSF.DFS.shouldTrackOrigins()) {
3463 OriginPN = PHINode::Create(DFSF.DFS.OriginTy, PN.getNumIncomingValues(), "",
3464 PN.getIterator());
3465 Value *UndefOrigin = UndefValue::get(DFSF.DFS.OriginTy);
3466 for (BasicBlock *BB : PN.blocks())
3467 OriginPN->addIncoming(UndefOrigin, BB);
3468 DFSF.setOrigin(&PN, OriginPN);
3469 }
3470
3471 DFSF.PHIFixups.push_back({&PN, ShadowPN, OriginPN});
3472}
3473
3476 // Return early if nosanitize_dataflow module flag is present for the module.
3477 if (checkIfAlreadyInstrumented(M, "nosanitize_dataflow"))
3478 return PreservedAnalyses::all();
3479 auto GetTLI = [&](Function &F) -> TargetLibraryInfo & {
3480 auto &FAM =
3483 };
3484 if (!DataFlowSanitizer(ABIListFiles).runImpl(M, GetTLI))
3485 return PreservedAnalyses::all();
3486
3488 // GlobalsAA is considered stateless and does not get invalidated unless
3489 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
3490 // make changes that require GlobalsAA to be invalidated.
3491 PA.abandon<GlobalsAA>();
3492 return PA;
3493}
static bool isConstant(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const MemoryMapParams Linux_LoongArch64_MemoryMapParams
const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< bool > ClTrackSelectControlFlow("dfsan-track-select-control-flow", cl::desc("Propagate labels from condition values of select instructions " "to results."), cl::Hidden, cl::init(true))
static cl::list< std::string > ClCombineTaintLookupTables("dfsan-combine-taint-lookup-table", cl::desc("When dfsan-combine-offset-labels-on-gep and/or " "dfsan-combine-pointer-labels-on-load are false, this flag can " "be used to re-enable combining offset and/or pointer taint when " "loading specific constant global variables (i.e. lookup tables)."), cl::Hidden)
static const Align MinOriginAlignment
static cl::opt< int > ClTrackOrigins("dfsan-track-origins", cl::desc("Track origins of labels"), cl::Hidden, cl::init(0))
static cl::list< std::string > ClABIListFiles("dfsan-abilist", cl::desc("File listing native ABI functions and how the pass treats them"), cl::Hidden)
static cl::opt< bool > ClReachesFunctionCallbacks("dfsan-reaches-function-callbacks", cl::desc("Insert calls to callback functions on data reaching a function."), cl::Hidden, cl::init(false))
static Value * expandFromPrimitiveShadowRecursive(Value *Shadow, SmallVector< unsigned, 4 > &Indices, Type *SubShadowTy, Value *PrimitiveShadow, IRBuilder<> &IRB)
static cl::opt< int > ClInstrumentWithCallThreshold("dfsan-instrument-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< bool > ClPreserveAlignment("dfsan-preserve-alignment", cl::desc("respect alignment requirements provided by input IR"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClDebugNonzeroLabels("dfsan-debug-nonzero-labels", cl::desc("Insert calls to __dfsan_nonzero_label on observing a parameter, " "load or return with a nonzero label"), cl::Hidden)
static cl::opt< bool > ClCombineOffsetLabelsOnGEP("dfsan-combine-offset-labels-on-gep", cl::desc("Combine the label of the offset with the label of the pointer when " "doing pointer arithmetic."), cl::Hidden, cl::init(true))
static cl::opt< bool > ClIgnorePersonalityRoutine("dfsan-ignore-personality-routine", cl::desc("If a personality routine is marked uninstrumented from the ABI " "list, do not create a wrapper for it."), cl::Hidden, cl::init(false))
static const Align ShadowTLSAlignment
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
Value * StripPointerGEPsAndCasts(Value *V)
const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< bool > ClConditionalCallbacks("dfsan-conditional-callbacks", cl::desc("Insert calls to callback functions on conditionals."), cl::Hidden, cl::init(false))
static cl::opt< bool > ClCombinePointerLabelsOnLoad("dfsan-combine-pointer-labels-on-load", cl::desc("Combine the label of the pointer with the label of the data when " "loading from memory."), cl::Hidden, cl::init(true))
static StringRef getGlobalTypeString(const GlobalValue &G)
static cl::opt< bool > ClCombinePointerLabelsOnStore("dfsan-combine-pointer-labels-on-store", cl::desc("Combine the label of the pointer with the label of the data when " "storing in memory."), cl::Hidden, cl::init(false))
static const unsigned ArgTLSSize
static const unsigned RetvalTLSSize
static bool isAMustTailRetVal(Value *RetVal)
static cl::opt< bool > ClEventCallbacks("dfsan-event-callbacks", cl::desc("Insert calls to __dfsan_*_callback functions on data events."), cl::Hidden, cl::init(false))
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
uint64_t Addr
std::string Name
uint64_t Size
static bool runImpl(Function &F, const TargetLowering &TLI)
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
Module.h This file contains the declarations for the Module class.
nvptx lower args
uint64_t IntrinsicInst * II
#define P(N)
Module * Mod
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
StringSet - A set-like wrapper for the StringMap.
Defines the virtual file system interface vfs::FileSystem.
Class for arbitrary precision integers.
Definition: APInt.h:78
an instruction to allocate memory on the stack
Definition: Instructions.h:61
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:405
This class represents an incoming formal argument to a Function.
Definition: Argument.h:31
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:495
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:696
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
unsigned getNumAttrSets() const
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition: AttributeMask.h:44
static Attribute getWithMemoryEffects(LLVMContext &Context, MemoryEffects ME)
Definition: Attributes.cpp:280
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
const Instruction & front() const
Definition: BasicBlock.h:471
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:212
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:177
This class represents a no-op cast from one type to another.
Conditional or Unconditional Branch instruction.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1236
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1532
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1527
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1465
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1523
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1385
Value * getCalledOperand() const
Definition: InstrTypes.h:1458
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1546
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Definition: InstrTypes.h:1584
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1410
void setArgOperand(unsigned i, Value *v)
Definition: InstrTypes.h:1415
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1391
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1323
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1401
unsigned arg_size() const
Definition: InstrTypes.h:1408
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1542
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1594
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition: InstrTypes.h:1504
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
bool isMustTailCall() const
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:530
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:747
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1650
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
Definition: Constants.cpp:2966
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1097
This is the shared class of boolean and integer constants.
Definition: Constants.h:81
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition: Constants.h:124
This is an important base class in LLVM.
Definition: Constant.h:42
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:90
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:103
A debug info location.
Definition: DebugLoc.h:33
unsigned getLine() const
Definition: DebugLoc.cpp:24
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:20
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
This instruction extracts a single (scalar) element from a VectorType value.
This instruction extracts a struct member or array element value from an aggregate value.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:680
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:172
void removeFnAttrs(const AttributeMask &Attrs)
Definition: Function.cpp:709
void removeFnAttr(Attribute::AttrKind Kind)
Remove function attributes from this function.
Definition: Function.cpp:701
arg_iterator arg_begin()
Definition: Function.h:866
void removeRetAttrs(const AttributeMask &Attrs)
removes the attributes from the return value list of attributes.
Definition: Function.cpp:721
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
Definition: Function.cpp:863
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:915
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:580
const GlobalObject * getAliaseeObject() const
Definition: Globals.cpp:588
static bool isExternalWeakLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:412
LinkageTypes getLinkage() const
Definition: GlobalValue.h:546
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:51
@ WeakODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:57
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:55
Type * getValueType() const
Definition: GlobalValue.h:296
Analysis pass providing a never-invalidated alias analysis result.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2480
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Definition: IRBuilder.h:1884
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Definition: IRBuilder.h:1778
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2531
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2468
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Definition: IRBuilder.h:1812
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2178
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2524
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1091
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:172
Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")
Definition: IRBuilder.h:1977
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2130
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1442
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:528
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2253
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition: IRBuilder.h:1871
Constant * CreateGlobalStringPtr(StringRef Str, const Twine &Name="", unsigned AddressSpace=0, Module *M=nullptr, bool AddNull=true)
Same as CreateGlobalString, but return a pointer with "i8*" type instead of a pointer to array of i8.
Definition: IRBuilder.h:2000
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1795
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1421
LLVMContext & getContext() const
Definition: IRBuilder.h:173
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1480
Value * CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, const Twine &Name="")
Definition: IRBuilder.h:1964
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1808
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1332
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Definition: IRBuilder.h:2015
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1502
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2204
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Definition: IRBuilder.h:1831
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2420
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1524
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1366
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2674
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:563
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
Definition: InstVisitor.h:78
RetTy visitCmpInst(CmpInst &I)
Definition: InstVisitor.h:262
RetTy visitExtractElementInst(ExtractElementInst &I)
Definition: InstVisitor.h:191
RetTy visitCallBase(CallBase &I)
Definition: InstVisitor.h:267
RetTy visitInsertValueInst(InsertValueInst &I)
Definition: InstVisitor.h:195
RetTy visitShuffleVectorInst(ShuffleVectorInst &I)
Definition: InstVisitor.h:193
RetTy visitLandingPadInst(LandingPadInst &I)
Definition: InstVisitor.h:196
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
Definition: InstVisitor.h:171
RetTy visitBitCastInst(BitCastInst &I)
Definition: InstVisitor.h:187
RetTy visitSwitchInst(SwitchInst &I)
Definition: InstVisitor.h:232
RetTy visitPHINode(PHINode &I)
Definition: InstVisitor.h:175
RetTy visitReturnInst(ReturnInst &I)
Definition: InstVisitor.h:226
RetTy visitExtractValueInst(ExtractValueInst &I)
Definition: InstVisitor.h:194
RetTy visitUnaryOperator(UnaryOperator &I)
Definition: InstVisitor.h:260
RetTy visitStoreInst(StoreInst &I)
Definition: InstVisitor.h:170
RetTy visitInsertElementInst(InsertElementInst &I)
Definition: InstVisitor.h:192
RetTy visitAtomicRMWInst(AtomicRMWInst &I)
Definition: InstVisitor.h:172
RetTy visitAllocaInst(AllocaInst &I)
Definition: InstVisitor.h:168
RetTy visitBinaryOperator(BinaryOperator &I)
Definition: InstVisitor.h:261
RetTy visitMemTransferInst(MemTransferInst &I)
Definition: InstVisitor.h:214
RetTy visitMemSetInst(MemSetInst &I)
Definition: InstVisitor.h:209
RetTy visitCastInst(CastInst &I)
Definition: InstVisitor.h:259
RetTy visitBranchInst(BranchInst &I)
Definition: InstVisitor.h:229
RetTy visitSelectInst(SelectInst &I)
Definition: InstVisitor.h:189
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
Definition: InstVisitor.h:174
RetTy visitLoadInst(LoadInst &I)
Definition: InstVisitor.h:169
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:466
bool isAtomic() const LLVM_READONLY
Return true if this instruction has an AtomicOrdering of unordered or higher.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:92
bool isTerminator() const
Definition: Instruction.h:277
bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:463
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Definition: Instruction.cpp:74
Class to represent integer types.
Definition: DerivedTypes.h:40
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:266
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:174
void setAlignment(Align Align)
Definition: Instructions.h:213
Value * getPointerOperand()
Definition: Instructions.h:253
void setOrdering(AtomicOrdering Ordering)
Sets the ordering constraint of this load instruction.
Definition: Instructions.h:223
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:218
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:209
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
Definition: MDBuilder.cpp:47
Metadata node.
Definition: Metadata.h:1069
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
This class wraps the llvm.memcpy/memmove intrinsics.
static MemoryEffectsBase readOnly()
Create MemoryEffectsBase that can read any memory.
Definition: ModRef.h:122
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const std::string & getModuleInlineAsm() const
Get any module-scope inline assembly blocks.
Definition: Module.h:303
void setModuleInlineAsm(StringRef Asm)
Set the module-scope inline assembly blocks.
Definition: Module.h:342
FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:169
Constant * getOrInsertGlobal(StringRef Name, Type *Ty, function_ref< GlobalVariable *()> CreateGlobalCallback)
Look up the specified global in the module symbol table.
Definition: Module.cpp:223
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition: Operator.h:42
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
iterator_range< const_block_iterator > blocks() const
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1852
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:114
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
void abandon()
Mark an analysis as abandoned.
Definition: Analysis.h:164
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, InsertPosition InsertBefore=nullptr)
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", InsertPosition InsertBefore=nullptr, Instruction *MDFrom=nullptr)
This instruction constructs a fixed permutation of two input vectors.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:436
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:442
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:503
size_t size() const
Definition: SmallVector.h:92
void push_back(const T &Elt)
Definition: SmallVector.h:427
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1210
static std::unique_ptr< SpecialCaseList > createOrDie(const std::vector< std::string > &Paths, llvm::vfs::FileSystem &FS)
Parses the special case list entries from files.
An instruction for storing to memory.
Definition: Instructions.h:290
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
StringSet - A wrapper for StringMap that provides set-like functionality.
Definition: StringSet.h:23
std::pair< typename Base::iterator, bool > insert(StringRef key)
Definition: StringSet.h:38
Class to represent struct types.
Definition: DerivedTypes.h:216
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:361
Multiway switch.
Value * getCondition() const
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
@ loongarch64
Definition: Triple.h:62
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:298
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:224
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1833
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:694
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
bool hasName() const
Definition: Value.h:261
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
Definition: ilist_node.h:32
self_iterator getIterator()
Definition: ilist_node.h:132
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:353
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Key
PAL metadata keys.
AttributeMask typeIncompatible(Type *Ty, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:1110
@ CE
Windows NT (Windows on ARM)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
NodeAddr< PhiNode * > Phi
Definition: RDFGraph.h:390
NodeAddr< BlockNode * > Block
Definition: RDFGraph.h:392
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:480
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2073
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
void erase(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition: STLExtras.h:2065
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, const LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
Definition: Alignment.h:111
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:3202
bool checkIfAlreadyInstrumented(Module &M, StringRef Flag)
Check if module has flag attached, if not add the flag.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85