LLVM 17.0.0git
DataFlowSanitizer.cpp
Go to the documentation of this file.
1//===- DataFlowSanitizer.cpp - dynamic data flow analysis -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This file is a part of DataFlowSanitizer, a generalised dynamic data flow
11/// analysis.
12///
13/// Unlike other Sanitizer tools, this tool is not designed to detect a specific
14/// class of bugs on its own. Instead, it provides a generic dynamic data flow
15/// analysis framework to be used by clients to help detect application-specific
16/// issues within their own code.
17///
18/// The analysis is based on automatic propagation of data flow labels (also
19/// known as taint labels) through a program as it performs computation.
20///
21/// Argument and return value labels are passed through TLS variables
22/// __dfsan_arg_tls and __dfsan_retval_tls.
23///
24/// Each byte of application memory is backed by a shadow memory byte. The
25/// shadow byte can represent up to 8 labels. On Linux/x86_64, memory is then
26/// laid out as follows:
27///
28/// +--------------------+ 0x800000000000 (top of memory)
29/// | application 3 |
30/// +--------------------+ 0x700000000000
31/// | invalid |
32/// +--------------------+ 0x610000000000
33/// | origin 1 |
34/// +--------------------+ 0x600000000000
35/// | application 2 |
36/// +--------------------+ 0x510000000000
37/// | shadow 1 |
38/// +--------------------+ 0x500000000000
39/// | invalid |
40/// +--------------------+ 0x400000000000
41/// | origin 3 |
42/// +--------------------+ 0x300000000000
43/// | shadow 3 |
44/// +--------------------+ 0x200000000000
45/// | origin 2 |
46/// +--------------------+ 0x110000000000
47/// | invalid |
48/// +--------------------+ 0x100000000000
49/// | shadow 2 |
50/// +--------------------+ 0x010000000000
51/// | application 1 |
52/// +--------------------+ 0x000000000000
53///
54/// MEM_TO_SHADOW(mem) = mem ^ 0x500000000000
55/// SHADOW_TO_ORIGIN(shadow) = shadow + 0x100000000000
56///
57/// For more information, please refer to the design document:
58/// http://clang.llvm.org/docs/DataFlowSanitizerDesign.html
59//
60//===----------------------------------------------------------------------===//
61
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DenseSet.h"
68#include "llvm/ADT/StringRef.h"
69#include "llvm/ADT/StringSet.h"
70#include "llvm/ADT/iterator.h"
74#include "llvm/IR/Argument.h"
75#include "llvm/IR/Attributes.h"
76#include "llvm/IR/BasicBlock.h"
77#include "llvm/IR/Constant.h"
78#include "llvm/IR/Constants.h"
79#include "llvm/IR/DataLayout.h"
81#include "llvm/IR/Dominators.h"
82#include "llvm/IR/Function.h"
83#include "llvm/IR/GlobalAlias.h"
84#include "llvm/IR/GlobalValue.h"
86#include "llvm/IR/IRBuilder.h"
87#include "llvm/IR/InstVisitor.h"
88#include "llvm/IR/InstrTypes.h"
89#include "llvm/IR/Instruction.h"
92#include "llvm/IR/MDBuilder.h"
93#include "llvm/IR/Module.h"
94#include "llvm/IR/PassManager.h"
95#include "llvm/IR/Type.h"
96#include "llvm/IR/User.h"
97#include "llvm/IR/Value.h"
99#include "llvm/Pass.h"
101#include "llvm/Support/Casting.h"
110#include <algorithm>
111#include <cassert>
112#include <cstddef>
113#include <cstdint>
114#include <memory>
115#include <set>
116#include <string>
117#include <utility>
118#include <vector>
119
120using namespace llvm;
121
122// This must be consistent with ShadowWidthBits.
124
126
127// The size of TLS variables. These constants must be kept in sync with the ones
128// in dfsan.cpp.
129static const unsigned ArgTLSSize = 800;
130static const unsigned RetvalTLSSize = 800;
131
132// The -dfsan-preserve-alignment flag controls whether this pass assumes that
133// alignment requirements provided by the input IR are correct. For example,
134// if the input IR contains a load with alignment 8, this flag will cause
135// the shadow load to have alignment 16. This flag is disabled by default as
136// we have unfortunately encountered too much code (including Clang itself;
137// see PR14291) which performs misaligned access.
139 "dfsan-preserve-alignment",
140 cl::desc("respect alignment requirements provided by input IR"), cl::Hidden,
141 cl::init(false));
142
143// The ABI list files control how shadow parameters are passed. The pass treats
144// every function labelled "uninstrumented" in the ABI list file as conforming
145// to the "native" (i.e. unsanitized) ABI. Unless the ABI list contains
146// additional annotations for those functions, a call to one of those functions
147// will produce a warning message, as the labelling behaviour of the function is
148// unknown. The other supported annotations for uninstrumented functions are
149// "functional" and "discard", which are described below under
150// DataFlowSanitizer::WrapperKind.
151// Functions will often be labelled with both "uninstrumented" and one of
152// "functional" or "discard". This will leave the function unchanged by this
153// pass, and create a wrapper function that will call the original.
154//
155// Instrumented functions can also be annotated as "force_zero_labels", which
156// will make all shadow and return values set zero labels.
157// Functions should never be labelled with both "force_zero_labels" and
158// "uninstrumented" or any of the unistrumented wrapper kinds.
160 "dfsan-abilist",
161 cl::desc("File listing native ABI functions and how the pass treats them"),
162 cl::Hidden);
163
164// Controls whether the pass includes or ignores the labels of pointers in load
165// instructions.
167 "dfsan-combine-pointer-labels-on-load",
168 cl::desc("Combine the label of the pointer with the label of the data when "
169 "loading from memory."),
170 cl::Hidden, cl::init(true));
171
172// Controls whether the pass includes or ignores the labels of pointers in
173// stores instructions.
175 "dfsan-combine-pointer-labels-on-store",
176 cl::desc("Combine the label of the pointer with the label of the data when "
177 "storing in memory."),
178 cl::Hidden, cl::init(false));
179
180// Controls whether the pass propagates labels of offsets in GEP instructions.
182 "dfsan-combine-offset-labels-on-gep",
183 cl::desc(
184 "Combine the label of the offset with the label of the pointer when "
185 "doing pointer arithmetic."),
186 cl::Hidden, cl::init(true));
187
189 "dfsan-combine-taint-lookup-table",
190 cl::desc(
191 "When dfsan-combine-offset-labels-on-gep and/or "
192 "dfsan-combine-pointer-labels-on-load are false, this flag can "
193 "be used to re-enable combining offset and/or pointer taint when "
194 "loading specific constant global variables (i.e. lookup tables)."),
195 cl::Hidden);
196
198 "dfsan-debug-nonzero-labels",
199 cl::desc("Insert calls to __dfsan_nonzero_label on observing a parameter, "
200 "load or return with a nonzero label"),
201 cl::Hidden);
202
203// Experimental feature that inserts callbacks for certain data events.
204// Currently callbacks are only inserted for loads, stores, memory transfers
205// (i.e. memcpy and memmove), and comparisons.
206//
207// If this flag is set to true, the user must provide definitions for the
208// following callback functions:
209// void __dfsan_load_callback(dfsan_label Label, void* addr);
210// void __dfsan_store_callback(dfsan_label Label, void* addr);
211// void __dfsan_mem_transfer_callback(dfsan_label *Start, size_t Len);
212// void __dfsan_cmp_callback(dfsan_label CombinedLabel);
214 "dfsan-event-callbacks",
215 cl::desc("Insert calls to __dfsan_*_callback functions on data events."),
216 cl::Hidden, cl::init(false));
217
218// Experimental feature that inserts callbacks for conditionals, including:
219// conditional branch, switch, select.
220// This must be true for dfsan_set_conditional_callback() to have effect.
222 "dfsan-conditional-callbacks",
223 cl::desc("Insert calls to callback functions on conditionals."), cl::Hidden,
224 cl::init(false));
225
226// Experimental feature that inserts callbacks for data reaching a function,
227// either via function arguments and loads.
228// This must be true for dfsan_set_reaches_function_callback() to have effect.
230 "dfsan-reaches-function-callbacks",
231 cl::desc("Insert calls to callback functions on data reaching a function."),
232 cl::Hidden, cl::init(false));
233
234// Controls whether the pass tracks the control flow of select instructions.
236 "dfsan-track-select-control-flow",
237 cl::desc("Propagate labels from condition values of select instructions "
238 "to results."),
239 cl::Hidden, cl::init(true));
240
241// TODO: This default value follows MSan. DFSan may use a different value.
243 "dfsan-instrument-with-call-threshold",
244 cl::desc("If the function being instrumented requires more than "
245 "this number of origin stores, use callbacks instead of "
246 "inline checks (-1 means never use callbacks)."),
247 cl::Hidden, cl::init(3500));
248
249// Controls how to track origins.
250// * 0: do not track origins.
251// * 1: track origins at memory store operations.
252// * 2: track origins at memory load and store operations.
253// TODO: track callsites.
254static cl::opt<int> ClTrackOrigins("dfsan-track-origins",
255 cl::desc("Track origins of labels"),
256 cl::Hidden, cl::init(0));
257
259 "dfsan-ignore-personality-routine",
260 cl::desc("If a personality routine is marked uninstrumented from the ABI "
261 "list, do not create a wrapper for it."),
262 cl::Hidden, cl::init(false));
263
265 // Types of GlobalVariables are always pointer types.
266 Type *GType = G.getValueType();
267 // For now we support excluding struct types only.
268 if (StructType *SGType = dyn_cast<StructType>(GType)) {
269 if (!SGType->isLiteral())
270 return SGType->getName();
271 }
272 return "<unknown type>";
273}
274
275namespace {
276
277// Memory map parameters used in application-to-shadow address calculation.
278// Offset = (Addr & ~AndMask) ^ XorMask
279// Shadow = ShadowBase + Offset
280// Origin = (OriginBase + Offset) & ~3ULL
281struct MemoryMapParams {
282 uint64_t AndMask;
283 uint64_t XorMask;
284 uint64_t ShadowBase;
285 uint64_t OriginBase;
286};
287
288} // end anonymous namespace
289
290// NOLINTBEGIN(readability-identifier-naming)
291// aarch64 Linux
292const MemoryMapParams Linux_AArch64_MemoryMapParams = {
293 0, // AndMask (not used)
294 0x0B00000000000, // XorMask
295 0, // ShadowBase (not used)
296 0x0200000000000, // OriginBase
297};
298
299// x86_64 Linux
300const MemoryMapParams Linux_X86_64_MemoryMapParams = {
301 0, // AndMask (not used)
302 0x500000000000, // XorMask
303 0, // ShadowBase (not used)
304 0x100000000000, // OriginBase
305};
306// NOLINTEND(readability-identifier-naming)
307
308namespace {
309
310class DFSanABIList {
311 std::unique_ptr<SpecialCaseList> SCL;
312
313public:
314 DFSanABIList() = default;
315
316 void set(std::unique_ptr<SpecialCaseList> List) { SCL = std::move(List); }
317
318 /// Returns whether either this function or its source file are listed in the
319 /// given category.
320 bool isIn(const Function &F, StringRef Category) const {
321 return isIn(*F.getParent(), Category) ||
322 SCL->inSection("dataflow", "fun", F.getName(), Category);
323 }
324
325 /// Returns whether this global alias is listed in the given category.
326 ///
327 /// If GA aliases a function, the alias's name is matched as a function name
328 /// would be. Similarly, aliases of globals are matched like globals.
329 bool isIn(const GlobalAlias &GA, StringRef Category) const {
330 if (isIn(*GA.getParent(), Category))
331 return true;
332
333 if (isa<FunctionType>(GA.getValueType()))
334 return SCL->inSection("dataflow", "fun", GA.getName(), Category);
335
336 return SCL->inSection("dataflow", "global", GA.getName(), Category) ||
337 SCL->inSection("dataflow", "type", getGlobalTypeString(GA),
338 Category);
339 }
340
341 /// Returns whether this module is listed in the given category.
342 bool isIn(const Module &M, StringRef Category) const {
343 return SCL->inSection("dataflow", "src", M.getModuleIdentifier(), Category);
344 }
345};
346
347/// TransformedFunction is used to express the result of transforming one
348/// function type into another. This struct is immutable. It holds metadata
349/// useful for updating calls of the old function to the new type.
350struct TransformedFunction {
351 TransformedFunction(FunctionType *OriginalType, FunctionType *TransformedType,
352 std::vector<unsigned> ArgumentIndexMapping)
353 : OriginalType(OriginalType), TransformedType(TransformedType),
354 ArgumentIndexMapping(ArgumentIndexMapping) {}
355
356 // Disallow copies.
357 TransformedFunction(const TransformedFunction &) = delete;
358 TransformedFunction &operator=(const TransformedFunction &) = delete;
359
360 // Allow moves.
361 TransformedFunction(TransformedFunction &&) = default;
362 TransformedFunction &operator=(TransformedFunction &&) = default;
363
364 /// Type of the function before the transformation.
365 FunctionType *OriginalType;
366
367 /// Type of the function after the transformation.
368 FunctionType *TransformedType;
369
370 /// Transforming a function may change the position of arguments. This
371 /// member records the mapping from each argument's old position to its new
372 /// position. Argument positions are zero-indexed. If the transformation
373 /// from F to F' made the first argument of F into the third argument of F',
374 /// then ArgumentIndexMapping[0] will equal 2.
375 std::vector<unsigned> ArgumentIndexMapping;
376};
377
378/// Given function attributes from a call site for the original function,
379/// return function attributes appropriate for a call to the transformed
380/// function.
382transformFunctionAttributes(const TransformedFunction &TransformedFunction,
383 LLVMContext &Ctx, AttributeList CallSiteAttrs) {
384
385 // Construct a vector of AttributeSet for each function argument.
386 std::vector<llvm::AttributeSet> ArgumentAttributes(
387 TransformedFunction.TransformedType->getNumParams());
388
389 // Copy attributes from the parameter of the original function to the
390 // transformed version. 'ArgumentIndexMapping' holds the mapping from
391 // old argument position to new.
392 for (unsigned I = 0, IE = TransformedFunction.ArgumentIndexMapping.size();
393 I < IE; ++I) {
394 unsigned TransformedIndex = TransformedFunction.ArgumentIndexMapping[I];
395 ArgumentAttributes[TransformedIndex] = CallSiteAttrs.getParamAttrs(I);
396 }
397
398 // Copy annotations on varargs arguments.
399 for (unsigned I = TransformedFunction.OriginalType->getNumParams(),
400 IE = CallSiteAttrs.getNumAttrSets();
401 I < IE; ++I) {
402 ArgumentAttributes.push_back(CallSiteAttrs.getParamAttrs(I));
403 }
404
405 return AttributeList::get(Ctx, CallSiteAttrs.getFnAttrs(),
406 CallSiteAttrs.getRetAttrs(),
407 llvm::ArrayRef(ArgumentAttributes));
408}
409
410class DataFlowSanitizer {
411 friend struct DFSanFunction;
412 friend class DFSanVisitor;
413
414 enum { ShadowWidthBits = 8, ShadowWidthBytes = ShadowWidthBits / 8 };
415
416 enum { OriginWidthBits = 32, OriginWidthBytes = OriginWidthBits / 8 };
417
418 /// How should calls to uninstrumented functions be handled?
419 enum WrapperKind {
420 /// This function is present in an uninstrumented form but we don't know
421 /// how it should be handled. Print a warning and call the function anyway.
422 /// Don't label the return value.
423 WK_Warning,
424
425 /// This function does not write to (user-accessible) memory, and its return
426 /// value is unlabelled.
427 WK_Discard,
428
429 /// This function does not write to (user-accessible) memory, and the label
430 /// of its return value is the union of the label of its arguments.
431 WK_Functional,
432
433 /// Instead of calling the function, a custom wrapper __dfsw_F is called,
434 /// where F is the name of the function. This function may wrap the
435 /// original function or provide its own implementation. WK_Custom uses an
436 /// extra pointer argument to return the shadow. This allows the wrapped
437 /// form of the function type to be expressed in C.
438 WK_Custom
439 };
440
441 Module *Mod;
442 LLVMContext *Ctx;
443 Type *Int8Ptr;
444 IntegerType *OriginTy;
445 PointerType *OriginPtrTy;
446 ConstantInt *ZeroOrigin;
447 /// The shadow type for all primitive types and vector types.
448 IntegerType *PrimitiveShadowTy;
449 PointerType *PrimitiveShadowPtrTy;
450 IntegerType *IntptrTy;
451 ConstantInt *ZeroPrimitiveShadow;
452 Constant *ArgTLS;
453 ArrayType *ArgOriginTLSTy;
454 Constant *ArgOriginTLS;
455 Constant *RetvalTLS;
456 Constant *RetvalOriginTLS;
457 FunctionType *DFSanUnionLoadFnTy;
458 FunctionType *DFSanLoadLabelAndOriginFnTy;
459 FunctionType *DFSanUnimplementedFnTy;
460 FunctionType *DFSanWrapperExternWeakNullFnTy;
461 FunctionType *DFSanSetLabelFnTy;
462 FunctionType *DFSanNonzeroLabelFnTy;
463 FunctionType *DFSanVarargWrapperFnTy;
464 FunctionType *DFSanConditionalCallbackFnTy;
465 FunctionType *DFSanConditionalCallbackOriginFnTy;
466 FunctionType *DFSanReachesFunctionCallbackFnTy;
467 FunctionType *DFSanReachesFunctionCallbackOriginFnTy;
468 FunctionType *DFSanCmpCallbackFnTy;
469 FunctionType *DFSanLoadStoreCallbackFnTy;
470 FunctionType *DFSanMemTransferCallbackFnTy;
471 FunctionType *DFSanChainOriginFnTy;
472 FunctionType *DFSanChainOriginIfTaintedFnTy;
473 FunctionType *DFSanMemOriginTransferFnTy;
474 FunctionType *DFSanMemShadowOriginTransferFnTy;
475 FunctionType *DFSanMemShadowOriginConditionalExchangeFnTy;
476 FunctionType *DFSanMaybeStoreOriginFnTy;
477 FunctionCallee DFSanUnionLoadFn;
478 FunctionCallee DFSanLoadLabelAndOriginFn;
479 FunctionCallee DFSanUnimplementedFn;
480 FunctionCallee DFSanWrapperExternWeakNullFn;
481 FunctionCallee DFSanSetLabelFn;
482 FunctionCallee DFSanNonzeroLabelFn;
483 FunctionCallee DFSanVarargWrapperFn;
484 FunctionCallee DFSanLoadCallbackFn;
485 FunctionCallee DFSanStoreCallbackFn;
486 FunctionCallee DFSanMemTransferCallbackFn;
487 FunctionCallee DFSanConditionalCallbackFn;
488 FunctionCallee DFSanConditionalCallbackOriginFn;
489 FunctionCallee DFSanReachesFunctionCallbackFn;
490 FunctionCallee DFSanReachesFunctionCallbackOriginFn;
491 FunctionCallee DFSanCmpCallbackFn;
492 FunctionCallee DFSanChainOriginFn;
493 FunctionCallee DFSanChainOriginIfTaintedFn;
494 FunctionCallee DFSanMemOriginTransferFn;
495 FunctionCallee DFSanMemShadowOriginTransferFn;
496 FunctionCallee DFSanMemShadowOriginConditionalExchangeFn;
497 FunctionCallee DFSanMaybeStoreOriginFn;
498 SmallPtrSet<Value *, 16> DFSanRuntimeFunctions;
499 MDNode *ColdCallWeights;
500 MDNode *OriginStoreWeights;
501 DFSanABIList ABIList;
502 DenseMap<Value *, Function *> UnwrappedFnMap;
503 AttributeMask ReadOnlyNoneAttrs;
504 StringSet<> CombineTaintLookupTableNames;
505
506 /// Memory map parameters used in calculation mapping application addresses
507 /// to shadow addresses and origin addresses.
508 const MemoryMapParams *MapParams;
509
510 Value *getShadowOffset(Value *Addr, IRBuilder<> &IRB);
511 Value *getShadowAddress(Value *Addr, Instruction *Pos);
512 Value *getShadowAddress(Value *Addr, Instruction *Pos, Value *ShadowOffset);
513 std::pair<Value *, Value *>
514 getShadowOriginAddress(Value *Addr, Align InstAlignment, Instruction *Pos);
515 bool isInstrumented(const Function *F);
516 bool isInstrumented(const GlobalAlias *GA);
517 bool isForceZeroLabels(const Function *F);
518 TransformedFunction getCustomFunctionType(FunctionType *T);
519 WrapperKind getWrapperKind(Function *F);
520 void addGlobalNameSuffix(GlobalValue *GV);
521 void buildExternWeakCheckIfNeeded(IRBuilder<> &IRB, Function *F);
522 Function *buildWrapperFunction(Function *F, StringRef NewFName,
524 FunctionType *NewFT);
525 void initializeCallbackFunctions(Module &M);
526 void initializeRuntimeFunctions(Module &M);
527 bool initializeModule(Module &M);
528
529 /// Advances \p OriginAddr to point to the next 32-bit origin and then loads
530 /// from it. Returns the origin's loaded value.
531 Value *loadNextOrigin(Instruction *Pos, Align OriginAlign,
532 Value **OriginAddr);
533
534 /// Returns whether the given load byte size is amenable to inlined
535 /// optimization patterns.
536 bool hasLoadSizeForFastPath(uint64_t Size);
537
538 /// Returns whether the pass tracks origins. Supports only TLS ABI mode.
539 bool shouldTrackOrigins();
540
541 /// Returns a zero constant with the shadow type of OrigTy.
542 ///
543 /// getZeroShadow({T1,T2,...}) = {getZeroShadow(T1),getZeroShadow(T2,...}
544 /// getZeroShadow([n x T]) = [n x getZeroShadow(T)]
545 /// getZeroShadow(other type) = i16(0)
546 Constant *getZeroShadow(Type *OrigTy);
547 /// Returns a zero constant with the shadow type of V's type.
548 Constant *getZeroShadow(Value *V);
549
550 /// Checks if V is a zero shadow.
551 bool isZeroShadow(Value *V);
552
553 /// Returns the shadow type of OrigTy.
554 ///
555 /// getShadowTy({T1,T2,...}) = {getShadowTy(T1),getShadowTy(T2),...}
556 /// getShadowTy([n x T]) = [n x getShadowTy(T)]
557 /// getShadowTy(other type) = i16
558 Type *getShadowTy(Type *OrigTy);
559 /// Returns the shadow type of of V's type.
560 Type *getShadowTy(Value *V);
561
562 const uint64_t NumOfElementsInArgOrgTLS = ArgTLSSize / OriginWidthBytes;
563
564public:
565 DataFlowSanitizer(const std::vector<std::string> &ABIListFiles);
566
567 bool runImpl(Module &M,
569};
570
571struct DFSanFunction {
572 DataFlowSanitizer &DFS;
573 Function *F;
574 DominatorTree DT;
575 bool IsNativeABI;
576 bool IsForceZeroLabels;
578 AllocaInst *LabelReturnAlloca = nullptr;
579 AllocaInst *OriginReturnAlloca = nullptr;
580 DenseMap<Value *, Value *> ValShadowMap;
581 DenseMap<Value *, Value *> ValOriginMap;
584
585 struct PHIFixupElement {
586 PHINode *Phi;
587 PHINode *ShadowPhi;
588 PHINode *OriginPhi;
589 };
590 std::vector<PHIFixupElement> PHIFixups;
591
592 DenseSet<Instruction *> SkipInsts;
593 std::vector<Value *> NonZeroChecks;
594
595 struct CachedShadow {
596 BasicBlock *Block; // The block where Shadow is defined.
597 Value *Shadow;
598 };
599 /// Maps a value to its latest shadow value in terms of domination tree.
600 DenseMap<std::pair<Value *, Value *>, CachedShadow> CachedShadows;
601 /// Maps a value to its latest collapsed shadow value it was converted to in
602 /// terms of domination tree. When ClDebugNonzeroLabels is on, this cache is
603 /// used at a post process where CFG blocks are split. So it does not cache
604 /// BasicBlock like CachedShadows, but uses domination between values.
605 DenseMap<Value *, Value *> CachedCollapsedShadows;
607
608 DFSanFunction(DataFlowSanitizer &DFS, Function *F, bool IsNativeABI,
609 bool IsForceZeroLabels, TargetLibraryInfo &TLI)
610 : DFS(DFS), F(F), IsNativeABI(IsNativeABI),
611 IsForceZeroLabels(IsForceZeroLabels), TLI(TLI) {
612 DT.recalculate(*F);
613 }
614
615 /// Computes the shadow address for a given function argument.
616 ///
617 /// Shadow = ArgTLS+ArgOffset.
618 Value *getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB);
619
620 /// Computes the shadow address for a return value.
621 Value *getRetvalTLS(Type *T, IRBuilder<> &IRB);
622
623 /// Computes the origin address for a given function argument.
624 ///
625 /// Origin = ArgOriginTLS[ArgNo].
626 Value *getArgOriginTLS(unsigned ArgNo, IRBuilder<> &IRB);
627
628 /// Computes the origin address for a return value.
629 Value *getRetvalOriginTLS();
630
631 Value *getOrigin(Value *V);
632 void setOrigin(Instruction *I, Value *Origin);
633 /// Generates IR to compute the origin of the last operand with a taint label.
634 Value *combineOperandOrigins(Instruction *Inst);
635 /// Before the instruction Pos, generates IR to compute the last origin with a
636 /// taint label. Labels and origins are from vectors Shadows and Origins
637 /// correspondingly. The generated IR is like
638 /// Sn-1 != Zero ? On-1: ... S2 != Zero ? O2: S1 != Zero ? O1: O0
639 /// When Zero is nullptr, it uses ZeroPrimitiveShadow. Otherwise it can be
640 /// zeros with other bitwidths.
641 Value *combineOrigins(const std::vector<Value *> &Shadows,
642 const std::vector<Value *> &Origins, Instruction *Pos,
643 ConstantInt *Zero = nullptr);
644
645 Value *getShadow(Value *V);
646 void setShadow(Instruction *I, Value *Shadow);
647 /// Generates IR to compute the union of the two given shadows, inserting it
648 /// before Pos. The combined value is with primitive type.
649 Value *combineShadows(Value *V1, Value *V2, Instruction *Pos);
650 /// Combines the shadow values of V1 and V2, then converts the combined value
651 /// with primitive type into a shadow value with the original type T.
652 Value *combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
653 Instruction *Pos);
654 Value *combineOperandShadows(Instruction *Inst);
655
656 /// Generates IR to load shadow and origin corresponding to bytes [\p
657 /// Addr, \p Addr + \p Size), where addr has alignment \p
658 /// InstAlignment, and take the union of each of those shadows. The returned
659 /// shadow always has primitive type.
660 ///
661 /// When tracking loads is enabled, the returned origin is a chain at the
662 /// current stack if the returned shadow is tainted.
663 std::pair<Value *, Value *> loadShadowOrigin(Value *Addr, uint64_t Size,
664 Align InstAlignment,
665 Instruction *Pos);
666
667 void storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
668 Align InstAlignment, Value *PrimitiveShadow,
669 Value *Origin, Instruction *Pos);
670 /// Applies PrimitiveShadow to all primitive subtypes of T, returning
671 /// the expanded shadow value.
672 ///
673 /// EFP({T1,T2, ...}, PS) = {EFP(T1,PS),EFP(T2,PS),...}
674 /// EFP([n x T], PS) = [n x EFP(T,PS)]
675 /// EFP(other types, PS) = PS
676 Value *expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
677 Instruction *Pos);
678 /// Collapses Shadow into a single primitive shadow value, unioning all
679 /// primitive shadow values in the process. Returns the final primitive
680 /// shadow value.
681 ///
682 /// CTP({V1,V2, ...}) = UNION(CFP(V1,PS),CFP(V2,PS),...)
683 /// CTP([V1,V2,...]) = UNION(CFP(V1,PS),CFP(V2,PS),...)
684 /// CTP(other types, PS) = PS
685 Value *collapseToPrimitiveShadow(Value *Shadow, Instruction *Pos);
686
687 void storeZeroPrimitiveShadow(Value *Addr, uint64_t Size, Align ShadowAlign,
688 Instruction *Pos);
689
690 Align getShadowAlign(Align InstAlignment);
691
692 // If ClConditionalCallbacks is enabled, insert a callback after a given
693 // branch instruction using the given conditional expression.
694 void addConditionalCallbacksIfEnabled(Instruction &I, Value *Condition);
695
696 // If ClReachesFunctionCallbacks is enabled, insert a callback for each
697 // argument and load instruction.
698 void addReachesFunctionCallbacksIfEnabled(IRBuilder<> &IRB, Instruction &I,
699 Value *Data);
700
701 bool isLookupTableConstant(Value *P);
702
703private:
704 /// Collapses the shadow with aggregate type into a single primitive shadow
705 /// value.
706 template <class AggregateType>
707 Value *collapseAggregateShadow(AggregateType *AT, Value *Shadow,
708 IRBuilder<> &IRB);
709
710 Value *collapseToPrimitiveShadow(Value *Shadow, IRBuilder<> &IRB);
711
712 /// Returns the shadow value of an argument A.
713 Value *getShadowForTLSArgument(Argument *A);
714
715 /// The fast path of loading shadows.
716 std::pair<Value *, Value *>
717 loadShadowFast(Value *ShadowAddr, Value *OriginAddr, uint64_t Size,
718 Align ShadowAlign, Align OriginAlign, Value *FirstOrigin,
719 Instruction *Pos);
720
721 Align getOriginAlign(Align InstAlignment);
722
723 /// Because 4 contiguous bytes share one 4-byte origin, the most accurate load
724 /// is __dfsan_load_label_and_origin. This function returns the union of all
725 /// labels and the origin of the first taint label. However this is an
726 /// additional call with many instructions. To ensure common cases are fast,
727 /// checks if it is possible to load labels and origins without using the
728 /// callback function.
729 ///
730 /// When enabling tracking load instructions, we always use
731 /// __dfsan_load_label_and_origin to reduce code size.
732 bool useCallbackLoadLabelAndOrigin(uint64_t Size, Align InstAlignment);
733
734 /// Returns a chain at the current stack with previous origin V.
735 Value *updateOrigin(Value *V, IRBuilder<> &IRB);
736
737 /// Returns a chain at the current stack with previous origin V if Shadow is
738 /// tainted.
739 Value *updateOriginIfTainted(Value *Shadow, Value *Origin, IRBuilder<> &IRB);
740
741 /// Creates an Intptr = Origin | Origin << 32 if Intptr's size is 64. Returns
742 /// Origin otherwise.
743 Value *originToIntptr(IRBuilder<> &IRB, Value *Origin);
744
745 /// Stores Origin into the address range [StoreOriginAddr, StoreOriginAddr +
746 /// Size).
747 void paintOrigin(IRBuilder<> &IRB, Value *Origin, Value *StoreOriginAddr,
748 uint64_t StoreOriginSize, Align Alignment);
749
750 /// Stores Origin in terms of its Shadow value.
751 /// * Do not write origins for zero shadows because we do not trace origins
752 /// for untainted sinks.
753 /// * Use __dfsan_maybe_store_origin if there are too many origin store
754 /// instrumentations.
755 void storeOrigin(Instruction *Pos, Value *Addr, uint64_t Size, Value *Shadow,
756 Value *Origin, Value *StoreOriginAddr, Align InstAlignment);
757
758 /// Convert a scalar value to an i1 by comparing with 0.
759 Value *convertToBool(Value *V, IRBuilder<> &IRB, const Twine &Name = "");
760
761 bool shouldInstrumentWithCall();
762
763 /// Generates IR to load shadow and origin corresponding to bytes [\p
764 /// Addr, \p Addr + \p Size), where addr has alignment \p
765 /// InstAlignment, and take the union of each of those shadows. The returned
766 /// shadow always has primitive type.
767 std::pair<Value *, Value *>
768 loadShadowOriginSansLoadTracking(Value *Addr, uint64_t Size,
769 Align InstAlignment, Instruction *Pos);
770 int NumOriginStores = 0;
771};
772
773class DFSanVisitor : public InstVisitor<DFSanVisitor> {
774public:
775 DFSanFunction &DFSF;
776
777 DFSanVisitor(DFSanFunction &DFSF) : DFSF(DFSF) {}
778
779 const DataLayout &getDataLayout() const {
780 return DFSF.F->getParent()->getDataLayout();
781 }
782
783 // Combines shadow values and origins for all of I's operands.
784 void visitInstOperands(Instruction &I);
785
788 void visitBitCastInst(BitCastInst &BCI);
789 void visitCastInst(CastInst &CI);
790 void visitCmpInst(CmpInst &CI);
793 void visitLoadInst(LoadInst &LI);
794 void visitStoreInst(StoreInst &SI);
797 void visitReturnInst(ReturnInst &RI);
798 void visitLibAtomicLoad(CallBase &CB);
799 void visitLibAtomicStore(CallBase &CB);
800 void visitLibAtomicExchange(CallBase &CB);
801 void visitLibAtomicCompareExchange(CallBase &CB);
802 void visitCallBase(CallBase &CB);
803 void visitPHINode(PHINode &PN);
813 void visitBranchInst(BranchInst &BR);
814 void visitSwitchInst(SwitchInst &SW);
815
816private:
817 void visitCASOrRMW(Align InstAlignment, Instruction &I);
818
819 // Returns false when this is an invoke of a custom function.
820 bool visitWrappedCallBase(Function &F, CallBase &CB);
821
822 // Combines origins for all of I's operands.
823 void visitInstOperandOrigins(Instruction &I);
824
825 void addShadowArguments(Function &F, CallBase &CB, std::vector<Value *> &Args,
826 IRBuilder<> &IRB);
827
828 void addOriginArguments(Function &F, CallBase &CB, std::vector<Value *> &Args,
829 IRBuilder<> &IRB);
830
831 Value *makeAddAcquireOrderingTable(IRBuilder<> &IRB);
832 Value *makeAddReleaseOrderingTable(IRBuilder<> &IRB);
833};
834
835bool LibAtomicFunction(const Function &F) {
836 // This is a bit of a hack because TargetLibraryInfo is a function pass.
837 // The DFSan pass would need to be refactored to be function pass oriented
838 // (like MSan is) in order to fit together nicely with TargetLibraryInfo.
839 // We need this check to prevent them from being instrumented, or wrapped.
840 // Match on name and number of arguments.
841 if (!F.hasName() || F.isVarArg())
842 return false;
843 switch (F.arg_size()) {
844 case 4:
845 return F.getName() == "__atomic_load" || F.getName() == "__atomic_store";
846 case 5:
847 return F.getName() == "__atomic_exchange";
848 case 6:
849 return F.getName() == "__atomic_compare_exchange";
850 default:
851 return false;
852 }
853}
854
855} // end anonymous namespace
856
857DataFlowSanitizer::DataFlowSanitizer(
858 const std::vector<std::string> &ABIListFiles) {
859 std::vector<std::string> AllABIListFiles(std::move(ABIListFiles));
860 llvm::append_range(AllABIListFiles, ClABIListFiles);
861 // FIXME: should we propagate vfs::FileSystem to this constructor?
862 ABIList.set(
864
866 CombineTaintLookupTableNames.insert(v);
867}
868
869TransformedFunction DataFlowSanitizer::getCustomFunctionType(FunctionType *T) {
870 SmallVector<Type *, 4> ArgTypes;
871
872 // Some parameters of the custom function being constructed are
873 // parameters of T. Record the mapping from parameters of T to
874 // parameters of the custom function, so that parameter attributes
875 // at call sites can be updated.
876 std::vector<unsigned> ArgumentIndexMapping;
877 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I) {
878 Type *ParamType = T->getParamType(I);
879 ArgumentIndexMapping.push_back(ArgTypes.size());
880 ArgTypes.push_back(ParamType);
881 }
882 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I)
883 ArgTypes.push_back(PrimitiveShadowTy);
884 if (T->isVarArg())
885 ArgTypes.push_back(PrimitiveShadowPtrTy);
886 Type *RetType = T->getReturnType();
887 if (!RetType->isVoidTy())
888 ArgTypes.push_back(PrimitiveShadowPtrTy);
889
890 if (shouldTrackOrigins()) {
891 for (unsigned I = 0, E = T->getNumParams(); I != E; ++I)
892 ArgTypes.push_back(OriginTy);
893 if (T->isVarArg())
894 ArgTypes.push_back(OriginPtrTy);
895 if (!RetType->isVoidTy())
896 ArgTypes.push_back(OriginPtrTy);
897 }
898
899 return TransformedFunction(
900 T, FunctionType::get(T->getReturnType(), ArgTypes, T->isVarArg()),
901 ArgumentIndexMapping);
902}
903
904bool DataFlowSanitizer::isZeroShadow(Value *V) {
905 Type *T = V->getType();
906 if (!isa<ArrayType>(T) && !isa<StructType>(T)) {
907 if (const ConstantInt *CI = dyn_cast<ConstantInt>(V))
908 return CI->isZero();
909 return false;
910 }
911
912 return isa<ConstantAggregateZero>(V);
913}
914
915bool DataFlowSanitizer::hasLoadSizeForFastPath(uint64_t Size) {
916 uint64_t ShadowSize = Size * ShadowWidthBytes;
917 return ShadowSize % 8 == 0 || ShadowSize == 4;
918}
919
920bool DataFlowSanitizer::shouldTrackOrigins() {
921 static const bool ShouldTrackOrigins = ClTrackOrigins;
922 return ShouldTrackOrigins;
923}
924
925Constant *DataFlowSanitizer::getZeroShadow(Type *OrigTy) {
926 if (!isa<ArrayType>(OrigTy) && !isa<StructType>(OrigTy))
927 return ZeroPrimitiveShadow;
928 Type *ShadowTy = getShadowTy(OrigTy);
929 return ConstantAggregateZero::get(ShadowTy);
930}
931
932Constant *DataFlowSanitizer::getZeroShadow(Value *V) {
933 return getZeroShadow(V->getType());
934}
935
937 Value *Shadow, SmallVector<unsigned, 4> &Indices, Type *SubShadowTy,
938 Value *PrimitiveShadow, IRBuilder<> &IRB) {
939 if (!isa<ArrayType>(SubShadowTy) && !isa<StructType>(SubShadowTy))
940 return IRB.CreateInsertValue(Shadow, PrimitiveShadow, Indices);
941
942 if (ArrayType *AT = dyn_cast<ArrayType>(SubShadowTy)) {
943 for (unsigned Idx = 0; Idx < AT->getNumElements(); Idx++) {
944 Indices.push_back(Idx);
946 Shadow, Indices, AT->getElementType(), PrimitiveShadow, IRB);
947 Indices.pop_back();
948 }
949 return Shadow;
950 }
951
952 if (StructType *ST = dyn_cast<StructType>(SubShadowTy)) {
953 for (unsigned Idx = 0; Idx < ST->getNumElements(); Idx++) {
954 Indices.push_back(Idx);
956 Shadow, Indices, ST->getElementType(Idx), PrimitiveShadow, IRB);
957 Indices.pop_back();
958 }
959 return Shadow;
960 }
961 llvm_unreachable("Unexpected shadow type");
962}
963
964bool DFSanFunction::shouldInstrumentWithCall() {
965 return ClInstrumentWithCallThreshold >= 0 &&
966 NumOriginStores >= ClInstrumentWithCallThreshold;
967}
968
969Value *DFSanFunction::expandFromPrimitiveShadow(Type *T, Value *PrimitiveShadow,
970 Instruction *Pos) {
971 Type *ShadowTy = DFS.getShadowTy(T);
972
973 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
974 return PrimitiveShadow;
975
976 if (DFS.isZeroShadow(PrimitiveShadow))
977 return DFS.getZeroShadow(ShadowTy);
978
979 IRBuilder<> IRB(Pos);
981 Value *Shadow = UndefValue::get(ShadowTy);
982 Shadow = expandFromPrimitiveShadowRecursive(Shadow, Indices, ShadowTy,
983 PrimitiveShadow, IRB);
984
985 // Caches the primitive shadow value that built the shadow value.
986 CachedCollapsedShadows[Shadow] = PrimitiveShadow;
987 return Shadow;
988}
989
990template <class AggregateType>
991Value *DFSanFunction::collapseAggregateShadow(AggregateType *AT, Value *Shadow,
992 IRBuilder<> &IRB) {
993 if (!AT->getNumElements())
994 return DFS.ZeroPrimitiveShadow;
995
996 Value *FirstItem = IRB.CreateExtractValue(Shadow, 0);
997 Value *Aggregator = collapseToPrimitiveShadow(FirstItem, IRB);
998
999 for (unsigned Idx = 1; Idx < AT->getNumElements(); Idx++) {
1000 Value *ShadowItem = IRB.CreateExtractValue(Shadow, Idx);
1001 Value *ShadowInner = collapseToPrimitiveShadow(ShadowItem, IRB);
1002 Aggregator = IRB.CreateOr(Aggregator, ShadowInner);
1003 }
1004 return Aggregator;
1005}
1006
1007Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
1008 IRBuilder<> &IRB) {
1009 Type *ShadowTy = Shadow->getType();
1010 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
1011 return Shadow;
1012 if (ArrayType *AT = dyn_cast<ArrayType>(ShadowTy))
1013 return collapseAggregateShadow<>(AT, Shadow, IRB);
1014 if (StructType *ST = dyn_cast<StructType>(ShadowTy))
1015 return collapseAggregateShadow<>(ST, Shadow, IRB);
1016 llvm_unreachable("Unexpected shadow type");
1017}
1018
1019Value *DFSanFunction::collapseToPrimitiveShadow(Value *Shadow,
1020 Instruction *Pos) {
1021 Type *ShadowTy = Shadow->getType();
1022 if (!isa<ArrayType>(ShadowTy) && !isa<StructType>(ShadowTy))
1023 return Shadow;
1024
1025 // Checks if the cached collapsed shadow value dominates Pos.
1026 Value *&CS = CachedCollapsedShadows[Shadow];
1027 if (CS && DT.dominates(CS, Pos))
1028 return CS;
1029
1030 IRBuilder<> IRB(Pos);
1031 Value *PrimitiveShadow = collapseToPrimitiveShadow(Shadow, IRB);
1032 // Caches the converted primitive shadow value.
1033 CS = PrimitiveShadow;
1034 return PrimitiveShadow;
1035}
1036
1037void DFSanFunction::addConditionalCallbacksIfEnabled(Instruction &I,
1038 Value *Condition) {
1040 return;
1041 }
1042 IRBuilder<> IRB(&I);
1043 Value *CondShadow = getShadow(Condition);
1044 CallInst *CI;
1045 if (DFS.shouldTrackOrigins()) {
1046 Value *CondOrigin = getOrigin(Condition);
1047 CI = IRB.CreateCall(DFS.DFSanConditionalCallbackOriginFn,
1048 {CondShadow, CondOrigin});
1049 } else {
1050 CI = IRB.CreateCall(DFS.DFSanConditionalCallbackFn, {CondShadow});
1051 }
1052 CI->addParamAttr(0, Attribute::ZExt);
1053}
1054
1055void DFSanFunction::addReachesFunctionCallbacksIfEnabled(IRBuilder<> &IRB,
1056 Instruction &I,
1057 Value *Data) {
1059 return;
1060 }
1061 const DebugLoc &dbgloc = I.getDebugLoc();
1062 Value *DataShadow = collapseToPrimitiveShadow(getShadow(Data), IRB);
1063 ConstantInt *CILine;
1064 llvm::Value *FilePathPtr;
1065
1066 if (dbgloc.get() == nullptr) {
1067 CILine = llvm::ConstantInt::get(I.getContext(), llvm::APInt(32, 0));
1068 FilePathPtr = IRB.CreateGlobalStringPtr(
1069 I.getFunction()->getParent()->getSourceFileName());
1070 } else {
1071 CILine = llvm::ConstantInt::get(I.getContext(),
1072 llvm::APInt(32, dbgloc.getLine()));
1073 FilePathPtr =
1074 IRB.CreateGlobalStringPtr(dbgloc->getFilename());
1075 }
1076
1077 llvm::Value *FunctionNamePtr =
1078 IRB.CreateGlobalStringPtr(I.getFunction()->getName());
1079
1080 CallInst *CB;
1081 std::vector<Value *> args;
1082
1083 if (DFS.shouldTrackOrigins()) {
1084 Value *DataOrigin = getOrigin(Data);
1085 args = { DataShadow, DataOrigin, FilePathPtr, CILine, FunctionNamePtr };
1086 CB = IRB.CreateCall(DFS.DFSanReachesFunctionCallbackOriginFn, args);
1087 } else {
1088 args = { DataShadow, FilePathPtr, CILine, FunctionNamePtr };
1089 CB = IRB.CreateCall(DFS.DFSanReachesFunctionCallbackFn, args);
1090 }
1091 CB->addParamAttr(0, Attribute::ZExt);
1092 CB->setDebugLoc(dbgloc);
1093}
1094
1095Type *DataFlowSanitizer::getShadowTy(Type *OrigTy) {
1096 if (!OrigTy->isSized())
1097 return PrimitiveShadowTy;
1098 if (isa<IntegerType>(OrigTy))
1099 return PrimitiveShadowTy;
1100 if (isa<VectorType>(OrigTy))
1101 return PrimitiveShadowTy;
1102 if (ArrayType *AT = dyn_cast<ArrayType>(OrigTy))
1103 return ArrayType::get(getShadowTy(AT->getElementType()),
1104 AT->getNumElements());
1105 if (StructType *ST = dyn_cast<StructType>(OrigTy)) {
1107 for (unsigned I = 0, N = ST->getNumElements(); I < N; ++I)
1108 Elements.push_back(getShadowTy(ST->getElementType(I)));
1109 return StructType::get(*Ctx, Elements);
1110 }
1111 return PrimitiveShadowTy;
1112}
1113
1114Type *DataFlowSanitizer::getShadowTy(Value *V) {
1115 return getShadowTy(V->getType());
1116}
1117
1118bool DataFlowSanitizer::initializeModule(Module &M) {
1119 Triple TargetTriple(M.getTargetTriple());
1120 const DataLayout &DL = M.getDataLayout();
1121
1122 if (TargetTriple.getOS() != Triple::Linux)
1123 report_fatal_error("unsupported operating system");
1124 switch (TargetTriple.getArch()) {
1125 case Triple::aarch64:
1126 MapParams = &Linux_AArch64_MemoryMapParams;
1127 break;
1128 case Triple::x86_64:
1129 MapParams = &Linux_X86_64_MemoryMapParams;
1130 break;
1131 default:
1132 report_fatal_error("unsupported architecture");
1133 }
1134
1135 Mod = &M;
1136 Ctx = &M.getContext();
1137 Int8Ptr = Type::getInt8PtrTy(*Ctx);
1138 OriginTy = IntegerType::get(*Ctx, OriginWidthBits);
1139 OriginPtrTy = PointerType::getUnqual(OriginTy);
1140 PrimitiveShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
1141 PrimitiveShadowPtrTy = PointerType::getUnqual(PrimitiveShadowTy);
1142 IntptrTy = DL.getIntPtrType(*Ctx);
1143 ZeroPrimitiveShadow = ConstantInt::getSigned(PrimitiveShadowTy, 0);
1144 ZeroOrigin = ConstantInt::getSigned(OriginTy, 0);
1145
1146 Type *DFSanUnionLoadArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1147 DFSanUnionLoadFnTy = FunctionType::get(PrimitiveShadowTy, DFSanUnionLoadArgs,
1148 /*isVarArg=*/false);
1149 Type *DFSanLoadLabelAndOriginArgs[2] = {Int8Ptr, IntptrTy};
1150 DFSanLoadLabelAndOriginFnTy =
1151 FunctionType::get(IntegerType::get(*Ctx, 64), DFSanLoadLabelAndOriginArgs,
1152 /*isVarArg=*/false);
1153 DFSanUnimplementedFnTy = FunctionType::get(
1154 Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
1155 Type *DFSanWrapperExternWeakNullArgs[2] = {Int8Ptr, Int8Ptr};
1156 DFSanWrapperExternWeakNullFnTy =
1157 FunctionType::get(Type::getVoidTy(*Ctx), DFSanWrapperExternWeakNullArgs,
1158 /*isVarArg=*/false);
1159 Type *DFSanSetLabelArgs[4] = {PrimitiveShadowTy, OriginTy,
1160 Type::getInt8PtrTy(*Ctx), IntptrTy};
1161 DFSanSetLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx),
1162 DFSanSetLabelArgs, /*isVarArg=*/false);
1163 DFSanNonzeroLabelFnTy = FunctionType::get(Type::getVoidTy(*Ctx), std::nullopt,
1164 /*isVarArg=*/false);
1165 DFSanVarargWrapperFnTy = FunctionType::get(
1166 Type::getVoidTy(*Ctx), Type::getInt8PtrTy(*Ctx), /*isVarArg=*/false);
1167 DFSanConditionalCallbackFnTy =
1168 FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
1169 /*isVarArg=*/false);
1170 Type *DFSanConditionalCallbackOriginArgs[2] = {PrimitiveShadowTy, OriginTy};
1171 DFSanConditionalCallbackOriginFnTy = FunctionType::get(
1172 Type::getVoidTy(*Ctx), DFSanConditionalCallbackOriginArgs,
1173 /*isVarArg=*/false);
1174 Type *DFSanReachesFunctionCallbackArgs[4] = {PrimitiveShadowTy, Int8Ptr,
1175 OriginTy, Int8Ptr};
1176 DFSanReachesFunctionCallbackFnTy =
1177 FunctionType::get(Type::getVoidTy(*Ctx), DFSanReachesFunctionCallbackArgs,
1178 /*isVarArg=*/false);
1179 Type *DFSanReachesFunctionCallbackOriginArgs[5] = {
1180 PrimitiveShadowTy, OriginTy, Int8Ptr, OriginTy, Int8Ptr};
1181 DFSanReachesFunctionCallbackOriginFnTy = FunctionType::get(
1182 Type::getVoidTy(*Ctx), DFSanReachesFunctionCallbackOriginArgs,
1183 /*isVarArg=*/false);
1184 DFSanCmpCallbackFnTy =
1185 FunctionType::get(Type::getVoidTy(*Ctx), PrimitiveShadowTy,
1186 /*isVarArg=*/false);
1187 DFSanChainOriginFnTy =
1188 FunctionType::get(OriginTy, OriginTy, /*isVarArg=*/false);
1189 Type *DFSanChainOriginIfTaintedArgs[2] = {PrimitiveShadowTy, OriginTy};
1190 DFSanChainOriginIfTaintedFnTy = FunctionType::get(
1191 OriginTy, DFSanChainOriginIfTaintedArgs, /*isVarArg=*/false);
1192 Type *DFSanMaybeStoreOriginArgs[4] = {IntegerType::get(*Ctx, ShadowWidthBits),
1193 Int8Ptr, IntptrTy, OriginTy};
1194 DFSanMaybeStoreOriginFnTy = FunctionType::get(
1195 Type::getVoidTy(*Ctx), DFSanMaybeStoreOriginArgs, /*isVarArg=*/false);
1196 Type *DFSanMemOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1197 DFSanMemOriginTransferFnTy = FunctionType::get(
1198 Type::getVoidTy(*Ctx), DFSanMemOriginTransferArgs, /*isVarArg=*/false);
1199 Type *DFSanMemShadowOriginTransferArgs[3] = {Int8Ptr, Int8Ptr, IntptrTy};
1200 DFSanMemShadowOriginTransferFnTy =
1201 FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemShadowOriginTransferArgs,
1202 /*isVarArg=*/false);
1203 Type *DFSanMemShadowOriginConditionalExchangeArgs[5] = {
1204 IntegerType::get(*Ctx, 8), Int8Ptr, Int8Ptr, Int8Ptr, IntptrTy};
1205 DFSanMemShadowOriginConditionalExchangeFnTy = FunctionType::get(
1206 Type::getVoidTy(*Ctx), DFSanMemShadowOriginConditionalExchangeArgs,
1207 /*isVarArg=*/false);
1208 Type *DFSanLoadStoreCallbackArgs[2] = {PrimitiveShadowTy, Int8Ptr};
1209 DFSanLoadStoreCallbackFnTy =
1210 FunctionType::get(Type::getVoidTy(*Ctx), DFSanLoadStoreCallbackArgs,
1211 /*isVarArg=*/false);
1212 Type *DFSanMemTransferCallbackArgs[2] = {PrimitiveShadowPtrTy, IntptrTy};
1213 DFSanMemTransferCallbackFnTy =
1214 FunctionType::get(Type::getVoidTy(*Ctx), DFSanMemTransferCallbackArgs,
1215 /*isVarArg=*/false);
1216
1217 ColdCallWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000);
1218 OriginStoreWeights = MDBuilder(*Ctx).createBranchWeights(1, 1000);
1219 return true;
1220}
1221
1222bool DataFlowSanitizer::isInstrumented(const Function *F) {
1223 return !ABIList.isIn(*F, "uninstrumented");
1224}
1225
1226bool DataFlowSanitizer::isInstrumented(const GlobalAlias *GA) {
1227 return !ABIList.isIn(*GA, "uninstrumented");
1228}
1229
1230bool DataFlowSanitizer::isForceZeroLabels(const Function *F) {
1231 return ABIList.isIn(*F, "force_zero_labels");
1232}
1233
1234DataFlowSanitizer::WrapperKind DataFlowSanitizer::getWrapperKind(Function *F) {
1235 if (ABIList.isIn(*F, "functional"))
1236 return WK_Functional;
1237 if (ABIList.isIn(*F, "discard"))
1238 return WK_Discard;
1239 if (ABIList.isIn(*F, "custom"))
1240 return WK_Custom;
1241
1242 return WK_Warning;
1243}
1244
1245void DataFlowSanitizer::addGlobalNameSuffix(GlobalValue *GV) {
1246 std::string GVName = std::string(GV->getName()), Suffix = ".dfsan";
1247 GV->setName(GVName + Suffix);
1248
1249 // Try to change the name of the function in module inline asm. We only do
1250 // this for specific asm directives, currently only ".symver", to try to avoid
1251 // corrupting asm which happens to contain the symbol name as a substring.
1252 // Note that the substitution for .symver assumes that the versioned symbol
1253 // also has an instrumented name.
1254 std::string Asm = GV->getParent()->getModuleInlineAsm();
1255 std::string SearchStr = ".symver " + GVName + ",";
1256 size_t Pos = Asm.find(SearchStr);
1257 if (Pos != std::string::npos) {
1258 Asm.replace(Pos, SearchStr.size(), ".symver " + GVName + Suffix + ",");
1259 Pos = Asm.find("@");
1260
1261 if (Pos == std::string::npos)
1262 report_fatal_error(Twine("unsupported .symver: ", Asm));
1263
1264 Asm.replace(Pos, 1, Suffix + "@");
1265 GV->getParent()->setModuleInlineAsm(Asm);
1266 }
1267}
1268
1269void DataFlowSanitizer::buildExternWeakCheckIfNeeded(IRBuilder<> &IRB,
1270 Function *F) {
1271 // If the function we are wrapping was ExternWeak, it may be null.
1272 // The original code before calling this wrapper may have checked for null,
1273 // but replacing with a known-to-not-be-null wrapper can break this check.
1274 // When replacing uses of the extern weak function with the wrapper we try
1275 // to avoid replacing uses in conditionals, but this is not perfect.
1276 // In the case where we fail, and accidentally optimize out a null check
1277 // for a extern weak function, add a check here to help identify the issue.
1278 if (GlobalValue::isExternalWeakLinkage(F->getLinkage())) {
1279 std::vector<Value *> Args;
1280 Args.push_back(IRB.CreatePointerCast(F, IRB.getInt8PtrTy()));
1281 Args.push_back(IRB.CreateGlobalStringPtr(F->getName()));
1282 IRB.CreateCall(DFSanWrapperExternWeakNullFn, Args);
1283 }
1284}
1285
1286Function *
1287DataFlowSanitizer::buildWrapperFunction(Function *F, StringRef NewFName,
1289 FunctionType *NewFT) {
1290 FunctionType *FT = F->getFunctionType();
1291 Function *NewF = Function::Create(NewFT, NewFLink, F->getAddressSpace(),
1292 NewFName, F->getParent());
1293 NewF->copyAttributesFrom(F);
1294 NewF->removeRetAttrs(
1295 AttributeFuncs::typeIncompatible(NewFT->getReturnType()));
1296
1297 BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", NewF);
1298 if (F->isVarArg()) {
1299 NewF->removeFnAttr("split-stack");
1300 CallInst::Create(DFSanVarargWrapperFn,
1301 IRBuilder<>(BB).CreateGlobalStringPtr(F->getName()), "",
1302 BB);
1303 new UnreachableInst(*Ctx, BB);
1304 } else {
1305 auto ArgIt = pointer_iterator<Argument *>(NewF->arg_begin());
1306 std::vector<Value *> Args(ArgIt, ArgIt + FT->getNumParams());
1307
1308 CallInst *CI = CallInst::Create(F, Args, "", BB);
1309 if (FT->getReturnType()->isVoidTy())
1310 ReturnInst::Create(*Ctx, BB);
1311 else
1312 ReturnInst::Create(*Ctx, CI, BB);
1313 }
1314
1315 return NewF;
1316}
1317
1318// Initialize DataFlowSanitizer runtime functions and declare them in the module
1319void DataFlowSanitizer::initializeRuntimeFunctions(Module &M) {
1320 LLVMContext &C = M.getContext();
1321 {
1323 AL = AL.addFnAttribute(C, Attribute::NoUnwind);
1324 AL = AL.addFnAttribute(
1326 AL = AL.addRetAttribute(C, Attribute::ZExt);
1327 DFSanUnionLoadFn =
1328 Mod->getOrInsertFunction("__dfsan_union_load", DFSanUnionLoadFnTy, AL);
1329 }
1330 {
1332 AL = AL.addFnAttribute(C, Attribute::NoUnwind);
1333 AL = AL.addFnAttribute(
1335 AL = AL.addRetAttribute(C, Attribute::ZExt);
1336 DFSanLoadLabelAndOriginFn = Mod->getOrInsertFunction(
1337 "__dfsan_load_label_and_origin", DFSanLoadLabelAndOriginFnTy, AL);
1338 }
1339 DFSanUnimplementedFn =
1340 Mod->getOrInsertFunction("__dfsan_unimplemented", DFSanUnimplementedFnTy);
1341 DFSanWrapperExternWeakNullFn = Mod->getOrInsertFunction(
1342 "__dfsan_wrapper_extern_weak_null", DFSanWrapperExternWeakNullFnTy);
1343 {
1345 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1346 AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
1347 DFSanSetLabelFn =
1348 Mod->getOrInsertFunction("__dfsan_set_label", DFSanSetLabelFnTy, AL);
1349 }
1350 DFSanNonzeroLabelFn =
1351 Mod->getOrInsertFunction("__dfsan_nonzero_label", DFSanNonzeroLabelFnTy);
1352 DFSanVarargWrapperFn = Mod->getOrInsertFunction("__dfsan_vararg_wrapper",
1353 DFSanVarargWrapperFnTy);
1354 {
1356 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1357 AL = AL.addRetAttribute(M.getContext(), Attribute::ZExt);
1358 DFSanChainOriginFn = Mod->getOrInsertFunction("__dfsan_chain_origin",
1359 DFSanChainOriginFnTy, AL);
1360 }
1361 {
1363 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1364 AL = AL.addParamAttribute(M.getContext(), 1, Attribute::ZExt);
1365 AL = AL.addRetAttribute(M.getContext(), Attribute::ZExt);
1366 DFSanChainOriginIfTaintedFn = Mod->getOrInsertFunction(
1367 "__dfsan_chain_origin_if_tainted", DFSanChainOriginIfTaintedFnTy, AL);
1368 }
1369 DFSanMemOriginTransferFn = Mod->getOrInsertFunction(
1370 "__dfsan_mem_origin_transfer", DFSanMemOriginTransferFnTy);
1371
1372 DFSanMemShadowOriginTransferFn = Mod->getOrInsertFunction(
1373 "__dfsan_mem_shadow_origin_transfer", DFSanMemShadowOriginTransferFnTy);
1374
1375 DFSanMemShadowOriginConditionalExchangeFn =
1376 Mod->getOrInsertFunction("__dfsan_mem_shadow_origin_conditional_exchange",
1377 DFSanMemShadowOriginConditionalExchangeFnTy);
1378
1379 {
1381 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1382 AL = AL.addParamAttribute(M.getContext(), 3, Attribute::ZExt);
1383 DFSanMaybeStoreOriginFn = Mod->getOrInsertFunction(
1384 "__dfsan_maybe_store_origin", DFSanMaybeStoreOriginFnTy, AL);
1385 }
1386
1387 DFSanRuntimeFunctions.insert(
1388 DFSanUnionLoadFn.getCallee()->stripPointerCasts());
1389 DFSanRuntimeFunctions.insert(
1390 DFSanLoadLabelAndOriginFn.getCallee()->stripPointerCasts());
1391 DFSanRuntimeFunctions.insert(
1392 DFSanUnimplementedFn.getCallee()->stripPointerCasts());
1393 DFSanRuntimeFunctions.insert(
1394 DFSanWrapperExternWeakNullFn.getCallee()->stripPointerCasts());
1395 DFSanRuntimeFunctions.insert(
1396 DFSanSetLabelFn.getCallee()->stripPointerCasts());
1397 DFSanRuntimeFunctions.insert(
1398 DFSanNonzeroLabelFn.getCallee()->stripPointerCasts());
1399 DFSanRuntimeFunctions.insert(
1400 DFSanVarargWrapperFn.getCallee()->stripPointerCasts());
1401 DFSanRuntimeFunctions.insert(
1402 DFSanLoadCallbackFn.getCallee()->stripPointerCasts());
1403 DFSanRuntimeFunctions.insert(
1404 DFSanStoreCallbackFn.getCallee()->stripPointerCasts());
1405 DFSanRuntimeFunctions.insert(
1406 DFSanMemTransferCallbackFn.getCallee()->stripPointerCasts());
1407 DFSanRuntimeFunctions.insert(
1408 DFSanConditionalCallbackFn.getCallee()->stripPointerCasts());
1409 DFSanRuntimeFunctions.insert(
1410 DFSanConditionalCallbackOriginFn.getCallee()->stripPointerCasts());
1411 DFSanRuntimeFunctions.insert(
1412 DFSanReachesFunctionCallbackFn.getCallee()->stripPointerCasts());
1413 DFSanRuntimeFunctions.insert(
1414 DFSanReachesFunctionCallbackOriginFn.getCallee()->stripPointerCasts());
1415 DFSanRuntimeFunctions.insert(
1416 DFSanCmpCallbackFn.getCallee()->stripPointerCasts());
1417 DFSanRuntimeFunctions.insert(
1418 DFSanChainOriginFn.getCallee()->stripPointerCasts());
1419 DFSanRuntimeFunctions.insert(
1420 DFSanChainOriginIfTaintedFn.getCallee()->stripPointerCasts());
1421 DFSanRuntimeFunctions.insert(
1422 DFSanMemOriginTransferFn.getCallee()->stripPointerCasts());
1423 DFSanRuntimeFunctions.insert(
1424 DFSanMemShadowOriginTransferFn.getCallee()->stripPointerCasts());
1425 DFSanRuntimeFunctions.insert(
1426 DFSanMemShadowOriginConditionalExchangeFn.getCallee()
1427 ->stripPointerCasts());
1428 DFSanRuntimeFunctions.insert(
1429 DFSanMaybeStoreOriginFn.getCallee()->stripPointerCasts());
1430}
1431
1432// Initializes event callback functions and declare them in the module
1433void DataFlowSanitizer::initializeCallbackFunctions(Module &M) {
1434 {
1436 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1437 DFSanLoadCallbackFn = Mod->getOrInsertFunction(
1438 "__dfsan_load_callback", DFSanLoadStoreCallbackFnTy, AL);
1439 }
1440 {
1442 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1443 DFSanStoreCallbackFn = Mod->getOrInsertFunction(
1444 "__dfsan_store_callback", DFSanLoadStoreCallbackFnTy, AL);
1445 }
1446 DFSanMemTransferCallbackFn = Mod->getOrInsertFunction(
1447 "__dfsan_mem_transfer_callback", DFSanMemTransferCallbackFnTy);
1448 {
1450 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1451 DFSanCmpCallbackFn = Mod->getOrInsertFunction("__dfsan_cmp_callback",
1452 DFSanCmpCallbackFnTy, AL);
1453 }
1454 {
1456 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1457 DFSanConditionalCallbackFn = Mod->getOrInsertFunction(
1458 "__dfsan_conditional_callback", DFSanConditionalCallbackFnTy, AL);
1459 }
1460 {
1462 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1463 DFSanConditionalCallbackOriginFn =
1464 Mod->getOrInsertFunction("__dfsan_conditional_callback_origin",
1465 DFSanConditionalCallbackOriginFnTy, AL);
1466 }
1467 {
1469 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1470 DFSanReachesFunctionCallbackFn =
1471 Mod->getOrInsertFunction("__dfsan_reaches_function_callback",
1472 DFSanReachesFunctionCallbackFnTy, AL);
1473 }
1474 {
1476 AL = AL.addParamAttribute(M.getContext(), 0, Attribute::ZExt);
1477 DFSanReachesFunctionCallbackOriginFn =
1478 Mod->getOrInsertFunction("__dfsan_reaches_function_callback_origin",
1479 DFSanReachesFunctionCallbackOriginFnTy, AL);
1480 }
1481}
1482
1483bool DataFlowSanitizer::runImpl(
1485 initializeModule(M);
1486
1487 if (ABIList.isIn(M, "skip"))
1488 return false;
1489
1490 const unsigned InitialGlobalSize = M.global_size();
1491 const unsigned InitialModuleSize = M.size();
1492
1493 bool Changed = false;
1494
1495 auto GetOrInsertGlobal = [this, &Changed](StringRef Name,
1496 Type *Ty) -> Constant * {
1498 if (GlobalVariable *G = dyn_cast<GlobalVariable>(C)) {
1499 Changed |= G->getThreadLocalMode() != GlobalVariable::InitialExecTLSModel;
1500 G->setThreadLocalMode(GlobalVariable::InitialExecTLSModel);
1501 }
1502 return C;
1503 };
1504
1505 // These globals must be kept in sync with the ones in dfsan.cpp.
1506 ArgTLS =
1507 GetOrInsertGlobal("__dfsan_arg_tls",
1508 ArrayType::get(Type::getInt64Ty(*Ctx), ArgTLSSize / 8));
1509 RetvalTLS = GetOrInsertGlobal(
1510 "__dfsan_retval_tls",
1511 ArrayType::get(Type::getInt64Ty(*Ctx), RetvalTLSSize / 8));
1512 ArgOriginTLSTy = ArrayType::get(OriginTy, NumOfElementsInArgOrgTLS);
1513 ArgOriginTLS = GetOrInsertGlobal("__dfsan_arg_origin_tls", ArgOriginTLSTy);
1514 RetvalOriginTLS = GetOrInsertGlobal("__dfsan_retval_origin_tls", OriginTy);
1515
1516 (void)Mod->getOrInsertGlobal("__dfsan_track_origins", OriginTy, [&] {
1517 Changed = true;
1518 return new GlobalVariable(
1519 M, OriginTy, true, GlobalValue::WeakODRLinkage,
1520 ConstantInt::getSigned(OriginTy,
1521 shouldTrackOrigins() ? ClTrackOrigins : 0),
1522 "__dfsan_track_origins");
1523 });
1524
1525 initializeCallbackFunctions(M);
1526 initializeRuntimeFunctions(M);
1527
1528 std::vector<Function *> FnsToInstrument;
1529 SmallPtrSet<Function *, 2> FnsWithNativeABI;
1530 SmallPtrSet<Function *, 2> FnsWithForceZeroLabel;
1531 SmallPtrSet<Constant *, 1> PersonalityFns;
1532 for (Function &F : M)
1533 if (!F.isIntrinsic() && !DFSanRuntimeFunctions.contains(&F) &&
1534 !LibAtomicFunction(F)) {
1535 FnsToInstrument.push_back(&F);
1536 if (F.hasPersonalityFn())
1537 PersonalityFns.insert(F.getPersonalityFn()->stripPointerCasts());
1538 }
1539
1541 for (auto *C : PersonalityFns) {
1542 assert(isa<Function>(C) && "Personality routine is not a function!");
1543 Function *F = cast<Function>(C);
1544 if (!isInstrumented(F))
1545 llvm::erase_value(FnsToInstrument, F);
1546 }
1547 }
1548
1549 // Give function aliases prefixes when necessary, and build wrappers where the
1550 // instrumentedness is inconsistent.
1551 for (GlobalAlias &GA : llvm::make_early_inc_range(M.aliases())) {
1552 // Don't stop on weak. We assume people aren't playing games with the
1553 // instrumentedness of overridden weak aliases.
1554 auto *F = dyn_cast<Function>(GA.getAliaseeObject());
1555 if (!F)
1556 continue;
1557
1558 bool GAInst = isInstrumented(&GA), FInst = isInstrumented(F);
1559 if (GAInst && FInst) {
1560 addGlobalNameSuffix(&GA);
1561 } else if (GAInst != FInst) {
1562 // Non-instrumented alias of an instrumented function, or vice versa.
1563 // Replace the alias with a native-ABI wrapper of the aliasee. The pass
1564 // below will take care of instrumenting it.
1565 Function *NewF =
1566 buildWrapperFunction(F, "", GA.getLinkage(), F->getFunctionType());
1568 NewF->takeName(&GA);
1569 GA.eraseFromParent();
1570 FnsToInstrument.push_back(NewF);
1571 }
1572 }
1573
1574 // TODO: This could be more precise.
1575 ReadOnlyNoneAttrs.addAttribute(Attribute::Memory);
1576
1577 // First, change the ABI of every function in the module. ABI-listed
1578 // functions keep their original ABI and get a wrapper function.
1579 for (std::vector<Function *>::iterator FI = FnsToInstrument.begin(),
1580 FE = FnsToInstrument.end();
1581 FI != FE; ++FI) {
1582 Function &F = **FI;
1583 FunctionType *FT = F.getFunctionType();
1584
1585 bool IsZeroArgsVoidRet = (FT->getNumParams() == 0 && !FT->isVarArg() &&
1586 FT->getReturnType()->isVoidTy());
1587
1588 if (isInstrumented(&F)) {
1589 if (isForceZeroLabels(&F))
1590 FnsWithForceZeroLabel.insert(&F);
1591
1592 // Instrumented functions get a '.dfsan' suffix. This allows us to more
1593 // easily identify cases of mismatching ABIs. This naming scheme is
1594 // mangling-compatible (see Itanium ABI), using a vendor-specific suffix.
1595 addGlobalNameSuffix(&F);
1596 } else if (!IsZeroArgsVoidRet || getWrapperKind(&F) == WK_Custom) {
1597 // Build a wrapper function for F. The wrapper simply calls F, and is
1598 // added to FnsToInstrument so that any instrumentation according to its
1599 // WrapperKind is done in the second pass below.
1600
1601 // If the function being wrapped has local linkage, then preserve the
1602 // function's linkage in the wrapper function.
1603 GlobalValue::LinkageTypes WrapperLinkage =
1604 F.hasLocalLinkage() ? F.getLinkage()
1606
1607 Function *NewF = buildWrapperFunction(
1608 &F,
1609 (shouldTrackOrigins() ? std::string("dfso$") : std::string("dfsw$")) +
1610 std::string(F.getName()),
1611 WrapperLinkage, FT);
1612 NewF->removeFnAttrs(ReadOnlyNoneAttrs);
1613
1614 Value *WrappedFnCst =
1615 ConstantExpr::getBitCast(NewF, PointerType::getUnqual(FT));
1616
1617 // Extern weak functions can sometimes be null at execution time.
1618 // Code will sometimes check if an extern weak function is null.
1619 // This could look something like:
1620 // declare extern_weak i8 @my_func(i8)
1621 // br i1 icmp ne (i8 (i8)* @my_func, i8 (i8)* null), label %use_my_func,
1622 // label %avoid_my_func
1623 // The @"dfsw$my_func" wrapper is never null, so if we replace this use
1624 // in the comparison, the icmp will simplify to false and we have
1625 // accidentally optimized away a null check that is necessary.
1626 // This can lead to a crash when the null extern_weak my_func is called.
1627 //
1628 // To prevent (the most common pattern of) this problem,
1629 // do not replace uses in comparisons with the wrapper.
1630 // We definitely want to replace uses in call instructions.
1631 // Other uses (e.g. store the function address somewhere) might be
1632 // called or compared or both - this case may not be handled correctly.
1633 // We will default to replacing with wrapper in cases we are unsure.
1634 auto IsNotCmpUse = [](Use &U) -> bool {
1635 User *Usr = U.getUser();
1636 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Usr)) {
1637 // This is the most common case for icmp ne null
1638 if (CE->getOpcode() == Instruction::ICmp) {
1639 return false;
1640 }
1641 }
1642 if (Instruction *I = dyn_cast<Instruction>(Usr)) {
1643 if (I->getOpcode() == Instruction::ICmp) {
1644 return false;
1645 }
1646 }
1647 return true;
1648 };
1649 F.replaceUsesWithIf(WrappedFnCst, IsNotCmpUse);
1650
1651 UnwrappedFnMap[WrappedFnCst] = &F;
1652 *FI = NewF;
1653
1654 if (!F.isDeclaration()) {
1655 // This function is probably defining an interposition of an
1656 // uninstrumented function and hence needs to keep the original ABI.
1657 // But any functions it may call need to use the instrumented ABI, so
1658 // we instrument it in a mode which preserves the original ABI.
1659 FnsWithNativeABI.insert(&F);
1660
1661 // This code needs to rebuild the iterators, as they may be invalidated
1662 // by the push_back, taking care that the new range does not include
1663 // any functions added by this code.
1664 size_t N = FI - FnsToInstrument.begin(),
1665 Count = FE - FnsToInstrument.begin();
1666 FnsToInstrument.push_back(&F);
1667 FI = FnsToInstrument.begin() + N;
1668 FE = FnsToInstrument.begin() + Count;
1669 }
1670 // Hopefully, nobody will try to indirectly call a vararg
1671 // function... yet.
1672 } else if (FT->isVarArg()) {
1673 UnwrappedFnMap[&F] = &F;
1674 *FI = nullptr;
1675 }
1676 }
1677
1678 for (Function *F : FnsToInstrument) {
1679 if (!F || F->isDeclaration())
1680 continue;
1681
1683
1684 DFSanFunction DFSF(*this, F, FnsWithNativeABI.count(F),
1685 FnsWithForceZeroLabel.count(F), GetTLI(*F));
1686
1688 // Add callback for arguments reaching this function.
1689 for (auto &FArg : F->args()) {
1690 Instruction *Next = &F->getEntryBlock().front();
1691 Value *FArgShadow = DFSF.getShadow(&FArg);
1692 if (isZeroShadow(FArgShadow))
1693 continue;
1694 if (Instruction *FArgShadowInst = dyn_cast<Instruction>(FArgShadow)) {
1695 Next = FArgShadowInst->getNextNode();
1696 }
1697 if (shouldTrackOrigins()) {
1698 if (Instruction *Origin =
1699 dyn_cast<Instruction>(DFSF.getOrigin(&FArg))) {
1700 // Ensure IRB insertion point is after loads for shadow and origin.
1701 Instruction *OriginNext = Origin->getNextNode();
1702 if (Next->comesBefore(OriginNext)) {
1703 Next = OriginNext;
1704 }
1705 }
1706 }
1707 IRBuilder<> IRB(Next);
1708 DFSF.addReachesFunctionCallbacksIfEnabled(IRB, *Next, &FArg);
1709 }
1710 }
1711
1712 // DFSanVisitor may create new basic blocks, which confuses df_iterator.
1713 // Build a copy of the list before iterating over it.
1714 SmallVector<BasicBlock *, 4> BBList(depth_first(&F->getEntryBlock()));
1715
1716 for (BasicBlock *BB : BBList) {
1717 Instruction *Inst = &BB->front();
1718 while (true) {
1719 // DFSanVisitor may split the current basic block, changing the current
1720 // instruction's next pointer and moving the next instruction to the
1721 // tail block from which we should continue.
1722 Instruction *Next = Inst->getNextNode();
1723 // DFSanVisitor may delete Inst, so keep track of whether it was a
1724 // terminator.
1725 bool IsTerminator = Inst->isTerminator();
1726 if (!DFSF.SkipInsts.count(Inst))
1727 DFSanVisitor(DFSF).visit(Inst);
1728 if (IsTerminator)
1729 break;
1730 Inst = Next;
1731 }
1732 }
1733
1734 // We will not necessarily be able to compute the shadow for every phi node
1735 // until we have visited every block. Therefore, the code that handles phi
1736 // nodes adds them to the PHIFixups list so that they can be properly
1737 // handled here.
1738 for (DFSanFunction::PHIFixupElement &P : DFSF.PHIFixups) {
1739 for (unsigned Val = 0, N = P.Phi->getNumIncomingValues(); Val != N;
1740 ++Val) {
1741 P.ShadowPhi->setIncomingValue(
1742 Val, DFSF.getShadow(P.Phi->getIncomingValue(Val)));
1743 if (P.OriginPhi)
1744 P.OriginPhi->setIncomingValue(
1745 Val, DFSF.getOrigin(P.Phi->getIncomingValue(Val)));
1746 }
1747 }
1748
1749 // -dfsan-debug-nonzero-labels will split the CFG in all kinds of crazy
1750 // places (i.e. instructions in basic blocks we haven't even begun visiting
1751 // yet). To make our life easier, do this work in a pass after the main
1752 // instrumentation.
1754 for (Value *V : DFSF.NonZeroChecks) {
1755 Instruction *Pos;
1756 if (Instruction *I = dyn_cast<Instruction>(V))
1757 Pos = I->getNextNode();
1758 else
1759 Pos = &DFSF.F->getEntryBlock().front();
1760 while (isa<PHINode>(Pos) || isa<AllocaInst>(Pos))
1761 Pos = Pos->getNextNode();
1762 IRBuilder<> IRB(Pos);
1763 Value *PrimitiveShadow = DFSF.collapseToPrimitiveShadow(V, Pos);
1764 Value *Ne =
1765 IRB.CreateICmpNE(PrimitiveShadow, DFSF.DFS.ZeroPrimitiveShadow);
1766 BranchInst *BI = cast<BranchInst>(SplitBlockAndInsertIfThen(
1767 Ne, Pos, /*Unreachable=*/false, ColdCallWeights));
1768 IRBuilder<> ThenIRB(BI);
1769 ThenIRB.CreateCall(DFSF.DFS.DFSanNonzeroLabelFn, {});
1770 }
1771 }
1772 }
1773
1774 return Changed || !FnsToInstrument.empty() ||
1775 M.global_size() != InitialGlobalSize || M.size() != InitialModuleSize;
1776}
1777
1778Value *DFSanFunction::getArgTLS(Type *T, unsigned ArgOffset, IRBuilder<> &IRB) {
1779 Value *Base = IRB.CreatePointerCast(DFS.ArgTLS, DFS.IntptrTy);
1780 if (ArgOffset)
1781 Base = IRB.CreateAdd(Base, ConstantInt::get(DFS.IntptrTy, ArgOffset));
1782 return IRB.CreateIntToPtr(Base, PointerType::get(DFS.getShadowTy(T), 0),
1783 "_dfsarg");
1784}
1785
1786Value *DFSanFunction::getRetvalTLS(Type *T, IRBuilder<> &IRB) {
1787 return IRB.CreatePointerCast(
1788 DFS.RetvalTLS, PointerType::get(DFS.getShadowTy(T), 0), "_dfsret");
1789}
1790
1791Value *DFSanFunction::getRetvalOriginTLS() { return DFS.RetvalOriginTLS; }
1792
1793Value *DFSanFunction::getArgOriginTLS(unsigned ArgNo, IRBuilder<> &IRB) {
1794 return IRB.CreateConstGEP2_64(DFS.ArgOriginTLSTy, DFS.ArgOriginTLS, 0, ArgNo,
1795 "_dfsarg_o");
1796}
1797
1798Value *DFSanFunction::getOrigin(Value *V) {
1799 assert(DFS.shouldTrackOrigins());
1800 if (!isa<Argument>(V) && !isa<Instruction>(V))
1801 return DFS.ZeroOrigin;
1802 Value *&Origin = ValOriginMap[V];
1803 if (!Origin) {
1804 if (Argument *A = dyn_cast<Argument>(V)) {
1805 if (IsNativeABI)
1806 return DFS.ZeroOrigin;
1807 if (A->getArgNo() < DFS.NumOfElementsInArgOrgTLS) {
1808 Instruction *ArgOriginTLSPos = &*F->getEntryBlock().begin();
1809 IRBuilder<> IRB(ArgOriginTLSPos);
1810 Value *ArgOriginPtr = getArgOriginTLS(A->getArgNo(), IRB);
1811 Origin = IRB.CreateLoad(DFS.OriginTy, ArgOriginPtr);
1812 } else {
1813 // Overflow
1814 Origin = DFS.ZeroOrigin;
1815 }
1816 } else {
1817 Origin = DFS.ZeroOrigin;
1818 }
1819 }
1820 return Origin;
1821}
1822
1823void DFSanFunction::setOrigin(Instruction *I, Value *Origin) {
1824 if (!DFS.shouldTrackOrigins())
1825 return;
1826 assert(!ValOriginMap.count(I));
1827 assert(Origin->getType() == DFS.OriginTy);
1828 ValOriginMap[I] = Origin;
1829}
1830
1831Value *DFSanFunction::getShadowForTLSArgument(Argument *A) {
1832 unsigned ArgOffset = 0;
1833 const DataLayout &DL = F->getParent()->getDataLayout();
1834 for (auto &FArg : F->args()) {
1835 if (!FArg.getType()->isSized()) {
1836 if (A == &FArg)
1837 break;
1838 continue;
1839 }
1840
1841 unsigned Size = DL.getTypeAllocSize(DFS.getShadowTy(&FArg));
1842 if (A != &FArg) {
1843 ArgOffset += alignTo(Size, ShadowTLSAlignment);
1844 if (ArgOffset > ArgTLSSize)
1845 break; // ArgTLS overflows, uses a zero shadow.
1846 continue;
1847 }
1848
1849 if (ArgOffset + Size > ArgTLSSize)
1850 break; // ArgTLS overflows, uses a zero shadow.
1851
1852 Instruction *ArgTLSPos = &*F->getEntryBlock().begin();
1853 IRBuilder<> IRB(ArgTLSPos);
1854 Value *ArgShadowPtr = getArgTLS(FArg.getType(), ArgOffset, IRB);
1855 return IRB.CreateAlignedLoad(DFS.getShadowTy(&FArg), ArgShadowPtr,
1857 }
1858
1859 return DFS.getZeroShadow(A);
1860}
1861
1862Value *DFSanFunction::getShadow(Value *V) {
1863 if (!isa<Argument>(V) && !isa<Instruction>(V))
1864 return DFS.getZeroShadow(V);
1865 if (IsForceZeroLabels)
1866 return DFS.getZeroShadow(V);
1867 Value *&Shadow = ValShadowMap[V];
1868 if (!Shadow) {
1869 if (Argument *A = dyn_cast<Argument>(V)) {
1870 if (IsNativeABI)
1871 return DFS.getZeroShadow(V);
1872 Shadow = getShadowForTLSArgument(A);
1873 NonZeroChecks.push_back(Shadow);
1874 } else {
1875 Shadow = DFS.getZeroShadow(V);
1876 }
1877 }
1878 return Shadow;
1879}
1880
1881void DFSanFunction::setShadow(Instruction *I, Value *Shadow) {
1882 assert(!ValShadowMap.count(I));
1883 ValShadowMap[I] = Shadow;
1884}
1885
1886/// Compute the integer shadow offset that corresponds to a given
1887/// application address.
1888///
1889/// Offset = (Addr & ~AndMask) ^ XorMask
1890Value *DataFlowSanitizer::getShadowOffset(Value *Addr, IRBuilder<> &IRB) {
1891 assert(Addr != RetvalTLS && "Reinstrumenting?");
1892 Value *OffsetLong = IRB.CreatePointerCast(Addr, IntptrTy);
1893
1894 uint64_t AndMask = MapParams->AndMask;
1895 if (AndMask)
1896 OffsetLong =
1897 IRB.CreateAnd(OffsetLong, ConstantInt::get(IntptrTy, ~AndMask));
1898
1899 uint64_t XorMask = MapParams->XorMask;
1900 if (XorMask)
1901 OffsetLong = IRB.CreateXor(OffsetLong, ConstantInt::get(IntptrTy, XorMask));
1902 return OffsetLong;
1903}
1904
1905std::pair<Value *, Value *>
1906DataFlowSanitizer::getShadowOriginAddress(Value *Addr, Align InstAlignment,
1907 Instruction *Pos) {
1908 // Returns ((Addr & shadow_mask) + origin_base - shadow_base) & ~4UL
1909 IRBuilder<> IRB(Pos);
1910 Value *ShadowOffset = getShadowOffset(Addr, IRB);
1911 Value *ShadowLong = ShadowOffset;
1912 uint64_t ShadowBase = MapParams->ShadowBase;
1913 if (ShadowBase != 0) {
1914 ShadowLong =
1915 IRB.CreateAdd(ShadowLong, ConstantInt::get(IntptrTy, ShadowBase));
1916 }
1917 IntegerType *ShadowTy = IntegerType::get(*Ctx, ShadowWidthBits);
1918 Value *ShadowPtr =
1919 IRB.CreateIntToPtr(ShadowLong, PointerType::get(ShadowTy, 0));
1920 Value *OriginPtr = nullptr;
1921 if (shouldTrackOrigins()) {
1922 Value *OriginLong = ShadowOffset;
1923 uint64_t OriginBase = MapParams->OriginBase;
1924 if (OriginBase != 0)
1925 OriginLong =
1926 IRB.CreateAdd(OriginLong, ConstantInt::get(IntptrTy, OriginBase));
1927 const Align Alignment = llvm::assumeAligned(InstAlignment.value());
1928 // When alignment is >= 4, Addr must be aligned to 4, otherwise it is UB.
1929 // So Mask is unnecessary.
1930 if (Alignment < MinOriginAlignment) {
1932 OriginLong = IRB.CreateAnd(OriginLong, ConstantInt::get(IntptrTy, ~Mask));
1933 }
1934 OriginPtr = IRB.CreateIntToPtr(OriginLong, OriginPtrTy);
1935 }
1936 return std::make_pair(ShadowPtr, OriginPtr);
1937}
1938
1939Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos,
1940 Value *ShadowOffset) {
1941 IRBuilder<> IRB(Pos);
1942 return IRB.CreateIntToPtr(ShadowOffset, PrimitiveShadowPtrTy);
1943}
1944
1945Value *DataFlowSanitizer::getShadowAddress(Value *Addr, Instruction *Pos) {
1946 IRBuilder<> IRB(Pos);
1947 Value *ShadowOffset = getShadowOffset(Addr, IRB);
1948 return getShadowAddress(Addr, Pos, ShadowOffset);
1949}
1950
1951Value *DFSanFunction::combineShadowsThenConvert(Type *T, Value *V1, Value *V2,
1952 Instruction *Pos) {
1953 Value *PrimitiveValue = combineShadows(V1, V2, Pos);
1954 return expandFromPrimitiveShadow(T, PrimitiveValue, Pos);
1955}
1956
1957// Generates IR to compute the union of the two given shadows, inserting it
1958// before Pos. The combined value is with primitive type.
1959Value *DFSanFunction::combineShadows(Value *V1, Value *V2, Instruction *Pos) {
1960 if (DFS.isZeroShadow(V1))
1961 return collapseToPrimitiveShadow(V2, Pos);
1962 if (DFS.isZeroShadow(V2))
1963 return collapseToPrimitiveShadow(V1, Pos);
1964 if (V1 == V2)
1965 return collapseToPrimitiveShadow(V1, Pos);
1966
1967 auto V1Elems = ShadowElements.find(V1);
1968 auto V2Elems = ShadowElements.find(V2);
1969 if (V1Elems != ShadowElements.end() && V2Elems != ShadowElements.end()) {
1970 if (std::includes(V1Elems->second.begin(), V1Elems->second.end(),
1971 V2Elems->second.begin(), V2Elems->second.end())) {
1972 return collapseToPrimitiveShadow(V1, Pos);
1973 }
1974 if (std::includes(V2Elems->second.begin(), V2Elems->second.end(),
1975 V1Elems->second.begin(), V1Elems->second.end())) {
1976 return collapseToPrimitiveShadow(V2, Pos);
1977 }
1978 } else if (V1Elems != ShadowElements.end()) {
1979 if (V1Elems->second.count(V2))
1980 return collapseToPrimitiveShadow(V1, Pos);
1981 } else if (V2Elems != ShadowElements.end()) {
1982 if (V2Elems->second.count(V1))
1983 return collapseToPrimitiveShadow(V2, Pos);
1984 }
1985
1986 auto Key = std::make_pair(V1, V2);
1987 if (V1 > V2)
1988 std::swap(Key.first, Key.second);
1989 CachedShadow &CCS = CachedShadows[Key];
1990 if (CCS.Block && DT.dominates(CCS.Block, Pos->getParent()))
1991 return CCS.Shadow;
1992
1993 // Converts inputs shadows to shadows with primitive types.
1994 Value *PV1 = collapseToPrimitiveShadow(V1, Pos);
1995 Value *PV2 = collapseToPrimitiveShadow(V2, Pos);
1996
1997 IRBuilder<> IRB(Pos);
1998 CCS.Block = Pos->getParent();
1999 CCS.Shadow = IRB.CreateOr(PV1, PV2);
2000
2001 std::set<Value *> UnionElems;
2002 if (V1Elems != ShadowElements.end()) {
2003 UnionElems = V1Elems->second;
2004 } else {
2005 UnionElems.insert(V1);
2006 }
2007 if (V2Elems != ShadowElements.end()) {
2008 UnionElems.insert(V2Elems->second.begin(), V2Elems->second.end());
2009 } else {
2010 UnionElems.insert(V2);
2011 }
2012 ShadowElements[CCS.Shadow] = std::move(UnionElems);
2013
2014 return CCS.Shadow;
2015}
2016
2017// A convenience function which folds the shadows of each of the operands
2018// of the provided instruction Inst, inserting the IR before Inst. Returns
2019// the computed union Value.
2020Value *DFSanFunction::combineOperandShadows(Instruction *Inst) {
2021 if (Inst->getNumOperands() == 0)
2022 return DFS.getZeroShadow(Inst);
2023
2024 Value *Shadow = getShadow(Inst->getOperand(0));
2025 for (unsigned I = 1, N = Inst->getNumOperands(); I < N; ++I)
2026 Shadow = combineShadows(Shadow, getShadow(Inst->getOperand(I)), Inst);
2027
2028 return expandFromPrimitiveShadow(Inst->getType(), Shadow, Inst);
2029}
2030
2031void DFSanVisitor::visitInstOperands(Instruction &I) {
2032 Value *CombinedShadow = DFSF.combineOperandShadows(&I);
2033 DFSF.setShadow(&I, CombinedShadow);
2034 visitInstOperandOrigins(I);
2035}
2036
2037Value *DFSanFunction::combineOrigins(const std::vector<Value *> &Shadows,
2038 const std::vector<Value *> &Origins,
2039 Instruction *Pos, ConstantInt *Zero) {
2040 assert(Shadows.size() == Origins.size());
2041 size_t Size = Origins.size();
2042 if (Size == 0)
2043 return DFS.ZeroOrigin;
2044 Value *Origin = nullptr;
2045 if (!Zero)
2046 Zero = DFS.ZeroPrimitiveShadow;
2047 for (size_t I = 0; I != Size; ++I) {
2048 Value *OpOrigin = Origins[I];
2049 Constant *ConstOpOrigin = dyn_cast<Constant>(OpOrigin);
2050 if (ConstOpOrigin && ConstOpOrigin->isNullValue())
2051 continue;
2052 if (!Origin) {
2053 Origin = OpOrigin;
2054 continue;
2055 }
2056 Value *OpShadow = Shadows[I];
2057 Value *PrimitiveShadow = collapseToPrimitiveShadow(OpShadow, Pos);
2058 IRBuilder<> IRB(Pos);
2059 Value *Cond = IRB.CreateICmpNE(PrimitiveShadow, Zero);
2060 Origin = IRB.CreateSelect(Cond, OpOrigin, Origin);
2061 }
2062 return Origin ? Origin : DFS.ZeroOrigin;
2063}
2064
2065Value *DFSanFunction::combineOperandOrigins(Instruction *Inst) {
2066 size_t Size = Inst->getNumOperands();
2067 std::vector<Value *> Shadows(Size);
2068 std::vector<Value *> Origins(Size);
2069 for (unsigned I = 0; I != Size; ++I) {
2070 Shadows[I] = getShadow(Inst->getOperand(I));
2071 Origins[I] = getOrigin(Inst->getOperand(I));
2072 }
2073 return combineOrigins(Shadows, Origins, Inst);
2074}
2075
2076void DFSanVisitor::visitInstOperandOrigins(Instruction &I) {
2077 if (!DFSF.DFS.shouldTrackOrigins())
2078 return;
2079 Value *CombinedOrigin = DFSF.combineOperandOrigins(&I);
2080 DFSF.setOrigin(&I, CombinedOrigin);
2081}
2082
2083Align DFSanFunction::getShadowAlign(Align InstAlignment) {
2084 const Align Alignment = ClPreserveAlignment ? InstAlignment : Align(1);
2085 return Align(Alignment.value() * DFS.ShadowWidthBytes);
2086}
2087
2088Align DFSanFunction::getOriginAlign(Align InstAlignment) {
2089 const Align Alignment = llvm::assumeAligned(InstAlignment.value());
2090 return Align(std::max(MinOriginAlignment, Alignment));
2091}
2092
2093bool DFSanFunction::isLookupTableConstant(Value *P) {
2094 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(P->stripPointerCasts()))
2095 if (GV->isConstant() && GV->hasName())
2096 return DFS.CombineTaintLookupTableNames.count(GV->getName());
2097
2098 return false;
2099}
2100
2101bool DFSanFunction::useCallbackLoadLabelAndOrigin(uint64_t Size,
2102 Align InstAlignment) {
2103 // When enabling tracking load instructions, we always use
2104 // __dfsan_load_label_and_origin to reduce code size.
2105 if (ClTrackOrigins == 2)
2106 return true;
2107
2108 assert(Size != 0);
2109 // * if Size == 1, it is sufficient to load its origin aligned at 4.
2110 // * if Size == 2, we assume most cases Addr % 2 == 0, so it is sufficient to
2111 // load its origin aligned at 4. If not, although origins may be lost, it
2112 // should not happen very often.
2113 // * if align >= 4, Addr must be aligned to 4, otherwise it is UB. When
2114 // Size % 4 == 0, it is more efficient to load origins without callbacks.
2115 // * Otherwise we use __dfsan_load_label_and_origin.
2116 // This should ensure that common cases run efficiently.
2117 if (Size <= 2)
2118 return false;
2119
2120 const Align Alignment = llvm::assumeAligned(InstAlignment.value());
2121 return Alignment < MinOriginAlignment || !DFS.hasLoadSizeForFastPath(Size);
2122}
2123
2124Value *DataFlowSanitizer::loadNextOrigin(Instruction *Pos, Align OriginAlign,
2125 Value **OriginAddr) {
2126 IRBuilder<> IRB(Pos);
2127 *OriginAddr =
2128 IRB.CreateGEP(OriginTy, *OriginAddr, ConstantInt::get(IntptrTy, 1));
2129 return IRB.CreateAlignedLoad(OriginTy, *OriginAddr, OriginAlign);
2130}
2131
2132std::pair<Value *, Value *> DFSanFunction::loadShadowFast(
2133 Value *ShadowAddr, Value *OriginAddr, uint64_t Size, Align ShadowAlign,
2134 Align OriginAlign, Value *FirstOrigin, Instruction *Pos) {
2135 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2136 const uint64_t ShadowSize = Size * DFS.ShadowWidthBytes;
2137
2138 assert(Size >= 4 && "Not large enough load size for fast path!");
2139
2140 // Used for origin tracking.
2141 std::vector<Value *> Shadows;
2142 std::vector<Value *> Origins;
2143
2144 // Load instructions in LLVM can have arbitrary byte sizes (e.g., 3, 12, 20)
2145 // but this function is only used in a subset of cases that make it possible
2146 // to optimize the instrumentation.
2147 //
2148 // Specifically, when the shadow size in bytes (i.e., loaded bytes x shadow
2149 // per byte) is either:
2150 // - a multiple of 8 (common)
2151 // - equal to 4 (only for load32)
2152 //
2153 // For the second case, we can fit the wide shadow in a 32-bit integer. In all
2154 // other cases, we use a 64-bit integer to hold the wide shadow.
2155 Type *WideShadowTy =
2156 ShadowSize == 4 ? Type::getInt32Ty(*DFS.Ctx) : Type::getInt64Ty(*DFS.Ctx);
2157
2158 IRBuilder<> IRB(Pos);
2159 Value *WideAddr = IRB.CreateBitCast(ShadowAddr, WideShadowTy->getPointerTo());
2160 Value *CombinedWideShadow =
2161 IRB.CreateAlignedLoad(WideShadowTy, WideAddr, ShadowAlign);
2162
2163 unsigned WideShadowBitWidth = WideShadowTy->getIntegerBitWidth();
2164 const uint64_t BytesPerWideShadow = WideShadowBitWidth / DFS.ShadowWidthBits;
2165
2166 auto AppendWideShadowAndOrigin = [&](Value *WideShadow, Value *Origin) {
2167 if (BytesPerWideShadow > 4) {
2168 assert(BytesPerWideShadow == 8);
2169 // The wide shadow relates to two origin pointers: one for the first four
2170 // application bytes, and one for the latest four. We use a left shift to
2171 // get just the shadow bytes that correspond to the first origin pointer,
2172 // and then the entire shadow for the second origin pointer (which will be
2173 // chosen by combineOrigins() iff the least-significant half of the wide
2174 // shadow was empty but the other half was not).
2175 Value *WideShadowLo = IRB.CreateShl(
2176 WideShadow, ConstantInt::get(WideShadowTy, WideShadowBitWidth / 2));
2177 Shadows.push_back(WideShadow);
2178 Origins.push_back(DFS.loadNextOrigin(Pos, OriginAlign, &OriginAddr));
2179
2180 Shadows.push_back(WideShadowLo);
2181 Origins.push_back(Origin);
2182 } else {
2183 Shadows.push_back(WideShadow);
2184 Origins.push_back(Origin);
2185 }
2186 };
2187
2188 if (ShouldTrackOrigins)
2189 AppendWideShadowAndOrigin(CombinedWideShadow, FirstOrigin);
2190
2191 // First OR all the WideShadows (i.e., 64bit or 32bit shadow chunks) linearly;
2192 // then OR individual shadows within the combined WideShadow by binary ORing.
2193 // This is fewer instructions than ORing shadows individually, since it
2194 // needs logN shift/or instructions (N being the bytes of the combined wide
2195 // shadow).
2196 for (uint64_t ByteOfs = BytesPerWideShadow; ByteOfs < Size;
2197 ByteOfs += BytesPerWideShadow) {
2198 WideAddr = IRB.CreateGEP(WideShadowTy, WideAddr,
2199 ConstantInt::get(DFS.IntptrTy, 1));
2200 Value *NextWideShadow =
2201 IRB.CreateAlignedLoad(WideShadowTy, WideAddr, ShadowAlign);
2202 CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, NextWideShadow);
2203 if (ShouldTrackOrigins) {
2204 Value *NextOrigin = DFS.loadNextOrigin(Pos, OriginAlign, &OriginAddr);
2205 AppendWideShadowAndOrigin(NextWideShadow, NextOrigin);
2206 }
2207 }
2208 for (unsigned Width = WideShadowBitWidth / 2; Width >= DFS.ShadowWidthBits;
2209 Width >>= 1) {
2210 Value *ShrShadow = IRB.CreateLShr(CombinedWideShadow, Width);
2211 CombinedWideShadow = IRB.CreateOr(CombinedWideShadow, ShrShadow);
2212 }
2213 return {IRB.CreateTrunc(CombinedWideShadow, DFS.PrimitiveShadowTy),
2214 ShouldTrackOrigins
2215 ? combineOrigins(Shadows, Origins, Pos,
2217 : DFS.ZeroOrigin};
2218}
2219
2220std::pair<Value *, Value *> DFSanFunction::loadShadowOriginSansLoadTracking(
2221 Value *Addr, uint64_t Size, Align InstAlignment, Instruction *Pos) {
2222 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins();
2223
2224 // Non-escaped loads.
2225 if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
2226 const auto SI = AllocaShadowMap.find(AI);
2227 if (SI != AllocaShadowMap.end()) {
2228 IRBuilder<> IRB(Pos);
2229 Value *ShadowLI = IRB.CreateLoad(DFS.PrimitiveShadowTy, SI->second);
2230 const auto OI = AllocaOriginMap.find(AI);
2231 assert(!ShouldTrackOrigins || OI != AllocaOriginMap.end());
2232 return {ShadowLI, ShouldTrackOrigins
2233 ? IRB.CreateLoad(DFS.OriginTy, OI->second)
2234 : nullptr};
2235 }
2236 }
2237
2238 // Load from constant addresses.
2241 bool AllConstants = true;
2242 for (const Value *Obj : Objs) {
2243 if (isa<Function>(Obj) || isa<BlockAddress>(Obj))
2244 continue;
2245 if (isa<GlobalVariable>(Obj) && cast<GlobalVariable>(Obj)->isConstant())
2246 continue;
2247
2248 AllConstants = false;
2249 break;
2250 }
2251 if (AllConstants)
2252 return {DFS.ZeroPrimitiveShadow,
2253 ShouldTrackOrigins ? DFS.ZeroOrigin : nullptr};
2254
2255 if (Size == 0)
2256 return {DFS.ZeroPrimitiveShadow,
2257 ShouldTrackOrigins ? DFS.ZeroOrigin : nullptr};
2258
2259 // Use callback to load if this is not an optimizable case for origin
2260 // tracking.
2261 if (ShouldTrackOrigins &&
2262 useCallbackLoadLabelAndOrigin(Size, InstAlignment)) {
2263 IRBuilder<> IRB(Pos);
2264 CallInst *Call =
2265 IRB.CreateCall(DFS.DFSanLoadLabelAndOriginFn,
2266 {IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
2267 ConstantInt::get(DFS.IntptrTy, Size)});
2268 Call->addRetAttr(Attribute::ZExt);
2269 return {IRB.CreateTrunc(IRB.CreateLShr(Call, DFS.OriginWidthBits),
2270 DFS.PrimitiveShadowTy),
2271 IRB.CreateTrunc(Call, DFS.OriginTy)};
2272 }
2273
2274 // Other cases that support loading shadows or origins in a fast way.
2275 Value *ShadowAddr, *OriginAddr;
2276 std::tie(ShadowAddr, OriginAddr) =
2277 DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
2278
2279 const Align ShadowAlign = getShadowAlign(InstAlignment);
2280 const Align OriginAlign = getOriginAlign(InstAlignment);
2281 Value *Origin = nullptr;
2282 if (ShouldTrackOrigins) {
2283 IRBuilder<> IRB(Pos);
2284 Origin = IRB.CreateAlignedLoad(DFS.OriginTy, OriginAddr, OriginAlign);
2285 }
2286
2287 // When the byte size is small enough, we can load the shadow directly with
2288 // just a few instructions.
2289 switch (Size) {
2290 case 1: {
2291 LoadInst *LI = new LoadInst(DFS.PrimitiveShadowTy, ShadowAddr, "", Pos);
2292 LI->setAlignment(ShadowAlign);
2293 return {LI, Origin};
2294 }
2295 case 2: {
2296 IRBuilder<> IRB(Pos);
2297 Value *ShadowAddr1 = IRB.CreateGEP(DFS.PrimitiveShadowTy, ShadowAddr,
2298 ConstantInt::get(DFS.IntptrTy, 1));
2299 Value *Load =
2300 IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr, ShadowAlign);
2301 Value *Load1 =
2302 IRB.CreateAlignedLoad(DFS.PrimitiveShadowTy, ShadowAddr1, ShadowAlign);
2303 return {combineShadows(Load, Load1, Pos), Origin};
2304 }
2305 }
2306 bool HasSizeForFastPath = DFS.hasLoadSizeForFastPath(Size);
2307
2308 if (HasSizeForFastPath)
2309 return loadShadowFast(ShadowAddr, OriginAddr, Size, ShadowAlign,
2310 OriginAlign, Origin, Pos);
2311
2312 IRBuilder<> IRB(Pos);
2313 CallInst *FallbackCall = IRB.CreateCall(
2314 DFS.DFSanUnionLoadFn, {ShadowAddr, ConstantInt::get(DFS.IntptrTy, Size)});
2315 FallbackCall->addRetAttr(Attribute::ZExt);
2316 return {FallbackCall, Origin};
2317}
2318
2319std::pair<Value *, Value *> DFSanFunction::loadShadowOrigin(Value *Addr,
2320 uint64_t Size,
2321 Align InstAlignment,
2322 Instruction *Pos) {
2323 Value *PrimitiveShadow, *Origin;
2324 std::tie(PrimitiveShadow, Origin) =
2325 loadShadowOriginSansLoadTracking(Addr, Size, InstAlignment, Pos);
2326 if (DFS.shouldTrackOrigins()) {
2327 if (ClTrackOrigins == 2) {
2328 IRBuilder<> IRB(Pos);
2329 auto *ConstantShadow = dyn_cast<Constant>(PrimitiveShadow);
2330 if (!ConstantShadow || !ConstantShadow->isZeroValue())
2331 Origin = updateOriginIfTainted(PrimitiveShadow, Origin, IRB);
2332 }
2333 }
2334 return {PrimitiveShadow, Origin};
2335}
2336
2338 switch (AO) {
2339 case AtomicOrdering::NotAtomic:
2340 return AtomicOrdering::NotAtomic;
2341 case AtomicOrdering::Unordered:
2342 case AtomicOrdering::Monotonic:
2343 case AtomicOrdering::Acquire:
2344 return AtomicOrdering::Acquire;
2345 case AtomicOrdering::Release:
2346 case AtomicOrdering::AcquireRelease:
2347 return AtomicOrdering::AcquireRelease;
2348 case AtomicOrdering::SequentiallyConsistent:
2349 return AtomicOrdering::SequentiallyConsistent;
2350 }
2351 llvm_unreachable("Unknown ordering");
2352}
2353
2355 if (!V->getType()->isPointerTy())
2356 return V;
2357
2358 // DFSan pass should be running on valid IR, but we'll
2359 // keep a seen set to ensure there are no issues.
2361 Visited.insert(V);
2362 do {
2363 if (auto *GEP = dyn_cast<GEPOperator>(V)) {
2364 V = GEP->getPointerOperand();
2365 } else if (Operator::getOpcode(V) == Instruction::BitCast) {
2366 V = cast<Operator>(V)->getOperand(0);
2367 if (!V->getType()->isPointerTy())
2368 return V;
2369 } else if (isa<GlobalAlias>(V)) {
2370 V = cast<GlobalAlias>(V)->getAliasee();
2371 }
2372 } while (Visited.insert(V).second);
2373
2374 return V;
2375}
2376
2377void DFSanVisitor::visitLoadInst(LoadInst &LI) {
2378 auto &DL = LI.getModule()->getDataLayout();
2379 uint64_t Size = DL.getTypeStoreSize(LI.getType());
2380 if (Size == 0) {
2381 DFSF.setShadow(&LI, DFSF.DFS.getZeroShadow(&LI));
2382 DFSF.setOrigin(&LI, DFSF.DFS.ZeroOrigin);
2383 return;
2384 }
2385
2386 // When an application load is atomic, increase atomic ordering between
2387 // atomic application loads and stores to ensure happen-before order; load
2388 // shadow data after application data; store zero shadow data before
2389 // application data. This ensure shadow loads return either labels of the
2390 // initial application data or zeros.
2391 if (LI.isAtomic())
2393
2394 Instruction *AfterLi = LI.getNextNode();
2395 Instruction *Pos = LI.isAtomic() ? LI.getNextNode() : &LI;
2396 std::vector<Value *> Shadows;
2397 std::vector<Value *> Origins;
2398 Value *PrimitiveShadow, *Origin;
2399 std::tie(PrimitiveShadow, Origin) =
2400 DFSF.loadShadowOrigin(LI.getPointerOperand(), Size, LI.getAlign(), Pos);
2401 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2402 if (ShouldTrackOrigins) {
2403 Shadows.push_back(PrimitiveShadow);
2404 Origins.push_back(Origin);
2405 }
2407 DFSF.isLookupTableConstant(
2409 Value *PtrShadow = DFSF.getShadow(LI.getPointerOperand());
2410 PrimitiveShadow = DFSF.combineShadows(PrimitiveShadow, PtrShadow, Pos);
2411 if (ShouldTrackOrigins) {
2412 Shadows.push_back(PtrShadow);
2413 Origins.push_back(DFSF.getOrigin(LI.getPointerOperand()));
2414 }
2415 }
2416 if (!DFSF.DFS.isZeroShadow(PrimitiveShadow))
2417 DFSF.NonZeroChecks.push_back(PrimitiveShadow);
2418
2419 Value *Shadow =
2420 DFSF.expandFromPrimitiveShadow(LI.getType(), PrimitiveShadow, Pos);
2421 DFSF.setShadow(&LI, Shadow);
2422
2423 if (ShouldTrackOrigins) {
2424 DFSF.setOrigin(&LI, DFSF.combineOrigins(Shadows, Origins, Pos));
2425 }
2426
2427 if (ClEventCallbacks) {
2428 IRBuilder<> IRB(Pos);
2429 Value *Addr8 = IRB.CreateBitCast(LI.getPointerOperand(), DFSF.DFS.Int8Ptr);
2430 CallInst *CI =
2431 IRB.CreateCall(DFSF.DFS.DFSanLoadCallbackFn, {PrimitiveShadow, Addr8});
2432 CI->addParamAttr(0, Attribute::ZExt);
2433 }
2434
2435 IRBuilder<> IRB(AfterLi);
2436 DFSF.addReachesFunctionCallbacksIfEnabled(IRB, LI, &LI);
2437}
2438
2439Value *DFSanFunction::updateOriginIfTainted(Value *Shadow, Value *Origin,
2440 IRBuilder<> &IRB) {
2441 assert(DFS.shouldTrackOrigins());
2442 return IRB.CreateCall(DFS.DFSanChainOriginIfTaintedFn, {Shadow, Origin});
2443}
2444
2445Value *DFSanFunction::updateOrigin(Value *V, IRBuilder<> &IRB) {
2446 if (!DFS.shouldTrackOrigins())
2447 return V;
2448 return IRB.CreateCall(DFS.DFSanChainOriginFn, V);
2449}
2450
2451Value *DFSanFunction::originToIntptr(IRBuilder<> &IRB, Value *Origin) {
2452 const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2453 const DataLayout &DL = F->getParent()->getDataLayout();
2454 unsigned IntptrSize = DL.getTypeStoreSize(DFS.IntptrTy);
2455 if (IntptrSize == OriginSize)
2456 return Origin;
2457 assert(IntptrSize == OriginSize * 2);
2458 Origin = IRB.CreateIntCast(Origin, DFS.IntptrTy, /* isSigned */ false);
2459 return IRB.CreateOr(Origin, IRB.CreateShl(Origin, OriginSize * 8));
2460}
2461
2462void DFSanFunction::paintOrigin(IRBuilder<> &IRB, Value *Origin,
2463 Value *StoreOriginAddr,
2464 uint64_t StoreOriginSize, Align Alignment) {
2465 const unsigned OriginSize = DataFlowSanitizer::OriginWidthBytes;
2466 const DataLayout &DL = F->getParent()->getDataLayout();
2467 const Align IntptrAlignment = DL.getABITypeAlign(DFS.IntptrTy);
2468 unsigned IntptrSize = DL.getTypeStoreSize(DFS.IntptrTy);
2469 assert(IntptrAlignment >= MinOriginAlignment);
2470 assert(IntptrSize >= OriginSize);
2471
2472 unsigned Ofs = 0;
2473 Align CurrentAlignment = Alignment;
2474 if (Alignment >= IntptrAlignment && IntptrSize > OriginSize) {
2475 Value *IntptrOrigin = originToIntptr(IRB, Origin);
2476 Value *IntptrStoreOriginPtr = IRB.CreatePointerCast(
2477 StoreOriginAddr, PointerType::get(DFS.IntptrTy, 0));
2478 for (unsigned I = 0; I < StoreOriginSize / IntptrSize; ++I) {
2479 Value *Ptr =
2480 I ? IRB.CreateConstGEP1_32(DFS.IntptrTy, IntptrStoreOriginPtr, I)
2481 : IntptrStoreOriginPtr;
2482 IRB.CreateAlignedStore(IntptrOrigin, Ptr, CurrentAlignment);
2483 Ofs += IntptrSize / OriginSize;
2484 CurrentAlignment = IntptrAlignment;
2485 }
2486 }
2487
2488 for (unsigned I = Ofs; I < (StoreOriginSize + OriginSize - 1) / OriginSize;
2489 ++I) {
2490 Value *GEP = I ? IRB.CreateConstGEP1_32(DFS.OriginTy, StoreOriginAddr, I)
2491 : StoreOriginAddr;
2492 IRB.CreateAlignedStore(Origin, GEP, CurrentAlignment);
2493 CurrentAlignment = MinOriginAlignment;
2494 }
2495}
2496
2497Value *DFSanFunction::convertToBool(Value *V, IRBuilder<> &IRB,
2498 const Twine &Name) {
2499 Type *VTy = V->getType();
2500 assert(VTy->isIntegerTy());
2501 if (VTy->getIntegerBitWidth() == 1)
2502 // Just converting a bool to a bool, so do nothing.
2503 return V;
2504 return IRB.CreateICmpNE(V, ConstantInt::get(VTy, 0), Name);
2505}
2506
2507void DFSanFunction::storeOrigin(Instruction *Pos, Value *Addr, uint64_t Size,
2508 Value *Shadow, Value *Origin,
2509 Value *StoreOriginAddr, Align InstAlignment) {
2510 // Do not write origins for zero shadows because we do not trace origins for
2511 // untainted sinks.
2512 const Align OriginAlignment = getOriginAlign(InstAlignment);
2513 Value *CollapsedShadow = collapseToPrimitiveShadow(Shadow, Pos);
2514 IRBuilder<> IRB(Pos);
2515 if (auto *ConstantShadow = dyn_cast<Constant>(CollapsedShadow)) {
2516 if (!ConstantShadow->isZeroValue())
2517 paintOrigin(IRB, updateOrigin(Origin, IRB), StoreOriginAddr, Size,
2518 OriginAlignment);
2519 return;
2520 }
2521
2522 if (shouldInstrumentWithCall()) {
2523 IRB.CreateCall(DFS.DFSanMaybeStoreOriginFn,
2524 {CollapsedShadow,
2525 IRB.CreatePointerCast(Addr, IRB.getInt8PtrTy()),
2526 ConstantInt::get(DFS.IntptrTy, Size), Origin});
2527 } else {
2528 Value *Cmp = convertToBool(CollapsedShadow, IRB, "_dfscmp");
2530 Cmp, &*IRB.GetInsertPoint(), false, DFS.OriginStoreWeights, &DT);
2531 IRBuilder<> IRBNew(CheckTerm);
2532 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), StoreOriginAddr, Size,
2533 OriginAlignment);
2534 ++NumOriginStores;
2535 }
2536}
2537
2538void DFSanFunction::storeZeroPrimitiveShadow(Value *Addr, uint64_t Size,
2539 Align ShadowAlign,
2540 Instruction *Pos) {
2541 IRBuilder<> IRB(Pos);
2542 IntegerType *ShadowTy =
2543 IntegerType::get(*DFS.Ctx, Size * DFS.ShadowWidthBits);
2544 Value *ExtZeroShadow = ConstantInt::get(ShadowTy, 0);
2545 Value *ShadowAddr = DFS.getShadowAddress(Addr, Pos);
2546 Value *ExtShadowAddr =
2547 IRB.CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowTy));
2548 IRB.CreateAlignedStore(ExtZeroShadow, ExtShadowAddr, ShadowAlign);
2549 // Do not write origins for 0 shadows because we do not trace origins for
2550 // untainted sinks.
2551}
2552
2553void DFSanFunction::storePrimitiveShadowOrigin(Value *Addr, uint64_t Size,
2554 Align InstAlignment,
2555 Value *PrimitiveShadow,
2556 Value *Origin,
2557 Instruction *Pos) {
2558 const bool ShouldTrackOrigins = DFS.shouldTrackOrigins() && Origin;
2559
2560 if (AllocaInst *AI = dyn_cast<AllocaInst>(Addr)) {
2561 const auto SI = AllocaShadowMap.find(AI);
2562 if (SI != AllocaShadowMap.end()) {
2563 IRBuilder<> IRB(Pos);
2564 IRB.CreateStore(PrimitiveShadow, SI->second);
2565
2566 // Do not write origins for 0 shadows because we do not trace origins for
2567 // untainted sinks.
2568 if (ShouldTrackOrigins && !DFS.isZeroShadow(PrimitiveShadow)) {
2569 const auto OI = AllocaOriginMap.find(AI);
2570 assert(OI != AllocaOriginMap.end() && Origin);
2571 IRB.CreateStore(Origin, OI->second);
2572 }
2573 return;
2574 }
2575 }
2576
2577 const Align ShadowAlign = getShadowAlign(InstAlignment);
2578 if (DFS.isZeroShadow(PrimitiveShadow)) {
2579 storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, Pos);
2580 return;
2581 }
2582
2583 IRBuilder<> IRB(Pos);
2584 Value *ShadowAddr, *OriginAddr;
2585 std::tie(ShadowAddr, OriginAddr) =
2586 DFS.getShadowOriginAddress(Addr, InstAlignment, Pos);
2587
2588 const unsigned ShadowVecSize = 8;
2589 assert(ShadowVecSize * DFS.ShadowWidthBits <= 128 &&
2590 "Shadow vector is too large!");
2591
2592 uint64_t Offset = 0;
2593 uint64_t LeftSize = Size;
2594 if (LeftSize >= ShadowVecSize) {
2595 auto *ShadowVecTy =
2596 FixedVectorType::get(DFS.PrimitiveShadowTy, ShadowVecSize);
2597 Value *ShadowVec = PoisonValue::get(ShadowVecTy);
2598 for (unsigned I = 0; I != ShadowVecSize; ++I) {
2599 ShadowVec = IRB.CreateInsertElement(
2600 ShadowVec, PrimitiveShadow,
2601 ConstantInt::get(Type::getInt32Ty(*DFS.Ctx), I));
2602 }
2603 Value *ShadowVecAddr =
2604 IRB.CreateBitCast(ShadowAddr, PointerType::getUnqual(ShadowVecTy));
2605 do {
2606 Value *CurShadowVecAddr =
2607 IRB.CreateConstGEP1_32(ShadowVecTy, ShadowVecAddr, Offset);
2608 IRB.CreateAlignedStore(ShadowVec, CurShadowVecAddr, ShadowAlign);
2609 LeftSize -= ShadowVecSize;
2610 ++Offset;
2611 } while (LeftSize >= ShadowVecSize);
2612 Offset *= ShadowVecSize;
2613 }
2614 while (LeftSize > 0) {
2615 Value *CurShadowAddr =
2616 IRB.CreateConstGEP1_32(DFS.PrimitiveShadowTy, ShadowAddr, Offset);
2617 IRB.CreateAlignedStore(PrimitiveShadow, CurShadowAddr, ShadowAlign);
2618 --LeftSize;
2619 ++Offset;
2620 }
2621
2622 if (ShouldTrackOrigins) {
2623 storeOrigin(Pos, Addr, Size, PrimitiveShadow, Origin, OriginAddr,
2624 InstAlignment);
2625 }
2626}
2627
2629 switch (AO) {
2630 case AtomicOrdering::NotAtomic:
2631 return AtomicOrdering::NotAtomic;
2632 case AtomicOrdering::Unordered:
2633 case AtomicOrdering::Monotonic:
2634 case AtomicOrdering::Release:
2635 return AtomicOrdering::Release;
2636 case AtomicOrdering::Acquire:
2637 case AtomicOrdering::AcquireRelease:
2638 return AtomicOrdering::AcquireRelease;
2639 case AtomicOrdering::SequentiallyConsistent:
2640 return AtomicOrdering::SequentiallyConsistent;
2641 }
2642 llvm_unreachable("Unknown ordering");
2643}
2644
2645void DFSanVisitor::visitStoreInst(StoreInst &SI) {
2646 auto &DL = SI.getModule()->getDataLayout();
2647 Value *Val = SI.getValueOperand();
2648 uint64_t Size = DL.getTypeStoreSize(Val->getType());
2649 if (Size == 0)
2650 return;
2651
2652 // When an application store is atomic, increase atomic ordering between
2653 // atomic application loads and stores to ensure happen-before order; load
2654 // shadow data after application data; store zero shadow data before
2655 // application data. This ensure shadow loads return either labels of the
2656 // initial application data or zeros.
2657 if (SI.isAtomic())
2658 SI.setOrdering(addReleaseOrdering(SI.getOrdering()));
2659
2660 const bool ShouldTrackOrigins =
2661 DFSF.DFS.shouldTrackOrigins() && !SI.isAtomic();
2662 std::vector<Value *> Shadows;
2663 std::vector<Value *> Origins;
2664
2665 Value *Shadow =
2666 SI.isAtomic() ? DFSF.DFS.getZeroShadow(Val) : DFSF.getShadow(Val);
2667
2668 if (ShouldTrackOrigins) {
2669 Shadows.push_back(Shadow);
2670 Origins.push_back(DFSF.getOrigin(Val));
2671 }
2672
2673 Value *PrimitiveShadow;
2675 Value *PtrShadow = DFSF.getShadow(SI.getPointerOperand());
2676 if (ShouldTrackOrigins) {
2677 Shadows.push_back(PtrShadow);
2678 Origins.push_back(DFSF.getOrigin(SI.getPointerOperand()));
2679 }
2680 PrimitiveShadow = DFSF.combineShadows(Shadow, PtrShadow, &SI);
2681 } else {
2682 PrimitiveShadow = DFSF.collapseToPrimitiveShadow(Shadow, &SI);
2683 }
2684 Value *Origin = nullptr;
2685 if (ShouldTrackOrigins)
2686 Origin = DFSF.combineOrigins(Shadows, Origins, &SI);
2687 DFSF.storePrimitiveShadowOrigin(SI.getPointerOperand(), Size, SI.getAlign(),
2688 PrimitiveShadow, Origin, &SI);
2689 if (ClEventCallbacks) {
2690 IRBuilder<> IRB(&SI);
2691 Value *Addr8 = IRB.CreateBitCast(SI.getPointerOperand(), DFSF.DFS.Int8Ptr);
2692 CallInst *CI =
2693 IRB.CreateCall(DFSF.DFS.DFSanStoreCallbackFn, {PrimitiveShadow, Addr8});
2694 CI->addParamAttr(0, Attribute::ZExt);
2695 }
2696}
2697
2698void DFSanVisitor::visitCASOrRMW(Align InstAlignment, Instruction &I) {
2699 assert(isa<AtomicRMWInst>(I) || isa<AtomicCmpXchgInst>(I));
2700
2701 Value *Val = I.getOperand(1);
2702 const auto &DL = I.getModule()->getDataLayout();
2703 uint64_t Size = DL.getTypeStoreSize(Val->getType());
2704 if (Size == 0)
2705 return;
2706
2707 // Conservatively set data at stored addresses and return with zero shadow to
2708 // prevent shadow data races.
2709 IRBuilder<> IRB(&I);
2710 Value *Addr = I.getOperand(0);
2711 const Align ShadowAlign = DFSF.getShadowAlign(InstAlignment);
2712 DFSF.storeZeroPrimitiveShadow(Addr, Size, ShadowAlign, &I);
2713 DFSF.setShadow(&I, DFSF.DFS.getZeroShadow(&I));
2714 DFSF.setOrigin(&I, DFSF.DFS.ZeroOrigin);
2715}
2716
2717void DFSanVisitor::visitAtomicRMWInst(AtomicRMWInst &I) {
2718 visitCASOrRMW(I.getAlign(), I);
2719 // TODO: The ordering change follows MSan. It is possible not to change
2720 // ordering because we always set and use 0 shadows.
2721 I.setOrdering(addReleaseOrdering(I.getOrdering()));
2722}
2723
2724void DFSanVisitor::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) {
2725 visitCASOrRMW(I.getAlign(), I);
2726 // TODO: The ordering change follows MSan. It is possible not to change
2727 // ordering because we always set and use 0 shadows.
2728 I.setSuccessOrdering(addReleaseOrdering(I.getSuccessOrdering()));
2729}
2730
2731void DFSanVisitor::visitUnaryOperator(UnaryOperator &UO) {
2732 visitInstOperands(UO);
2733}
2734
2735void DFSanVisitor::visitBinaryOperator(BinaryOperator &BO) {
2736 visitInstOperands(BO);
2737}
2738
2739void DFSanVisitor::visitBitCastInst(BitCastInst &BCI) {
2740 // Special case: if this is the bitcast (there is exactly 1 allowed) between
2741 // a musttail call and a ret, don't instrument. New instructions are not
2742 // allowed after a musttail call.
2743 if (auto *CI = dyn_cast<CallInst>(BCI.getOperand(0)))
2744 if (CI->isMustTailCall())
2745 return;
2746 visitInstOperands(BCI);
2747}
2748
2749void DFSanVisitor::visitCastInst(CastInst &CI) { visitInstOperands(CI); }
2750
2751void DFSanVisitor::visitCmpInst(CmpInst &CI) {
2752 visitInstOperands(CI);
2753 if (ClEventCallbacks) {
2754 IRBuilder<> IRB(&CI);
2755 Value *CombinedShadow = DFSF.getShadow(&CI);
2756 CallInst *CallI =
2757 IRB.CreateCall(DFSF.DFS.DFSanCmpCallbackFn, CombinedShadow);
2758 CallI->addParamAttr(0, Attribute::ZExt);
2759 }
2760}
2761
2762void DFSanVisitor::visitLandingPadInst(LandingPadInst &LPI) {
2763 // We do not need to track data through LandingPadInst.
2764 //
2765 // For the C++ exceptions, if a value is thrown, this value will be stored
2766 // in a memory location provided by __cxa_allocate_exception(...) (on the
2767 // throw side) or __cxa_begin_catch(...) (on the catch side).
2768 // This memory will have a shadow, so with the loads and stores we will be
2769 // able to propagate labels on data thrown through exceptions, without any
2770 // special handling of the LandingPadInst.
2771 //
2772 // The second element in the pair result of the LandingPadInst is a
2773 // register value, but it is for a type ID and should never be tainted.
2774 DFSF.setShadow(&LPI, DFSF.DFS.getZeroShadow(&LPI));
2775 DFSF.setOrigin(&LPI, DFSF.DFS.ZeroOrigin);
2776}
2777
2778void DFSanVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
2780 DFSF.isLookupTableConstant(
2782 visitInstOperands(GEPI);
2783 return;
2784 }
2785
2786 // Only propagate shadow/origin of base pointer value but ignore those of
2787 // offset operands.
2788 Value *BasePointer = GEPI.getPointerOperand();
2789 DFSF.setShadow(&GEPI, DFSF.getShadow(BasePointer));
2790 if (DFSF.DFS.shouldTrackOrigins())
2791 DFSF.setOrigin(&GEPI, DFSF.getOrigin(BasePointer));
2792}
2793
2794void DFSanVisitor::visitExtractElementInst(ExtractElementInst &I) {
2795 visitInstOperands(I);
2796}
2797
2798void DFSanVisitor::visitInsertElementInst(InsertElementInst &I) {
2799 visitInstOperands(I);
2800}
2801
2802void DFSanVisitor::visitShuffleVectorInst(ShuffleVectorInst &I) {
2803 visitInstOperands(I);
2804}
2805
2806void DFSanVisitor::visitExtractValueInst(ExtractValueInst &I) {
2807 IRBuilder<> IRB(&I);
2808 Value *Agg = I.getAggregateOperand();
2809 Value *AggShadow = DFSF.getShadow(Agg);
2810 Value *ResShadow = IRB.CreateExtractValue(AggShadow, I.getIndices());
2811 DFSF.setShadow(&I, ResShadow);
2812 visitInstOperandOrigins(I);
2813}
2814
2815void DFSanVisitor::visitInsertValueInst(InsertValueInst &I) {
2816 IRBuilder<> IRB(&I);
2817 Value *AggShadow = DFSF.getShadow(I.getAggregateOperand());
2818 Value *InsShadow = DFSF.getShadow(I.getInsertedValueOperand());
2819 Value *Res = IRB.CreateInsertValue(AggShadow, InsShadow, I.getIndices());
2820 DFSF.setShadow(&I, Res);
2821 visitInstOperandOrigins(I);
2822}
2823
2824void DFSanVisitor::visitAllocaInst(AllocaInst &I) {
2825 bool AllLoadsStores = true;
2826 for (User *U : I.users()) {
2827 if (isa<LoadInst>(U))
2828 continue;
2829
2830 if (StoreInst *SI = dyn_cast<StoreInst>(U)) {
2831 if (SI->getPointerOperand() == &I)
2832 continue;
2833 }
2834
2835 AllLoadsStores = false;
2836 break;
2837 }
2838 if (AllLoadsStores) {
2839 IRBuilder<> IRB(&I);
2840 DFSF.AllocaShadowMap[&I] = IRB.CreateAlloca(DFSF.DFS.PrimitiveShadowTy);
2841 if (DFSF.DFS.shouldTrackOrigins()) {
2842 DFSF.AllocaOriginMap[&I] =
2843 IRB.CreateAlloca(DFSF.DFS.OriginTy, nullptr, "_dfsa");
2844 }
2845 }
2846 DFSF.setShadow(&I, DFSF.DFS.ZeroPrimitiveShadow);
2847 DFSF.setOrigin(&I, DFSF.DFS.ZeroOrigin);
2848}
2849
2850void DFSanVisitor::visitSelectInst(SelectInst &I) {
2851 Value *CondShadow = DFSF.getShadow(I.getCondition());
2852 Value *TrueShadow = DFSF.getShadow(I.getTrueValue());
2853 Value *FalseShadow = DFSF.getShadow(I.getFalseValue());
2854 Value *ShadowSel = nullptr;
2855 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
2856 std::vector<Value *> Shadows;
2857 std::vector<Value *> Origins;
2858 Value *TrueOrigin =
2859 ShouldTrackOrigins ? DFSF.getOrigin(I.getTrueValue()) : nullptr;
2860 Value *FalseOrigin =
2861 ShouldTrackOrigins ? DFSF.getOrigin(I.getFalseValue()) : nullptr;
2862
2863 DFSF.addConditionalCallbacksIfEnabled(I, I.getCondition());
2864
2865 if (isa<VectorType>(I.getCondition()->getType())) {
2866 ShadowSel = DFSF.combineShadowsThenConvert(I.getType(), TrueShadow,
2867 FalseShadow, &I);
2868 if (ShouldTrackOrigins) {
2869 Shadows.push_back(TrueShadow);
2870 Shadows.push_back(FalseShadow);
2871 Origins.push_back(TrueOrigin);
2872 Origins.push_back(FalseOrigin);
2873 }
2874 } else {
2875 if (TrueShadow == FalseShadow) {
2876 ShadowSel = TrueShadow;
2877 if (ShouldTrackOrigins) {
2878 Shadows.push_back(TrueShadow);
2879 Origins.push_back(TrueOrigin);
2880 }
2881 } else {
2882 ShadowSel =
2883 SelectInst::Create(I.getCondition(), TrueShadow, FalseShadow, "", &I);
2884 if (ShouldTrackOrigins) {
2885 Shadows.push_back(ShadowSel);
2886 Origins.push_back(SelectInst::Create(I.getCondition(), TrueOrigin,
2887 FalseOrigin, "", &I));
2888 }
2889 }
2890 }
2891 DFSF.setShadow(&I, ClTrackSelectControlFlow
2892 ? DFSF.combineShadowsThenConvert(
2893 I.getType(), CondShadow, ShadowSel, &I)
2894 : ShadowSel);
2895 if (ShouldTrackOrigins) {
2897 Shadows.push_back(CondShadow);
2898 Origins.push_back(DFSF.getOrigin(I.getCondition()));
2899 }
2900 DFSF.setOrigin(&I, DFSF.combineOrigins(Shadows, Origins, &I));
2901 }
2902}
2903
2904void DFSanVisitor::visitMemSetInst(MemSetInst &I) {
2905 IRBuilder<> IRB(&I);
2906 Value *ValShadow = DFSF.getShadow(I.getValue());
2907 Value *ValOrigin = DFSF.DFS.shouldTrackOrigins()
2908 ? DFSF.getOrigin(I.getValue())
2909 : DFSF.DFS.ZeroOrigin;
2910 IRB.CreateCall(
2911 DFSF.DFS.DFSanSetLabelFn,
2912 {ValShadow, ValOrigin,
2913 IRB.CreateBitCast(I.getDest(), Type::getInt8PtrTy(*DFSF.DFS.Ctx)),
2914 IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
2915}
2916
2917void DFSanVisitor::visitMemTransferInst(MemTransferInst &I) {
2918 IRBuilder<> IRB(&I);
2919
2920 // CopyOrMoveOrigin transfers origins by refering to their shadows. So we
2921 // need to move origins before moving shadows.
2922 if (DFSF.DFS.shouldTrackOrigins()) {
2923 IRB.CreateCall(
2924 DFSF.DFS.DFSanMemOriginTransferFn,
2925 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2926 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2927 IRB.CreateIntCast(I.getArgOperand(2), DFSF.DFS.IntptrTy, false)});
2928 }
2929
2930 Value *RawDestShadow = DFSF.DFS.getShadowAddress(I.getDest(), &I);
2931 Value *SrcShadow = DFSF.DFS.getShadowAddress(I.getSource(), &I);
2932 Value *LenShadow =
2933 IRB.CreateMul(I.getLength(), ConstantInt::get(I.getLength()->getType(),
2934 DFSF.DFS.ShadowWidthBytes));
2935 Type *Int8Ptr = Type::getInt8PtrTy(*DFSF.DFS.Ctx);
2936 Value *DestShadow = IRB.CreateBitCast(RawDestShadow, Int8Ptr);
2937 SrcShadow = IRB.CreateBitCast(SrcShadow, Int8Ptr);
2938 auto *MTI = cast<MemTransferInst>(
2939 IRB.CreateCall(I.getFunctionType(), I.getCalledOperand(),
2940 {DestShadow, SrcShadow, LenShadow, I.getVolatileCst()}));
2941 MTI->setDestAlignment(DFSF.getShadowAlign(I.getDestAlign().valueOrOne()));
2942 MTI->setSourceAlignment(DFSF.getShadowAlign(I.getSourceAlign().valueOrOne()));
2943 if (ClEventCallbacks) {
2944 IRB.CreateCall(DFSF.DFS.DFSanMemTransferCallbackFn,
2945 {RawDestShadow,
2946 IRB.CreateZExtOrTrunc(I.getLength(), DFSF.DFS.IntptrTy)});
2947 }
2948}
2949
2950void DFSanVisitor::visitBranchInst(BranchInst &BR) {
2951 if (!BR.isConditional())
2952 return;
2953
2954 DFSF.addConditionalCallbacksIfEnabled(BR, BR.getCondition());
2955}
2956
2957void DFSanVisitor::visitSwitchInst(SwitchInst &SW) {
2958 DFSF.addConditionalCallbacksIfEnabled(SW, SW.getCondition());
2959}
2960
2961static bool isAMustTailRetVal(Value *RetVal) {
2962 // Tail call may have a bitcast between return.
2963 if (auto *I = dyn_cast<BitCastInst>(RetVal)) {
2964 RetVal = I->getOperand(0);
2965 }
2966 if (auto *I = dyn_cast<CallInst>(RetVal)) {
2967 return I->isMustTailCall();
2968 }
2969 return false;
2970}
2971
2972void DFSanVisitor::visitReturnInst(ReturnInst &RI) {
2973 if (!DFSF.IsNativeABI && RI.getReturnValue()) {
2974 // Don't emit the instrumentation for musttail call returns.
2976 return;
2977
2978 Value *S = DFSF.getShadow(RI.getReturnValue());
2979 IRBuilder<> IRB(&RI);
2980 Type *RT = DFSF.F->getFunctionType()->getReturnType();
2981 unsigned Size = getDataLayout().getTypeAllocSize(DFSF.DFS.getShadowTy(RT));
2982 if (Size <= RetvalTLSSize) {
2983 // If the size overflows, stores nothing. At callsite, oversized return
2984 // shadows are set to zero.
2985 IRB.CreateAlignedStore(S, DFSF.getRetvalTLS(RT, IRB), ShadowTLSAlignment);
2986 }
2987 if (DFSF.DFS.shouldTrackOrigins()) {
2988 Value *O = DFSF.getOrigin(RI.getReturnValue());
2989 IRB.CreateStore(O, DFSF.getRetvalOriginTLS());
2990 }
2991 }
2992}
2993
2994void DFSanVisitor::addShadowArguments(Function &F, CallBase &CB,
2995 std::vector<Value *> &Args,
2996 IRBuilder<> &IRB) {
2997 FunctionType *FT = F.getFunctionType();
2998
2999 auto *I = CB.arg_begin();
3000
3001 // Adds non-variable argument shadows.
3002 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N)
3003 Args.push_back(DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), &CB));
3004
3005 // Adds variable argument shadows.
3006 if (FT->isVarArg()) {
3007 auto *LabelVATy = ArrayType::get(DFSF.DFS.PrimitiveShadowTy,
3008 CB.arg_size() - FT->getNumParams());
3009 auto *LabelVAAlloca =
3010 new AllocaInst(LabelVATy, getDataLayout().getAllocaAddrSpace(),
3011 "labelva", &DFSF.F->getEntryBlock().front());
3012
3013 for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) {
3014 auto *LabelVAPtr = IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, N);
3015 IRB.CreateStore(DFSF.collapseToPrimitiveShadow(DFSF.getShadow(*I), &CB),
3016 LabelVAPtr);
3017 }
3018
3019 Args.push_back(IRB.CreateStructGEP(LabelVATy, LabelVAAlloca, 0));
3020 }
3021
3022 // Adds the return value shadow.
3023 if (!FT->getReturnType()->isVoidTy()) {
3024 if (!DFSF.LabelReturnAlloca) {
3025 DFSF.LabelReturnAlloca = new AllocaInst(
3026 DFSF.DFS.PrimitiveShadowTy, getDataLayout().getAllocaAddrSpace(),
3027 "labelreturn", &DFSF.F->getEntryBlock().front());
3028 }
3029 Args.push_back(DFSF.LabelReturnAlloca);
3030 }
3031}
3032
3033void DFSanVisitor::addOriginArguments(Function &F, CallBase &CB,
3034 std::vector<Value *> &Args,
3035 IRBuilder<> &IRB) {
3036 FunctionType *FT = F.getFunctionType();
3037
3038 auto *I = CB.arg_begin();
3039
3040 // Add non-variable argument origins.
3041 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N)
3042 Args.push_back(DFSF.getOrigin(*I));
3043
3044 // Add variable argument origins.
3045 if (FT->isVarArg()) {
3046 auto *OriginVATy =
3047 ArrayType::get(DFSF.DFS.OriginTy, CB.arg_size() - FT->getNumParams());
3048 auto *OriginVAAlloca =
3049 new AllocaInst(OriginVATy, getDataLayout().getAllocaAddrSpace(),
3050 "originva", &DFSF.F->getEntryBlock().front());
3051
3052 for (unsigned N = 0; I != CB.arg_end(); ++I, ++N) {
3053 auto *OriginVAPtr = IRB.CreateStructGEP(OriginVATy, OriginVAAlloca, N);
3054 IRB.CreateStore(DFSF.getOrigin(*I), OriginVAPtr);
3055 }
3056
3057 Args.push_back(IRB.CreateStructGEP(OriginVATy, OriginVAAlloca, 0));
3058 }
3059
3060 // Add the return value origin.
3061 if (!FT->getReturnType()->isVoidTy()) {
3062 if (!DFSF.OriginReturnAlloca) {
3063 DFSF.OriginReturnAlloca = new AllocaInst(
3064 DFSF.DFS.OriginTy, getDataLayout().getAllocaAddrSpace(),
3065 "originreturn", &DFSF.F->getEntryBlock().front());
3066 }
3067 Args.push_back(DFSF.OriginReturnAlloca);
3068 }
3069}
3070
3071bool DFSanVisitor::visitWrappedCallBase(Function &F, CallBase &CB) {
3072 IRBuilder<> IRB(&CB);
3073 switch (DFSF.DFS.getWrapperKind(&F)) {
3074 case DataFlowSanitizer::WK_Warning:
3075 CB.setCalledFunction(&F);
3076 IRB.CreateCall(DFSF.DFS.DFSanUnimplementedFn,
3077 IRB.CreateGlobalStringPtr(F.getName()));
3078 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3079 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3080 DFSF.setOrigin(&CB, DFSF.DFS.ZeroOrigin);
3081 return true;
3082 case DataFlowSanitizer::WK_Discard:
3083 CB.setCalledFunction(&F);
3084 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3085 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3086 DFSF.setOrigin(&CB, DFSF.DFS.ZeroOrigin);
3087 return true;
3088 case DataFlowSanitizer::WK_Functional:
3089 CB.setCalledFunction(&F);
3090 DFSF.DFS.buildExternWeakCheckIfNeeded(IRB, &F);
3091 visitInstOperands(CB);
3092 return true;
3093 case DataFlowSanitizer::WK_Custom:
3094 // Don't try to handle invokes of custom functions, it's too complicated.
3095 // Instead, invoke the dfsw$ wrapper, which will in turn call the __dfsw_
3096 // wrapper.
3097 CallInst *CI = dyn_cast<CallInst>(&CB);
3098 if (!CI)
3099 return false;
3100
3101 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
3102 FunctionType *FT = F.getFunctionType();
3103 TransformedFunction CustomFn = DFSF.DFS.getCustomFunctionType(FT);
3104 std::string CustomFName = ShouldTrackOrigins ? "__dfso_" : "__dfsw_";
3105 CustomFName += F.getName();
3106 FunctionCallee CustomF = DFSF.DFS.Mod->getOrInsertFunction(
3107 CustomFName, CustomFn.TransformedType);
3108 if (Function *CustomFn = dyn_cast<Function>(CustomF.getCallee())) {
3109 CustomFn->copyAttributesFrom(&F);
3110
3111 // Custom functions returning non-void will write to the return label.
3112 if (!FT->getReturnType()->isVoidTy()) {
3113 CustomFn->removeFnAttrs(DFSF.DFS.ReadOnlyNoneAttrs);
3114 }
3115 }
3116
3117 std::vector<Value *> Args;
3118
3119 // Adds non-variable arguments.
3120 auto *I = CB.arg_begin();
3121 for (unsigned N = FT->getNumParams(); N != 0; ++I, --N) {
3122 Args.push_back(*I);
3123 }
3124
3125 // Adds shadow arguments.
3126 const unsigned ShadowArgStart = Args.size();
3127 addShadowArguments(F, CB, Args, IRB);
3128
3129 // Adds origin arguments.
3130 const unsigned OriginArgStart = Args.size();
3131 if (ShouldTrackOrigins)
3132 addOriginArguments(F, CB, Args, IRB);
3133
3134 // Adds variable arguments.
3135 append_range(Args, drop_begin(CB.args(), FT->getNumParams()));
3136
3137 CallInst *CustomCI = IRB.CreateCall(CustomF, Args);
3138 CustomCI->setCallingConv(CI->getCallingConv());
3139 CustomCI->setAttributes(transformFunctionAttributes(
3140 CustomFn, CI->getContext(), CI->getAttributes()));
3141
3142 // Update the parameter attributes of the custom call instruction to
3143 // zero extend the shadow parameters. This is required for targets
3144 // which consider PrimitiveShadowTy an illegal type.
3145 for (unsigned N = 0; N < FT->getNumParams(); N++) {
3146 const unsigned ArgNo = ShadowArgStart + N;
3147 if (CustomCI->getArgOperand(ArgNo)->getType() ==
3148 DFSF.DFS.PrimitiveShadowTy)
3149 CustomCI->addParamAttr(ArgNo, Attribute::ZExt);
3150 if (ShouldTrackOrigins) {
3151 const unsigned OriginArgNo = OriginArgStart + N;
3152 if (CustomCI->getArgOperand(OriginArgNo)->getType() ==
3153 DFSF.DFS.OriginTy)
3154 CustomCI->addParamAttr(OriginArgNo, Attribute::ZExt);
3155 }
3156 }
3157
3158 // Loads the return value shadow and origin.
3159 if (!FT->getReturnType()->isVoidTy()) {
3160 LoadInst *LabelLoad =
3161 IRB.CreateLoad(DFSF.DFS.PrimitiveShadowTy, DFSF.LabelReturnAlloca);
3162 DFSF.setShadow(CustomCI, DFSF.expandFromPrimitiveShadow(
3163 FT->getReturnType(), LabelLoad, &CB));
3164 if (ShouldTrackOrigins) {
3165 LoadInst *OriginLoad =
3166 IRB.CreateLoad(DFSF.DFS.OriginTy, DFSF.OriginReturnAlloca);
3167 DFSF.setOrigin(CustomCI, OriginLoad);
3168 }
3169 }
3170
3171 CI->replaceAllUsesWith(CustomCI);
3172 CI->eraseFromParent();
3173 return true;
3174 }
3175 return false;
3176}
3177
3178Value *DFSanVisitor::makeAddAcquireOrderingTable(IRBuilder<> &IRB) {
3179 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
3180 uint32_t OrderingTable[NumOrderings] = {};
3181
3182 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
3183 OrderingTable[(int)AtomicOrderingCABI::acquire] =
3184 OrderingTable[(int)AtomicOrderingCABI::consume] =
3185 (int)AtomicOrderingCABI::acquire;
3186 OrderingTable[(int)AtomicOrderingCABI::release] =
3187 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
3188 (int)AtomicOrderingCABI::acq_rel;
3189 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
3190 (int)AtomicOrderingCABI::seq_cst;
3191
3193 ArrayRef(OrderingTable, NumOrderings));
3194}
3195
3196void DFSanVisitor::visitLibAtomicLoad(CallBase &CB) {
3197 // Since we use getNextNode here, we can't have CB terminate the BB.
3198 assert(isa<CallInst>(CB));
3199
3200 IRBuilder<> IRB(&CB);
3201 Value *Size = CB.getArgOperand(0);
3202 Value *SrcPtr = CB.getArgOperand(1);
3203 Value *DstPtr = CB.getArgOperand(2);
3204 Value *Ordering = CB.getArgOperand(3);
3205 // Convert the call to have at least Acquire ordering to make sure
3206 // the shadow operations aren't reordered before it.
3207 Value *NewOrdering =
3208 IRB.CreateExtractElement(makeAddAcquireOrderingTable(IRB), Ordering);
3209 CB.setArgOperand(3, NewOrdering);
3210
3211 IRBuilder<> NextIRB(CB.getNextNode());
3212 NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3213
3214 // TODO: Support ClCombinePointerLabelsOnLoad
3215 // TODO: Support ClEventCallbacks
3216
3217 NextIRB.CreateCall(DFSF.DFS.DFSanMemShadowOriginTransferFn,
3218 {NextIRB.CreatePointerCast(DstPtr, NextIRB.getInt8PtrTy()),
3219 NextIRB.CreatePointerCast(SrcPtr, NextIRB.getInt8PtrTy()),
3220 NextIRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3221}
3222
3223Value *DFSanVisitor::makeAddReleaseOrderingTable(IRBuilder<> &IRB) {
3224 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
3225 uint32_t OrderingTable[NumOrderings] = {};
3226
3227 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
3228 OrderingTable[(int)AtomicOrderingCABI::release] =
3229 (int)AtomicOrderingCABI::release;
3230 OrderingTable[(int)AtomicOrderingCABI::consume] =
3231 OrderingTable[(int)AtomicOrderingCABI::acquire] =
3232 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
3233 (int)AtomicOrderingCABI::acq_rel;
3234 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
3235 (int)AtomicOrderingCABI::seq_cst;
3236
3238 ArrayRef(OrderingTable, NumOrderings));
3239}
3240
3241void DFSanVisitor::visitLibAtomicStore(CallBase &CB) {
3242 IRBuilder<> IRB(&CB);
3243 Value *Size = CB.getArgOperand(0);
3244 Value *SrcPtr = CB.getArgOperand(1);
3245 Value *DstPtr = CB.getArgOperand(2);
3246 Value *Ordering = CB.getArgOperand(3);
3247 // Convert the call to have at least Release ordering to make sure
3248 // the shadow operations aren't reordered after it.
3249 Value *NewOrdering =
3250 IRB.CreateExtractElement(makeAddReleaseOrderingTable(IRB), Ordering);
3251 CB.setArgOperand(3, NewOrdering);
3252
3253 // TODO: Support ClCombinePointerLabelsOnStore
3254 // TODO: Support ClEventCallbacks
3255
3256 IRB.CreateCall(DFSF.DFS.DFSanMemShadowOriginTransferFn,
3257 {IRB.CreatePointerCast(DstPtr, IRB.getInt8PtrTy()),
3258 IRB.CreatePointerCast(SrcPtr, IRB.getInt8PtrTy()),
3259 IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3260}
3261
3262void DFSanVisitor::visitLibAtomicExchange(CallBase &CB) {
3263 // void __atomic_exchange(size_t size, void *ptr, void *val, void *ret, int
3264 // ordering)
3265 IRBuilder<> IRB(&CB);
3266 Value *Size = CB.getArgOperand(0);
3267 Value *TargetPtr = CB.getArgOperand(1);
3268 Value *SrcPtr = CB.getArgOperand(2);
3269 Value *DstPtr = CB.getArgOperand(3);
3270
3271 // This operation is not atomic for the shadow and origin memory.
3272 // This could result in DFSan false positives or false negatives.
3273 // For now we will assume these operations are rare, and
3274 // the additional complexity to address this is not warrented.
3275
3276 // Current Target to Dest
3277 IRB.CreateCall(DFSF.DFS.DFSanMemShadowOriginTransferFn,
3278 {IRB.CreatePointerCast(DstPtr, IRB.getInt8PtrTy()),
3279 IRB.CreatePointerCast(TargetPtr, IRB.getInt8PtrTy()),
3280 IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3281
3282 // Current Src to Target (overriding)
3283 IRB.CreateCall(DFSF.DFS.DFSanMemShadowOriginTransferFn,
3284 {IRB.CreatePointerCast(TargetPtr, IRB.getInt8PtrTy()),
3285 IRB.CreatePointerCast(SrcPtr, IRB.getInt8PtrTy()),
3286 IRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3287}
3288
3289void DFSanVisitor::visitLibAtomicCompareExchange(CallBase &CB) {
3290 // bool __atomic_compare_exchange(size_t size, void *ptr, void *expected, void
3291 // *desired, int success_order, int failure_order)
3292 Value *Size = CB.getArgOperand(0);
3293 Value *TargetPtr = CB.getArgOperand(1);
3294 Value *ExpectedPtr = CB.getArgOperand(2);
3295 Value *DesiredPtr = CB.getArgOperand(3);
3296
3297 // This operation is not atomic for the shadow and origin memory.
3298 // This could result in DFSan false positives or false negatives.
3299 // For now we will assume these operations are rare, and
3300 // the additional complexity to address this is not warrented.
3301
3302 IRBuilder<> NextIRB(CB.getNextNode());
3303 NextIRB.SetCurrentDebugLocation(CB.getDebugLoc());
3304
3305 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3306
3307 // If original call returned true, copy Desired to Target.
3308 // If original call returned false, copy Target to Expected.
3309 NextIRB.CreateCall(
3310 DFSF.DFS.DFSanMemShadowOriginConditionalExchangeFn,
3311 {NextIRB.CreateIntCast(&CB, NextIRB.getInt8Ty(), false),
3312 NextIRB.CreatePointerCast(TargetPtr, NextIRB.getInt8PtrTy()),
3313 NextIRB.CreatePointerCast(ExpectedPtr, NextIRB.getInt8PtrTy()),
3314 NextIRB.CreatePointerCast(DesiredPtr, NextIRB.getInt8PtrTy()),
3315 NextIRB.CreateIntCast(Size, DFSF.DFS.IntptrTy, false)});
3316}
3317
3318void DFSanVisitor::visitCallBase(CallBase &CB) {
3320 if ((F && F->isIntrinsic()) || CB.isInlineAsm()) {
3321 visitInstOperands(CB);
3322 return;
3323 }
3324
3325 // Calls to this function are synthesized in wrappers, and we shouldn't
3326 // instrument them.
3327 if (F == DFSF.DFS.DFSanVarargWrapperFn.getCallee()->stripPointerCasts())
3328 return;
3329
3330 LibFunc LF;
3331 if (DFSF.TLI.getLibFunc(CB, LF)) {
3332 // libatomic.a functions need to have special handling because there isn't
3333 // a good way to intercept them or compile the library with
3334 // instrumentation.
3335 switch (LF) {
3336 case LibFunc_atomic_load:
3337 if (!isa<CallInst>(CB)) {
3338 llvm::errs() << "DFSAN -- cannot instrument invoke of libatomic load. "
3339 "Ignoring!\n";
3340 break;
3341 }
3342 visitLibAtomicLoad(CB);
3343 return;
3344 case LibFunc_atomic_store:
3345 visitLibAtomicStore(CB);
3346 return;
3347 default:
3348 break;
3349 }
3350 }
3351
3352 // TODO: These are not supported by TLI? They are not in the enum.
3353 if (F && F->hasName() && !F->isVarArg()) {
3354 if (F->getName() == "__atomic_exchange") {
3355 visitLibAtomicExchange(CB);
3356 return;
3357 }
3358 if (F->getName() == "__atomic_compare_exchange") {
3359 visitLibAtomicCompareExchange(CB);
3360 return;
3361 }
3362 }
3363
3365 DFSF.DFS.UnwrappedFnMap.find(CB.getCalledOperand());
3366 if (UnwrappedFnIt != DFSF.DFS.UnwrappedFnMap.end())
3367 if (visitWrappedCallBase(*UnwrappedFnIt->second, CB))
3368 return;
3369
3370 IRBuilder<> IRB(&CB);
3371
3372 const bool ShouldTrackOrigins = DFSF.DFS.shouldTrackOrigins();
3373 FunctionType *FT = CB.getFunctionType();
3374 const DataLayout &DL = getDataLayout();
3375
3376 // Stores argument shadows.
3377 unsigned ArgOffset = 0;
3378 for (unsigned I = 0, N = FT->getNumParams(); I != N; ++I) {
3379 if (ShouldTrackOrigins) {
3380 // Ignore overflowed origins
3381 Value *ArgShadow = DFSF.getShadow(CB.getArgOperand(I));
3382 if (I < DFSF.DFS.NumOfElementsInArgOrgTLS &&
3383 !DFSF.DFS.isZeroShadow(ArgShadow))
3384 IRB.CreateStore(DFSF.getOrigin(CB.getArgOperand(I)),
3385 DFSF.getArgOriginTLS(I, IRB));
3386 }
3387
3388 unsigned Size =
3389 DL.getTypeAllocSize(DFSF.DFS.getShadowTy(FT->getParamType(I)));
3390 // Stop storing if arguments' size overflows. Inside a function, arguments
3391 // after overflow have zero shadow values.
3392 if (ArgOffset + Size > ArgTLSSize)
3393 break;
3394 IRB.CreateAlignedStore(DFSF.getShadow(CB.getArgOperand(I)),
3395 DFSF.getArgTLS(FT->getParamType(I), ArgOffset, IRB),
3397 ArgOffset += alignTo(Size, ShadowTLSAlignment);
3398 }
3399
3400 Instruction *Next = nullptr;
3401 if (!CB.getType()->isVoidTy()) {
3402 if (InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
3403 if (II->getNormalDest()->getSinglePredecessor()) {
3404 Next = &II->getNormalDest()->front();
3405 } else {
3406 BasicBlock *NewBB =
3407 SplitEdge(II->getParent(), II->getNormalDest(), &DFSF.DT);
3408 Next = &NewBB->front();
3409 }
3410 } else {
3411 assert(CB.getIterator() != CB.getParent()->end());
3412 Next = CB.getNextNode();
3413 }
3414
3415 // Don't emit the epilogue for musttail call returns.
3416 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
3417 return;
3418
3419 // Loads the return value shadow.
3420 IRBuilder<> NextIRB(Next);
3421 unsigned Size = DL.getTypeAllocSize(DFSF.DFS.getShadowTy(&CB));
3422 if (Size > RetvalTLSSize) {
3423 // Set overflowed return shadow to be zero.
3424 DFSF.setShadow(&CB, DFSF.DFS.getZeroShadow(&CB));
3425 } else {
3426 LoadInst *LI = NextIRB.CreateAlignedLoad(
3427 DFSF.DFS.getShadowTy(&CB), DFSF.getRetvalTLS(CB.getType(), NextIRB),
3428 ShadowTLSAlignment, "_dfsret");
3429 DFSF.SkipInsts.insert(LI);
3430 DFSF.setShadow(&CB, LI);
3431 DFSF.NonZeroChecks.push_back(LI);
3432 }
3433
3434 if (ShouldTrackOrigins) {
3435 LoadInst *LI = NextIRB.CreateLoad(DFSF.DFS.OriginTy,
3436 DFSF.getRetvalOriginTLS(), "_dfsret_o");
3437 DFSF.SkipInsts.insert(LI);
3438 DFSF.setOrigin(&CB, LI);
3439 }
3440
3441 DFSF.addReachesFunctionCallbacksIfEnabled(NextIRB, CB, &CB);
3442 }
3443}
3444
3445void DFSanVisitor::visitPHINode(PHINode &PN) {
3446 Type *ShadowTy = DFSF.DFS.getShadowTy(&PN);
3447 PHINode *ShadowPN =
3448 PHINode::Create(ShadowTy, PN.getNumIncomingValues(), "", &PN);
3449
3450 // Give the shadow phi node valid predecessors to fool SplitEdge into working.
3451 Value *UndefShadow = UndefValue::get(ShadowTy);
3452 for (BasicBlock *BB : PN.blocks())
3453 ShadowPN->addIncoming(UndefShadow, BB);
3454
3455 DFSF.setShadow(&PN, ShadowPN);
3456
3457 PHINode *OriginPN = nullptr;
3458 if (DFSF.DFS.shouldTrackOrigins()) {
3459 OriginPN =
3460 PHINode::Create(DFSF.DFS.OriginTy, PN.getNumIncomingValues(), "", &PN);
3461 Value *UndefOrigin = UndefValue::get(DFSF.DFS.OriginTy);
3462 for (BasicBlock *BB : PN.blocks())
3463 OriginPN->addIncoming(UndefOrigin, BB);
3464 DFSF.setOrigin(&PN, OriginPN);
3465 }
3466
3467 DFSF.PHIFixups.push_back({&PN, ShadowPN, OriginPN});
3468}
3469
3472 auto GetTLI = [&](Function &F) -> TargetLibraryInfo & {
3473 auto &FAM =
3476 };
3477 if (!DataFlowSanitizer(ABIListFiles).runImpl(M, GetTLI))
3478 return PreservedAnalyses::all();
3479
3481 // GlobalsAA is considered stateless and does not get invalidated unless
3482 // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers
3483 // make changes that require GlobalsAA to be invalidated.
3484 PA.abandon<GlobalsAA>();
3485 return PA;
3486}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file contains the simple types necessary to represent the attributes associated with functions a...
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< bool > ClTrackSelectControlFlow("dfsan-track-select-control-flow", cl::desc("Propagate labels from condition values of select instructions " "to results."), cl::Hidden, cl::init(true))
static cl::list< std::string > ClCombineTaintLookupTables("dfsan-combine-taint-lookup-table", cl::desc("When dfsan-combine-offset-labels-on-gep and/or " "dfsan-combine-pointer-labels-on-load are false, this flag can " "be used to re-enable combining offset and/or pointer taint when " "loading specific constant global variables (i.e. lookup tables)."), cl::Hidden)
static const Align MinOriginAlignment
static cl::opt< int > ClTrackOrigins("dfsan-track-origins", cl::desc("Track origins of labels"), cl::Hidden, cl::init(0))
static cl::list< std::string > ClABIListFiles("dfsan-abilist", cl::desc("File listing native ABI functions and how the pass treats them"), cl::Hidden)
static cl::opt< bool > ClReachesFunctionCallbacks("dfsan-reaches-function-callbacks", cl::desc("Insert calls to callback functions on data reaching a function."), cl::Hidden, cl::init(false))
static Value * expandFromPrimitiveShadowRecursive(Value *Shadow, SmallVector< unsigned, 4 > &Indices, Type *SubShadowTy, Value *PrimitiveShadow, IRBuilder<> &IRB)
static cl::opt< int > ClInstrumentWithCallThreshold("dfsan-instrument-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< bool > ClPreserveAlignment("dfsan-preserve-alignment", cl::desc("respect alignment requirements provided by input IR"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClDebugNonzeroLabels("dfsan-debug-nonzero-labels", cl::desc("Insert calls to __dfsan_nonzero_label on observing a parameter, " "load or return with a nonzero label"), cl::Hidden)
static cl::opt< bool > ClCombineOffsetLabelsOnGEP("dfsan-combine-offset-labels-on-gep", cl::desc("Combine the label of the offset with the label of the pointer when " "doing pointer arithmetic."), cl::Hidden, cl::init(true))
static cl::opt< bool > ClIgnorePersonalityRoutine("dfsan-ignore-personality-routine", cl::desc("If a personality routine is marked uninstrumented from the ABI " "list, do not create a wrapper for it."), cl::Hidden, cl::init(false))
static const Align ShadowTLSAlignment
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
Value * StripPointerGEPsAndCasts(Value *V)
const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< bool > ClConditionalCallbacks("dfsan-conditional-callbacks", cl::desc("Insert calls to callback functions on conditionals."), cl::Hidden, cl::init(false))
static cl::opt< bool > ClCombinePointerLabelsOnLoad("dfsan-combine-pointer-labels-on-load", cl::desc("Combine the label of the pointer with the label of the data when " "loading from memory."), cl::Hidden, cl::init(true))
static StringRef getGlobalTypeString(const GlobalValue &G)
static cl::opt< bool > ClCombinePointerLabelsOnStore("dfsan-combine-pointer-labels-on-store", cl::desc("Combine the label of the pointer with the label of the data when " "storing in memory."), cl::Hidden, cl::init(false))
static const unsigned ArgTLSSize
static const unsigned RetvalTLSSize
static bool isAMustTailRetVal(Value *RetVal)
static cl::opt< bool > ClEventCallbacks("dfsan-event-callbacks", cl::desc("Insert calls to __dfsan_*_callback functions on data events."), cl::Hidden, cl::init(false))
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
This file defines the DenseSet and SmallDenseSet classes.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
uint64_t Addr
std::string Name
uint64_t Size
static bool runImpl(Function &F, const TargetLowering &TLI)
This is the interface for a simple mod/ref and alias analysis over globals.
Hexagon Common GEP
static bool isMustTailCall(Value *V)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
Module.h This file contains the declarations for the Module class.
nvptx lower args
#define P(N)
Module * Mod
FunctionAnalysisManager FAM
This header defines various interfaces for pass management in LLVM.
@ SI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
StringSet - A set-like wrapper for the StringMap.
Defines the virtual file system interface vfs::FileSystem.
Class for arbitrary precision integers.
Definition: APInt.h:75
an instruction to allocate memory on the stack
Definition: Instructions.h:58
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:620
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:513
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:718
AttributeSet getFnAttrs() const
The function attributes are returned.
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute > > Attrs)
Create an AttributeList with the specified parameters in it.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
unsigned getNumAttrSets() const
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
AttributeMask & addAttribute(Attribute::AttrKind Val)
Add an attribute to the mask.
Definition: Attributes.h:1006
static Attribute getWithMemoryEffects(LLVMContext &Context, MemoryEffects ME)
Definition: Attributes.cpp:214
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
iterator end()
Definition: BasicBlock.h:325
const Instruction & front() const
Definition: BasicBlock.h:335
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:105
This class represents a no-op cast from one type to another.
Conditional or Unconditional Branch instruction.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1186
bool isInlineAsm() const
Check if this call is an inline asm statement.
Definition: InstrTypes.h:1476
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1471
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1408
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1467
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1328
Value * getCalledOperand() const
Definition: InstrTypes.h:1401
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1490
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Definition: InstrTypes.h:1528
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1353
void setArgOperand(unsigned i, Value *v)
Definition: InstrTypes.h:1358
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1334
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1266
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
Definition: InstrTypes.h:1344
unsigned arg_size() const
Definition: InstrTypes.h:1351
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1486
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1538
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition: InstrTypes.h:1447
This class represents a function call, abstracting a target machine's calling convention.
bool isMustTailCall() const
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:428
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:708
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1586
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
Definition: Constants.cpp:2962
A constant value that is initialized with an expression using other constant values.
Definition: Constants.h:1002
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2220
This is the shared class of boolean and integer constants.
Definition: Constants.h:78
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition: Constants.h:114
This is an important base class in LLVM.
Definition: Constant.h:41
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:76
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
unsigned getLine() const
Definition: DebugLoc.cpp:24
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:20
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
bool dominates(const BasicBlock *BB, const Use &U) const
Return true if the (end of the) basic block BB dominates the use U.
Definition: Dominators.cpp:122
This instruction extracts a single (scalar) element from a VectorType value.
This instruction extracts a struct member or array element value from an aggregate value.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:704
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:136
void removeFnAttrs(const AttributeMask &Attrs)
Definition: Function.cpp:610
void removeFnAttr(Attribute::AttrKind Kind)
Remove function attributes from this function.
Definition: Function.cpp:602
arg_iterator arg_begin()
Definition: Function.h:771
void removeRetAttrs(const AttributeMask &Attrs)
removes the attributes from the return value list of attributes.
Definition: Function.cpp:622
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
Definition: Function.cpp:743
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:940
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:552
const GlobalObject * getAliaseeObject() const
Definition: Globals.cpp:562
static bool isExternalWeakLinkage(LinkageTypes Linkage)
Definition: GlobalValue.h:407
LinkageTypes getLinkage() const
Definition: GlobalValue.h:541
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:652
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:290
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
@ WeakODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:53
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
Type * getValueType() const
Definition: GlobalValue.h:292
Analysis pass providing a never-invalidated alias analysis result.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1926
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2367
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Definition: IRBuilder.h:1810
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Definition: IRBuilder.h:1702
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2418
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2355
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Definition: IRBuilder.h:1736
Constant * CreateGlobalStringPtr(StringRef Str, const Twine &Name="", unsigned AddressSpace=0, Module *M=nullptr)
Same as CreateGlobalString, but return a pointer with "i8*" type instead of a pointer to array of i8.
Definition: IRBuilder.h:1912
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2068
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
Definition: IRBuilder.h:2411
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1134
Value * CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, const Twine &Name="")
Definition: IRBuilder.h:1876
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:175
Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")
Definition: IRBuilder.h:1902
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2017
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1366
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:517
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2140
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2022
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1719
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1345
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Definition: IRBuilder.h:560
LLVMContext & getContext() const
Definition: IRBuilder.h:176
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1404
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1732
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1256
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1426
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2091
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
Definition: IRBuilder.h:1755
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2307
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1448
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1795
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1290
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2564
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:933
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
Definition: InstVisitor.h:78
RetTy visitCmpInst(CmpInst &I)
Definition: InstVisitor.h:262
RetTy visitExtractElementInst(ExtractElementInst &I)
Definition: InstVisitor.h:191
RetTy visitCallBase(CallBase &I)
Definition: InstVisitor.h:267
RetTy visitInsertValueInst(InsertValueInst &I)
Definition: InstVisitor.h:195
RetTy visitShuffleVectorInst(ShuffleVectorInst &I)
Definition: InstVisitor.h:193
RetTy visitLandingPadInst(LandingPadInst &I)
Definition: InstVisitor.h:196
RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I)
Definition: InstVisitor.h:171
RetTy visitBitCastInst(BitCastInst &I)
Definition: InstVisitor.h:187
RetTy visitSwitchInst(SwitchInst &I)
Definition: InstVisitor.h:232
RetTy visitPHINode(PHINode &I)
Definition: InstVisitor.h:175
RetTy visitReturnInst(ReturnInst &I)
Definition: InstVisitor.h:226
RetTy visitExtractValueInst(ExtractValueInst &I)
Definition: InstVisitor.h:194
RetTy visitUnaryOperator(UnaryOperator &I)
Definition: InstVisitor.h:260
RetTy visitStoreInst(StoreInst &I)
Definition: InstVisitor.h:170
RetTy visitInsertElementInst(InsertElementInst &I)
Definition: InstVisitor.h:192
RetTy visitAtomicRMWInst(AtomicRMWInst &I)
Definition: InstVisitor.h:172
RetTy visitAllocaInst(AllocaInst &I)
Definition: InstVisitor.h:168
RetTy visitBinaryOperator(BinaryOperator &I)
Definition: InstVisitor.h:261
RetTy visitMemTransferInst(MemTransferInst &I)
Definition: InstVisitor.h:214
RetTy visitMemSetInst(MemSetInst &I)
Definition: InstVisitor.h:209
RetTy visitCastInst(CastInst &I)
Definition: InstVisitor.h:259
RetTy visitBranchInst(BranchInst &I)
Definition: InstVisitor.h:229
RetTy visitSelectInst(SelectInst &I)
Definition: InstVisitor.h:189
RetTy visitGetElementPtrInst(GetElementPtrInst &I)
Definition: InstVisitor.h:174
RetTy visitLoadInst(LoadInst &I)
Definition: InstVisitor.h:169
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:358
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:70
bool isAtomic() const LLVM_READONLY
Return true if this instruction has an AtomicOrdering of unordered or higher.
const BasicBlock * getParent() const
Definition: Instruction.h:90
bool isTerminator() const
Definition: Instruction.h:171
bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:82
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:355
Class to represent integer types.
Definition: DerivedTypes.h:40
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:331
Invoke instruction.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
Definition: Instructions.h:177
void setAlignment(Align Align)
Definition: Instructions.h:224
Value * getPointerOperand()
Definition: Instructions.h:264
void setOrdering(AtomicOrdering Ordering)
Sets the ordering constraint of this load instruction.
Definition: Instructions.h:234
AtomicOrdering getOrdering() const
Returns the ordering constraint of this load instruction.
Definition: Instructions.h:229
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:220
MDNode * createBranchWeights(uint32_t TrueWeight, uint32_t FalseWeight)
Return metadata containing two branch weights.
Definition: MDBuilder.cpp:37
Metadata node.
Definition: Metadata.h:943
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
This class wraps the llvm.memcpy/memmove intrinsics.
static MemoryEffects readOnly()
Create MemoryEffects that can read any memory.
Definition: ModRef.h:123
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
const std::string & getModuleInlineAsm() const
Get any module-scope inline assembly blocks.
Definition: Module.h:266
void setModuleInlineAsm(StringRef Asm)
Set the module-scope inline assembly blocks.
Definition: Module.h:305
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:144
Constant * getOrInsertGlobal(StringRef Name, Type *Ty, function_ref< GlobalVariable *()> CreateGlobalCallback)
Look up the specified global in the module symbol table.
Definition: Module.cpp:205
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition: Operator.h:41
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
iterator_range< const_block_iterator > blocks() const
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1750
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
void abandon()
Mark an analysis as abandoned.
Definition: PassManager.h:206
Return a value (possibly void), from a function.
Value * getReturnValue() const
Convenience accessor. Returns null if there is no return value.
static ReturnInst * Create(LLVMContext &C, Value *retVal=nullptr, Instruction *InsertBefore=nullptr)
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
This instruction constructs a fixed permutation of two input vectors.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:383
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:389
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
size_t size() const
Definition: SmallVector.h:91
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
static std::unique_ptr< SpecialCaseList > createOrDie(const std::vector< std::string > &Paths, llvm::vfs::FileSystem &FS)
Parses the special case list entries from files.
An instruction for storing to memory.
Definition: Instructions.h:301
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
StringSet - A wrapper for StringMap that provides set-like functionality.
Definition: StringSet.h:23
std::pair< typename Base::iterator, bool > insert(StringRef key)
Definition: StringSet.h:34
Class to represent struct types.
Definition: DerivedTypes.h:213
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:426
Multiway switch.
Value * getCondition() const
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
unsigned getIntegerBitWidth() const
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:304
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:231
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:140
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1731
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:375
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:685
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:994
bool hasName() const
Definition: Value.h:261
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:381
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
Definition: ilist_node.h:82
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:289
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Key
PAL metadata keys.
AttributeMask typeIncompatible(Type *Ty, AttributeSafetyKind ASK=ASK_ALL)
Which attributes cannot be applied to a type.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ BR
Control flow instructions. These all have token chains.
Definition: ISDOpcodes.h:994
@ CE
Windows NT (Windows on ARM)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
IntrusiveRefCntPtr< FileSystem > getRealFileSystem()
Gets an vfs::FileSystem for the 'real' file system, as seen by the operating system.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:413
@ Offset
Definition: DWP.cpp:406
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:2129
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:748
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
void erase_value(Container &C, ValueType V)
Wrapper function to remove a value from a container:
Definition: STLExtras.h:2121
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:155
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
Definition: Alignment.h:111
iterator_range< df_iterator< T > > depth_first(const T &G)
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
Definition: Local.cpp:2607
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85