181#include "llvm/IR/IntrinsicsX86.h"
210#define DEBUG_TYPE "msan"
213 "Controls which checks to insert");
231 "msan-track-origins",
236 cl::desc(
"keep going after reporting a UMR"),
245 "msan-poison-stack-with-call",
250 "msan-poison-stack-pattern",
251 cl::desc(
"poison uninitialized stack variables with the given pattern"),
256 cl::desc(
"Print name of local stack variable"),
265 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
270 cl::desc(
"exact handling of relational integer ICmp"),
274 "msan-handle-lifetime-intrinsics",
276 "when possible, poison scoped variables at the beginning of the scope "
277 "(slower, but more precise)"),
288 "msan-handle-asm-conservative",
299 "msan-check-access-address",
300 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
305 cl::desc(
"check arguments and return values at function call boundaries"),
309 "msan-dump-strict-instructions",
310 cl::desc(
"print out instructions with default strict semantics"),
314 "msan-instrumentation-with-call-threshold",
316 "If the function being instrumented requires more than "
317 "this number of checks and origin stores, use callbacks instead of "
318 "inline checks (-1 means never use callbacks)."),
323 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
333 cl::desc(
"Insert checks for constant shadow values"),
340 cl::desc(
"Place MSan constructors in comdat sections"),
346 cl::desc(
"Define custom MSan AndMask"),
350 cl::desc(
"Define custom MSan XorMask"),
354 cl::desc(
"Define custom MSan ShadowBase"),
358 cl::desc(
"Define custom MSan OriginBase"),
363 cl::desc(
"Define threshold for number of checks per "
364 "debug location to force origin update."),
376struct MemoryMapParams {
383struct PlatformMemoryMapParams {
384 const MemoryMapParams *bits32;
385 const MemoryMapParams *bits64;
531class MemorySanitizer {
540 MemorySanitizer(MemorySanitizer &&) =
delete;
541 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
542 MemorySanitizer(
const MemorySanitizer &) =
delete;
543 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
548 friend struct MemorySanitizerVisitor;
549 friend struct VarArgHelperBase;
550 friend struct VarArgAMD64Helper;
551 friend struct VarArgMIPS64Helper;
552 friend struct VarArgAArch64Helper;
553 friend struct VarArgPowerPC64Helper;
554 friend struct VarArgSystemZHelper;
556 void initializeModule(
Module &M);
561 template <
typename... ArgsTy>
588 Value *ParamOriginTLS;
594 Value *RetvalOriginTLS;
600 Value *VAArgOriginTLS;
603 Value *VAArgOverflowSizeTLS;
606 bool CallbacksInitialized =
false;
651 Value *MsanMetadataAlloca;
657 const MemoryMapParams *MapParams;
661 MemoryMapParams CustomMapParams;
666 MDNode *OriginStoreWeights;
669void insertModuleCtor(
Module &M) {
697 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
712 MemorySanitizer Msan(*
F.getParent(),
Options);
731 OS, MapClassName2PassName);
738 OS <<
"eager-checks;";
739 OS <<
"track-origins=" <<
Options.TrackOrigins;
755template <
typename... ArgsTy>
763 std::forward<ArgsTy>(Args)...);
766 return M.getOrInsertFunction(
Name, MsanMetadata,
767 std::forward<ArgsTy>(Args)...);
776 RetvalOriginTLS =
nullptr;
778 ParamOriginTLS =
nullptr;
780 VAArgOriginTLS =
nullptr;
781 VAArgOverflowSizeTLS =
nullptr;
783 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
785 IRB.getVoidTy(), IRB.getInt32Ty());
796 MsanGetContextStateFn =
M.getOrInsertFunction(
802 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
803 std::string name_load =
804 "__msan_metadata_ptr_for_load_" + std::to_string(size);
805 std::string name_store =
806 "__msan_metadata_ptr_for_store_" + std::to_string(size);
807 MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
809 MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
813 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
816 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
817 M,
"__msan_metadata_ptr_for_store_n",
821 MsanPoisonAllocaFn =
M.getOrInsertFunction(
822 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
823 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
824 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
828 return M.getOrInsertGlobal(
Name, Ty, [&] {
830 nullptr,
Name,
nullptr,
843 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
844 :
"__msan_warning_with_origin_noreturn";
845 WarningFn =
M.getOrInsertFunction(WarningFnName,
847 IRB.getVoidTy(), IRB.getInt32Ty());
850 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
851 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
877 VAArgOverflowSizeTLS =
882 unsigned AccessSize = 1 << AccessSizeIndex;
883 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
884 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
886 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
888 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
889 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
891 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
895 MsanSetAllocaOriginWithDescriptionFn =
896 M.getOrInsertFunction(
"__msan_set_alloca_origin_with_descr",
897 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
898 MsanSetAllocaOriginNoDescriptionFn =
899 M.getOrInsertFunction(
"__msan_set_alloca_origin_no_descr",
900 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
901 MsanPoisonStackFn =
M.getOrInsertFunction(
"__msan_poison_stack",
902 IRB.getVoidTy(), PtrTy, IntptrTy);
908 if (CallbacksInitialized)
914 MsanChainOriginFn =
M.getOrInsertFunction(
915 "__msan_chain_origin",
918 MsanSetOriginFn =
M.getOrInsertFunction(
920 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
922 M.getOrInsertFunction(
"__msan_memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
924 M.getOrInsertFunction(
"__msan_memcpy", PtrTy, PtrTy, PtrTy, IntptrTy);
925 MemsetFn =
M.getOrInsertFunction(
"__msan_memset",
927 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
929 MsanInstrumentAsmStoreFn =
930 M.getOrInsertFunction(
"__msan_instrument_asm_store", IRB.getVoidTy(),
934 createKernelApi(M, TLI);
936 createUserspaceApi(M, TLI);
938 CallbacksInitialized =
true;
944 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
962void MemorySanitizer::initializeModule(
Module &M) {
963 auto &
DL =
M.getDataLayout();
965 TargetTriple =
Triple(
M.getTargetTriple());
967 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
968 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
970 if (ShadowPassed || OriginPassed) {
975 MapParams = &CustomMapParams;
977 switch (TargetTriple.getOS()) {
979 switch (TargetTriple.getArch()) {
994 switch (TargetTriple.getArch()) {
1003 switch (TargetTriple.getArch()) {
1037 C = &(
M.getContext());
1039 IntptrTy = IRB.getIntPtrTy(
DL);
1040 OriginTy = IRB.getInt32Ty();
1041 PtrTy = IRB.getPtrTy();
1046 if (!CompileKernel) {
1048 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1049 return new GlobalVariable(
1050 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1051 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1055 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1056 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1057 GlobalValue::WeakODRLinkage,
1058 IRB.getInt32(Recover),
"__msan_keep_going");
1073struct VarArgHelper {
1074 virtual ~VarArgHelper() =
default;
1089 virtual void finalizeInstrumentation() = 0;
1092struct MemorySanitizerVisitor;
1097 MemorySanitizerVisitor &Visitor);
1104 if (TypeSizeFixed <= 8)
1113class NextNodeIRBuilder :
public IRBuilder<> {
1126struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1128 MemorySanitizer &MS;
1131 std::unique_ptr<VarArgHelper> VAHelper;
1138 bool PropagateShadow;
1142 struct ShadowOriginAndInsertPoint {
1148 : Shadow(S), Origin(
O), OrigIns(
I) {}
1156 int64_t SplittableBlocksCount = 0;
1158 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1161 bool SanitizeFunction =
1163 InsertChecks = SanitizeFunction;
1164 PropagateShadow = SanitizeFunction;
1174 MS.initializeCallbacks(*
F.getParent(), TLI);
1175 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1178 if (MS.CompileKernel) {
1180 insertKmsanPrologue(IRB);
1184 <<
"MemorySanitizer is not inserting checks into '"
1185 <<
F.getName() <<
"'\n");
1188 bool instrumentWithCalls(
Value *V) {
1190 if (isa<Constant>(V))
1193 ++SplittableBlocksCount;
1199 return I.getParent() == FnPrologueEnd->
getParent() &&
1200 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1208 if (MS.TrackOrigins <= 1)
1210 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1215 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1227 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1228 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1238 auto [InsertPt,
Index] =
1250 Align CurrentAlignment = Alignment;
1251 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1252 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1253 Value *IntptrOriginPtr =
1255 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1260 CurrentAlignment = IntptrAlignment;
1277 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1278 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1286 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1295 if (instrumentWithCalls(ConvertedShadow) &&
1298 Value *ConvertedShadow2 =
1304 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1308 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1313 void materializeStores() {
1316 Value *Val =
SI->getValueOperand();
1318 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1319 Value *ShadowPtr, *OriginPtr;
1321 const Align Alignment =
SI->getAlign();
1323 std::tie(ShadowPtr, OriginPtr) =
1324 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1333 if (MS.TrackOrigins && !
SI->isAtomic())
1334 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1341 if (MS.TrackOrigins < 2)
1344 if (LazyWarningDebugLocationCount.
empty())
1345 for (
const auto &
I : InstrumentationList)
1346 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1360 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1362 auto NewDebugLoc = OI->getDebugLoc();
1369 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1370 Origin = updateOrigin(Origin, IRBOrigin);
1375 if (MS.CompileKernel || MS.TrackOrigins)
1389 if (instrumentWithCalls(ConvertedShadow) &&
1392 Value *ConvertedShadow2 =
1395 Fn, {ConvertedShadow2,
1396 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1400 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1403 !MS.Recover, MS.ColdCallWeights);
1406 insertWarningFn(IRB, Origin);
1411 void materializeInstructionChecks(
1416 bool Combine = !MS.TrackOrigins;
1418 Value *Shadow =
nullptr;
1419 for (
const auto &ShadowData : InstructionChecks) {
1423 Value *ConvertedShadow = ShadowData.Shadow;
1425 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1432 insertWarningFn(IRB, ShadowData.Origin);
1442 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1447 Shadow = ConvertedShadow;
1451 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1452 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1453 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1459 materializeOneCheck(IRB, Shadow,
nullptr);
1463 void materializeChecks() {
1465 [](
const ShadowOriginAndInsertPoint &L,
1466 const ShadowOriginAndInsertPoint &R) {
1467 return L.OrigIns <
R.OrigIns;
1470 for (
auto I = InstrumentationList.begin();
1471 I != InstrumentationList.end();) {
1473 std::find_if(
I + 1, InstrumentationList.end(),
1474 [L =
I->OrigIns](
const ShadowOriginAndInsertPoint &R) {
1475 return L != R.OrigIns;
1489 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1490 {Zero, IRB.getInt32(0)},
"param_shadow");
1491 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1492 {Zero, IRB.getInt32(1)},
"retval_shadow");
1493 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1494 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1495 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1496 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1497 MS.VAArgOverflowSizeTLS =
1498 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1499 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1500 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1501 {Zero, IRB.getInt32(5)},
"param_origin");
1502 MS.RetvalOriginTLS =
1503 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1504 {Zero, IRB.getInt32(6)},
"retval_origin");
1506 MS.MsanMetadataAlloca = IRB.
CreateAlloca(MS.MsanMetadata, 0u);
1518 for (
PHINode *PN : ShadowPHINodes) {
1519 PHINode *PNS = cast<PHINode>(getShadow(PN));
1520 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1521 size_t NumValues = PN->getNumIncomingValues();
1522 for (
size_t v = 0;
v < NumValues;
v++) {
1523 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1525 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1529 VAHelper->finalizeInstrumentation();
1533 if (InstrumentLifetimeStart) {
1534 for (
auto Item : LifetimeStartList) {
1535 instrumentAlloca(*Item.second, Item.first);
1536 AllocaSet.
remove(Item.second);
1542 instrumentAlloca(*AI);
1545 materializeChecks();
1549 materializeStores();
1555 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1567 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1568 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1570 VT->getElementCount());
1572 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1573 return ArrayType::get(getShadowTy(AT->getElementType()),
1574 AT->getNumElements());
1576 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1578 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1579 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1581 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1597 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1599 if (Aggregator != FalseVal)
1600 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1602 Aggregator = ShadowBool;
1611 if (!
Array->getNumElements())
1615 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1619 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1620 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1630 return collapseStructShadow(
Struct, V, IRB);
1631 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1632 return collapseArrayShadow(Array, V, IRB);
1633 if (isa<VectorType>(
V->getType())) {
1634 if (isa<ScalableVectorType>(
V->getType()))
1637 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1645 Type *VTy =
V->getType();
1647 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1654 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1655 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1656 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1657 VectTy->getElementCount());
1663 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1664 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1665 return VectorType::get(
1666 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1667 VectTy->getElementCount());
1669 assert(IntPtrTy == MS.IntptrTy);
1670 return PointerType::get(*MS.C, 0);
1674 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1676 VectTy->getElementCount(), constToIntPtr(VectTy->getElementType(),
C));
1678 assert(IntPtrTy == MS.IntptrTy);
1679 return ConstantInt::get(MS.IntptrTy,
C);
1690 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1693 if (
uint64_t AndMask = MS.MapParams->AndMask)
1694 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1696 if (
uint64_t XorMask = MS.MapParams->XorMask)
1697 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1709 std::pair<Value *, Value *>
1716 assert(VectTy->getElementType()->isPointerTy());
1718 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1719 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1720 Value *ShadowLong = ShadowOffset;
1721 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1723 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1726 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1728 Value *OriginPtr =
nullptr;
1729 if (MS.TrackOrigins) {
1730 Value *OriginLong = ShadowOffset;
1731 uint64_t OriginBase = MS.MapParams->OriginBase;
1732 if (OriginBase != 0)
1734 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1737 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1740 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1742 return std::make_pair(ShadowPtr, OriginPtr);
1745 template <
typename... ArgsTy>
1750 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1751 return IRB.
CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1754 return IRB.
CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1757 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1761 Value *ShadowOriginPtrs;
1769 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1771 Value *SizeVal = ConstantInt::get(MS.IntptrTy,
Size);
1772 ShadowOriginPtrs = createMetadataCall(
1774 isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1781 return std::make_pair(ShadowPtr, OriginPtr);
1787 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1794 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1798 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1799 Value *ShadowPtrs = ConstantInt::getNullValue(
1801 Value *OriginPtrs =
nullptr;
1802 if (MS.TrackOrigins)
1803 OriginPtrs = ConstantInt::getNullValue(
1805 for (
unsigned i = 0; i < NumElements; ++i) {
1808 auto [ShadowPtr, OriginPtr] =
1809 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1812 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1813 if (MS.TrackOrigins)
1815 OriginPtrs, OriginPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1817 return {ShadowPtrs, OriginPtrs};
1824 if (MS.CompileKernel)
1825 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1826 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1841 if (!MS.TrackOrigins)
1855 Value *getOriginPtrForRetval() {
1857 return MS.RetvalOriginTLS;
1862 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1863 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1868 if (!MS.TrackOrigins)
1870 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1871 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1872 OriginMap[
V] = Origin;
1876 Type *ShadowTy = getShadowTy(OrigTy);
1886 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1891 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1893 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1895 getPoisonedShadow(AT->getElementType()));
1898 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1900 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1901 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1909 Type *ShadowTy = getShadowTy(V);
1912 return getPoisonedShadow(ShadowTy);
1924 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1925 return getCleanShadow(V);
1927 Value *Shadow = ShadowMap[
V];
1929 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1931 assert(Shadow &&
"No shadow for a value");
1935 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1936 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1937 : getCleanShadow(V);
1942 if (
Argument *
A = dyn_cast<Argument>(V)) {
1944 Value *&ShadowPtr = ShadowMap[
V];
1949 unsigned ArgOffset = 0;
1951 for (
auto &FArg :
F->args()) {
1952 if (!FArg.getType()->isSized()) {
1957 unsigned Size = FArg.hasByValAttr()
1958 ?
DL.getTypeAllocSize(FArg.getParamByValType())
1959 :
DL.getTypeAllocSize(FArg.getType());
1963 if (FArg.hasByValAttr()) {
1967 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
1968 FArg.getParamAlign(), FArg.getParamByValType());
1969 Value *CpShadowPtr, *CpOriginPtr;
1970 std::tie(CpShadowPtr, CpOriginPtr) =
1971 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1973 if (!PropagateShadow || Overflow) {
1975 EntryIRB.CreateMemSet(
1979 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
1981 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
1986 if (MS.TrackOrigins) {
1988 getOriginPtrForArgument(EntryIRB, ArgOffset);
1992 EntryIRB.CreateMemCpy(
2001 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2002 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
2003 ShadowPtr = getCleanShadow(V);
2004 setOrigin(
A, getCleanOrigin());
2007 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2008 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
2010 if (MS.TrackOrigins) {
2012 getOriginPtrForArgument(EntryIRB, ArgOffset);
2013 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
2017 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
2023 assert(ShadowPtr &&
"Could not find shadow for an argument");
2027 return getCleanShadow(V);
2032 return getShadow(
I->getOperand(i));
2037 if (!MS.TrackOrigins)
2039 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
2040 return getCleanOrigin();
2041 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2042 "Unexpected value type in getOrigin()");
2044 if (
I->getMetadata(LLVMContext::MD_nosanitize))
2045 return getCleanOrigin();
2047 Value *Origin = OriginMap[
V];
2048 assert(Origin &&
"Missing origin");
2054 return getOrigin(
I->getOperand(i));
2067 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
2068 << *OrigIns <<
"\n");
2073 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2074 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2075 "Can only insert checks for integer, vector, and aggregate shadow "
2078 InstrumentationList.push_back(
2079 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2088 Value *Shadow, *Origin;
2090 Shadow = getShadow(Val);
2093 Origin = getOrigin(Val);
2095 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2098 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2100 insertShadowCheck(Shadow, Origin, OrigIns);
2105 case AtomicOrdering::NotAtomic:
2106 return AtomicOrdering::NotAtomic;
2107 case AtomicOrdering::Unordered:
2108 case AtomicOrdering::Monotonic:
2109 case AtomicOrdering::Release:
2110 return AtomicOrdering::Release;
2111 case AtomicOrdering::Acquire:
2112 case AtomicOrdering::AcquireRelease:
2113 return AtomicOrdering::AcquireRelease;
2114 case AtomicOrdering::SequentiallyConsistent:
2115 return AtomicOrdering::SequentiallyConsistent;
2121 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2122 uint32_t OrderingTable[NumOrderings] = {};
2124 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2125 OrderingTable[(
int)AtomicOrderingCABI::release] =
2126 (int)AtomicOrderingCABI::release;
2127 OrderingTable[(int)AtomicOrderingCABI::consume] =
2128 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2129 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2130 (
int)AtomicOrderingCABI::acq_rel;
2131 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2132 (
int)AtomicOrderingCABI::seq_cst;
2139 case AtomicOrdering::NotAtomic:
2140 return AtomicOrdering::NotAtomic;
2141 case AtomicOrdering::Unordered:
2142 case AtomicOrdering::Monotonic:
2143 case AtomicOrdering::Acquire:
2144 return AtomicOrdering::Acquire;
2145 case AtomicOrdering::Release:
2146 case AtomicOrdering::AcquireRelease:
2147 return AtomicOrdering::AcquireRelease;
2148 case AtomicOrdering::SequentiallyConsistent:
2149 return AtomicOrdering::SequentiallyConsistent;
2155 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2156 uint32_t OrderingTable[NumOrderings] = {};
2158 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2159 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2160 OrderingTable[(int)AtomicOrderingCABI::consume] =
2161 (
int)AtomicOrderingCABI::acquire;
2162 OrderingTable[(int)AtomicOrderingCABI::release] =
2163 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2164 (int)AtomicOrderingCABI::acq_rel;
2165 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2166 (
int)AtomicOrderingCABI::seq_cst;
2174 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2177 if (isInPrologue(
I))
2187 assert(
I.getType()->isSized() &&
"Load type must have size");
2188 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2189 NextNodeIRBuilder IRB(&
I);
2190 Type *ShadowTy = getShadowTy(&
I);
2192 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2193 const Align Alignment =
I.getAlign();
2194 if (PropagateShadow) {
2195 std::tie(ShadowPtr, OriginPtr) =
2196 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2200 setShadow(&
I, getCleanShadow(&
I));
2204 insertShadowCheck(
I.getPointerOperand(), &
I);
2209 if (MS.TrackOrigins) {
2210 if (PropagateShadow) {
2215 setOrigin(&
I, getCleanOrigin());
2225 StoreList.push_back(&
I);
2227 insertShadowCheck(
I.getPointerOperand(), &
I);
2231 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2235 Value *Val =
I.getOperand(1);
2236 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2241 insertShadowCheck(
Addr, &
I);
2246 if (isa<AtomicCmpXchgInst>(
I))
2247 insertShadowCheck(Val, &
I);
2251 setShadow(&
I, getCleanShadow(&
I));
2252 setOrigin(&
I, getCleanOrigin());
2267 insertShadowCheck(
I.getOperand(1), &
I);
2271 setOrigin(&
I, getOrigin(&
I, 0));
2275 insertShadowCheck(
I.getOperand(2), &
I);
2277 auto *Shadow0 = getShadow(&
I, 0);
2278 auto *Shadow1 = getShadow(&
I, 1);
2281 setOriginForNaryOp(
I);
2286 auto *Shadow0 = getShadow(&
I, 0);
2287 auto *Shadow1 = getShadow(&
I, 1);
2290 setOriginForNaryOp(
I);
2296 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2297 setOrigin(&
I, getOrigin(&
I, 0));
2302 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2303 setOrigin(&
I, getOrigin(&
I, 0));
2308 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2309 setOrigin(&
I, getOrigin(&
I, 0));
2316 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2317 if (CI->isMustTailCall())
2321 setOrigin(&
I, getOrigin(&
I, 0));
2327 "_msprop_ptrtoint"));
2328 setOrigin(&
I, getOrigin(&
I, 0));
2334 "_msprop_inttoptr"));
2335 setOrigin(&
I, getOrigin(&
I, 0));
2338 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2339 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2340 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2341 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2342 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2343 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2358 Value *S2 = getShadow(&
I, 1);
2359 Value *V1 =
I.getOperand(0);
2368 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2369 setOriginForNaryOp(
I);
2380 Value *S2 = getShadow(&
I, 1);
2390 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2391 setOriginForNaryOp(
I);
2409 template <
bool CombineShadow>
class Combiner {
2410 Value *Shadow =
nullptr;
2411 Value *Origin =
nullptr;
2413 MemorySanitizerVisitor *MSV;
2417 : IRB(IRB), MSV(MSV) {}
2421 if (CombineShadow) {
2426 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2427 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2431 if (MSV->MS.TrackOrigins) {
2436 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2438 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2439 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2449 Value *OpShadow = MSV->getShadow(V);
2450 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2451 return Add(OpShadow, OpOrigin);
2457 if (CombineShadow) {
2459 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2460 MSV->setShadow(
I, Shadow);
2462 if (MSV->MS.TrackOrigins) {
2464 MSV->setOrigin(
I, Origin);
2474 if (!MS.TrackOrigins)
2477 OriginCombiner
OC(
this, IRB);
2478 for (
Use &
Op :
I.operands())
2483 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2485 "Vector of pointers is not a valid shadow type");
2486 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2495 Type *srcTy =
V->getType();
2496 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2497 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2498 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2504 cast<VectorType>(dstTy)->getElementCount() ==
2505 cast<VectorType>(srcTy)->getElementCount())
2516 Type *ShadowTy = getShadowTy(V);
2517 if (
V->getType() == ShadowTy)
2519 if (
V->getType()->isPtrOrPtrVectorTy())
2528 ShadowAndOriginCombiner
SC(
this, IRB);
2529 for (
Use &
Op :
I.operands())
2549 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2550 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2551 Type *EltTy = VTy->getElementType();
2553 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2556 const APInt &
V = Elt->getValue();
2558 Elements.push_back(ConstantInt::get(EltTy, V2));
2560 Elements.push_back(ConstantInt::get(EltTy, 1));
2565 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2566 const APInt &
V = Elt->getValue();
2568 ShadowMul = ConstantInt::get(Ty, V2);
2570 ShadowMul = ConstantInt::get(Ty, 1);
2576 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2577 setOrigin(&
I, getOrigin(OtherArg));
2581 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2582 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2583 if (constOp0 && !constOp1)
2584 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2585 else if (constOp1 && !constOp0)
2586 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2601 insertShadowCheck(
I.getOperand(1), &
I);
2602 setShadow(&
I, getShadow(&
I, 0));
2603 setOrigin(&
I, getOrigin(&
I, 0));
2620 void handleEqualityComparison(
ICmpInst &
I) {
2624 Value *Sa = getShadow(
A);
2625 Value *Sb = getShadow(
B);
2651 setOriginForNaryOp(
I);
2693 void handleRelationalComparisonExact(
ICmpInst &
I) {
2697 Value *Sa = getShadow(
A);
2698 Value *Sb = getShadow(
B);
2709 bool IsSigned =
I.isSigned();
2711 getLowestPossibleValue(IRB,
A, Sa, IsSigned),
2712 getHighestPossibleValue(IRB,
B, Sb, IsSigned));
2714 getHighestPossibleValue(IRB,
A, Sa, IsSigned),
2715 getLowestPossibleValue(IRB,
B, Sb, IsSigned));
2718 setOriginForNaryOp(
I);
2725 void handleSignedRelationalComparison(
ICmpInst &
I) {
2729 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2730 op =
I.getOperand(0);
2731 pre =
I.getPredicate();
2732 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2733 op =
I.getOperand(1);
2734 pre =
I.getSwappedPredicate();
2747 setShadow(&
I, Shadow);
2748 setOrigin(&
I, getOrigin(
op));
2759 if (
I.isEquality()) {
2760 handleEqualityComparison(
I);
2766 handleRelationalComparisonExact(
I);
2770 handleSignedRelationalComparison(
I);
2775 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2776 handleRelationalComparisonExact(
I);
2783 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2790 Value *S2 = getShadow(&
I, 1);
2795 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2796 setOriginForNaryOp(
I);
2807 Value *S0 = getShadow(&
I, 0);
2809 Value *S2 = getShadow(&
I, 2);
2814 I.getModule(),
I.getIntrinsicID(), S2Conv->
getType());
2816 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2817 setOriginForNaryOp(
I);
2831 getShadow(
I.getArgOperand(1));
2834 {I.getArgOperand(0), I.getArgOperand(1),
2835 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2836 I.eraseFromParent();
2854 getShadow(
I.getArgOperand(1));
2857 {I.getArgOperand(0), I.getArgOperand(1),
2858 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2859 I.eraseFromParent();
2867 {I.getArgOperand(0),
2868 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2869 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2870 I.eraseFromParent();
2873 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2875 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2884 Value *Shadow = getShadow(&
I, 1);
2885 Value *ShadowPtr, *OriginPtr;
2889 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2894 insertShadowCheck(
Addr, &
I);
2897 if (MS.TrackOrigins)
2910 Type *ShadowTy = getShadowTy(&
I);
2911 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2912 if (PropagateShadow) {
2916 std::tie(ShadowPtr, OriginPtr) =
2917 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2921 setShadow(&
I, getCleanShadow(&
I));
2925 insertShadowCheck(
Addr, &
I);
2927 if (MS.TrackOrigins) {
2928 if (PropagateShadow)
2929 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2931 setOrigin(&
I, getCleanOrigin());
2944 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy() ||
2945 RetTy->isX86_MMXTy()))
2948 unsigned NumArgOperands =
I.arg_size();
2949 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2950 Type *Ty =
I.getArgOperand(i)->getType();
2956 ShadowAndOriginCombiner
SC(
this, IRB);
2957 for (
unsigned i = 0; i < NumArgOperands; ++i)
2958 SC.Add(
I.getArgOperand(i));
2975 unsigned NumArgOperands =
I.arg_size();
2976 if (NumArgOperands == 0)
2979 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
2980 I.getArgOperand(1)->getType()->isVectorTy() &&
2981 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
2983 return handleVectorStoreIntrinsic(
I);
2986 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
2987 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
2989 return handleVectorLoadIntrinsic(
I);
2992 if (
I.doesNotAccessMemory())
2993 if (maybeHandleSimpleNomemIntrinsic(
I))
3001 setShadow(&
I, getShadow(&
I, 0));
3002 setOrigin(&
I, getOrigin(&
I, 0));
3010 InstrumentLifetimeStart =
false;
3011 LifetimeStartList.push_back(std::make_pair(&
I, AI));
3017 Type *OpType =
Op->getType();
3019 F.getParent(), Intrinsic::bswap,
ArrayRef(&OpType, 1));
3021 setOrigin(&
I, getOrigin(
Op));
3026 Value *Src =
I.getArgOperand(0);
3032 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
3035 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
3038 Value *OutputShadow =
3039 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
3041 setShadow(&
I, OutputShadow);
3042 setOriginForNaryOp(
I);
3060 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
3061 bool HasRoundingMode =
false) {
3063 Value *CopyOp, *ConvertOp;
3065 assert((!HasRoundingMode ||
3066 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
3067 "Invalid rounding mode");
3069 switch (
I.arg_size() - HasRoundingMode) {
3071 CopyOp =
I.getArgOperand(0);
3072 ConvertOp =
I.getArgOperand(1);
3075 ConvertOp =
I.getArgOperand(0);
3089 Value *ConvertShadow = getShadow(ConvertOp);
3090 Value *AggShadow =
nullptr;
3093 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), 0));
3094 for (
int i = 1; i < NumUsedElements; ++i) {
3096 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), i));
3097 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3100 AggShadow = ConvertShadow;
3103 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3110 Value *ResultShadow = getShadow(CopyOp);
3111 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3112 for (
int i = 0; i < NumUsedElements; ++i) {
3114 ResultShadow, ConstantInt::getNullValue(EltTy),
3117 setShadow(&
I, ResultShadow);
3118 setOrigin(&
I, getOrigin(CopyOp));
3120 setShadow(&
I, getCleanShadow(&
I));
3121 setOrigin(&
I, getCleanOrigin());
3129 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3132 return CreateShadowCast(IRB, S2,
T,
true);
3140 return CreateShadowCast(IRB, S2,
T,
true);
3157 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3163 Value *S2 = getShadow(&
I, 1);
3164 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3165 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3166 Value *V1 =
I.getOperand(0);
3169 {IRB.CreateBitCast(S1, V1->getType()), V2});
3171 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3172 setOriginForNaryOp(
I);
3176 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3177 const unsigned X86_MMXSizeInBits = 64;
3178 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3179 "Illegal MMX vector element size");
3181 X86_MMXSizeInBits / EltSizeInBits);
3188 case Intrinsic::x86_sse2_packsswb_128:
3189 case Intrinsic::x86_sse2_packuswb_128:
3190 return Intrinsic::x86_sse2_packsswb_128;
3192 case Intrinsic::x86_sse2_packssdw_128:
3193 case Intrinsic::x86_sse41_packusdw:
3194 return Intrinsic::x86_sse2_packssdw_128;
3196 case Intrinsic::x86_avx2_packsswb:
3197 case Intrinsic::x86_avx2_packuswb:
3198 return Intrinsic::x86_avx2_packsswb;
3200 case Intrinsic::x86_avx2_packssdw:
3201 case Intrinsic::x86_avx2_packusdw:
3202 return Intrinsic::x86_avx2_packssdw;
3204 case Intrinsic::x86_mmx_packsswb:
3205 case Intrinsic::x86_mmx_packuswb:
3206 return Intrinsic::x86_mmx_packsswb;
3208 case Intrinsic::x86_mmx_packssdw:
3209 return Intrinsic::x86_mmx_packssdw;
3222 void handleVectorPackIntrinsic(
IntrinsicInst &
I,
unsigned EltSizeInBits = 0) {
3224 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3227 Value *S2 = getShadow(&
I, 1);
3228 assert(isX86_MMX ||
S1->getType()->isVectorTy());
3233 Type *
T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) :
S1->
getType();
3249 F.getParent(), getSignedPackIntrinsic(
I.getIntrinsicID()));
3252 IRB.
CreateCall(ShadowFn, {S1_ext, S2_ext},
"_msprop_vector_pack");
3256 setOriginForNaryOp(
I);
3261 const unsigned SignificantBitsPerResultElement = 16;
3262 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3264 unsigned ZeroBitsPerResultElement =
3268 auto *Shadow0 = getShadow(&
I, 0);
3269 auto *Shadow1 = getShadow(&
I, 1);
3274 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3277 setOriginForNaryOp(
I);
3282 unsigned EltSizeInBits = 0) {
3283 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3284 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) :
I.
getType();
3286 auto *Shadow0 = getShadow(&
I, 0);
3287 auto *Shadow1 = getShadow(&
I, 1);
3294 setOriginForNaryOp(
I);
3302 Type *ResTy = getShadowTy(&
I);
3303 auto *Shadow0 = getShadow(&
I, 0);
3304 auto *Shadow1 = getShadow(&
I, 1);
3309 setOriginForNaryOp(
I);
3317 auto *Shadow0 = getShadow(&
I, 0);
3318 auto *Shadow1 = getShadow(&
I, 1);
3320 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3322 setOriginForNaryOp(
I);
3331 setOrigin(&
I, getOrigin(&
I, 0));
3339 Value *OperandShadow = getShadow(&
I, 0);
3341 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3349 setOrigin(&
I, getOrigin(&
I, 0));
3357 Value *OperandShadow = getShadow(&
I, 0);
3358 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3366 setOrigin(&
I, getOrigin(&
I, 0));
3374 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3379 insertShadowCheck(
Addr, &
I);
3390 Value *ShadowPtr, *OriginPtr;
3391 std::tie(ShadowPtr, OriginPtr) =
3392 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3395 insertShadowCheck(
Addr, &
I);
3398 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3400 insertShadowCheck(Shadow, Origin, &
I);
3407 Value *PassThru =
I.getArgOperand(2);
3410 insertShadowCheck(
Ptr, &
I);
3411 insertShadowCheck(Mask, &
I);
3414 if (!PropagateShadow) {
3415 setShadow(&
I, getCleanShadow(&
I));
3416 setOrigin(&
I, getCleanOrigin());
3420 Type *ShadowTy = getShadowTy(&
I);
3421 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3422 auto [ShadowPtr, OriginPtr] =
3423 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3426 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3428 setShadow(&
I, Shadow);
3431 setOrigin(&
I, getCleanOrigin());
3436 Value *Values =
I.getArgOperand(0);
3441 insertShadowCheck(
Ptr, &
I);
3442 insertShadowCheck(Mask, &
I);
3445 Value *Shadow = getShadow(Values);
3446 Type *ElementShadowTy =
3447 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3448 auto [ShadowPtr, OriginPtrs] =
3449 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3458 Value *Ptrs =
I.getArgOperand(0);
3459 const Align Alignment(
3460 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3462 Value *PassThru =
I.getArgOperand(3);
3464 Type *PtrsShadowTy = getShadowTy(Ptrs);
3466 insertShadowCheck(Mask, &
I);
3470 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3473 if (!PropagateShadow) {
3474 setShadow(&
I, getCleanShadow(&
I));
3475 setOrigin(&
I, getCleanOrigin());
3479 Type *ShadowTy = getShadowTy(&
I);
3480 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3481 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3482 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3486 getShadow(PassThru),
"_msmaskedgather");
3488 setShadow(&
I, Shadow);
3491 setOrigin(&
I, getCleanOrigin());
3496 Value *Values =
I.getArgOperand(0);
3497 Value *Ptrs =
I.getArgOperand(1);
3498 const Align Alignment(
3499 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3502 Type *PtrsShadowTy = getShadowTy(Ptrs);
3504 insertShadowCheck(Mask, &
I);
3508 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3511 Value *Shadow = getShadow(Values);
3512 Type *ElementShadowTy =
3513 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3514 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3515 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3524 Value *
V =
I.getArgOperand(0);
3526 const Align Alignment(
3527 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3529 Value *Shadow = getShadow(V);
3532 insertShadowCheck(
Ptr, &
I);
3533 insertShadowCheck(Mask, &
I);
3538 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3539 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3543 if (!MS.TrackOrigins)
3546 auto &
DL =
F.getParent()->getDataLayout();
3547 paintOrigin(IRB, getOrigin(V), OriginPtr,
3555 const Align Alignment(
3556 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3558 Value *PassThru =
I.getArgOperand(3);
3561 insertShadowCheck(
Ptr, &
I);
3562 insertShadowCheck(Mask, &
I);
3565 if (!PropagateShadow) {
3566 setShadow(&
I, getCleanShadow(&
I));
3567 setOrigin(&
I, getCleanOrigin());
3571 Type *ShadowTy = getShadowTy(&
I);
3572 Value *ShadowPtr, *OriginPtr;
3573 std::tie(ShadowPtr, OriginPtr) =
3574 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3576 getShadow(PassThru),
"_msmaskedld"));
3578 if (!MS.TrackOrigins)
3585 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3590 setOrigin(&
I, Origin);
3600 Type *ShadowTy = getShadowTy(&
I);
3603 Value *SMask = getShadow(&
I, 1);
3608 {getShadow(&I, 0), I.getOperand(1)});
3611 setOriginForNaryOp(
I);
3616 for (
unsigned X = OddElements ? 1 : 0;
X < Width;
X += 2) {
3633 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3634 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3635 "pclmul 3rd operand must be a constant");
3636 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3638 getPclmulMask(Width, Imm & 0x01));
3640 getPclmulMask(Width, Imm & 0x10));
3641 ShadowAndOriginCombiner SOC(
this, IRB);
3642 SOC.Add(Shuf0, getOrigin(&
I, 0));
3643 SOC.Add(Shuf1, getOrigin(&
I, 1));
3651 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3653 Value *Second = getShadow(&
I, 1);
3656 Mask.push_back(Width);
3657 for (
unsigned i = 1; i < Width; i++)
3661 setShadow(&
I, Shadow);
3662 setOriginForNaryOp(
I);
3667 Value *Shadow0 = getShadow(&
I, 0);
3668 Value *Shadow1 = getShadow(&
I, 1);
3674 setShadow(&
I, Shadow);
3675 setOriginForNaryOp(
I);
3681 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3683 Value *Second = getShadow(&
I, 1);
3687 Mask.push_back(Width);
3688 for (
unsigned i = 1; i < Width; i++)
3692 setShadow(&
I, Shadow);
3693 setOriginForNaryOp(
I);
3700 assert(
I.getType()->isIntOrIntVectorTy());
3701 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3705 setShadow(&
I, getShadow(&
I, 0));
3706 setOrigin(&
I, getOrigin(&
I, 0));
3711 Value *Shadow = getShadow(&
I, 0);
3712 setShadow(&
I, IRB.
CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3713 setOrigin(&
I, getOrigin(&
I, 0));
3718 Value *Shadow0 = getShadow(&
I, 0);
3719 Value *Shadow1 = getShadow(&
I, 1);
3722 IRB.
CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
3728 setShadow(&
I, Shadow);
3729 setOriginForNaryOp(
I);
3733 switch (
I.getIntrinsicID()) {
3734 case Intrinsic::uadd_with_overflow:
3735 case Intrinsic::sadd_with_overflow:
3736 case Intrinsic::usub_with_overflow:
3737 case Intrinsic::ssub_with_overflow:
3738 case Intrinsic::umul_with_overflow:
3739 case Intrinsic::smul_with_overflow:
3740 handleArithmeticWithOverflow(
I);
3742 case Intrinsic::abs:
3743 handleAbsIntrinsic(
I);
3745 case Intrinsic::is_fpclass:
3748 case Intrinsic::lifetime_start:
3749 handleLifetimeStart(
I);
3751 case Intrinsic::launder_invariant_group:
3752 case Intrinsic::strip_invariant_group:
3753 handleInvariantGroup(
I);
3755 case Intrinsic::bswap:
3758 case Intrinsic::ctlz:
3759 case Intrinsic::cttz:
3760 handleCountZeroes(
I);
3762 case Intrinsic::masked_compressstore:
3763 handleMaskedCompressStore(
I);
3765 case Intrinsic::masked_expandload:
3766 handleMaskedExpandLoad(
I);
3768 case Intrinsic::masked_gather:
3769 handleMaskedGather(
I);
3771 case Intrinsic::masked_scatter:
3772 handleMaskedScatter(
I);
3774 case Intrinsic::masked_store:
3775 handleMaskedStore(
I);
3777 case Intrinsic::masked_load:
3778 handleMaskedLoad(
I);
3780 case Intrinsic::vector_reduce_and:
3781 handleVectorReduceAndIntrinsic(
I);
3783 case Intrinsic::vector_reduce_or:
3784 handleVectorReduceOrIntrinsic(
I);
3786 case Intrinsic::vector_reduce_add:
3787 case Intrinsic::vector_reduce_xor:
3788 case Intrinsic::vector_reduce_mul:
3789 handleVectorReduceIntrinsic(
I);
3791 case Intrinsic::x86_sse_stmxcsr:
3794 case Intrinsic::x86_sse_ldmxcsr:
3797 case Intrinsic::x86_avx512_vcvtsd2usi64:
3798 case Intrinsic::x86_avx512_vcvtsd2usi32:
3799 case Intrinsic::x86_avx512_vcvtss2usi64:
3800 case Intrinsic::x86_avx512_vcvtss2usi32:
3801 case Intrinsic::x86_avx512_cvttss2usi64:
3802 case Intrinsic::x86_avx512_cvttss2usi:
3803 case Intrinsic::x86_avx512_cvttsd2usi64:
3804 case Intrinsic::x86_avx512_cvttsd2usi:
3805 case Intrinsic::x86_avx512_cvtusi2ss:
3806 case Intrinsic::x86_avx512_cvtusi642sd:
3807 case Intrinsic::x86_avx512_cvtusi642ss:
3808 handleVectorConvertIntrinsic(
I, 1,
true);
3810 case Intrinsic::x86_sse2_cvtsd2si64:
3811 case Intrinsic::x86_sse2_cvtsd2si:
3812 case Intrinsic::x86_sse2_cvtsd2ss:
3813 case Intrinsic::x86_sse2_cvttsd2si64:
3814 case Intrinsic::x86_sse2_cvttsd2si:
3815 case Intrinsic::x86_sse_cvtss2si64:
3816 case Intrinsic::x86_sse_cvtss2si:
3817 case Intrinsic::x86_sse_cvttss2si64:
3818 case Intrinsic::x86_sse_cvttss2si:
3819 handleVectorConvertIntrinsic(
I, 1);
3821 case Intrinsic::x86_sse_cvtps2pi:
3822 case Intrinsic::x86_sse_cvttps2pi:
3823 handleVectorConvertIntrinsic(
I, 2);
3826 case Intrinsic::x86_avx512_psll_w_512:
3827 case Intrinsic::x86_avx512_psll_d_512:
3828 case Intrinsic::x86_avx512_psll_q_512:
3829 case Intrinsic::x86_avx512_pslli_w_512:
3830 case Intrinsic::x86_avx512_pslli_d_512:
3831 case Intrinsic::x86_avx512_pslli_q_512:
3832 case Intrinsic::x86_avx512_psrl_w_512:
3833 case Intrinsic::x86_avx512_psrl_d_512:
3834 case Intrinsic::x86_avx512_psrl_q_512:
3835 case Intrinsic::x86_avx512_psra_w_512:
3836 case Intrinsic::x86_avx512_psra_d_512:
3837 case Intrinsic::x86_avx512_psra_q_512:
3838 case Intrinsic::x86_avx512_psrli_w_512:
3839 case Intrinsic::x86_avx512_psrli_d_512:
3840 case Intrinsic::x86_avx512_psrli_q_512:
3841 case Intrinsic::x86_avx512_psrai_w_512:
3842 case Intrinsic::x86_avx512_psrai_d_512:
3843 case Intrinsic::x86_avx512_psrai_q_512:
3844 case Intrinsic::x86_avx512_psra_q_256:
3845 case Intrinsic::x86_avx512_psra_q_128:
3846 case Intrinsic::x86_avx512_psrai_q_256:
3847 case Intrinsic::x86_avx512_psrai_q_128:
3848 case Intrinsic::x86_avx2_psll_w:
3849 case Intrinsic::x86_avx2_psll_d:
3850 case Intrinsic::x86_avx2_psll_q:
3851 case Intrinsic::x86_avx2_pslli_w:
3852 case Intrinsic::x86_avx2_pslli_d:
3853 case Intrinsic::x86_avx2_pslli_q:
3854 case Intrinsic::x86_avx2_psrl_w:
3855 case Intrinsic::x86_avx2_psrl_d:
3856 case Intrinsic::x86_avx2_psrl_q:
3857 case Intrinsic::x86_avx2_psra_w:
3858 case Intrinsic::x86_avx2_psra_d:
3859 case Intrinsic::x86_avx2_psrli_w:
3860 case Intrinsic::x86_avx2_psrli_d:
3861 case Intrinsic::x86_avx2_psrli_q:
3862 case Intrinsic::x86_avx2_psrai_w:
3863 case Intrinsic::x86_avx2_psrai_d:
3864 case Intrinsic::x86_sse2_psll_w:
3865 case Intrinsic::x86_sse2_psll_d:
3866 case Intrinsic::x86_sse2_psll_q:
3867 case Intrinsic::x86_sse2_pslli_w:
3868 case Intrinsic::x86_sse2_pslli_d:
3869 case Intrinsic::x86_sse2_pslli_q:
3870 case Intrinsic::x86_sse2_psrl_w:
3871 case Intrinsic::x86_sse2_psrl_d:
3872 case Intrinsic::x86_sse2_psrl_q:
3873 case Intrinsic::x86_sse2_psra_w:
3874 case Intrinsic::x86_sse2_psra_d:
3875 case Intrinsic::x86_sse2_psrli_w:
3876 case Intrinsic::x86_sse2_psrli_d:
3877 case Intrinsic::x86_sse2_psrli_q:
3878 case Intrinsic::x86_sse2_psrai_w:
3879 case Intrinsic::x86_sse2_psrai_d:
3880 case Intrinsic::x86_mmx_psll_w:
3881 case Intrinsic::x86_mmx_psll_d:
3882 case Intrinsic::x86_mmx_psll_q:
3883 case Intrinsic::x86_mmx_pslli_w:
3884 case Intrinsic::x86_mmx_pslli_d:
3885 case Intrinsic::x86_mmx_pslli_q:
3886 case Intrinsic::x86_mmx_psrl_w:
3887 case Intrinsic::x86_mmx_psrl_d:
3888 case Intrinsic::x86_mmx_psrl_q:
3889 case Intrinsic::x86_mmx_psra_w:
3890 case Intrinsic::x86_mmx_psra_d:
3891 case Intrinsic::x86_mmx_psrli_w:
3892 case Intrinsic::x86_mmx_psrli_d:
3893 case Intrinsic::x86_mmx_psrli_q:
3894 case Intrinsic::x86_mmx_psrai_w:
3895 case Intrinsic::x86_mmx_psrai_d:
3896 handleVectorShiftIntrinsic(
I,
false);
3898 case Intrinsic::x86_avx2_psllv_d:
3899 case Intrinsic::x86_avx2_psllv_d_256:
3900 case Intrinsic::x86_avx512_psllv_d_512:
3901 case Intrinsic::x86_avx2_psllv_q:
3902 case Intrinsic::x86_avx2_psllv_q_256:
3903 case Intrinsic::x86_avx512_psllv_q_512:
3904 case Intrinsic::x86_avx2_psrlv_d:
3905 case Intrinsic::x86_avx2_psrlv_d_256:
3906 case Intrinsic::x86_avx512_psrlv_d_512:
3907 case Intrinsic::x86_avx2_psrlv_q:
3908 case Intrinsic::x86_avx2_psrlv_q_256:
3909 case Intrinsic::x86_avx512_psrlv_q_512:
3910 case Intrinsic::x86_avx2_psrav_d:
3911 case Intrinsic::x86_avx2_psrav_d_256:
3912 case Intrinsic::x86_avx512_psrav_d_512:
3913 case Intrinsic::x86_avx512_psrav_q_128:
3914 case Intrinsic::x86_avx512_psrav_q_256:
3915 case Intrinsic::x86_avx512_psrav_q_512:
3916 handleVectorShiftIntrinsic(
I,
true);
3919 case Intrinsic::x86_sse2_packsswb_128:
3920 case Intrinsic::x86_sse2_packssdw_128:
3921 case Intrinsic::x86_sse2_packuswb_128:
3922 case Intrinsic::x86_sse41_packusdw:
3923 case Intrinsic::x86_avx2_packsswb:
3924 case Intrinsic::x86_avx2_packssdw:
3925 case Intrinsic::x86_avx2_packuswb:
3926 case Intrinsic::x86_avx2_packusdw:
3927 handleVectorPackIntrinsic(
I);
3930 case Intrinsic::x86_mmx_packsswb:
3931 case Intrinsic::x86_mmx_packuswb:
3932 handleVectorPackIntrinsic(
I, 16);
3935 case Intrinsic::x86_mmx_packssdw:
3936 handleVectorPackIntrinsic(
I, 32);
3939 case Intrinsic::x86_mmx_psad_bw:
3940 case Intrinsic::x86_sse2_psad_bw:
3941 case Intrinsic::x86_avx2_psad_bw:
3942 handleVectorSadIntrinsic(
I);
3945 case Intrinsic::x86_sse2_pmadd_wd:
3946 case Intrinsic::x86_avx2_pmadd_wd:
3947 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3948 case Intrinsic::x86_avx2_pmadd_ub_sw:
3949 handleVectorPmaddIntrinsic(
I);
3952 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3953 handleVectorPmaddIntrinsic(
I, 8);
3956 case Intrinsic::x86_mmx_pmadd_wd:
3957 handleVectorPmaddIntrinsic(
I, 16);
3960 case Intrinsic::x86_sse_cmp_ss:
3961 case Intrinsic::x86_sse2_cmp_sd:
3962 case Intrinsic::x86_sse_comieq_ss:
3963 case Intrinsic::x86_sse_comilt_ss:
3964 case Intrinsic::x86_sse_comile_ss:
3965 case Intrinsic::x86_sse_comigt_ss:
3966 case Intrinsic::x86_sse_comige_ss:
3967 case Intrinsic::x86_sse_comineq_ss:
3968 case Intrinsic::x86_sse_ucomieq_ss:
3969 case Intrinsic::x86_sse_ucomilt_ss:
3970 case Intrinsic::x86_sse_ucomile_ss:
3971 case Intrinsic::x86_sse_ucomigt_ss:
3972 case Intrinsic::x86_sse_ucomige_ss:
3973 case Intrinsic::x86_sse_ucomineq_ss:
3974 case Intrinsic::x86_sse2_comieq_sd:
3975 case Intrinsic::x86_sse2_comilt_sd:
3976 case Intrinsic::x86_sse2_comile_sd:
3977 case Intrinsic::x86_sse2_comigt_sd:
3978 case Intrinsic::x86_sse2_comige_sd:
3979 case Intrinsic::x86_sse2_comineq_sd:
3980 case Intrinsic::x86_sse2_ucomieq_sd:
3981 case Intrinsic::x86_sse2_ucomilt_sd:
3982 case Intrinsic::x86_sse2_ucomile_sd:
3983 case Intrinsic::x86_sse2_ucomigt_sd:
3984 case Intrinsic::x86_sse2_ucomige_sd:
3985 case Intrinsic::x86_sse2_ucomineq_sd:
3986 handleVectorCompareScalarIntrinsic(
I);
3989 case Intrinsic::x86_avx_cmp_pd_256:
3990 case Intrinsic::x86_avx_cmp_ps_256:
3991 case Intrinsic::x86_sse2_cmp_pd:
3992 case Intrinsic::x86_sse_cmp_ps:
3993 handleVectorComparePackedIntrinsic(
I);
3996 case Intrinsic::x86_bmi_bextr_32:
3997 case Intrinsic::x86_bmi_bextr_64:
3998 case Intrinsic::x86_bmi_bzhi_32:
3999 case Intrinsic::x86_bmi_bzhi_64:
4000 case Intrinsic::x86_bmi_pdep_32:
4001 case Intrinsic::x86_bmi_pdep_64:
4002 case Intrinsic::x86_bmi_pext_32:
4003 case Intrinsic::x86_bmi_pext_64:
4004 handleBmiIntrinsic(
I);
4007 case Intrinsic::x86_pclmulqdq:
4008 case Intrinsic::x86_pclmulqdq_256:
4009 case Intrinsic::x86_pclmulqdq_512:
4010 handlePclmulIntrinsic(
I);
4013 case Intrinsic::x86_sse41_round_sd:
4014 case Intrinsic::x86_sse41_round_ss:
4015 handleUnarySdSsIntrinsic(
I);
4017 case Intrinsic::x86_sse2_max_sd:
4018 case Intrinsic::x86_sse_max_ss:
4019 case Intrinsic::x86_sse2_min_sd:
4020 case Intrinsic::x86_sse_min_ss:
4021 handleBinarySdSsIntrinsic(
I);
4024 case Intrinsic::x86_avx_vtestc_pd:
4025 case Intrinsic::x86_avx_vtestc_pd_256:
4026 case Intrinsic::x86_avx_vtestc_ps:
4027 case Intrinsic::x86_avx_vtestc_ps_256:
4028 case Intrinsic::x86_avx_vtestnzc_pd:
4029 case Intrinsic::x86_avx_vtestnzc_pd_256:
4030 case Intrinsic::x86_avx_vtestnzc_ps:
4031 case Intrinsic::x86_avx_vtestnzc_ps_256:
4032 case Intrinsic::x86_avx_vtestz_pd:
4033 case Intrinsic::x86_avx_vtestz_pd_256:
4034 case Intrinsic::x86_avx_vtestz_ps:
4035 case Intrinsic::x86_avx_vtestz_ps_256:
4036 case Intrinsic::x86_avx_ptestc_256:
4037 case Intrinsic::x86_avx_ptestnzc_256:
4038 case Intrinsic::x86_avx_ptestz_256:
4039 case Intrinsic::x86_sse41_ptestc:
4040 case Intrinsic::x86_sse41_ptestnzc:
4041 case Intrinsic::x86_sse41_ptestz:
4042 handleVtestIntrinsic(
I);
4045 case Intrinsic::fshl:
4046 case Intrinsic::fshr:
4047 handleFunnelShift(
I);
4050 case Intrinsic::is_constant:
4052 setShadow(&
I, getCleanShadow(&
I));
4053 setOrigin(&
I, getCleanOrigin());
4057 if (!handleUnknownIntrinsic(
I))
4058 visitInstruction(
I);
4063 void visitLibAtomicLoad(
CallBase &CB) {
4065 assert(isa<CallInst>(CB));
4074 Value *NewOrdering =
4078 NextNodeIRBuilder NextIRB(&CB);
4079 Value *SrcShadowPtr, *SrcOriginPtr;
4080 std::tie(SrcShadowPtr, SrcOriginPtr) =
4081 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4083 Value *DstShadowPtr =
4084 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4088 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
4089 if (MS.TrackOrigins) {
4090 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
4092 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
4093 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
4097 void visitLibAtomicStore(
CallBase &CB) {
4104 Value *NewOrdering =
4108 Value *DstShadowPtr =
4126 visitAsmInstruction(CB);
4128 visitInstruction(CB);
4137 case LibFunc_atomic_load:
4138 if (!isa<CallInst>(CB)) {
4139 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4143 visitLibAtomicLoad(CB);
4145 case LibFunc_atomic_store:
4146 visitLibAtomicStore(CB);
4153 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4154 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4162 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4164 Call->removeFnAttrs(
B);
4166 Func->removeFnAttrs(
B);
4172 bool MayCheckCall = MS.EagerChecks;
4176 MayCheckCall &= !
Func->getName().starts_with(
"__sanitizer_unaligned_");
4179 unsigned ArgOffset = 0;
4182 if (!
A->getType()->isSized()) {
4183 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4191 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4194 insertShadowCheck(
A, &CB);
4195 Size =
DL.getTypeAllocSize(
A->getType());
4201 Value *ArgShadow = getShadow(
A);
4202 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4204 <<
" Shadow: " << *ArgShadow <<
"\n");
4208 assert(
A->getType()->isPointerTy() &&
4209 "ByVal argument is not a pointer!");
4217 Value *AShadowPtr, *AOriginPtr;
4218 std::tie(AShadowPtr, AOriginPtr) =
4219 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4221 if (!PropagateShadow) {
4228 if (MS.TrackOrigins) {
4229 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4243 Size =
DL.getTypeAllocSize(
A->getType());
4248 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4249 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4251 getOriginPtrForArgument(IRB, ArgOffset));
4255 assert(Store !=
nullptr);
4264 if (FT->isVarArg()) {
4265 VAHelper->visitCallBase(CB, IRB);
4272 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4275 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4276 setShadow(&CB, getCleanShadow(&CB));
4277 setOrigin(&CB, getCleanOrigin());
4283 Value *
Base = getShadowPtrForRetval(IRBBefore);
4284 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4287 if (isa<CallInst>(CB)) {
4291 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4296 setShadow(&CB, getCleanShadow(&CB));
4297 setOrigin(&CB, getCleanOrigin());
4304 "Could not find insertion point for retval shadow load");
4307 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4308 getShadowTy(&CB), getShadowPtrForRetval(IRBAfter),
4310 setShadow(&CB, RetvalShadow);
4311 if (MS.TrackOrigins)
4312 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
4313 getOriginPtrForRetval()));
4317 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4318 RetVal =
I->getOperand(0);
4320 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4321 return I->isMustTailCall();
4328 Value *RetVal =
I.getReturnValue();
4334 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4335 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4336 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4339 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4341 Value *Shadow = getShadow(RetVal);
4342 bool StoreOrigin =
true;
4344 insertShadowCheck(RetVal, &
I);
4345 Shadow = getCleanShadow(RetVal);
4346 StoreOrigin =
false;
4353 if (MS.TrackOrigins && StoreOrigin)
4354 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4360 if (!PropagateShadow) {
4361 setShadow(&
I, getCleanShadow(&
I));
4362 setOrigin(&
I, getCleanOrigin());
4366 ShadowPHINodes.push_back(&
I);
4367 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4369 if (MS.TrackOrigins)
4371 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4388 IRB.
CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4390 Value *ShadowBase, *OriginBase;
4391 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4398 if (PoisonStack && MS.TrackOrigins) {
4399 Value *Idptr = getLocalVarIdptr(
I);
4401 Value *Descr = getLocalVarDescription(
I);
4402 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4403 {&I, Len, Idptr, Descr});
4405 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4411 Value *Descr = getLocalVarDescription(
I);
4413 IRB.
CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4415 IRB.
CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4422 NextNodeIRBuilder IRB(InsPoint);
4426 if (
I.isArrayAllocation())
4430 if (MS.CompileKernel)
4431 poisonAllocaKmsan(
I, IRB, Len);
4433 poisonAllocaUserspace(
I, IRB, Len);
4437 setShadow(&
I, getCleanShadow(&
I));
4438 setOrigin(&
I, getCleanOrigin());
4450 Value *Sb = getShadow(
B);
4451 Value *Sc = getShadow(
C);
4452 Value *Sd = getShadow(
D);
4457 if (
I.getType()->isAggregateType()) {
4461 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4469 C = CreateAppToShadowCast(IRB,
C);
4470 D = CreateAppToShadowCast(IRB,
D);
4477 if (MS.TrackOrigins) {
4480 if (
B->getType()->isVectorTy()) {
4481 B = convertToBool(
B, IRB);
4482 Sb = convertToBool(Sb, IRB);
4489 getOrigin(
I.getFalseValue()))));
4496 setShadow(&
I, getCleanShadow(&
I));
4497 setOrigin(&
I, getCleanOrigin());
4501 setShadow(&
I, getCleanShadow(&
I));
4502 setOrigin(&
I, getCleanOrigin());
4506 setShadow(&
I, getCleanShadow(&
I));
4507 setOrigin(&
I, getCleanOrigin());
4514 Value *Agg =
I.getAggregateOperand();
4516 Value *AggShadow = getShadow(Agg);
4520 setShadow(&
I, ResShadow);
4521 setOriginForNaryOp(
I);
4527 Value *AggShadow = getShadow(
I.getAggregateOperand());
4528 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4534 setOriginForNaryOp(
I);
4538 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4539 errs() <<
"ZZZ call " << CI->getCalledFunction()->getName() <<
"\n";
4541 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4543 errs() <<
"QQQ " <<
I <<
"\n";
4570 insertShadowCheck(Operand, &
I);
4577 auto Size =
DL.getTypeStoreSize(ElemTy);
4579 if (MS.CompileKernel) {
4580 IRB.
CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
4586 auto [ShadowPtr,
_] =
4587 getShadowOriginPtrUserspace(Operand, IRB, IRB.
getInt8Ty(),
Align(1));
4598 int NumRetOutputs = 0;
4600 Type *
RetTy = cast<Value>(CB)->getType();
4601 if (!
RetTy->isVoidTy()) {
4603 auto *
ST = dyn_cast<StructType>(
RetTy);
4605 NumRetOutputs =
ST->getNumElements();
4611 switch (
Info.Type) {
4619 return NumOutputs - NumRetOutputs;
4642 int OutputArgs = getNumOutputArgs(IA, CB);
4648 for (
int i = OutputArgs; i < NumOperands; i++) {
4656 for (
int i = 0; i < OutputArgs; i++) {
4662 setShadow(&
I, getCleanShadow(&
I));
4663 setOrigin(&
I, getCleanOrigin());
4668 setShadow(&
I, getCleanShadow(&
I));
4669 setOrigin(&
I, getCleanOrigin());
4677 for (
size_t i = 0, n =
I.getNumOperands(); i < n; i++) {
4678 Value *Operand =
I.getOperand(i);
4680 insertShadowCheck(Operand, &
I);
4682 setShadow(&
I, getCleanShadow(&
I));
4683 setOrigin(&
I, getCleanOrigin());
4687struct VarArgHelperBase :
public VarArgHelper {
4689 MemorySanitizer &MS;
4690 MemorySanitizerVisitor &MSV;
4692 const unsigned VAListTagSize;
4694 VarArgHelperBase(
Function &
F, MemorySanitizer &MS,
4695 MemorySanitizerVisitor &MSV,
unsigned VAListTagSize)
4696 :
F(
F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
4700 return IRB.
CreateAdd(
Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4705 unsigned ArgOffset) {
4714 unsigned ArgOffset,
unsigned ArgSize) {
4718 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4733 unsigned BaseOffset) {
4742 TailSize,
Align(8));
4747 Value *VAListTag =
I.getArgOperand(0);
4749 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
4750 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4753 VAListTagSize, Alignment,
false);
4760 unpoisonVAListTagForInst(
I);
4766 unpoisonVAListTagForInst(
I);
4771struct VarArgAMD64Helper :
public VarArgHelperBase {
4774 static const unsigned AMD64GpEndOffset = 48;
4775 static const unsigned AMD64FpEndOffsetSSE = 176;
4777 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4779 unsigned AMD64FpEndOffset;
4782 Value *VAArgOverflowSize =
nullptr;
4784 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4786 VarArgAMD64Helper(
Function &
F, MemorySanitizer &MS,
4787 MemorySanitizerVisitor &MSV)
4788 : VarArgHelperBase(
F, MS, MSV, 24) {
4789 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4790 for (
const auto &Attr :
F.getAttributes().getFnAttrs()) {
4791 if (Attr.isStringAttribute() &&
4792 (Attr.getKindAsString() ==
"target-features")) {
4793 if (Attr.getValueAsString().contains(
"-sse"))
4794 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4800 ArgKind classifyArgument(
Value *arg) {
4803 if (
T->isX86_FP80Ty())
4805 if (
T->isFPOrFPVectorTy() ||
T->isX86_MMXTy())
4806 return AK_FloatingPoint;
4807 if (
T->isIntegerTy() &&
T->getPrimitiveSizeInBits() <= 64)
4808 return AK_GeneralPurpose;
4809 if (
T->isPointerTy())
4810 return AK_GeneralPurpose;
4823 unsigned GpOffset = 0;
4824 unsigned FpOffset = AMD64GpEndOffset;
4825 unsigned OverflowOffset = AMD64FpEndOffset;
4830 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
4837 assert(
A->getType()->isPointerTy());
4839 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
4841 unsigned BaseOffset = OverflowOffset;
4843 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
4844 Value *OriginBase =
nullptr;
4845 if (MS.TrackOrigins)
4846 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4847 OverflowOffset += AlignedSize;
4850 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4854 Value *ShadowPtr, *OriginPtr;
4855 std::tie(ShadowPtr, OriginPtr) =
4860 if (MS.TrackOrigins)
4864 ArgKind AK = classifyArgument(
A);
4865 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4867 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4869 Value *ShadowBase, *OriginBase =
nullptr;
4871 case AK_GeneralPurpose:
4872 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, GpOffset);
4873 if (MS.TrackOrigins)
4874 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
4878 case AK_FloatingPoint:
4879 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, FpOffset);
4880 if (MS.TrackOrigins)
4881 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
4888 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
4890 unsigned BaseOffset = OverflowOffset;
4892 getShadowPtrForVAArgument(
A->getType(), IRB, OverflowOffset);
4893 if (MS.TrackOrigins) {
4894 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4896 OverflowOffset += AlignedSize;
4899 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4908 Value *Shadow = MSV.getShadow(
A);
4910 if (MS.TrackOrigins) {
4911 Value *Origin = MSV.getOrigin(
A);
4913 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4919 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4920 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4923 void finalizeInstrumentation()
override {
4924 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4925 "finalizeInstrumentation called twice");
4926 if (!VAStartInstrumentationList.
empty()) {
4933 ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
4940 Intrinsic::umin, CopySize,
4944 if (MS.TrackOrigins) {
4954 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
4955 CallInst *OrigInst = VAStartInstrumentationList[i];
4956 NextNodeIRBuilder IRB(OrigInst);
4959 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
4962 ConstantInt::get(MS.IntptrTy, 16)),
4963 PointerType::get(RegSaveAreaPtrTy, 0));
4964 Value *RegSaveAreaPtr =
4965 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4966 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4968 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4969 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
4971 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4973 if (MS.TrackOrigins)
4974 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
4975 Alignment, AMD64FpEndOffset);
4976 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
4979 ConstantInt::get(MS.IntptrTy, 8)),
4980 PointerType::get(OverflowArgAreaPtrTy, 0));
4981 Value *OverflowArgAreaPtr =
4982 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
4983 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4984 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
4985 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
4989 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4991 if (MS.TrackOrigins) {
4994 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5003struct VarArgMIPS64Helper :
public VarArgHelperBase {
5005 Value *VAArgSize =
nullptr;
5007 VarArgMIPS64Helper(
Function &
F, MemorySanitizer &MS,
5008 MemorySanitizerVisitor &MSV)
5009 : VarArgHelperBase(
F, MS, MSV, 8) {}
5012 unsigned VAArgOffset = 0;
5016 Triple TargetTriple(
F.getParent()->getTargetTriple());
5018 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5023 VAArgOffset += (8 - ArgSize);
5025 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VAArgOffset, ArgSize);
5026 VAArgOffset += ArgSize;
5027 VAArgOffset =
alignTo(VAArgOffset, 8);
5036 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5039 void finalizeInstrumentation()
override {
5040 assert(!VAArgSize && !VAArgTLSCopy &&
5041 "finalizeInstrumentation called twice");
5045 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5047 if (!VAStartInstrumentationList.
empty()) {
5056 Intrinsic::umin, CopySize,
5064 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5065 CallInst *OrigInst = VAStartInstrumentationList[i];
5066 NextNodeIRBuilder IRB(OrigInst);
5068 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5069 Value *RegSaveAreaPtrPtr =
5071 PointerType::get(RegSaveAreaPtrTy, 0));
5072 Value *RegSaveAreaPtr =
5073 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5074 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5076 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5077 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5079 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5086struct VarArgAArch64Helper :
public VarArgHelperBase {
5087 static const unsigned kAArch64GrArgSize = 64;
5088 static const unsigned kAArch64VrArgSize = 128;
5090 static const unsigned AArch64GrBegOffset = 0;
5091 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
5093 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
5094 static const unsigned AArch64VrEndOffset =
5095 AArch64VrBegOffset + kAArch64VrArgSize;
5096 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
5099 Value *VAArgOverflowSize =
nullptr;
5101 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5103 VarArgAArch64Helper(
Function &
F, MemorySanitizer &MS,
5104 MemorySanitizerVisitor &MSV)
5105 : VarArgHelperBase(
F, MS, MSV, 32) {}
5108 std::pair<ArgKind, uint64_t> classifyArgument(
Type *
T) {
5109 if (
T->isIntOrPtrTy() &&
T->getPrimitiveSizeInBits() <= 64)
5110 return {AK_GeneralPurpose, 1};
5111 if (
T->isFloatingPointTy() &&
T->getPrimitiveSizeInBits() <= 128)
5112 return {AK_FloatingPoint, 1};
5114 if (
T->isArrayTy()) {
5115 auto R = classifyArgument(
T->getArrayElementType());
5116 R.second *=
T->getScalarType()->getArrayNumElements();
5121 auto R = classifyArgument(FV->getScalarType());
5122 R.second *= FV->getNumElements();
5127 return {AK_Memory, 0};
5140 unsigned GrOffset = AArch64GrBegOffset;
5141 unsigned VrOffset = AArch64VrBegOffset;
5142 unsigned OverflowOffset = AArch64VAEndOffset;
5147 auto [AK, RegNum] = classifyArgument(
A->getType());
5148 if (AK == AK_GeneralPurpose &&
5149 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
5151 if (AK == AK_FloatingPoint &&
5152 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
5156 case AK_GeneralPurpose:
5157 Base = getShadowPtrForVAArgument(
A->getType(), IRB, GrOffset);
5158 GrOffset += 8 * RegNum;
5160 case AK_FloatingPoint:
5161 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VrOffset);
5162 VrOffset += 16 * RegNum;
5169 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5171 unsigned BaseOffset = OverflowOffset;
5172 Base = getShadowPtrForVAArgument(
A->getType(), IRB, BaseOffset);
5173 OverflowOffset += AlignedSize;
5176 CleanUnusedTLS(IRB,
Base, BaseOffset);
5188 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5189 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5196 ConstantInt::get(MS.IntptrTy, offset)),
5197 PointerType::get(*MS.C, 0));
5205 ConstantInt::get(MS.IntptrTy, offset)),
5206 PointerType::get(*MS.C, 0));
5208 return IRB.
CreateSExt(SaveArea32, MS.IntptrTy);
5211 void finalizeInstrumentation()
override {
5212 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5213 "finalizeInstrumentation called twice");
5214 if (!VAStartInstrumentationList.
empty()) {
5221 ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
5228 Intrinsic::umin, CopySize,
5234 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
5235 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
5239 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5240 CallInst *OrigInst = VAStartInstrumentationList[i];
5241 NextNodeIRBuilder IRB(OrigInst);
5260 Value *StackSaveAreaPtr =
5261 IRB.
CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5264 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5265 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5268 IRB.
CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5271 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5272 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5275 IRB.
CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5281 Value *GrRegSaveAreaShadowPtrOff =
5282 IRB.
CreateAdd(GrArgSize, GrOffSaveArea);
5284 Value *GrRegSaveAreaShadowPtr =
5285 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5291 Value *GrCopySize = IRB.
CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5297 Value *VrRegSaveAreaShadowPtrOff =
5298 IRB.
CreateAdd(VrArgSize, VrOffSaveArea);
5300 Value *VrRegSaveAreaShadowPtr =
5301 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5308 VrRegSaveAreaShadowPtrOff);
5309 Value *VrCopySize = IRB.
CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5315 Value *StackSaveAreaShadowPtr =
5316 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5321 VAArgTLSCopy, IRB.
getInt32(AArch64VAEndOffset));
5324 Align(16), VAArgOverflowSize);
5330struct VarArgPowerPC64Helper :
public VarArgHelperBase {
5332 Value *VAArgSize =
nullptr;
5334 VarArgPowerPC64Helper(
Function &
F, MemorySanitizer &MS,
5335 MemorySanitizerVisitor &MSV)
5336 : VarArgHelperBase(
F, MS, MSV, 8) {}
5346 Triple TargetTriple(
F.getParent()->getTargetTriple());
5354 unsigned VAArgOffset = VAArgBase;
5358 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5360 assert(
A->getType()->isPointerTy());
5362 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5365 ArgAlign =
Align(8);
5366 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5368 Value *
Base = getShadowPtrForVAArgument(
5369 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
5371 Value *AShadowPtr, *AOriginPtr;
5372 std::tie(AShadowPtr, AOriginPtr) =
5373 MSV.getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(),
5383 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5385 if (
A->getType()->isArrayTy()) {
5388 Type *ElementTy =
A->getType()->getArrayElementType();
5390 ArgAlign =
Align(
DL.getTypeAllocSize(ElementTy));
5391 }
else if (
A->getType()->isVectorTy()) {
5393 ArgAlign =
Align(ArgSize);
5396 ArgAlign =
Align(8);
5397 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5398 if (
DL.isBigEndian()) {
5402 VAArgOffset += (8 - ArgSize);
5405 Base = getShadowPtrForVAArgument(
A->getType(), IRB,
5406 VAArgOffset - VAArgBase, ArgSize);
5410 VAArgOffset += ArgSize;
5414 VAArgBase = VAArgOffset;
5418 ConstantInt::get(IRB.
getInt64Ty(), VAArgOffset - VAArgBase);
5421 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5424 void finalizeInstrumentation()
override {
5425 assert(!VAArgSize && !VAArgTLSCopy &&
5426 "finalizeInstrumentation called twice");
5430 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5432 if (!VAStartInstrumentationList.
empty()) {
5442 Intrinsic::umin, CopySize,
5450 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5451 CallInst *OrigInst = VAStartInstrumentationList[i];
5452 NextNodeIRBuilder IRB(OrigInst);
5454 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5455 Value *RegSaveAreaPtrPtr =
5457 PointerType::get(RegSaveAreaPtrTy, 0));
5458 Value *RegSaveAreaPtr =
5459 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5460 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5462 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5463 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5465 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5472struct VarArgSystemZHelper :
public VarArgHelperBase {
5473 static const unsigned SystemZGpOffset = 16;
5474 static const unsigned SystemZGpEndOffset = 56;
5475 static const unsigned SystemZFpOffset = 128;
5476 static const unsigned SystemZFpEndOffset = 160;
5477 static const unsigned SystemZMaxVrArgs = 8;
5478 static const unsigned SystemZRegSaveAreaSize = 160;
5479 static const unsigned SystemZOverflowOffset = 160;
5480 static const unsigned SystemZVAListTagSize = 32;
5481 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5482 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5484 bool IsSoftFloatABI;
5487 Value *VAArgOverflowSize =
nullptr;
5489 enum class ArgKind {
5497 enum class ShadowExtension {
None,
Zero, Sign };
5499 VarArgSystemZHelper(
Function &
F, MemorySanitizer &MS,
5500 MemorySanitizerVisitor &MSV)
5501 : VarArgHelperBase(
F, MS, MSV, SystemZVAListTagSize),
5502 IsSoftFloatABI(
F.getFnAttribute(
"use-soft-float").getValueAsBool()) {}
5504 ArgKind classifyArgument(
Type *
T) {
5511 if (
T->isIntegerTy(128) ||
T->isFP128Ty())
5512 return ArgKind::Indirect;
5513 if (
T->isFloatingPointTy())
5514 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5515 if (
T->isIntegerTy() ||
T->isPointerTy())
5516 return ArgKind::GeneralPurpose;
5517 if (
T->isVectorTy())
5518 return ArgKind::Vector;
5519 return ArgKind::Memory;
5522 ShadowExtension getShadowExtension(
const CallBase &CB,
unsigned ArgNo) {
5532 return ShadowExtension::Zero;
5536 return ShadowExtension::Sign;
5538 return ShadowExtension::None;
5542 unsigned GpOffset = SystemZGpOffset;
5543 unsigned FpOffset = SystemZFpOffset;
5544 unsigned VrIndex = 0;
5545 unsigned OverflowOffset = SystemZOverflowOffset;
5552 ArgKind AK = classifyArgument(
T);
5553 if (AK == ArgKind::Indirect) {
5554 T = PointerType::get(
T, 0);
5555 AK = ArgKind::GeneralPurpose;
5557 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5558 AK = ArgKind::Memory;
5559 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5560 AK = ArgKind::Memory;
5561 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5562 AK = ArgKind::Memory;
5563 Value *ShadowBase =
nullptr;
5564 Value *OriginBase =
nullptr;
5565 ShadowExtension SE = ShadowExtension::None;
5567 case ArgKind::GeneralPurpose: {
5572 SE = getShadowExtension(CB, ArgNo);
5574 if (SE == ShadowExtension::None) {
5576 assert(ArgAllocSize <= ArgSize);
5577 GapSize = ArgSize - ArgAllocSize;
5579 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5580 if (MS.TrackOrigins)
5581 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5583 GpOffset += ArgSize;
5589 case ArgKind::FloatingPoint: {
5598 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5599 if (MS.TrackOrigins)
5600 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5602 FpOffset += ArgSize;
5608 case ArgKind::Vector: {
5615 case ArgKind::Memory: {
5623 SE = getShadowExtension(CB, ArgNo);
5625 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5627 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5628 if (MS.TrackOrigins)
5630 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5631 OverflowOffset += ArgSize;
5638 case ArgKind::Indirect:
5641 if (ShadowBase ==
nullptr)
5643 Value *Shadow = MSV.getShadow(
A);
5644 if (SE != ShadowExtension::None)
5645 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.
getInt64Ty(),
5646 SE == ShadowExtension::Sign);
5648 ShadowBase, PointerType::get(Shadow->
getType(), 0),
"_msarg_va_s");
5650 if (MS.TrackOrigins) {
5651 Value *Origin = MSV.getOrigin(
A);
5653 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5657 Constant *OverflowSize = ConstantInt::get(
5658 IRB.
getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5659 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5663 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5667 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5668 PointerType::get(RegSaveAreaPtrTy, 0));
5669 Value *RegSaveAreaPtr = IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5670 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5672 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5673 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(), Alignment,
5678 unsigned RegSaveAreaSize =
5679 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
5680 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5682 if (MS.TrackOrigins)
5683 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5684 Alignment, RegSaveAreaSize);
5690 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5694 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5695 PointerType::get(OverflowArgAreaPtrTy, 0));
5696 Value *OverflowArgAreaPtr =
5697 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5698 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5700 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5701 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5704 SystemZOverflowOffset);
5705 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5707 if (MS.TrackOrigins) {
5709 SystemZOverflowOffset);
5710 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5715 void finalizeInstrumentation()
override {
5716 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5717 "finalizeInstrumentation called twice");
5718 if (!VAStartInstrumentationList.
empty()) {
5725 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5733 Intrinsic::umin, CopySize,
5737 if (MS.TrackOrigins) {
5747 for (
size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.
size();
5748 VaStartNo < VaStartNum; VaStartNo++) {
5749 CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5750 NextNodeIRBuilder IRB(OrigInst);
5752 copyRegSaveArea(IRB, VAListTag);
5753 copyOverflowArea(IRB, VAListTag);
5760using VarArgLoongArch64Helper = VarArgMIPS64Helper;
5763struct VarArgNoOpHelper :
public VarArgHelper {
5764 VarArgNoOpHelper(
Function &
F, MemorySanitizer &MS,
5765 MemorySanitizerVisitor &MSV) {}
5773 void finalizeInstrumentation()
override {}
5779 MemorySanitizerVisitor &Visitor) {
5782 Triple TargetTriple(Func.getParent()->getTargetTriple());
5784 return new VarArgAMD64Helper(Func, Msan, Visitor);
5786 return new VarArgMIPS64Helper(Func, Msan, Visitor);
5788 return new VarArgAArch64Helper(Func, Msan, Visitor);
5791 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5793 return new VarArgSystemZHelper(Func, Msan, Visitor);
5795 return new VarArgLoongArch64Helper(Func, Msan, Visitor);
5797 return new VarArgNoOpHelper(Func, Msan, Visitor);
5804 if (
F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5807 MemorySanitizerVisitor Visitor(
F, *
this, TLI);
5811 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
5814 return Visitor.runOnFunction();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
static bool isStore(int Opcode)
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
static const size_t kNumberOfAccessSizes
VarLocInsertPt getNextNode(const DbgRecord *DVR)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
static bool isAMustTailRetVal(Value *RetVal)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool runOnFunction(Function &F, bool PostInlining)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
static size_t TypeSizeToSizeIndex(uint32_t TypeSize)
static const MemoryMapParams Linux_LoongArch64_MemoryMapParams
static const PlatformMemoryMapParams Linux_S390_MemoryMapParams
static const Align kMinOriginAlignment
static const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< uint64_t > ClShadowBase("msan-shadow-base", cl::desc("Define custom MSan ShadowBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClDumpStrictInstructions("msan-dump-strict-instructions", cl::desc("print out instructions with default strict semantics"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClHandleICmpExact("msan-handle-icmp-exact", cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams Linux_X86_MemoryMapParams
static cl::opt< uint64_t > ClOriginBase("msan-origin-base", cl::desc("Define custom MSan OriginBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClCheckConstantShadow("msan-check-constant-shadow", cl::desc("Insert checks for constant shadow values"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams
static const MemoryMapParams NetBSD_X86_64_MemoryMapParams
static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams
static const unsigned kOriginSize
static cl::opt< bool > ClWithComdat("msan-with-comdat", cl::desc("Place MSan constructors in comdat sections"), cl::Hidden, cl::init(false))
static cl::opt< int > ClTrackOrigins("msan-track-origins", cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden, cl::init(0))
Track origins of uninitialized values.
static cl::opt< int > ClInstrumentationWithCallThreshold("msan-instrumentation-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of checks and origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< int > ClPoisonStackPattern("msan-poison-stack-pattern", cl::desc("poison uninitialized stack variables with the given pattern"), cl::Hidden, cl::init(0xff))
static const Align kShadowTLSAlignment
static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams
static Constant * getOrInsertGlobal(Module &M, StringRef Name, Type *Ty)
static const MemoryMapParams Linux_S390X_MemoryMapParams
static cl::opt< bool > ClPoisonUndef("msan-poison-undef", cl::desc("poison undef temps"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClPoisonStack("msan-poison-stack", cl::desc("poison uninitialized stack variables"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_I386_MemoryMapParams
const char kMsanInitName[]
static cl::opt< bool > ClPrintStackNames("msan-print-stack-names", cl::desc("Print name of local stack variable"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< uint64_t > ClAndMask("msan-and-mask", cl::desc("Define custom MSan AndMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleLifetimeIntrinsics("msan-handle-lifetime-intrinsics", cl::desc("when possible, poison scoped variables at the beginning of the scope " "(slower, but more precise)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClKeepGoing("msan-keep-going", cl::desc("keep going after reporting a UMR"), cl::Hidden, cl::init(false))
static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams
static GlobalVariable * createPrivateConstGlobalForString(Module &M, StringRef Str)
Create a non-const global initialized with the given string.
static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams
static const size_t kNumberOfAccessSizes
static cl::opt< bool > ClEagerChecks("msan-eager-checks", cl::desc("check arguments and return values at function call boundaries"), cl::Hidden, cl::init(false))
static cl::opt< int > ClDisambiguateWarning("msan-disambiguate-warning-threshold", cl::desc("Define threshold for number of checks per " "debug location to force origin update."), cl::Hidden, cl::init(3))
static VarArgHelper * CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, MemorySanitizerVisitor &Visitor)
static const MemoryMapParams Linux_MIPS64_MemoryMapParams
static const MemoryMapParams Linux_PowerPC64_MemoryMapParams
static cl::opt< uint64_t > ClXorMask("msan-xor-mask", cl::desc("Define custom MSan XorMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleAsmConservative("msan-handle-asm-conservative", cl::desc("conservative handling of inline assembly"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams
static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams
static const unsigned kParamTLSSize
static cl::opt< bool > ClHandleICmp("msan-handle-icmp", cl::desc("propagate shadow through ICmpEQ and ICmpNE"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClEnableKmsan("msan-kernel", cl::desc("Enable KernelMemorySanitizer instrumentation"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClPoisonStackWithCall("msan-poison-stack-with-call", cl::desc("poison uninitialized stack variables with a call"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams
static const unsigned kRetvalTLSSize
static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams
const char kMsanModuleCtorName[]
static const MemoryMapParams FreeBSD_I386_MemoryMapParams
static cl::opt< bool > ClCheckAccessAddress("msan-check-access-address", cl::desc("report accesses through a pointer which has poisoned shadow"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClDisableChecks("msan-disable-checks", cl::desc("Apply no_sanitize to the whole file"), cl::Hidden, cl::init(false))
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
Value * getCalledOperand() const
Type * getParamElementType(unsigned ArgNo) const
Extract the elementtype type for a parameter.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_SGT
signed greater than
@ ICMP_SGE
signed greater or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
bool isZeroValue() const
Return true if the value is negative zero or null value.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
static bool shouldExecute(unsigned CounterName)
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void setComdat(Comdat *C)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ ExternalLinkage
Externally visible function.
Analysis pass providing a never-invalidated alias analysis result.
This instruction compares its operands according to the predicate given to the constructor.
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::vector< ConstraintInfo > ConstraintInfoVector
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
This class represents a cast from an integer to a pointer.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
This class wraps the llvm.memcpy intrinsic.
This class wraps the llvm.memmove intrinsic.
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
This class represents a cast from a pointer to an integer.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This instruction constructs a fixed permutation of two input vectors.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
AttributeList getAttrList(LLVMContext *C, ArrayRef< unsigned > ArgNos, bool Signed, bool Ret=false, AttributeList AL=AttributeList()) const
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Triple - Helper class for working with autoconf configuration names.
bool isMIPS64() const
Tests whether the target is MIPS 64-bit (little and big endian).
ArchType getArch() const
Get the parsed architecture type of this triple.
bool isLoongArch64() const
Tests whether the target is 64-bit LoongArch.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
static Type * getX86_MMXTy(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
'undef' values are things that do not have specified contents.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This represents the llvm.va_copy intrinsic.
This represents the llvm.va_start intrinsic.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
This class represents zero extension of integer types.
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
This class provides various memory handling functions that manipulate MemoryBlock instances.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
initializer< Ty > init(const Ty &Val)
Function * Kernel
Summary of a kernel (=entry point for target offloading).
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
void stable_sort(R &&Range)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
AllocaInst * findAllocaForValue(Value *V, bool OffsetZero=false)
Returns unique alloca where the value comes from, or nullptr.
std::pair< Function *, FunctionCallee > getOrCreateSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, function_ref< void(Function *, FunctionCallee)> FunctionsCreatedCallback, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function lazily.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Or
Bitwise or logical OR of integers.
std::pair< Instruction *, Value * > SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore)
Insert a for (int i = 0; i < End; i++) loop structure (with the exception that End is assumed > 0,...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, const TargetLibraryInfo *TLI)
Given a CallInst, check if it calls a string function known to CodeGen, and mark it with NoBuiltin if...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A CRTP mix-in to automatically provide informational APIs needed for passes.