184#include "llvm/IR/IntrinsicsAArch64.h"
185#include "llvm/IR/IntrinsicsX86.h"
215#define DEBUG_TYPE "msan"
218 "Controls which checks to insert");
221 "Controls which instruction to instrument");
239 "msan-track-origins",
244 cl::desc(
"keep going after reporting a UMR"),
253 "msan-poison-stack-with-call",
258 "msan-poison-stack-pattern",
259 cl::desc(
"poison uninitialized stack variables with the given pattern"),
264 cl::desc(
"Print name of local stack variable"),
273 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
278 cl::desc(
"exact handling of relational integer ICmp"),
282 "msan-handle-lifetime-intrinsics",
284 "when possible, poison scoped variables at the beginning of the scope "
285 "(slower, but more precise)"),
296 "msan-handle-asm-conservative",
307 "msan-check-access-address",
308 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
313 cl::desc(
"check arguments and return values at function call boundaries"),
317 "msan-dump-strict-instructions",
318 cl::desc(
"print out instructions with default strict semantics"),
322 "msan-instrumentation-with-call-threshold",
324 "If the function being instrumented requires more than "
325 "this number of checks and origin stores, use callbacks instead of "
326 "inline checks (-1 means never use callbacks)."),
331 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
341 cl::desc(
"Insert checks for constant shadow values"),
348 cl::desc(
"Place MSan constructors in comdat sections"),
354 cl::desc(
"Define custom MSan AndMask"),
358 cl::desc(
"Define custom MSan XorMask"),
362 cl::desc(
"Define custom MSan ShadowBase"),
366 cl::desc(
"Define custom MSan OriginBase"),
371 cl::desc(
"Define threshold for number of checks per "
372 "debug location to force origin update."),
384struct MemoryMapParams {
391struct PlatformMemoryMapParams {
392 const MemoryMapParams *bits32;
393 const MemoryMapParams *bits64;
555class MemorySanitizer {
564 MemorySanitizer(MemorySanitizer &&) =
delete;
565 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
566 MemorySanitizer(
const MemorySanitizer &) =
delete;
567 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
572 friend struct MemorySanitizerVisitor;
573 friend struct VarArgHelperBase;
574 friend struct VarArgAMD64Helper;
575 friend struct VarArgAArch64Helper;
576 friend struct VarArgPowerPCHelper;
577 friend struct VarArgSystemZHelper;
578 friend struct VarArgI386Helper;
579 friend struct VarArgGenericHelper;
581 void initializeModule(
Module &M);
586 template <
typename... ArgsTy>
613 Value *ParamOriginTLS;
619 Value *RetvalOriginTLS;
625 Value *VAArgOriginTLS;
628 Value *VAArgOverflowSizeTLS;
631 bool CallbacksInitialized =
false;
676 Value *MsanMetadataAlloca;
682 const MemoryMapParams *MapParams;
686 MemoryMapParams CustomMapParams;
691 MDNode *OriginStoreWeights;
694void insertModuleCtor(
Module &M) {
722 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
740 MemorySanitizer Msan(*
F.getParent(),
Options);
759 OS, MapClassName2PassName);
766 OS <<
"eager-checks;";
767 OS <<
"track-origins=" <<
Options.TrackOrigins;
783template <
typename... ArgsTy>
790 std::forward<ArgsTy>(Args)...);
793 return M.getOrInsertFunction(
Name, MsanMetadata,
794 std::forward<ArgsTy>(Args)...);
803 RetvalOriginTLS =
nullptr;
805 ParamOriginTLS =
nullptr;
807 VAArgOriginTLS =
nullptr;
808 VAArgOverflowSizeTLS =
nullptr;
810 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
812 IRB.getVoidTy(), IRB.getInt32Ty());
823 MsanGetContextStateFn =
824 M.getOrInsertFunction(
"__msan_get_context_state", PtrTy);
828 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
829 std::string name_load =
830 "__msan_metadata_ptr_for_load_" + std::to_string(size);
831 std::string name_store =
832 "__msan_metadata_ptr_for_store_" + std::to_string(size);
833 MsanMetadataPtrForLoad_1_8[ind] =
834 getOrInsertMsanMetadataFunction(M, name_load, PtrTy);
835 MsanMetadataPtrForStore_1_8[ind] =
836 getOrInsertMsanMetadataFunction(M, name_store, PtrTy);
839 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
840 M,
"__msan_metadata_ptr_for_load_n", PtrTy, IRB.getInt64Ty());
841 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
842 M,
"__msan_metadata_ptr_for_store_n", PtrTy, IRB.getInt64Ty());
845 MsanPoisonAllocaFn =
M.getOrInsertFunction(
846 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
847 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
848 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
852 return M.getOrInsertGlobal(
Name, Ty, [&] {
854 nullptr,
Name,
nullptr,
860void MemorySanitizer::createUserspaceApi(
Module &M,
868 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
869 :
"__msan_warning_with_origin_noreturn";
870 WarningFn =
M.getOrInsertFunction(WarningFnName,
872 IRB.getVoidTy(), IRB.getInt32Ty());
875 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
876 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
902 VAArgOverflowSizeTLS =
907 unsigned AccessSize = 1 << AccessSizeIndex;
908 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
909 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
911 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
913 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
914 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
916 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
920 MsanSetAllocaOriginWithDescriptionFn =
921 M.getOrInsertFunction(
"__msan_set_alloca_origin_with_descr",
922 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
923 MsanSetAllocaOriginNoDescriptionFn =
924 M.getOrInsertFunction(
"__msan_set_alloca_origin_no_descr",
925 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
926 MsanPoisonStackFn =
M.getOrInsertFunction(
"__msan_poison_stack",
927 IRB.getVoidTy(), PtrTy, IntptrTy);
931void MemorySanitizer::initializeCallbacks(
Module &M,
934 if (CallbacksInitialized)
940 MsanChainOriginFn =
M.getOrInsertFunction(
941 "__msan_chain_origin",
944 MsanSetOriginFn =
M.getOrInsertFunction(
946 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
948 M.getOrInsertFunction(
"__msan_memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
950 M.getOrInsertFunction(
"__msan_memcpy", PtrTy, PtrTy, PtrTy, IntptrTy);
951 MemsetFn =
M.getOrInsertFunction(
"__msan_memset",
953 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
955 MsanInstrumentAsmStoreFn =
M.getOrInsertFunction(
956 "__msan_instrument_asm_store", IRB.getVoidTy(), PtrTy, IntptrTy);
959 createKernelApi(M, TLI);
961 createUserspaceApi(M, TLI);
963 CallbacksInitialized =
true;
969 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
987void MemorySanitizer::initializeModule(
Module &M) {
988 auto &
DL =
M.getDataLayout();
990 TargetTriple =
Triple(
M.getTargetTriple());
992 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
993 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
995 if (ShadowPassed || OriginPassed) {
1000 MapParams = &CustomMapParams;
1002 switch (TargetTriple.getOS()) {
1004 switch (TargetTriple.getArch()) {
1019 switch (TargetTriple.getArch()) {
1028 switch (TargetTriple.getArch()) {
1062 C = &(
M.getContext());
1064 IntptrTy = IRB.getIntPtrTy(
DL);
1065 OriginTy = IRB.getInt32Ty();
1066 PtrTy = IRB.getPtrTy();
1071 if (!CompileKernel) {
1073 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1074 return new GlobalVariable(
1075 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1076 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1080 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1081 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1082 GlobalValue::WeakODRLinkage,
1083 IRB.getInt32(Recover),
"__msan_keep_going");
1098struct VarArgHelper {
1099 virtual ~VarArgHelper() =
default;
1114 virtual void finalizeInstrumentation() = 0;
1117struct MemorySanitizerVisitor;
1122 MemorySanitizerVisitor &Visitor);
1129 if (TypeSizeFixed <= 8)
1138class NextNodeIRBuilder :
public IRBuilder<> {
1151struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1153 MemorySanitizer &MS;
1156 std::unique_ptr<VarArgHelper> VAHelper;
1164 bool PropagateShadow;
1168 struct ShadowOriginAndInsertPoint {
1174 : Shadow(S), Origin(
O), OrigIns(
I) {}
1182 int64_t SplittableBlocksCount = 0;
1184 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1187 bool SanitizeFunction =
1189 InsertChecks = SanitizeFunction;
1190 PropagateShadow = SanitizeFunction;
1200 MS.initializeCallbacks(*
F.getParent(), TLI);
1201 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1204 if (MS.CompileKernel) {
1206 insertKmsanPrologue(IRB);
1210 <<
"MemorySanitizer is not inserting checks into '"
1211 <<
F.getName() <<
"'\n");
1214 bool instrumentWithCalls(
Value *V) {
1216 if (isa<Constant>(V))
1219 ++SplittableBlocksCount;
1225 return I.getParent() == FnPrologueEnd->
getParent() &&
1226 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1234 if (MS.TrackOrigins <= 1)
1236 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1241 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1253 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1254 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1266 auto [InsertPt,
Index] =
1278 Align CurrentAlignment = Alignment;
1279 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1280 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1282 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1287 CurrentAlignment = IntptrAlignment;
1305 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1306 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1314 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1323 if (instrumentWithCalls(ConvertedShadow) &&
1326 Value *ConvertedShadow2 =
1332 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1336 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1341 void materializeStores() {
1344 Value *Val =
SI->getValueOperand();
1346 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1347 Value *ShadowPtr, *OriginPtr;
1349 const Align Alignment =
SI->getAlign();
1351 std::tie(ShadowPtr, OriginPtr) =
1352 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1361 if (MS.TrackOrigins && !
SI->isAtomic())
1362 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1369 if (MS.TrackOrigins < 2)
1372 if (LazyWarningDebugLocationCount.
empty())
1373 for (
const auto &
I : InstrumentationList)
1374 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1388 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1390 auto NewDebugLoc = OI->getDebugLoc();
1397 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1398 Origin = updateOrigin(Origin, IRBOrigin);
1403 if (MS.CompileKernel || MS.TrackOrigins)
1417 if (instrumentWithCalls(ConvertedShadow) &&
1421 ConvertedShadow = convertShadowToScalar(ConvertedShadow, IRB);
1422 Value *ConvertedShadow2 =
1425 Fn, {ConvertedShadow2,
1426 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1430 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1433 !MS.Recover, MS.ColdCallWeights);
1436 insertWarningFn(IRB, Origin);
1441 void materializeInstructionChecks(
1446 bool Combine = !MS.TrackOrigins;
1448 Value *Shadow =
nullptr;
1449 for (
const auto &ShadowData : InstructionChecks) {
1453 Value *ConvertedShadow = ShadowData.Shadow;
1455 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1462 insertWarningFn(IRB, ShadowData.Origin);
1472 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1477 Shadow = ConvertedShadow;
1481 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1482 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1483 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1489 materializeOneCheck(IRB, Shadow,
nullptr);
1493 void materializeChecks() {
1499 for (
auto I = InstrumentationList.begin();
1500 I != InstrumentationList.end();) {
1501 auto OrigIns =
I->OrigIns;
1505 auto J = std::find_if(
I + 1, InstrumentationList.end(),
1506 [OrigIns](
const ShadowOriginAndInsertPoint &R) {
1507 return OrigIns != R.OrigIns;
1521 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1522 {Zero, IRB.getInt32(0)},
"param_shadow");
1523 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1524 {Zero, IRB.getInt32(1)},
"retval_shadow");
1525 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1526 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1527 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1528 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1529 MS.VAArgOverflowSizeTLS =
1530 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1531 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1532 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1533 {Zero, IRB.getInt32(5)},
"param_origin");
1534 MS.RetvalOriginTLS =
1535 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1536 {Zero, IRB.getInt32(6)},
"retval_origin");
1538 MS.MsanMetadataAlloca = IRB.
CreateAlloca(MS.MsanMetadata, 0u);
1555 for (
PHINode *PN : ShadowPHINodes) {
1556 PHINode *PNS = cast<PHINode>(getShadow(PN));
1557 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1558 size_t NumValues = PN->getNumIncomingValues();
1559 for (
size_t v = 0;
v < NumValues;
v++) {
1560 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1562 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1566 VAHelper->finalizeInstrumentation();
1570 if (InstrumentLifetimeStart) {
1571 for (
auto Item : LifetimeStartList) {
1572 instrumentAlloca(*Item.second, Item.first);
1573 AllocaSet.
remove(Item.second);
1579 instrumentAlloca(*AI);
1582 materializeChecks();
1586 materializeStores();
1592 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1604 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1605 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1607 VT->getElementCount());
1609 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1610 return ArrayType::get(getShadowTy(AT->getElementType()),
1611 AT->getNumElements());
1613 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1615 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1616 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1618 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1634 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1636 if (Aggregator != FalseVal)
1637 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1639 Aggregator = ShadowBool;
1648 if (!
Array->getNumElements())
1652 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1656 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1657 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1667 return collapseStructShadow(
Struct, V, IRB);
1668 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1669 return collapseArrayShadow(Array, V, IRB);
1670 if (isa<VectorType>(
V->getType())) {
1671 if (isa<ScalableVectorType>(
V->getType()))
1674 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1682 Type *VTy =
V->getType();
1684 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1691 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1692 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1693 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1694 VectTy->getElementCount());
1700 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1701 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1702 return VectorType::get(
1703 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1704 VectTy->getElementCount());
1706 assert(IntPtrTy == MS.IntptrTy);
1711 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1713 VectTy->getElementCount(),
1714 constToIntPtr(VectTy->getElementType(),
C));
1716 assert(IntPtrTy == MS.IntptrTy);
1717 return ConstantInt::get(MS.IntptrTy,
C);
1728 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1731 if (
uint64_t AndMask = MS.MapParams->AndMask)
1732 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1734 if (
uint64_t XorMask = MS.MapParams->XorMask)
1735 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1747 std::pair<Value *, Value *>
1754 assert(VectTy->getElementType()->isPointerTy());
1756 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1757 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1758 Value *ShadowLong = ShadowOffset;
1759 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1761 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1764 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1766 Value *OriginPtr =
nullptr;
1767 if (MS.TrackOrigins) {
1768 Value *OriginLong = ShadowOffset;
1769 uint64_t OriginBase = MS.MapParams->OriginBase;
1770 if (OriginBase != 0)
1772 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1775 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1778 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1780 return std::make_pair(ShadowPtr, OriginPtr);
1783 template <
typename... ArgsTy>
1788 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1789 return IRB.
CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1792 return IRB.
CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1795 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1799 Value *ShadowOriginPtrs;
1806 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1808 Value *SizeVal = ConstantInt::get(MS.IntptrTy,
Size);
1809 ShadowOriginPtrs = createMetadataCall(
1811 isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1818 return std::make_pair(ShadowPtr, OriginPtr);
1824 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1831 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1835 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1836 Value *ShadowPtrs = ConstantInt::getNullValue(
1838 Value *OriginPtrs =
nullptr;
1839 if (MS.TrackOrigins)
1840 OriginPtrs = ConstantInt::getNullValue(
1842 for (
unsigned i = 0; i < NumElements; ++i) {
1845 auto [ShadowPtr, OriginPtr] =
1846 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1849 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1850 if (MS.TrackOrigins)
1852 OriginPtrs, OriginPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1854 return {ShadowPtrs, OriginPtrs};
1861 if (MS.CompileKernel)
1862 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1863 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1878 if (!MS.TrackOrigins)
1892 Value *getOriginPtrForRetval() {
1894 return MS.RetvalOriginTLS;
1899 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1900 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1905 if (!MS.TrackOrigins)
1907 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1908 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1909 OriginMap[
V] = Origin;
1913 Type *ShadowTy = getShadowTy(OrigTy);
1923 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1928 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1930 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1932 getPoisonedShadow(AT->getElementType()));
1935 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1937 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1938 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1946 Type *ShadowTy = getShadowTy(V);
1949 return getPoisonedShadow(ShadowTy);
1961 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1962 return getCleanShadow(V);
1964 Value *Shadow = ShadowMap[
V];
1966 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1968 assert(Shadow &&
"No shadow for a value");
1972 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1973 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1974 : getCleanShadow(V);
1979 if (
Argument *
A = dyn_cast<Argument>(V)) {
1981 Value *&ShadowPtr = ShadowMap[
V];
1986 unsigned ArgOffset = 0;
1988 for (
auto &FArg :
F->args()) {
1989 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
1991 ?
"vscale not fully supported\n"
1992 :
"Arg is not sized\n"));
1994 ShadowPtr = getCleanShadow(V);
1995 setOrigin(
A, getCleanOrigin());
2001 unsigned Size = FArg.hasByValAttr()
2002 ?
DL.getTypeAllocSize(FArg.getParamByValType())
2003 :
DL.getTypeAllocSize(FArg.getType());
2007 if (FArg.hasByValAttr()) {
2011 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
2012 FArg.getParamAlign(), FArg.getParamByValType());
2013 Value *CpShadowPtr, *CpOriginPtr;
2014 std::tie(CpShadowPtr, CpOriginPtr) =
2015 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
2017 if (!PropagateShadow || Overflow) {
2019 EntryIRB.CreateMemSet(
2023 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2025 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
2030 if (MS.TrackOrigins) {
2031 Value *OriginPtr = getOriginPtrForArgument(EntryIRB, ArgOffset);
2035 EntryIRB.CreateMemCpy(
2044 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2045 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
2046 ShadowPtr = getCleanShadow(V);
2047 setOrigin(
A, getCleanOrigin());
2050 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2051 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
2053 if (MS.TrackOrigins) {
2054 Value *OriginPtr = getOriginPtrForArgument(EntryIRB, ArgOffset);
2055 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
2059 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
2065 assert(ShadowPtr &&
"Could not find shadow for an argument");
2069 return getCleanShadow(V);
2074 return getShadow(
I->getOperand(i));
2079 if (!MS.TrackOrigins)
2081 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
2082 return getCleanOrigin();
2083 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2084 "Unexpected value type in getOrigin()");
2086 if (
I->getMetadata(LLVMContext::MD_nosanitize))
2087 return getCleanOrigin();
2089 Value *Origin = OriginMap[
V];
2090 assert(Origin &&
"Missing origin");
2096 return getOrigin(
I->getOperand(i));
2109 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
2110 << *OrigIns <<
"\n");
2115 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2116 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2117 "Can only insert checks for integer, vector, and aggregate shadow "
2120 InstrumentationList.push_back(
2121 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2130 Value *Shadow, *Origin;
2132 Shadow = getShadow(Val);
2135 Origin = getOrigin(Val);
2137 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2140 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2142 insertShadowCheck(Shadow, Origin, OrigIns);
2147 case AtomicOrdering::NotAtomic:
2148 return AtomicOrdering::NotAtomic;
2149 case AtomicOrdering::Unordered:
2150 case AtomicOrdering::Monotonic:
2151 case AtomicOrdering::Release:
2152 return AtomicOrdering::Release;
2153 case AtomicOrdering::Acquire:
2154 case AtomicOrdering::AcquireRelease:
2155 return AtomicOrdering::AcquireRelease;
2156 case AtomicOrdering::SequentiallyConsistent:
2157 return AtomicOrdering::SequentiallyConsistent;
2163 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2164 uint32_t OrderingTable[NumOrderings] = {};
2166 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2167 OrderingTable[(
int)AtomicOrderingCABI::release] =
2168 (int)AtomicOrderingCABI::release;
2169 OrderingTable[(int)AtomicOrderingCABI::consume] =
2170 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2171 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2172 (
int)AtomicOrderingCABI::acq_rel;
2173 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2174 (
int)AtomicOrderingCABI::seq_cst;
2181 case AtomicOrdering::NotAtomic:
2182 return AtomicOrdering::NotAtomic;
2183 case AtomicOrdering::Unordered:
2184 case AtomicOrdering::Monotonic:
2185 case AtomicOrdering::Acquire:
2186 return AtomicOrdering::Acquire;
2187 case AtomicOrdering::Release:
2188 case AtomicOrdering::AcquireRelease:
2189 return AtomicOrdering::AcquireRelease;
2190 case AtomicOrdering::SequentiallyConsistent:
2191 return AtomicOrdering::SequentiallyConsistent;
2197 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2198 uint32_t OrderingTable[NumOrderings] = {};
2200 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2201 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2202 OrderingTable[(int)AtomicOrderingCABI::consume] =
2203 (
int)AtomicOrderingCABI::acquire;
2204 OrderingTable[(int)AtomicOrderingCABI::release] =
2205 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2206 (int)AtomicOrderingCABI::acq_rel;
2207 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2208 (
int)AtomicOrderingCABI::seq_cst;
2216 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2219 if (isInPrologue(
I))
2224 setShadow(&
I, getCleanShadow(&
I));
2225 setOrigin(&
I, getCleanOrigin());
2237 assert(
I.getType()->isSized() &&
"Load type must have size");
2238 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2239 NextNodeIRBuilder IRB(&
I);
2240 Type *ShadowTy = getShadowTy(&
I);
2242 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2243 const Align Alignment =
I.getAlign();
2244 if (PropagateShadow) {
2245 std::tie(ShadowPtr, OriginPtr) =
2246 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2250 setShadow(&
I, getCleanShadow(&
I));
2254 insertShadowCheck(
I.getPointerOperand(), &
I);
2259 if (MS.TrackOrigins) {
2260 if (PropagateShadow) {
2265 setOrigin(&
I, getCleanOrigin());
2275 StoreList.push_back(&
I);
2277 insertShadowCheck(
I.getPointerOperand(), &
I);
2281 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2285 Value *Val =
I.getOperand(1);
2286 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2291 insertShadowCheck(
Addr, &
I);
2296 if (isa<AtomicCmpXchgInst>(
I))
2297 insertShadowCheck(Val, &
I);
2301 setShadow(&
I, getCleanShadow(&
I));
2302 setOrigin(&
I, getCleanOrigin());
2317 insertShadowCheck(
I.getOperand(1), &
I);
2321 setOrigin(&
I, getOrigin(&
I, 0));
2325 insertShadowCheck(
I.getOperand(2), &
I);
2327 auto *Shadow0 = getShadow(&
I, 0);
2328 auto *Shadow1 = getShadow(&
I, 1);
2331 setOriginForNaryOp(
I);
2336 auto *Shadow0 = getShadow(&
I, 0);
2337 auto *Shadow1 = getShadow(&
I, 1);
2340 setOriginForNaryOp(
I);
2346 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2347 setOrigin(&
I, getOrigin(&
I, 0));
2352 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2353 setOrigin(&
I, getOrigin(&
I, 0));
2358 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2359 setOrigin(&
I, getOrigin(&
I, 0));
2366 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2367 if (CI->isMustTailCall())
2371 setOrigin(&
I, getOrigin(&
I, 0));
2377 "_msprop_ptrtoint"));
2378 setOrigin(&
I, getOrigin(&
I, 0));
2384 "_msprop_inttoptr"));
2385 setOrigin(&
I, getOrigin(&
I, 0));
2388 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2389 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2390 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2391 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2392 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2393 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2408 Value *S2 = getShadow(&
I, 1);
2409 Value *V1 =
I.getOperand(0);
2418 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2419 setOriginForNaryOp(
I);
2430 Value *S2 = getShadow(&
I, 1);
2440 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2441 setOriginForNaryOp(
I);
2459 template <
bool CombineShadow>
class Combiner {
2460 Value *Shadow =
nullptr;
2461 Value *Origin =
nullptr;
2463 MemorySanitizerVisitor *MSV;
2467 : IRB(IRB), MSV(MSV) {}
2471 if (CombineShadow) {
2476 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2477 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2481 if (MSV->MS.TrackOrigins) {
2486 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2488 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2489 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2499 Value *OpShadow = MSV->getShadow(V);
2500 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2501 return Add(OpShadow, OpOrigin);
2507 if (CombineShadow) {
2509 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2510 MSV->setShadow(
I, Shadow);
2512 if (MSV->MS.TrackOrigins) {
2514 MSV->setOrigin(
I, Origin);
2521 if (MSV->MS.TrackOrigins) {
2533 if (!MS.TrackOrigins)
2536 OriginCombiner
OC(
this, IRB);
2537 for (
Use &
Op :
I.operands())
2542 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2544 "Vector of pointers is not a valid shadow type");
2545 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2554 Type *srcTy =
V->getType();
2557 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2558 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2559 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2565 cast<VectorType>(dstTy)->getElementCount() ==
2566 cast<VectorType>(srcTy)->getElementCount())
2577 Type *ShadowTy = getShadowTy(V);
2578 if (
V->getType() == ShadowTy)
2580 if (
V->getType()->isPtrOrPtrVectorTy())
2589 ShadowAndOriginCombiner
SC(
this, IRB);
2590 for (
Use &
Op :
I.operands())
2610 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2611 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2612 Type *EltTy = VTy->getElementType();
2614 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2617 const APInt &
V = Elt->getValue();
2619 Elements.push_back(ConstantInt::get(EltTy, V2));
2621 Elements.push_back(ConstantInt::get(EltTy, 1));
2626 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2627 const APInt &
V = Elt->getValue();
2629 ShadowMul = ConstantInt::get(Ty, V2);
2631 ShadowMul = ConstantInt::get(Ty, 1);
2637 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2638 setOrigin(&
I, getOrigin(OtherArg));
2642 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2643 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2644 if (constOp0 && !constOp1)
2645 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2646 else if (constOp1 && !constOp0)
2647 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2662 insertShadowCheck(
I.getOperand(1), &
I);
2663 setShadow(&
I, getShadow(&
I, 0));
2664 setOrigin(&
I, getOrigin(&
I, 0));
2681 void handleEqualityComparison(
ICmpInst &
I) {
2685 Value *Sa = getShadow(
A);
2686 Value *Sb = getShadow(
B);
2712 setOriginForNaryOp(
I);
2720 void handleRelationalComparisonExact(
ICmpInst &
I) {
2724 Value *Sa = getShadow(
A);
2725 Value *Sb = getShadow(
B);
2736 bool IsSigned =
I.isSigned();
2738 auto GetMinMaxUnsigned = [&](
Value *
V,
Value *S) {
2748 V = IRB.
CreateXor(V, ConstantInt::get(
V->getType(), MinVal));
2753 return std::make_pair(Min, Max);
2756 auto [Amin, Amax] = GetMinMaxUnsigned(
A, Sa);
2757 auto [Bmin, Bmax] = GetMinMaxUnsigned(
B, Sb);
2763 setOriginForNaryOp(
I);
2770 void handleSignedRelationalComparison(
ICmpInst &
I) {
2774 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2775 op =
I.getOperand(0);
2776 pre =
I.getPredicate();
2777 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2778 op =
I.getOperand(1);
2779 pre =
I.getSwappedPredicate();
2792 setShadow(&
I, Shadow);
2793 setOrigin(&
I, getOrigin(
op));
2804 if (
I.isEquality()) {
2805 handleEqualityComparison(
I);
2811 handleRelationalComparisonExact(
I);
2815 handleSignedRelationalComparison(
I);
2820 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2821 handleRelationalComparisonExact(
I);
2828 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2835 Value *S2 = getShadow(&
I, 1);
2840 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2841 setOriginForNaryOp(
I);
2852 Value *S0 = getShadow(&
I, 0);
2854 Value *S2 = getShadow(&
I, 2);
2860 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2861 setOriginForNaryOp(
I);
2875 getShadow(
I.getArgOperand(1));
2878 {I.getArgOperand(0), I.getArgOperand(1),
2879 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2880 I.eraseFromParent();
2898 getShadow(
I.getArgOperand(1));
2901 {I.getArgOperand(0), I.getArgOperand(1),
2902 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2903 I.eraseFromParent();
2911 {I.getArgOperand(0),
2912 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2913 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2914 I.eraseFromParent();
2917 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2919 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2928 Value *Shadow = getShadow(&
I, 1);
2929 Value *ShadowPtr, *OriginPtr;
2933 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2938 insertShadowCheck(
Addr, &
I);
2941 if (MS.TrackOrigins)
2954 Type *ShadowTy = getShadowTy(&
I);
2955 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2956 if (PropagateShadow) {
2960 std::tie(ShadowPtr, OriginPtr) =
2961 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2965 setShadow(&
I, getCleanShadow(&
I));
2969 insertShadowCheck(
Addr, &
I);
2971 if (MS.TrackOrigins) {
2972 if (PropagateShadow)
2973 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2975 setOrigin(&
I, getCleanOrigin());
2988 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy()))
2991 unsigned NumArgOperands =
I.arg_size();
2992 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2993 Type *Ty =
I.getArgOperand(i)->getType();
2999 ShadowAndOriginCombiner
SC(
this, IRB);
3000 for (
unsigned i = 0; i < NumArgOperands; ++i)
3001 SC.Add(
I.getArgOperand(i));
3018 unsigned NumArgOperands =
I.arg_size();
3019 if (NumArgOperands == 0)
3022 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3023 I.getArgOperand(1)->getType()->isVectorTy() &&
3024 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
3026 return handleVectorStoreIntrinsic(
I);
3029 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3030 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
3032 return handleVectorLoadIntrinsic(
I);
3035 if (
I.doesNotAccessMemory())
3036 if (maybeHandleSimpleNomemIntrinsic(
I))
3044 setShadow(&
I, getShadow(&
I, 0));
3045 setOrigin(&
I, getOrigin(&
I, 0));
3053 InstrumentLifetimeStart =
false;
3054 LifetimeStartList.push_back(std::make_pair(&
I, AI));
3060 Type *OpType =
Op->getType();
3063 setOrigin(&
I, getOrigin(
Op));
3068 Value *Src =
I.getArgOperand(0);
3074 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
3077 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
3080 Value *OutputShadow =
3081 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
3083 setShadow(&
I, OutputShadow);
3084 setOriginForNaryOp(
I);
3102 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
3103 bool HasRoundingMode =
false) {
3105 Value *CopyOp, *ConvertOp;
3107 assert((!HasRoundingMode ||
3108 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
3109 "Invalid rounding mode");
3111 switch (
I.arg_size() - HasRoundingMode) {
3113 CopyOp =
I.getArgOperand(0);
3114 ConvertOp =
I.getArgOperand(1);
3117 ConvertOp =
I.getArgOperand(0);
3131 Value *ConvertShadow = getShadow(ConvertOp);
3132 Value *AggShadow =
nullptr;
3135 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), 0));
3136 for (
int i = 1; i < NumUsedElements; ++i) {
3138 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), i));
3139 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3142 AggShadow = ConvertShadow;
3145 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3152 Value *ResultShadow = getShadow(CopyOp);
3153 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3154 for (
int i = 0; i < NumUsedElements; ++i) {
3156 ResultShadow, ConstantInt::getNullValue(EltTy),
3159 setShadow(&
I, ResultShadow);
3160 setOrigin(&
I, getOrigin(CopyOp));
3162 setShadow(&
I, getCleanShadow(&
I));
3163 setOrigin(&
I, getCleanOrigin());
3171 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3174 return CreateShadowCast(IRB, S2,
T,
true);
3182 return CreateShadowCast(IRB, S2,
T,
true);
3199 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3205 Value *S2 = getShadow(&
I, 1);
3206 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3207 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3208 Value *V1 =
I.getOperand(0);
3211 {IRB.CreateBitCast(S1, V1->getType()), V2});
3213 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3214 setOriginForNaryOp(
I);
3218 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3219 const unsigned X86_MMXSizeInBits = 64;
3220 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3221 "Illegal MMX vector element size");
3223 X86_MMXSizeInBits / EltSizeInBits);
3230 case Intrinsic::x86_sse2_packsswb_128:
3231 case Intrinsic::x86_sse2_packuswb_128:
3232 return Intrinsic::x86_sse2_packsswb_128;
3234 case Intrinsic::x86_sse2_packssdw_128:
3235 case Intrinsic::x86_sse41_packusdw:
3236 return Intrinsic::x86_sse2_packssdw_128;
3238 case Intrinsic::x86_avx2_packsswb:
3239 case Intrinsic::x86_avx2_packuswb:
3240 return Intrinsic::x86_avx2_packsswb;
3242 case Intrinsic::x86_avx2_packssdw:
3243 case Intrinsic::x86_avx2_packusdw:
3244 return Intrinsic::x86_avx2_packssdw;
3246 case Intrinsic::x86_mmx_packsswb:
3247 case Intrinsic::x86_mmx_packuswb:
3248 return Intrinsic::x86_mmx_packsswb;
3250 case Intrinsic::x86_mmx_packssdw:
3251 return Intrinsic::x86_mmx_packssdw;
3265 unsigned MMXEltSizeInBits = 0) {
3269 Value *S2 = getShadow(&
I, 1);
3270 assert(
S1->getType()->isVectorTy());
3276 MMXEltSizeInBits ? getMMXVectorTy(MMXEltSizeInBits) :
S1->
getType();
3277 if (MMXEltSizeInBits) {
3285 if (MMXEltSizeInBits) {
3291 {}, {S1_ext, S2_ext},
nullptr,
3292 "_msprop_vector_pack");
3293 if (MMXEltSizeInBits)
3296 setOriginForNaryOp(
I);
3300 Constant *createDppMask(
unsigned Width,
unsigned Mask) {
3313 const unsigned Width =
3314 cast<FixedVectorType>(S->
getType())->getNumElements();
3320 Value *DstMaskV = createDppMask(Width, DstMask);
3340 Value *S0 = getShadow(&
I, 0);
3344 const unsigned Width =
3345 cast<FixedVectorType>(S->
getType())->getNumElements();
3346 assert(Width == 2 || Width == 4 || Width == 8);
3348 const unsigned Mask = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3349 const unsigned SrcMask =
Mask >> 4;
3350 const unsigned DstMask =
Mask & 0xf;
3353 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
3358 SI1, findDppPoisonedOutput(IRB, S, SrcMask << 4, DstMask << 4));
3365 setOriginForNaryOp(
I);
3369 C = CreateAppToShadowCast(IRB,
C);
3383 Value *Sc = getShadow(&
I, 2);
3384 Value *Oc = MS.TrackOrigins ? getOrigin(
C) : nullptr;
3389 C = convertBlendvToSelectMask(IRB,
C);
3390 Sc = convertBlendvToSelectMask(IRB, Sc);
3396 handleSelectLikeInst(
I,
C,
T,
F);
3400 void handleVectorSadIntrinsic(
IntrinsicInst &
I,
bool IsMMX =
false) {
3401 const unsigned SignificantBitsPerResultElement = 16;
3403 unsigned ZeroBitsPerResultElement =
3407 auto *Shadow0 = getShadow(&
I, 0);
3408 auto *Shadow1 = getShadow(&
I, 1);
3413 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3416 setOriginForNaryOp(
I);
3421 unsigned MMXEltSizeInBits = 0) {
3423 MMXEltSizeInBits ? getMMXVectorTy(MMXEltSizeInBits * 2) :
I.
getType();
3425 auto *Shadow0 = getShadow(&
I, 0);
3426 auto *Shadow1 = getShadow(&
I, 1);
3433 setOriginForNaryOp(
I);
3441 Type *ResTy = getShadowTy(&
I);
3442 auto *Shadow0 = getShadow(&
I, 0);
3443 auto *Shadow1 = getShadow(&
I, 1);
3448 setOriginForNaryOp(
I);
3456 auto *Shadow0 = getShadow(&
I, 0);
3457 auto *Shadow1 = getShadow(&
I, 1);
3459 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3461 setOriginForNaryOp(
I);
3470 setOrigin(&
I, getOrigin(&
I, 0));
3478 Value *OperandShadow = getShadow(&
I, 0);
3480 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3488 setOrigin(&
I, getOrigin(&
I, 0));
3496 Value *OperandShadow = getShadow(&
I, 0);
3497 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3505 setOrigin(&
I, getOrigin(&
I, 0));
3513 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3518 insertShadowCheck(
Addr, &
I);
3529 Value *ShadowPtr, *OriginPtr;
3530 std::tie(ShadowPtr, OriginPtr) =
3531 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3534 insertShadowCheck(
Addr, &
I);
3537 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3539 insertShadowCheck(Shadow, Origin, &
I);
3546 Value *PassThru =
I.getArgOperand(2);
3549 insertShadowCheck(
Ptr, &
I);
3550 insertShadowCheck(Mask, &
I);
3553 if (!PropagateShadow) {
3554 setShadow(&
I, getCleanShadow(&
I));
3555 setOrigin(&
I, getCleanOrigin());
3559 Type *ShadowTy = getShadowTy(&
I);
3560 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3561 auto [ShadowPtr, OriginPtr] =
3562 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3565 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3567 setShadow(&
I, Shadow);
3570 setOrigin(&
I, getCleanOrigin());
3575 Value *Values =
I.getArgOperand(0);
3580 insertShadowCheck(
Ptr, &
I);
3581 insertShadowCheck(Mask, &
I);
3584 Value *Shadow = getShadow(Values);
3585 Type *ElementShadowTy =
3586 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3587 auto [ShadowPtr, OriginPtrs] =
3588 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3597 Value *Ptrs =
I.getArgOperand(0);
3598 const Align Alignment(
3599 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3601 Value *PassThru =
I.getArgOperand(3);
3603 Type *PtrsShadowTy = getShadowTy(Ptrs);
3605 insertShadowCheck(Mask, &
I);
3609 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3612 if (!PropagateShadow) {
3613 setShadow(&
I, getCleanShadow(&
I));
3614 setOrigin(&
I, getCleanOrigin());
3618 Type *ShadowTy = getShadowTy(&
I);
3619 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3620 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3621 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3625 getShadow(PassThru),
"_msmaskedgather");
3627 setShadow(&
I, Shadow);
3630 setOrigin(&
I, getCleanOrigin());
3635 Value *Values =
I.getArgOperand(0);
3636 Value *Ptrs =
I.getArgOperand(1);
3637 const Align Alignment(
3638 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3641 Type *PtrsShadowTy = getShadowTy(Ptrs);
3643 insertShadowCheck(Mask, &
I);
3647 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3650 Value *Shadow = getShadow(Values);
3651 Type *ElementShadowTy =
3652 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3653 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3654 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3663 Value *
V =
I.getArgOperand(0);
3665 const Align Alignment(
3666 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3668 Value *Shadow = getShadow(V);
3671 insertShadowCheck(
Ptr, &
I);
3672 insertShadowCheck(Mask, &
I);
3677 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3678 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3682 if (!MS.TrackOrigins)
3685 auto &
DL =
F.getDataLayout();
3686 paintOrigin(IRB, getOrigin(V), OriginPtr,
3694 const Align Alignment(
3695 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3697 Value *PassThru =
I.getArgOperand(3);
3700 insertShadowCheck(
Ptr, &
I);
3701 insertShadowCheck(Mask, &
I);
3704 if (!PropagateShadow) {
3705 setShadow(&
I, getCleanShadow(&
I));
3706 setOrigin(&
I, getCleanOrigin());
3710 Type *ShadowTy = getShadowTy(&
I);
3711 Value *ShadowPtr, *OriginPtr;
3712 std::tie(ShadowPtr, OriginPtr) =
3713 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3715 getShadow(PassThru),
"_msmaskedld"));
3717 if (!MS.TrackOrigins)
3724 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3729 setOrigin(&
I, Origin);
3739 Type *ShadowTy = getShadowTy(&
I);
3742 Value *SMask = getShadow(&
I, 1);
3747 {getShadow(&I, 0), I.getOperand(1)});
3750 setOriginForNaryOp(
I);
3755 for (
unsigned X = OddElements ? 1 : 0;
X < Width;
X += 2) {
3772 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3773 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3774 "pclmul 3rd operand must be a constant");
3775 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3777 getPclmulMask(Width, Imm & 0x01));
3779 getPclmulMask(Width, Imm & 0x10));
3780 ShadowAndOriginCombiner SOC(
this, IRB);
3781 SOC.Add(Shuf0, getOrigin(&
I, 0));
3782 SOC.Add(Shuf1, getOrigin(&
I, 1));
3790 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3792 Value *Second = getShadow(&
I, 1);
3795 Mask.push_back(Width);
3796 for (
unsigned i = 1; i < Width; i++)
3800 setShadow(&
I, Shadow);
3801 setOriginForNaryOp(
I);
3806 Value *Shadow0 = getShadow(&
I, 0);
3807 Value *Shadow1 = getShadow(&
I, 1);
3813 setShadow(&
I, Shadow);
3814 setOriginForNaryOp(
I);
3820 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3822 Value *Second = getShadow(&
I, 1);
3826 Mask.push_back(Width);
3827 for (
unsigned i = 1; i < Width; i++)
3831 setShadow(&
I, Shadow);
3832 setOriginForNaryOp(
I);
3839 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3841 assert(isa<ConstantInt>(
I.getArgOperand(1)));
3844 ShadowAndOriginCombiner
SC(
this, IRB);
3845 SC.Add(
I.getArgOperand(0));
3853 assert(
I.getType()->isIntOrIntVectorTy());
3854 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3858 setShadow(&
I, getShadow(&
I, 0));
3859 setOrigin(&
I, getOrigin(&
I, 0));
3864 Value *Shadow = getShadow(&
I, 0);
3865 setShadow(&
I, IRB.
CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3866 setOrigin(&
I, getOrigin(&
I, 0));
3871 Value *Shadow0 = getShadow(&
I, 0);
3872 Value *Shadow1 = getShadow(&
I, 1);
3875 IRB.
CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
3881 setShadow(&
I, Shadow);
3882 setOriginForNaryOp(
I);
3899 void handleNEONVectorStoreIntrinsic(
IntrinsicInst &
I,
bool useLane) {
3903 int numArgOperands =
I.arg_size();
3906 assert(numArgOperands >= 1);
3907 Value *
Addr =
I.getArgOperand(numArgOperands - 1);
3909 int skipTrailingOperands = 1;
3912 insertShadowCheck(
Addr, &
I);
3916 skipTrailingOperands++;
3917 assert(numArgOperands >=
static_cast<int>(skipTrailingOperands));
3919 I.getArgOperand(numArgOperands - skipTrailingOperands)->getType()));
3924 for (
int i = 0; i < numArgOperands - skipTrailingOperands; i++) {
3925 assert(isa<FixedVectorType>(
I.getArgOperand(i)->getType()));
3926 Value *Shadow = getShadow(&
I, i);
3927 ShadowArgs.
append(1, Shadow);
3942 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getElementType(),
3943 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements() *
3944 (numArgOperands - skipTrailingOperands));
3945 Type *OutputShadowTy = getShadowTy(OutputVectorTy);
3949 I.getArgOperand(numArgOperands - skipTrailingOperands));
3951 Value *OutputShadowPtr, *OutputOriginPtr;
3953 std::tie(OutputShadowPtr, OutputOriginPtr) = getShadowOriginPtr(
3954 Addr, IRB, OutputShadowTy,
Align(1),
true);
3955 ShadowArgs.
append(1, OutputShadowPtr);
3961 if (MS.TrackOrigins) {
3969 OriginCombiner
OC(
this, IRB);
3970 for (
int i = 0; i < numArgOperands - skipTrailingOperands; i++)
3971 OC.Add(
I.getArgOperand(i));
3974 OC.DoneAndStoreOrigin(
DL.getTypeStoreSize(OutputVectorTy),
3994 unsigned int trailingVerbatimArgs) {
3997 assert(trailingVerbatimArgs <
I.arg_size());
4001 for (
unsigned int i = 0; i <
I.arg_size() - trailingVerbatimArgs; i++) {
4002 Value *Shadow = getShadow(&
I, i);
4006 for (
unsigned int i =
I.arg_size() - trailingVerbatimArgs; i <
I.arg_size();
4008 Value *Arg =
I.getArgOperand(i);
4014 Value *CombinedShadow = CI;
4017 for (
unsigned int i =
I.arg_size() - trailingVerbatimArgs; i <
I.arg_size();
4020 CreateShadowCast(IRB, getShadow(&
I, i), CombinedShadow->
getType());
4021 CombinedShadow = IRB.
CreateOr(Shadow, CombinedShadow,
"_msprop");
4024 setShadow(&
I, CombinedShadow);
4026 setOriginForNaryOp(
I);
4035 switch (
I.getIntrinsicID()) {
4036 case Intrinsic::uadd_with_overflow:
4037 case Intrinsic::sadd_with_overflow:
4038 case Intrinsic::usub_with_overflow:
4039 case Intrinsic::ssub_with_overflow:
4040 case Intrinsic::umul_with_overflow:
4041 case Intrinsic::smul_with_overflow:
4042 handleArithmeticWithOverflow(
I);
4044 case Intrinsic::abs:
4045 handleAbsIntrinsic(
I);
4047 case Intrinsic::is_fpclass:
4050 case Intrinsic::lifetime_start:
4051 handleLifetimeStart(
I);
4053 case Intrinsic::launder_invariant_group:
4054 case Intrinsic::strip_invariant_group:
4055 handleInvariantGroup(
I);
4057 case Intrinsic::bswap:
4060 case Intrinsic::ctlz:
4061 case Intrinsic::cttz:
4062 handleCountZeroes(
I);
4064 case Intrinsic::masked_compressstore:
4065 handleMaskedCompressStore(
I);
4067 case Intrinsic::masked_expandload:
4068 handleMaskedExpandLoad(
I);
4070 case Intrinsic::masked_gather:
4071 handleMaskedGather(
I);
4073 case Intrinsic::masked_scatter:
4074 handleMaskedScatter(
I);
4076 case Intrinsic::masked_store:
4077 handleMaskedStore(
I);
4079 case Intrinsic::masked_load:
4080 handleMaskedLoad(
I);
4082 case Intrinsic::vector_reduce_and:
4083 handleVectorReduceAndIntrinsic(
I);
4085 case Intrinsic::vector_reduce_or:
4086 handleVectorReduceOrIntrinsic(
I);
4088 case Intrinsic::vector_reduce_add:
4089 case Intrinsic::vector_reduce_xor:
4090 case Intrinsic::vector_reduce_mul:
4091 handleVectorReduceIntrinsic(
I);
4093 case Intrinsic::x86_sse_stmxcsr:
4096 case Intrinsic::x86_sse_ldmxcsr:
4099 case Intrinsic::x86_avx512_vcvtsd2usi64:
4100 case Intrinsic::x86_avx512_vcvtsd2usi32:
4101 case Intrinsic::x86_avx512_vcvtss2usi64:
4102 case Intrinsic::x86_avx512_vcvtss2usi32:
4103 case Intrinsic::x86_avx512_cvttss2usi64:
4104 case Intrinsic::x86_avx512_cvttss2usi:
4105 case Intrinsic::x86_avx512_cvttsd2usi64:
4106 case Intrinsic::x86_avx512_cvttsd2usi:
4107 case Intrinsic::x86_avx512_cvtusi2ss:
4108 case Intrinsic::x86_avx512_cvtusi642sd:
4109 case Intrinsic::x86_avx512_cvtusi642ss:
4110 handleVectorConvertIntrinsic(
I, 1,
true);
4112 case Intrinsic::x86_sse2_cvtsd2si64:
4113 case Intrinsic::x86_sse2_cvtsd2si:
4114 case Intrinsic::x86_sse2_cvtsd2ss:
4115 case Intrinsic::x86_sse2_cvttsd2si64:
4116 case Intrinsic::x86_sse2_cvttsd2si:
4117 case Intrinsic::x86_sse_cvtss2si64:
4118 case Intrinsic::x86_sse_cvtss2si:
4119 case Intrinsic::x86_sse_cvttss2si64:
4120 case Intrinsic::x86_sse_cvttss2si:
4121 handleVectorConvertIntrinsic(
I, 1);
4123 case Intrinsic::x86_sse_cvtps2pi:
4124 case Intrinsic::x86_sse_cvttps2pi:
4125 handleVectorConvertIntrinsic(
I, 2);
4128 case Intrinsic::x86_avx512_psll_w_512:
4129 case Intrinsic::x86_avx512_psll_d_512:
4130 case Intrinsic::x86_avx512_psll_q_512:
4131 case Intrinsic::x86_avx512_pslli_w_512:
4132 case Intrinsic::x86_avx512_pslli_d_512:
4133 case Intrinsic::x86_avx512_pslli_q_512:
4134 case Intrinsic::x86_avx512_psrl_w_512:
4135 case Intrinsic::x86_avx512_psrl_d_512:
4136 case Intrinsic::x86_avx512_psrl_q_512:
4137 case Intrinsic::x86_avx512_psra_w_512:
4138 case Intrinsic::x86_avx512_psra_d_512:
4139 case Intrinsic::x86_avx512_psra_q_512:
4140 case Intrinsic::x86_avx512_psrli_w_512:
4141 case Intrinsic::x86_avx512_psrli_d_512:
4142 case Intrinsic::x86_avx512_psrli_q_512:
4143 case Intrinsic::x86_avx512_psrai_w_512:
4144 case Intrinsic::x86_avx512_psrai_d_512:
4145 case Intrinsic::x86_avx512_psrai_q_512:
4146 case Intrinsic::x86_avx512_psra_q_256:
4147 case Intrinsic::x86_avx512_psra_q_128:
4148 case Intrinsic::x86_avx512_psrai_q_256:
4149 case Intrinsic::x86_avx512_psrai_q_128:
4150 case Intrinsic::x86_avx2_psll_w:
4151 case Intrinsic::x86_avx2_psll_d:
4152 case Intrinsic::x86_avx2_psll_q:
4153 case Intrinsic::x86_avx2_pslli_w:
4154 case Intrinsic::x86_avx2_pslli_d:
4155 case Intrinsic::x86_avx2_pslli_q:
4156 case Intrinsic::x86_avx2_psrl_w:
4157 case Intrinsic::x86_avx2_psrl_d:
4158 case Intrinsic::x86_avx2_psrl_q:
4159 case Intrinsic::x86_avx2_psra_w:
4160 case Intrinsic::x86_avx2_psra_d:
4161 case Intrinsic::x86_avx2_psrli_w:
4162 case Intrinsic::x86_avx2_psrli_d:
4163 case Intrinsic::x86_avx2_psrli_q:
4164 case Intrinsic::x86_avx2_psrai_w:
4165 case Intrinsic::x86_avx2_psrai_d:
4166 case Intrinsic::x86_sse2_psll_w:
4167 case Intrinsic::x86_sse2_psll_d:
4168 case Intrinsic::x86_sse2_psll_q:
4169 case Intrinsic::x86_sse2_pslli_w:
4170 case Intrinsic::x86_sse2_pslli_d:
4171 case Intrinsic::x86_sse2_pslli_q:
4172 case Intrinsic::x86_sse2_psrl_w:
4173 case Intrinsic::x86_sse2_psrl_d:
4174 case Intrinsic::x86_sse2_psrl_q:
4175 case Intrinsic::x86_sse2_psra_w:
4176 case Intrinsic::x86_sse2_psra_d:
4177 case Intrinsic::x86_sse2_psrli_w:
4178 case Intrinsic::x86_sse2_psrli_d:
4179 case Intrinsic::x86_sse2_psrli_q:
4180 case Intrinsic::x86_sse2_psrai_w:
4181 case Intrinsic::x86_sse2_psrai_d:
4182 case Intrinsic::x86_mmx_psll_w:
4183 case Intrinsic::x86_mmx_psll_d:
4184 case Intrinsic::x86_mmx_psll_q:
4185 case Intrinsic::x86_mmx_pslli_w:
4186 case Intrinsic::x86_mmx_pslli_d:
4187 case Intrinsic::x86_mmx_pslli_q:
4188 case Intrinsic::x86_mmx_psrl_w:
4189 case Intrinsic::x86_mmx_psrl_d:
4190 case Intrinsic::x86_mmx_psrl_q:
4191 case Intrinsic::x86_mmx_psra_w:
4192 case Intrinsic::x86_mmx_psra_d:
4193 case Intrinsic::x86_mmx_psrli_w:
4194 case Intrinsic::x86_mmx_psrli_d:
4195 case Intrinsic::x86_mmx_psrli_q:
4196 case Intrinsic::x86_mmx_psrai_w:
4197 case Intrinsic::x86_mmx_psrai_d:
4198 case Intrinsic::aarch64_neon_rshrn:
4199 case Intrinsic::aarch64_neon_sqrshl:
4200 case Intrinsic::aarch64_neon_sqrshrn:
4201 case Intrinsic::aarch64_neon_sqrshrun:
4202 case Intrinsic::aarch64_neon_sqshl:
4203 case Intrinsic::aarch64_neon_sqshlu:
4204 case Intrinsic::aarch64_neon_sqshrn:
4205 case Intrinsic::aarch64_neon_sqshrun:
4206 case Intrinsic::aarch64_neon_srshl:
4207 case Intrinsic::aarch64_neon_sshl:
4208 case Intrinsic::aarch64_neon_uqrshl:
4209 case Intrinsic::aarch64_neon_uqrshrn:
4210 case Intrinsic::aarch64_neon_uqshl:
4211 case Intrinsic::aarch64_neon_uqshrn:
4212 case Intrinsic::aarch64_neon_urshl:
4213 case Intrinsic::aarch64_neon_ushl:
4215 handleVectorShiftIntrinsic(
I,
false);
4217 case Intrinsic::x86_avx2_psllv_d:
4218 case Intrinsic::x86_avx2_psllv_d_256:
4219 case Intrinsic::x86_avx512_psllv_d_512:
4220 case Intrinsic::x86_avx2_psllv_q:
4221 case Intrinsic::x86_avx2_psllv_q_256:
4222 case Intrinsic::x86_avx512_psllv_q_512:
4223 case Intrinsic::x86_avx2_psrlv_d:
4224 case Intrinsic::x86_avx2_psrlv_d_256:
4225 case Intrinsic::x86_avx512_psrlv_d_512:
4226 case Intrinsic::x86_avx2_psrlv_q:
4227 case Intrinsic::x86_avx2_psrlv_q_256:
4228 case Intrinsic::x86_avx512_psrlv_q_512:
4229 case Intrinsic::x86_avx2_psrav_d:
4230 case Intrinsic::x86_avx2_psrav_d_256:
4231 case Intrinsic::x86_avx512_psrav_d_512:
4232 case Intrinsic::x86_avx512_psrav_q_128:
4233 case Intrinsic::x86_avx512_psrav_q_256:
4234 case Intrinsic::x86_avx512_psrav_q_512:
4235 handleVectorShiftIntrinsic(
I,
true);
4238 case Intrinsic::x86_sse2_packsswb_128:
4239 case Intrinsic::x86_sse2_packssdw_128:
4240 case Intrinsic::x86_sse2_packuswb_128:
4241 case Intrinsic::x86_sse41_packusdw:
4242 case Intrinsic::x86_avx2_packsswb:
4243 case Intrinsic::x86_avx2_packssdw:
4244 case Intrinsic::x86_avx2_packuswb:
4245 case Intrinsic::x86_avx2_packusdw:
4246 handleVectorPackIntrinsic(
I);
4249 case Intrinsic::x86_sse41_pblendvb:
4250 case Intrinsic::x86_sse41_blendvpd:
4251 case Intrinsic::x86_sse41_blendvps:
4252 case Intrinsic::x86_avx_blendv_pd_256:
4253 case Intrinsic::x86_avx_blendv_ps_256:
4254 case Intrinsic::x86_avx2_pblendvb:
4255 handleBlendvIntrinsic(
I);
4258 case Intrinsic::x86_avx_dp_ps_256:
4259 case Intrinsic::x86_sse41_dppd:
4260 case Intrinsic::x86_sse41_dpps:
4261 handleDppIntrinsic(
I);
4264 case Intrinsic::x86_mmx_packsswb:
4265 case Intrinsic::x86_mmx_packuswb:
4266 handleVectorPackIntrinsic(
I, 16);
4269 case Intrinsic::x86_mmx_packssdw:
4270 handleVectorPackIntrinsic(
I, 32);
4273 case Intrinsic::x86_mmx_psad_bw:
4274 handleVectorSadIntrinsic(
I,
true);
4276 case Intrinsic::x86_sse2_psad_bw:
4277 case Intrinsic::x86_avx2_psad_bw:
4278 handleVectorSadIntrinsic(
I);
4281 case Intrinsic::x86_sse2_pmadd_wd:
4282 case Intrinsic::x86_avx2_pmadd_wd:
4283 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
4284 case Intrinsic::x86_avx2_pmadd_ub_sw:
4285 handleVectorPmaddIntrinsic(
I);
4288 case Intrinsic::x86_ssse3_pmadd_ub_sw:
4289 handleVectorPmaddIntrinsic(
I, 8);
4292 case Intrinsic::x86_mmx_pmadd_wd:
4293 handleVectorPmaddIntrinsic(
I, 16);
4296 case Intrinsic::x86_sse_cmp_ss:
4297 case Intrinsic::x86_sse2_cmp_sd:
4298 case Intrinsic::x86_sse_comieq_ss:
4299 case Intrinsic::x86_sse_comilt_ss:
4300 case Intrinsic::x86_sse_comile_ss:
4301 case Intrinsic::x86_sse_comigt_ss:
4302 case Intrinsic::x86_sse_comige_ss:
4303 case Intrinsic::x86_sse_comineq_ss:
4304 case Intrinsic::x86_sse_ucomieq_ss:
4305 case Intrinsic::x86_sse_ucomilt_ss:
4306 case Intrinsic::x86_sse_ucomile_ss:
4307 case Intrinsic::x86_sse_ucomigt_ss:
4308 case Intrinsic::x86_sse_ucomige_ss:
4309 case Intrinsic::x86_sse_ucomineq_ss:
4310 case Intrinsic::x86_sse2_comieq_sd:
4311 case Intrinsic::x86_sse2_comilt_sd:
4312 case Intrinsic::x86_sse2_comile_sd:
4313 case Intrinsic::x86_sse2_comigt_sd:
4314 case Intrinsic::x86_sse2_comige_sd:
4315 case Intrinsic::x86_sse2_comineq_sd:
4316 case Intrinsic::x86_sse2_ucomieq_sd:
4317 case Intrinsic::x86_sse2_ucomilt_sd:
4318 case Intrinsic::x86_sse2_ucomile_sd:
4319 case Intrinsic::x86_sse2_ucomigt_sd:
4320 case Intrinsic::x86_sse2_ucomige_sd:
4321 case Intrinsic::x86_sse2_ucomineq_sd:
4322 handleVectorCompareScalarIntrinsic(
I);
4325 case Intrinsic::x86_avx_cmp_pd_256:
4326 case Intrinsic::x86_avx_cmp_ps_256:
4327 case Intrinsic::x86_sse2_cmp_pd:
4328 case Intrinsic::x86_sse_cmp_ps:
4329 handleVectorComparePackedIntrinsic(
I);
4332 case Intrinsic::x86_bmi_bextr_32:
4333 case Intrinsic::x86_bmi_bextr_64:
4334 case Intrinsic::x86_bmi_bzhi_32:
4335 case Intrinsic::x86_bmi_bzhi_64:
4336 case Intrinsic::x86_bmi_pdep_32:
4337 case Intrinsic::x86_bmi_pdep_64:
4338 case Intrinsic::x86_bmi_pext_32:
4339 case Intrinsic::x86_bmi_pext_64:
4340 handleBmiIntrinsic(
I);
4343 case Intrinsic::x86_pclmulqdq:
4344 case Intrinsic::x86_pclmulqdq_256:
4345 case Intrinsic::x86_pclmulqdq_512:
4346 handlePclmulIntrinsic(
I);
4349 case Intrinsic::x86_avx_round_pd_256:
4350 case Intrinsic::x86_avx_round_ps_256:
4351 case Intrinsic::x86_sse41_round_pd:
4352 case Intrinsic::x86_sse41_round_ps:
4353 handleRoundPdPsIntrinsic(
I);
4356 case Intrinsic::x86_sse41_round_sd:
4357 case Intrinsic::x86_sse41_round_ss:
4358 handleUnarySdSsIntrinsic(
I);
4361 case Intrinsic::x86_sse2_max_sd:
4362 case Intrinsic::x86_sse_max_ss:
4363 case Intrinsic::x86_sse2_min_sd:
4364 case Intrinsic::x86_sse_min_ss:
4365 handleBinarySdSsIntrinsic(
I);
4368 case Intrinsic::x86_avx_vtestc_pd:
4369 case Intrinsic::x86_avx_vtestc_pd_256:
4370 case Intrinsic::x86_avx_vtestc_ps:
4371 case Intrinsic::x86_avx_vtestc_ps_256:
4372 case Intrinsic::x86_avx_vtestnzc_pd:
4373 case Intrinsic::x86_avx_vtestnzc_pd_256:
4374 case Intrinsic::x86_avx_vtestnzc_ps:
4375 case Intrinsic::x86_avx_vtestnzc_ps_256:
4376 case Intrinsic::x86_avx_vtestz_pd:
4377 case Intrinsic::x86_avx_vtestz_pd_256:
4378 case Intrinsic::x86_avx_vtestz_ps:
4379 case Intrinsic::x86_avx_vtestz_ps_256:
4380 case Intrinsic::x86_avx_ptestc_256:
4381 case Intrinsic::x86_avx_ptestnzc_256:
4382 case Intrinsic::x86_avx_ptestz_256:
4383 case Intrinsic::x86_sse41_ptestc:
4384 case Intrinsic::x86_sse41_ptestnzc:
4385 case Intrinsic::x86_sse41_ptestz:
4386 handleVtestIntrinsic(
I);
4389 case Intrinsic::fshl:
4390 case Intrinsic::fshr:
4391 handleFunnelShift(
I);
4394 case Intrinsic::is_constant:
4396 setShadow(&
I, getCleanShadow(&
I));
4397 setOrigin(&
I, getCleanOrigin());
4400 case Intrinsic::aarch64_neon_st1x2:
4401 case Intrinsic::aarch64_neon_st1x3:
4402 case Intrinsic::aarch64_neon_st1x4:
4403 case Intrinsic::aarch64_neon_st2:
4404 case Intrinsic::aarch64_neon_st3:
4405 case Intrinsic::aarch64_neon_st4: {
4406 handleNEONVectorStoreIntrinsic(
I,
false);
4410 case Intrinsic::aarch64_neon_st2lane:
4411 case Intrinsic::aarch64_neon_st3lane:
4412 case Intrinsic::aarch64_neon_st4lane: {
4413 handleNEONVectorStoreIntrinsic(
I,
true);
4426 case Intrinsic::aarch64_neon_tbl1:
4427 case Intrinsic::aarch64_neon_tbl2:
4428 case Intrinsic::aarch64_neon_tbl3:
4429 case Intrinsic::aarch64_neon_tbl4:
4430 case Intrinsic::aarch64_neon_tbx1:
4431 case Intrinsic::aarch64_neon_tbx2:
4432 case Intrinsic::aarch64_neon_tbx3:
4433 case Intrinsic::aarch64_neon_tbx4: {
4435 handleIntrinsicByApplyingToShadow(
I, 1);
4439 case Intrinsic::aarch64_neon_fmulx:
4440 case Intrinsic::aarch64_neon_pmul:
4441 case Intrinsic::aarch64_neon_pmull:
4442 case Intrinsic::aarch64_neon_smull:
4443 case Intrinsic::aarch64_neon_pmull64:
4444 case Intrinsic::aarch64_neon_umull: {
4445 handleNEONVectorMultiplyIntrinsic(
I);
4450 if (!handleUnknownIntrinsic(
I))
4451 visitInstruction(
I);
4456 void visitLibAtomicLoad(
CallBase &CB) {
4458 assert(isa<CallInst>(CB));
4467 Value *NewOrdering =
4471 NextNodeIRBuilder NextIRB(&CB);
4472 Value *SrcShadowPtr, *SrcOriginPtr;
4473 std::tie(SrcShadowPtr, SrcOriginPtr) =
4474 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4476 Value *DstShadowPtr =
4477 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4481 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
4482 if (MS.TrackOrigins) {
4483 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
4485 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
4486 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
4490 void visitLibAtomicStore(
CallBase &CB) {
4497 Value *NewOrdering =
4501 Value *DstShadowPtr =
4519 visitAsmInstruction(CB);
4521 visitInstruction(CB);
4530 case LibFunc_atomic_load:
4531 if (!isa<CallInst>(CB)) {
4532 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4536 visitLibAtomicLoad(CB);
4538 case LibFunc_atomic_store:
4539 visitLibAtomicStore(CB);
4546 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4547 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4555 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4557 Call->removeFnAttrs(
B);
4559 Func->removeFnAttrs(
B);
4565 bool MayCheckCall = MS.EagerChecks;
4569 MayCheckCall &= !
Func->getName().starts_with(
"__sanitizer_unaligned_");
4572 unsigned ArgOffset = 0;
4575 if (!
A->getType()->isSized()) {
4576 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4580 if (
A->getType()->isScalableTy()) {
4581 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is vscale: " << CB <<
"\n");
4583 insertShadowCheck(
A, &CB);
4592 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4595 insertShadowCheck(
A, &CB);
4596 Size =
DL.getTypeAllocSize(
A->getType());
4602 Value *ArgShadow = getShadow(
A);
4603 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4605 <<
" Shadow: " << *ArgShadow <<
"\n");
4609 assert(
A->getType()->isPointerTy() &&
4610 "ByVal argument is not a pointer!");
4618 Value *AShadowPtr, *AOriginPtr;
4619 std::tie(AShadowPtr, AOriginPtr) =
4620 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4622 if (!PropagateShadow) {
4629 if (MS.TrackOrigins) {
4630 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4644 Size =
DL.getTypeAllocSize(
A->getType());
4649 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4650 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4652 getOriginPtrForArgument(IRB, ArgOffset));
4656 assert(Store !=
nullptr);
4665 if (FT->isVarArg()) {
4666 VAHelper->visitCallBase(CB, IRB);
4673 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4676 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4677 setShadow(&CB, getCleanShadow(&CB));
4678 setOrigin(&CB, getCleanOrigin());
4684 Value *
Base = getShadowPtrForRetval(IRBBefore);
4685 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4688 if (isa<CallInst>(CB)) {
4692 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4697 setShadow(&CB, getCleanShadow(&CB));
4698 setOrigin(&CB, getCleanOrigin());
4705 "Could not find insertion point for retval shadow load");
4708 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4711 setShadow(&CB, RetvalShadow);
4712 if (MS.TrackOrigins)
4713 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy, getOriginPtrForRetval()));
4717 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4718 RetVal =
I->getOperand(0);
4720 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4721 return I->isMustTailCall();
4728 Value *RetVal =
I.getReturnValue();
4734 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4735 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4736 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4739 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4741 Value *Shadow = getShadow(RetVal);
4742 bool StoreOrigin =
true;
4744 insertShadowCheck(RetVal, &
I);
4745 Shadow = getCleanShadow(RetVal);
4746 StoreOrigin =
false;
4753 if (MS.TrackOrigins && StoreOrigin)
4754 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4760 if (!PropagateShadow) {
4761 setShadow(&
I, getCleanShadow(&
I));
4762 setOrigin(&
I, getCleanOrigin());
4766 ShadowPHINodes.push_back(&
I);
4767 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4769 if (MS.TrackOrigins)
4771 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4788 IRB.
CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4790 Value *ShadowBase, *OriginBase;
4791 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4798 if (PoisonStack && MS.TrackOrigins) {
4799 Value *Idptr = getLocalVarIdptr(
I);
4801 Value *Descr = getLocalVarDescription(
I);
4802 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4803 {&I, Len, Idptr, Descr});
4805 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4811 Value *Descr = getLocalVarDescription(
I);
4813 IRB.
CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4815 IRB.
CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4822 NextNodeIRBuilder IRB(InsPoint);
4824 TypeSize TS =
DL.getTypeAllocSize(
I.getAllocatedType());
4826 if (
I.isArrayAllocation())
4830 if (MS.CompileKernel)
4831 poisonAllocaKmsan(
I, IRB, Len);
4833 poisonAllocaUserspace(
I, IRB, Len);
4837 setShadow(&
I, getCleanShadow(&
I));
4838 setOrigin(&
I, getCleanOrigin());
4850 handleSelectLikeInst(
I,
B,
C,
D);
4856 Value *Sb = getShadow(
B);
4857 Value *Sc = getShadow(
C);
4858 Value *Sd = getShadow(
D);
4860 Value *Ob = MS.TrackOrigins ? getOrigin(
B) : nullptr;
4861 Value *Oc = MS.TrackOrigins ? getOrigin(
C) : nullptr;
4862 Value *Od = MS.TrackOrigins ? getOrigin(
D) : nullptr;
4867 if (
I.getType()->isAggregateType()) {
4871 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4879 C = CreateAppToShadowCast(IRB,
C);
4880 D = CreateAppToShadowCast(IRB,
D);
4887 if (MS.TrackOrigins) {
4890 if (
B->getType()->isVectorTy()) {
4891 B = convertToBool(
B, IRB);
4892 Sb = convertToBool(Sb, IRB);
4903 setShadow(&
I, getCleanShadow(&
I));
4904 setOrigin(&
I, getCleanOrigin());
4908 setShadow(&
I, getCleanShadow(&
I));
4909 setOrigin(&
I, getCleanOrigin());
4913 setShadow(&
I, getCleanShadow(&
I));
4914 setOrigin(&
I, getCleanOrigin());
4921 Value *Agg =
I.getAggregateOperand();
4923 Value *AggShadow = getShadow(Agg);
4927 setShadow(&
I, ResShadow);
4928 setOriginForNaryOp(
I);
4934 Value *AggShadow = getShadow(
I.getAggregateOperand());
4935 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4941 setOriginForNaryOp(
I);
4945 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4948 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4950 errs() <<
"QQQ " <<
I <<
"\n";
4977 insertShadowCheck(Operand, &
I);