177#include "llvm/IR/IntrinsicsX86.h"
206#define DEBUG_TYPE "msan"
209 "Controls which checks to insert");
227 "msan-track-origins",
232 cl::desc(
"keep going after reporting a UMR"),
241 "msan-poison-stack-with-call",
246 "msan-poison-stack-pattern",
247 cl::desc(
"poison uninitialized stack variables with the given pattern"),
252 cl::desc(
"Print name of local stack variable"),
261 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
266 cl::desc(
"exact handling of relational integer ICmp"),
270 "msan-handle-lifetime-intrinsics",
272 "when possible, poison scoped variables at the beginning of the scope "
273 "(slower, but more precise)"),
287 "msan-handle-asm-conservative",
298 "msan-check-access-address",
299 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
304 cl::desc(
"check arguments and return values at function call boundaries"),
308 "msan-dump-strict-instructions",
309 cl::desc(
"print out instructions with default strict semantics"),
313 "msan-instrumentation-with-call-threshold",
315 "If the function being instrumented requires more than "
316 "this number of checks and origin stores, use callbacks instead of "
317 "inline checks (-1 means never use callbacks)."),
322 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
332 cl::desc(
"Insert checks for constant shadow values"),
339 cl::desc(
"Place MSan constructors in comdat sections"),
345 cl::desc(
"Define custom MSan AndMask"),
349 cl::desc(
"Define custom MSan XorMask"),
353 cl::desc(
"Define custom MSan ShadowBase"),
357 cl::desc(
"Define custom MSan OriginBase"),
362 cl::desc(
"Define threshold for number of checks per "
363 "debug location to force origin update."),
375struct MemoryMapParams {
382struct PlatformMemoryMapParams {
383 const MemoryMapParams *bits32;
384 const MemoryMapParams *bits64;
517class MemorySanitizer {
526 MemorySanitizer(MemorySanitizer &&) =
delete;
527 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
528 MemorySanitizer(
const MemorySanitizer &) =
delete;
529 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
534 friend struct MemorySanitizerVisitor;
535 friend struct VarArgAMD64Helper;
536 friend struct VarArgMIPS64Helper;
537 friend struct VarArgAArch64Helper;
538 friend struct VarArgPowerPC64Helper;
539 friend struct VarArgSystemZHelper;
541 void initializeModule(
Module &M);
567 Value *ParamOriginTLS;
573 Value *RetvalOriginTLS;
581 Value *VAArgOriginTLS;
585 Value *VAArgOverflowSizeTLS;
588 bool CallbacksInitialized =
false;
634 const MemoryMapParams *MapParams;
638 MemoryMapParams CustomMapParams;
643 MDNode *OriginStoreWeights;
646void insertModuleCtor(
Module &M) {
674 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
689 MemorySanitizer Msan(*
F.getParent(),
Options);
708 OS, MapClassName2PassName);
715 OS <<
"eager-checks;";
716 OS <<
"track-origins=" <<
Options.TrackOrigins;
738 RetvalOriginTLS =
nullptr;
740 ParamOriginTLS =
nullptr;
742 VAArgOriginTLS =
nullptr;
743 VAArgOverflowSizeTLS =
nullptr;
745 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
747 IRB.getVoidTy(), IRB.getInt32Ty());
758 MsanGetContextStateFn =
M.getOrInsertFunction(
764 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
765 std::string name_load =
766 "__msan_metadata_ptr_for_load_" + std::to_string(size);
767 std::string name_store =
768 "__msan_metadata_ptr_for_store_" + std::to_string(size);
769 MsanMetadataPtrForLoad_1_8[ind] =
M.getOrInsertFunction(
771 MsanMetadataPtrForStore_1_8[ind] =
M.getOrInsertFunction(
775 MsanMetadataPtrForLoadN =
M.getOrInsertFunction(
776 "__msan_metadata_ptr_for_load_n",
RetTy,
778 MsanMetadataPtrForStoreN =
M.getOrInsertFunction(
779 "__msan_metadata_ptr_for_store_n",
RetTy,
784 M.getOrInsertFunction(
"__msan_poison_alloca", IRB.getVoidTy(),
785 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy());
786 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
787 "__msan_unpoison_alloca", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
791 return M.getOrInsertGlobal(
Name, Ty, [&] {
793 nullptr,
Name,
nullptr,
806 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
807 :
"__msan_warning_with_origin_noreturn";
808 WarningFn =
M.getOrInsertFunction(WarningFnName,
810 IRB.getVoidTy(), IRB.getInt32Ty());
813 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
814 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
840 VAArgOverflowSizeTLS =
845 unsigned AccessSize = 1 << AccessSizeIndex;
846 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
847 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
849 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
851 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
852 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
854 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt8PtrTy(),
858 MsanSetAllocaOriginWithDescriptionFn =
M.getOrInsertFunction(
859 "__msan_set_alloca_origin_with_descr", IRB.getVoidTy(),
860 IRB.getInt8PtrTy(), IntptrTy, IRB.getInt8PtrTy(), IRB.getInt8PtrTy());
861 MsanSetAllocaOriginNoDescriptionFn =
M.getOrInsertFunction(
862 "__msan_set_alloca_origin_no_descr", IRB.getVoidTy(), IRB.getInt8PtrTy(),
863 IntptrTy, IRB.getInt8PtrTy());
864 MsanPoisonStackFn =
M.getOrInsertFunction(
865 "__msan_poison_stack", IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy);
871 if (CallbacksInitialized)
877 MsanChainOriginFn =
M.getOrInsertFunction(
878 "__msan_chain_origin",
881 MsanSetOriginFn =
M.getOrInsertFunction(
883 IRB.getVoidTy(), IRB.getInt8PtrTy(), IntptrTy, IRB.getInt32Ty());
885 M.getOrInsertFunction(
"__msan_memmove", IRB.getInt8PtrTy(),
886 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
888 M.getOrInsertFunction(
"__msan_memcpy", IRB.getInt8PtrTy(),
889 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
890 MemsetFn =
M.getOrInsertFunction(
892 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IRB.getInt32Ty(), IntptrTy);
894 MsanInstrumentAsmStoreFn =
895 M.getOrInsertFunction(
"__msan_instrument_asm_store", IRB.getVoidTy(),
899 createKernelApi(M, TLI);
901 createUserspaceApi(M, TLI);
903 CallbacksInitialized =
true;
909 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
927void MemorySanitizer::initializeModule(
Module &M) {
928 auto &
DL =
M.getDataLayout();
930 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
931 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
933 if (ShadowPassed || OriginPassed) {
938 MapParams = &CustomMapParams;
940 Triple TargetTriple(
M.getTargetTriple());
941 switch (TargetTriple.getOS()) {
943 switch (TargetTriple.getArch()) {
958 switch (TargetTriple.getArch()) {
967 switch (TargetTriple.getArch()) {
998 C = &(
M.getContext());
1000 IntptrTy = IRB.getIntPtrTy(
DL);
1001 OriginTy = IRB.getInt32Ty();
1006 if (!CompileKernel) {
1008 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1009 return new GlobalVariable(
1010 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1011 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1015 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1016 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1017 GlobalValue::WeakODRLinkage,
1018 IRB.getInt32(Recover),
"__msan_keep_going");
1033struct VarArgHelper {
1034 virtual ~VarArgHelper() =
default;
1049 virtual void finalizeInstrumentation() = 0;
1052struct MemorySanitizerVisitor;
1057 MemorySanitizerVisitor &Visitor);
1064 if (TypeSizeFixed <= 8)
1073class NextNodeIRBuilder :
public IRBuilder<> {
1086struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1088 MemorySanitizer &MS;
1091 std::unique_ptr<VarArgHelper> VAHelper;
1098 bool PropagateShadow;
1102 struct ShadowOriginAndInsertPoint {
1108 : Shadow(S), Origin(
O), OrigIns(
I) {}
1116 int64_t SplittableBlocksCount = 0;
1118 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1121 bool SanitizeFunction =
1123 InsertChecks = SanitizeFunction;
1124 PropagateShadow = SanitizeFunction;
1134 MS.initializeCallbacks(*
F.getParent(), TLI);
1135 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1138 if (MS.CompileKernel) {
1140 insertKmsanPrologue(IRB);
1144 <<
"MemorySanitizer is not inserting checks into '"
1145 <<
F.getName() <<
"'\n");
1148 bool instrumentWithCalls(
Value *V) {
1150 if (isa<Constant>(V))
1153 ++SplittableBlocksCount;
1159 return I.getParent() == FnPrologueEnd->
getParent() &&
1160 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1168 if (MS.TrackOrigins <= 1)
1170 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1175 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1188 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1189 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1194 Align CurrentAlignment = Alignment;
1195 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1196 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1197 Value *IntptrOriginPtr =
1199 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1204 CurrentAlignment = IntptrAlignment;
1221 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1222 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1230 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1239 if (instrumentWithCalls(ConvertedShadow) &&
1242 Value *ConvertedShadow2 =
1245 Fn, {ConvertedShadow2,
1250 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1254 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1259 void materializeStores() {
1262 Value *Val =
SI->getValueOperand();
1264 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1265 Value *ShadowPtr, *OriginPtr;
1267 const Align Alignment =
SI->getAlign();
1269 std::tie(ShadowPtr, OriginPtr) =
1270 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1279 if (MS.TrackOrigins && !
SI->isAtomic())
1280 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1287 if (MS.TrackOrigins < 2)
1290 if (LazyWarningDebugLocationCount.
empty())
1291 for (
const auto &
I : InstrumentationList)
1292 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1306 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1308 auto NewDebugLoc = OI->getDebugLoc();
1315 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1316 Origin = updateOrigin(Origin, IRBOrigin);
1321 if (MS.CompileKernel || MS.TrackOrigins)
1335 if (instrumentWithCalls(ConvertedShadow) &&
1338 Value *ConvertedShadow2 =
1341 Fn, {ConvertedShadow2,
1342 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1346 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1349 !MS.Recover, MS.ColdCallWeights);
1352 insertWarningFn(IRB, Origin);
1357 void materializeInstructionChecks(
1362 bool Combine = !MS.TrackOrigins;
1364 Value *Shadow =
nullptr;
1365 for (
const auto &ShadowData : InstructionChecks) {
1369 Value *ConvertedShadow = ShadowData.Shadow;
1371 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1378 insertWarningFn(IRB, ShadowData.Origin);
1388 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1393 Shadow = ConvertedShadow;
1397 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1398 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1399 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1405 materializeOneCheck(IRB, Shadow,
nullptr);
1409 void materializeChecks() {
1411 [](
const ShadowOriginAndInsertPoint &L,
1412 const ShadowOriginAndInsertPoint &R) {
1413 return L.OrigIns <
R.OrigIns;
1416 for (
auto I = InstrumentationList.begin();
1417 I != InstrumentationList.end();) {
1419 std::find_if(
I + 1, InstrumentationList.end(),
1420 [L =
I->OrigIns](
const ShadowOriginAndInsertPoint &R) {
1421 return L != R.OrigIns;
1435 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1436 {Zero, IRB.getInt32(0)},
"param_shadow");
1437 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1438 {Zero, IRB.getInt32(1)},
"retval_shadow");
1439 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1440 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1441 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1442 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1443 MS.VAArgOverflowSizeTLS =
1444 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1445 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1446 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1447 {Zero, IRB.getInt32(5)},
"param_origin");
1448 MS.RetvalOriginTLS =
1449 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1450 {Zero, IRB.getInt32(6)},
"retval_origin");
1462 for (
PHINode *PN : ShadowPHINodes) {
1463 PHINode *PNS = cast<PHINode>(getShadow(PN));
1464 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1465 size_t NumValues = PN->getNumIncomingValues();
1466 for (
size_t v = 0;
v < NumValues;
v++) {
1467 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1469 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1473 VAHelper->finalizeInstrumentation();
1477 if (InstrumentLifetimeStart) {
1478 for (
auto Item : LifetimeStartList) {
1479 instrumentAlloca(*Item.second, Item.first);
1480 AllocaSet.
remove(Item.second);
1486 instrumentAlloca(*AI);
1489 materializeChecks();
1493 materializeStores();
1499 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1511 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1512 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1514 VT->getElementCount());
1516 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1517 return ArrayType::get(getShadowTy(AT->getElementType()),
1518 AT->getNumElements());
1520 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1522 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1523 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1525 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1541 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1543 if (Aggregator != FalseVal)
1544 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1546 Aggregator = ShadowBool;
1555 if (!
Array->getNumElements())
1559 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1563 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1564 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1574 return collapseStructShadow(
Struct, V, IRB);
1575 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1576 return collapseArrayShadow(Array, V, IRB);
1577 if (isa<VectorType>(
V->getType())) {
1579 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1587 Type *VTy =
V->getType();
1589 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1596 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1597 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1598 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1599 VectTy->getElementCount());
1605 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1606 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1607 return VectorType::get(
1608 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1609 VectTy->getElementCount());
1611 assert(IntPtrTy == MS.IntptrTy);
1616 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1618 VectTy->getElementCount(), constToIntPtr(VectTy->getElementType(),
C));
1620 assert(IntPtrTy == MS.IntptrTy);
1632 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1635 if (
uint64_t AndMask = MS.MapParams->AndMask)
1636 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1638 if (
uint64_t XorMask = MS.MapParams->XorMask)
1639 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1651 std::pair<Value *, Value *>
1654 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1655 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1656 Value *ShadowLong = ShadowOffset;
1657 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1659 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1662 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1664 Value *OriginPtr =
nullptr;
1665 if (MS.TrackOrigins) {
1666 Value *OriginLong = ShadowOffset;
1667 uint64_t OriginBase = MS.MapParams->OriginBase;
1668 if (OriginBase != 0)
1670 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1673 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1676 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1678 return std::make_pair(ShadowPtr, OriginPtr);
1681 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1685 Value *ShadowOriginPtrs;
1693 ShadowOriginPtrs = IRB.
CreateCall(Getter, AddrCast);
1697 : MS.MsanMetadataPtrForLoadN,
1698 {AddrCast, SizeVal});
1704 return std::make_pair(ShadowPtr, OriginPtr);
1710 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1717 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1721 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1722 Value *ShadowPtrs = ConstantInt::getNullValue(
1724 Value *OriginPtrs =
nullptr;
1725 if (MS.TrackOrigins)
1726 OriginPtrs = ConstantInt::getNullValue(
1728 for (
unsigned i = 0; i < NumElements; ++i) {
1731 auto [ShadowPtr, OriginPtr] =
1732 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1736 if (MS.TrackOrigins)
1740 return {ShadowPtrs, OriginPtrs};
1747 if (MS.CompileKernel)
1748 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1749 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1765 if (!MS.TrackOrigins)
1777 PointerType::get(getShadowTy(
A), 0),
"_msret");
1783 return MS.RetvalOriginTLS;
1788 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1789 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1794 if (!MS.TrackOrigins)
1796 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1797 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1798 OriginMap[
V] = Origin;
1802 Type *ShadowTy = getShadowTy(OrigTy);
1812 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1817 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1819 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1821 getPoisonedShadow(AT->getElementType()));
1824 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1826 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1827 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1835 Type *ShadowTy = getShadowTy(V);
1838 return getPoisonedShadow(ShadowTy);
1850 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1851 return getCleanShadow(V);
1853 Value *Shadow = ShadowMap[
V];
1855 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1857 assert(Shadow &&
"No shadow for a value");
1861 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1862 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1863 : getCleanShadow(V);
1868 if (
Argument *
A = dyn_cast<Argument>(V)) {
1870 Value *&ShadowPtr = ShadowMap[
V];
1875 unsigned ArgOffset = 0;
1877 for (
auto &FArg :
F->args()) {
1878 if (!FArg.getType()->isSized()) {
1883 unsigned Size = FArg.hasByValAttr()
1884 ?
DL.getTypeAllocSize(FArg.getParamByValType())
1885 :
DL.getTypeAllocSize(FArg.getType());
1889 if (FArg.hasByValAttr()) {
1893 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
1894 FArg.getParamAlign(), FArg.getParamByValType());
1895 Value *CpShadowPtr, *CpOriginPtr;
1896 std::tie(CpShadowPtr, CpOriginPtr) =
1897 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1899 if (!PropagateShadow || Overflow) {
1901 EntryIRB.CreateMemSet(
1905 Value *
Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1907 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
1912 if (MS.TrackOrigins) {
1914 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1918 EntryIRB.CreateMemCpy(
1927 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
1928 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
1929 ShadowPtr = getCleanShadow(V);
1930 setOrigin(
A, getCleanOrigin());
1933 Value *
Base = getShadowPtrForArgument(&FArg, EntryIRB, ArgOffset);
1934 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
1936 if (MS.TrackOrigins) {
1938 getOriginPtrForArgument(&FArg, EntryIRB, ArgOffset);
1939 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
1943 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
1949 assert(ShadowPtr &&
"Could not find shadow for an argument");
1953 return getCleanShadow(V);
1958 return getShadow(
I->getOperand(i));
1963 if (!MS.TrackOrigins)
1965 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
1966 return getCleanOrigin();
1967 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
1968 "Unexpected value type in getOrigin()");
1970 if (
I->getMetadata(LLVMContext::MD_nosanitize))
1971 return getCleanOrigin();
1973 Value *Origin = OriginMap[
V];
1974 assert(Origin &&
"Missing origin");
1980 return getOrigin(
I->getOperand(i));
1993 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
1994 << *OrigIns <<
"\n");
1999 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2000 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2001 "Can only insert checks for integer, vector, and aggregate shadow "
2004 InstrumentationList.push_back(
2005 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2014 Value *Shadow, *Origin;
2016 Shadow = getShadow(Val);
2019 Origin = getOrigin(Val);
2021 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2024 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2026 insertShadowCheck(Shadow, Origin, OrigIns);
2031 case AtomicOrdering::NotAtomic:
2032 return AtomicOrdering::NotAtomic;
2033 case AtomicOrdering::Unordered:
2034 case AtomicOrdering::Monotonic:
2035 case AtomicOrdering::Release:
2036 return AtomicOrdering::Release;
2037 case AtomicOrdering::Acquire:
2038 case AtomicOrdering::AcquireRelease:
2039 return AtomicOrdering::AcquireRelease;
2040 case AtomicOrdering::SequentiallyConsistent:
2041 return AtomicOrdering::SequentiallyConsistent;
2047 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2048 uint32_t OrderingTable[NumOrderings] = {};
2050 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2051 OrderingTable[(
int)AtomicOrderingCABI::release] =
2052 (int)AtomicOrderingCABI::release;
2053 OrderingTable[(int)AtomicOrderingCABI::consume] =
2054 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2055 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2056 (
int)AtomicOrderingCABI::acq_rel;
2057 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2058 (
int)AtomicOrderingCABI::seq_cst;
2061 ArrayRef(OrderingTable, NumOrderings));
2066 case AtomicOrdering::NotAtomic:
2067 return AtomicOrdering::NotAtomic;
2068 case AtomicOrdering::Unordered:
2069 case AtomicOrdering::Monotonic:
2070 case AtomicOrdering::Acquire:
2071 return AtomicOrdering::Acquire;
2072 case AtomicOrdering::Release:
2073 case AtomicOrdering::AcquireRelease:
2074 return AtomicOrdering::AcquireRelease;
2075 case AtomicOrdering::SequentiallyConsistent:
2076 return AtomicOrdering::SequentiallyConsistent;
2082 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2083 uint32_t OrderingTable[NumOrderings] = {};
2085 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2086 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2087 OrderingTable[(int)AtomicOrderingCABI::consume] =
2088 (
int)AtomicOrderingCABI::acquire;
2089 OrderingTable[(int)AtomicOrderingCABI::release] =
2090 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2091 (int)AtomicOrderingCABI::acq_rel;
2092 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2093 (
int)AtomicOrderingCABI::seq_cst;
2096 ArrayRef(OrderingTable, NumOrderings));
2102 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2105 if (isInPrologue(
I))
2115 assert(
I.getType()->isSized() &&
"Load type must have size");
2116 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2117 NextNodeIRBuilder IRB(&
I);
2118 Type *ShadowTy = getShadowTy(&
I);
2120 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2121 const Align Alignment =
I.getAlign();
2122 if (PropagateShadow) {
2123 std::tie(ShadowPtr, OriginPtr) =
2124 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2128 setShadow(&
I, getCleanShadow(&
I));
2132 insertShadowCheck(
I.getPointerOperand(), &
I);
2137 if (MS.TrackOrigins) {
2138 if (PropagateShadow) {
2143 setOrigin(&
I, getCleanOrigin());
2153 StoreList.push_back(&
I);
2155 insertShadowCheck(
I.getPointerOperand(), &
I);
2159 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2163 Value *Val =
I.getOperand(1);
2164 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2169 insertShadowCheck(
Addr, &
I);
2174 if (isa<AtomicCmpXchgInst>(
I))
2175 insertShadowCheck(Val, &
I);
2179 setShadow(&
I, getCleanShadow(&
I));
2180 setOrigin(&
I, getCleanOrigin());
2195 insertShadowCheck(
I.getOperand(1), &
I);
2199 setOrigin(&
I, getOrigin(&
I, 0));
2203 insertShadowCheck(
I.getOperand(2), &
I);
2205 auto *Shadow0 = getShadow(&
I, 0);
2206 auto *Shadow1 = getShadow(&
I, 1);
2209 setOriginForNaryOp(
I);
2214 auto *Shadow0 = getShadow(&
I, 0);
2215 auto *Shadow1 = getShadow(&
I, 1);
2218 setOriginForNaryOp(
I);
2224 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2225 setOrigin(&
I, getOrigin(&
I, 0));
2230 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2231 setOrigin(&
I, getOrigin(&
I, 0));
2236 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2237 setOrigin(&
I, getOrigin(&
I, 0));
2244 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2245 if (CI->isMustTailCall())
2249 setOrigin(&
I, getOrigin(&
I, 0));
2255 "_msprop_ptrtoint"));
2256 setOrigin(&
I, getOrigin(&
I, 0));
2262 "_msprop_inttoptr"));
2263 setOrigin(&
I, getOrigin(&
I, 0));
2266 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2267 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2268 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2269 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2270 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2271 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2285 Value *S1 = getShadow(&
I, 0);
2286 Value *S2 = getShadow(&
I, 1);
2287 Value *V1 =
I.getOperand(0);
2296 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2297 setOriginForNaryOp(
I);
2307 Value *S1 = getShadow(&
I, 0);
2308 Value *S2 = getShadow(&
I, 1);
2318 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2319 setOriginForNaryOp(
I);
2337 template <
bool CombineShadow>
class Combiner {
2338 Value *Shadow =
nullptr;
2339 Value *Origin =
nullptr;
2341 MemorySanitizerVisitor *MSV;
2345 : IRB(IRB), MSV(MSV) {}
2349 if (CombineShadow) {
2354 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2355 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2359 if (MSV->MS.TrackOrigins) {
2364 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2366 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2367 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2377 Value *OpShadow = MSV->getShadow(V);
2378 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2379 return Add(OpShadow, OpOrigin);
2385 if (CombineShadow) {
2387 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2388 MSV->setShadow(
I, Shadow);
2390 if (MSV->MS.TrackOrigins) {
2392 MSV->setOrigin(
I, Origin);
2402 if (!MS.TrackOrigins)
2405 OriginCombiner
OC(
this, IRB);
2406 for (
Use &Op :
I.operands())
2411 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2413 "Vector of pointers is not a valid shadow type");
2414 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2423 Type *srcTy =
V->getType();
2424 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2425 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2426 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2432 cast<VectorType>(dstTy)->getElementCount() ==
2433 cast<VectorType>(srcTy)->getElementCount())
2444 Type *ShadowTy = getShadowTy(V);
2445 if (
V->getType() == ShadowTy)
2447 if (
V->getType()->isPtrOrPtrVectorTy())
2456 ShadowAndOriginCombiner
SC(
this, IRB);
2457 for (
Use &Op :
I.operands())
2477 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2478 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2479 Type *EltTy = VTy->getElementType();
2481 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2484 const APInt &
V = Elt->getValue();
2493 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2494 const APInt &
V = Elt->getValue();
2504 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2505 setOrigin(&
I, getOrigin(OtherArg));
2509 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2510 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2511 if (constOp0 && !constOp1)
2512 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2513 else if (constOp1 && !constOp0)
2514 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2529 insertShadowCheck(
I.getOperand(1), &
I);
2530 setShadow(&
I, getShadow(&
I, 0));
2531 setOrigin(&
I, getOrigin(&
I, 0));
2548 void handleEqualityComparison(
ICmpInst &
I) {
2552 Value *Sa = getShadow(
A);
2553 Value *Sb = getShadow(
B);
2579 setOriginForNaryOp(
I);
2621 void handleRelationalComparisonExact(
ICmpInst &
I) {
2625 Value *Sa = getShadow(
A);
2626 Value *Sb = getShadow(
B);
2637 bool IsSigned =
I.isSigned();
2639 getLowestPossibleValue(IRB,
A, Sa, IsSigned),
2640 getHighestPossibleValue(IRB,
B, Sb, IsSigned));
2642 getHighestPossibleValue(IRB,
A, Sa, IsSigned),
2643 getLowestPossibleValue(IRB,
B, Sb, IsSigned));
2646 setOriginForNaryOp(
I);
2653 void handleSignedRelationalComparison(
ICmpInst &
I) {
2657 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2658 op =
I.getOperand(0);
2659 pre =
I.getPredicate();
2660 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2661 op =
I.getOperand(1);
2662 pre =
I.getSwappedPredicate();
2675 setShadow(&
I, Shadow);
2676 setOrigin(&
I, getOrigin(
op));
2687 if (
I.isEquality()) {
2688 handleEqualityComparison(
I);
2694 handleRelationalComparisonExact(
I);
2698 handleSignedRelationalComparison(
I);
2703 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2704 handleRelationalComparisonExact(
I);
2711 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2717 Value *S1 = getShadow(&
I, 0);
2718 Value *S2 = getShadow(&
I, 1);
2723 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2724 setOriginForNaryOp(
I);
2735 Value *S0 = getShadow(&
I, 0);
2736 Value *S1 = getShadow(&
I, 1);
2737 Value *S2 = getShadow(&
I, 2);
2742 I.getModule(),
I.getIntrinsicID(), S2Conv->
getType());
2744 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2745 setOriginForNaryOp(
I);
2759 getShadow(
I.getArgOperand(1));
2763 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2764 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2765 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2766 I.eraseFromParent();
2784 getShadow(
I.getArgOperand(1));
2788 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2789 IRB.CreatePointerCast(I.getArgOperand(1), IRB.getInt8PtrTy()),
2790 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2791 I.eraseFromParent();
2799 {IRB.CreatePointerCast(I.getArgOperand(0), IRB.getInt8PtrTy()),
2800 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2801 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2802 I.eraseFromParent();
2805 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2807 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2816 Value *Shadow = getShadow(&
I, 1);
2817 Value *ShadowPtr, *OriginPtr;
2821 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2826 insertShadowCheck(
Addr, &
I);
2829 if (MS.TrackOrigins)
2842 Type *ShadowTy = getShadowTy(&
I);
2843 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2844 if (PropagateShadow) {
2848 std::tie(ShadowPtr, OriginPtr) =
2849 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2853 setShadow(&
I, getCleanShadow(&
I));
2857 insertShadowCheck(
Addr, &
I);
2859 if (MS.TrackOrigins) {
2860 if (PropagateShadow)
2861 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2863 setOrigin(&
I, getCleanOrigin());
2876 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy() ||
2877 RetTy->isX86_MMXTy()))
2880 unsigned NumArgOperands =
I.arg_size();
2881 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2882 Type *Ty =
I.getArgOperand(i)->getType();
2888 ShadowAndOriginCombiner
SC(
this, IRB);
2889 for (
unsigned i = 0; i < NumArgOperands; ++i)
2890 SC.Add(
I.getArgOperand(i));
2907 unsigned NumArgOperands =
I.arg_size();
2908 if (NumArgOperands == 0)
2911 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
2912 I.getArgOperand(1)->getType()->isVectorTy() &&
2913 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
2915 return handleVectorStoreIntrinsic(
I);
2918 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
2919 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
2921 return handleVectorLoadIntrinsic(
I);
2924 if (
I.doesNotAccessMemory())
2925 if (maybeHandleSimpleNomemIntrinsic(
I))
2933 setShadow(&
I, getShadow(&
I, 0));
2934 setOrigin(&
I, getOrigin(&
I, 0));
2942 InstrumentLifetimeStart =
false;
2943 LifetimeStartList.push_back(std::make_pair(&
I, AI));
2949 Type *OpType =
Op->getType();
2951 F.getParent(), Intrinsic::bswap,
ArrayRef(&OpType, 1));
2952 setShadow(&
I, IRB.
CreateCall(BswapFunc, getShadow(Op)));
2953 setOrigin(&
I, getOrigin(Op));
2958 Value *Src =
I.getArgOperand(0);
2964 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
2967 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
2970 Value *OutputShadow =
2971 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
2973 setShadow(&
I, OutputShadow);
2974 setOriginForNaryOp(
I);
2992 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
2993 bool HasRoundingMode =
false) {
2995 Value *CopyOp, *ConvertOp;
2997 assert((!HasRoundingMode ||
2998 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
2999 "Invalid rounding mode");
3001 switch (
I.arg_size() - HasRoundingMode) {
3003 CopyOp =
I.getArgOperand(0);
3004 ConvertOp =
I.getArgOperand(1);
3007 ConvertOp =
I.getArgOperand(0);
3021 Value *ConvertShadow = getShadow(ConvertOp);
3022 Value *AggShadow =
nullptr;
3026 for (
int i = 1; i < NumUsedElements; ++i) {
3029 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3032 AggShadow = ConvertShadow;
3035 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3042 Value *ResultShadow = getShadow(CopyOp);
3043 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3044 for (
int i = 0; i < NumUsedElements; ++i) {
3046 ResultShadow, ConstantInt::getNullValue(EltTy),
3049 setShadow(&
I, ResultShadow);
3050 setOrigin(&
I, getOrigin(CopyOp));
3052 setShadow(&
I, getCleanShadow(&
I));
3053 setOrigin(&
I, getCleanOrigin());
3061 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3064 return CreateShadowCast(IRB, S2,
T,
true);
3072 return CreateShadowCast(IRB, S2,
T,
true);
3089 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3094 Value *S1 = getShadow(&
I, 0);
3095 Value *S2 = getShadow(&
I, 1);
3096 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3097 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3098 Value *V1 =
I.getOperand(0);
3101 {IRB.CreateBitCast(S1, V1->getType()), V2});
3103 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3104 setOriginForNaryOp(
I);
3108 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3109 const unsigned X86_MMXSizeInBits = 64;
3110 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3111 "Illegal MMX vector element size");
3113 X86_MMXSizeInBits / EltSizeInBits);
3120 case Intrinsic::x86_sse2_packsswb_128:
3121 case Intrinsic::x86_sse2_packuswb_128:
3122 return Intrinsic::x86_sse2_packsswb_128;
3124 case Intrinsic::x86_sse2_packssdw_128:
3125 case Intrinsic::x86_sse41_packusdw:
3126 return Intrinsic::x86_sse2_packssdw_128;
3128 case Intrinsic::x86_avx2_packsswb:
3129 case Intrinsic::x86_avx2_packuswb:
3130 return Intrinsic::x86_avx2_packsswb;
3132 case Intrinsic::x86_avx2_packssdw:
3133 case Intrinsic::x86_avx2_packusdw:
3134 return Intrinsic::x86_avx2_packssdw;
3136 case Intrinsic::x86_mmx_packsswb:
3137 case Intrinsic::x86_mmx_packuswb:
3138 return Intrinsic::x86_mmx_packsswb;
3140 case Intrinsic::x86_mmx_packssdw:
3141 return Intrinsic::x86_mmx_packssdw;
3154 void handleVectorPackIntrinsic(
IntrinsicInst &
I,
unsigned EltSizeInBits = 0) {
3156 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3158 Value *S1 = getShadow(&
I, 0);
3159 Value *S2 = getShadow(&
I, 1);
3165 Type *
T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) : S1->
getType();
3181 F.getParent(), getSignedPackIntrinsic(
I.getIntrinsicID()));
3184 IRB.
CreateCall(ShadowFn, {S1_ext, S2_ext},
"_msprop_vector_pack");
3188 setOriginForNaryOp(
I);
3193 const unsigned SignificantBitsPerResultElement = 16;
3194 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3196 unsigned ZeroBitsPerResultElement =
3200 auto *Shadow0 = getShadow(&
I, 0);
3201 auto *Shadow1 = getShadow(&
I, 1);
3206 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3209 setOriginForNaryOp(
I);
3214 unsigned EltSizeInBits = 0) {
3215 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3216 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) :
I.
getType();
3218 auto *Shadow0 = getShadow(&
I, 0);
3219 auto *Shadow1 = getShadow(&
I, 1);
3226 setOriginForNaryOp(
I);
3234 Type *ResTy = getShadowTy(&
I);
3235 auto *Shadow0 = getShadow(&
I, 0);
3236 auto *Shadow1 = getShadow(&
I, 1);
3241 setOriginForNaryOp(
I);
3249 auto *Shadow0 = getShadow(&
I, 0);
3250 auto *Shadow1 = getShadow(&
I, 1);
3252 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3254 setOriginForNaryOp(
I);
3263 setOrigin(&
I, getOrigin(&
I, 0));
3271 Value *OperandShadow = getShadow(&
I, 0);
3273 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3281 setOrigin(&
I, getOrigin(&
I, 0));
3289 Value *OperandShadow = getShadow(&
I, 0);
3290 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3298 setOrigin(&
I, getOrigin(&
I, 0));
3306 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3312 insertShadowCheck(
Addr, &
I);
3323 Value *ShadowPtr, *OriginPtr;
3324 std::tie(ShadowPtr, OriginPtr) =
3325 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3328 insertShadowCheck(
Addr, &
I);
3331 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3333 insertShadowCheck(Shadow, Origin, &
I);
3340 Value *PassThru =
I.getArgOperand(2);
3343 insertShadowCheck(
Ptr, &
I);
3344 insertShadowCheck(Mask, &
I);
3347 if (!PropagateShadow) {
3348 setShadow(&
I, getCleanShadow(&
I));
3349 setOrigin(&
I, getCleanOrigin());
3353 Type *ShadowTy = getShadowTy(&
I);
3354 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3355 auto [ShadowPtr, OriginPtr] =
3356 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3359 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3361 setShadow(&
I, Shadow);
3364 setOrigin(&
I, getCleanOrigin());
3369 Value *Values =
I.getArgOperand(0);
3374 insertShadowCheck(
Ptr, &
I);
3375 insertShadowCheck(Mask, &
I);
3378 Value *Shadow = getShadow(Values);
3379 Type *ElementShadowTy =
3380 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3381 auto [ShadowPtr, OriginPtrs] =
3382 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3391 Value *Ptrs =
I.getArgOperand(0);
3392 const Align Alignment(
3393 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3395 Value *PassThru =
I.getArgOperand(3);
3397 Type *PtrsShadowTy = getShadowTy(Ptrs);
3399 insertShadowCheck(Mask, &
I);
3403 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3406 if (!PropagateShadow) {
3407 setShadow(&
I, getCleanShadow(&
I));
3408 setOrigin(&
I, getCleanOrigin());
3412 Type *ShadowTy = getShadowTy(&
I);
3413 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3414 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3415 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3419 getShadow(PassThru),
"_msmaskedgather");
3421 setShadow(&
I, Shadow);
3424 setOrigin(&
I, getCleanOrigin());
3429 Value *Values =
I.getArgOperand(0);
3430 Value *Ptrs =
I.getArgOperand(1);
3431 const Align Alignment(
3432 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3435 Type *PtrsShadowTy = getShadowTy(Ptrs);
3437 insertShadowCheck(Mask, &
I);
3441 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3444 Value *Shadow = getShadow(Values);
3445 Type *ElementShadowTy =
3446 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3447 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3448 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3457 Value *
V =
I.getArgOperand(0);
3459 const Align Alignment(
3460 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3462 Value *Shadow = getShadow(V);
3465 insertShadowCheck(
Ptr, &
I);
3466 insertShadowCheck(Mask, &
I);
3471 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3472 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3476 if (!MS.TrackOrigins)
3479 auto &
DL =
F.getParent()->getDataLayout();
3480 paintOrigin(IRB, getOrigin(V), OriginPtr,
3488 const Align Alignment(
3489 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3491 Value *PassThru =
I.getArgOperand(3);
3494 insertShadowCheck(
Ptr, &
I);
3495 insertShadowCheck(Mask, &
I);
3498 if (!PropagateShadow) {
3499 setShadow(&
I, getCleanShadow(&
I));
3500 setOrigin(&
I, getCleanOrigin());
3504 Type *ShadowTy = getShadowTy(&
I);
3505 Value *ShadowPtr, *OriginPtr;
3506 std::tie(ShadowPtr, OriginPtr) =
3507 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3509 getShadow(PassThru),
"_msmaskedld"));
3511 if (!MS.TrackOrigins)
3518 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3523 setOrigin(&
I, Origin);
3533 Type *ShadowTy = getShadowTy(&
I);
3536 Value *SMask = getShadow(&
I, 1);
3541 {getShadow(&I, 0), I.getOperand(1)});
3544 setOriginForNaryOp(
I);
3549 for (
unsigned X = OddElements ? 1 : 0;
X <
Width;
X += 2) {
3566 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3567 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3568 "pclmul 3rd operand must be a constant");
3569 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3571 getPclmulMask(Width, Imm & 0x01));
3573 getPclmulMask(Width, Imm & 0x10));
3574 ShadowAndOriginCombiner SOC(
this, IRB);
3575 SOC.Add(Shuf0, getOrigin(&
I, 0));
3576 SOC.Add(Shuf1, getOrigin(&
I, 1));
3584 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3586 Value *Second = getShadow(&
I, 1);
3589 Mask.push_back(Width);
3590 for (
unsigned i = 1; i <
Width; i++)
3594 setShadow(&
I, Shadow);
3595 setOriginForNaryOp(
I);
3600 Value *Shadow0 = getShadow(&
I, 0);
3601 Value *Shadow1 = getShadow(&
I, 1);
3607 setShadow(&
I, Shadow);
3608 setOriginForNaryOp(
I);
3614 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3616 Value *Second = getShadow(&
I, 1);
3620 Mask.push_back(Width);
3621 for (
unsigned i = 1; i <
Width; i++)
3625 setShadow(&
I, Shadow);
3626 setOriginForNaryOp(
I);
3633 assert(
I.getType()->isIntOrIntVectorTy());
3634 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3638 setShadow(&
I, getShadow(&
I, 0));
3639 setOrigin(&
I, getOrigin(&
I, 0));
3643 switch (
I.getIntrinsicID()) {
3644 case Intrinsic::abs:
3645 handleAbsIntrinsic(
I);
3647 case Intrinsic::lifetime_start:
3648 handleLifetimeStart(
I);
3650 case Intrinsic::launder_invariant_group:
3651 case Intrinsic::strip_invariant_group:
3652 handleInvariantGroup(
I);
3654 case Intrinsic::bswap:
3657 case Intrinsic::ctlz:
3658 case Intrinsic::cttz:
3659 handleCountZeroes(
I);
3661 case Intrinsic::masked_compressstore:
3662 handleMaskedCompressStore(
I);
3664 case Intrinsic::masked_expandload:
3665 handleMaskedExpandLoad(
I);
3667 case Intrinsic::masked_gather:
3668 handleMaskedGather(
I);
3670 case Intrinsic::masked_scatter:
3671 handleMaskedScatter(
I);
3673 case Intrinsic::masked_store:
3674 handleMaskedStore(
I);
3676 case Intrinsic::masked_load:
3677 handleMaskedLoad(
I);
3679 case Intrinsic::vector_reduce_and:
3680 handleVectorReduceAndIntrinsic(
I);
3682 case Intrinsic::vector_reduce_or:
3683 handleVectorReduceOrIntrinsic(
I);
3685 case Intrinsic::vector_reduce_add:
3686 case Intrinsic::vector_reduce_xor:
3687 case Intrinsic::vector_reduce_mul:
3688 handleVectorReduceIntrinsic(
I);
3690 case Intrinsic::x86_sse_stmxcsr:
3693 case Intrinsic::x86_sse_ldmxcsr:
3696 case Intrinsic::x86_avx512_vcvtsd2usi64:
3697 case Intrinsic::x86_avx512_vcvtsd2usi32:
3698 case Intrinsic::x86_avx512_vcvtss2usi64:
3699 case Intrinsic::x86_avx512_vcvtss2usi32:
3700 case Intrinsic::x86_avx512_cvttss2usi64:
3701 case Intrinsic::x86_avx512_cvttss2usi:
3702 case Intrinsic::x86_avx512_cvttsd2usi64:
3703 case Intrinsic::x86_avx512_cvttsd2usi:
3704 case Intrinsic::x86_avx512_cvtusi2ss:
3705 case Intrinsic::x86_avx512_cvtusi642sd:
3706 case Intrinsic::x86_avx512_cvtusi642ss:
3707 handleVectorConvertIntrinsic(
I, 1,
true);
3709 case Intrinsic::x86_sse2_cvtsd2si64:
3710 case Intrinsic::x86_sse2_cvtsd2si:
3711 case Intrinsic::x86_sse2_cvtsd2ss:
3712 case Intrinsic::x86_sse2_cvttsd2si64:
3713 case Intrinsic::x86_sse2_cvttsd2si:
3714 case Intrinsic::x86_sse_cvtss2si64:
3715 case Intrinsic::x86_sse_cvtss2si:
3716 case Intrinsic::x86_sse_cvttss2si64:
3717 case Intrinsic::x86_sse_cvttss2si:
3718 handleVectorConvertIntrinsic(
I, 1);
3720 case Intrinsic::x86_sse_cvtps2pi:
3721 case Intrinsic::x86_sse_cvttps2pi:
3722 handleVectorConvertIntrinsic(
I, 2);
3725 case Intrinsic::x86_avx512_psll_w_512:
3726 case Intrinsic::x86_avx512_psll_d_512:
3727 case Intrinsic::x86_avx512_psll_q_512:
3728 case Intrinsic::x86_avx512_pslli_w_512:
3729 case Intrinsic::x86_avx512_pslli_d_512:
3730 case Intrinsic::x86_avx512_pslli_q_512:
3731 case Intrinsic::x86_avx512_psrl_w_512:
3732 case Intrinsic::x86_avx512_psrl_d_512:
3733 case Intrinsic::x86_avx512_psrl_q_512:
3734 case Intrinsic::x86_avx512_psra_w_512:
3735 case Intrinsic::x86_avx512_psra_d_512:
3736 case Intrinsic::x86_avx512_psra_q_512:
3737 case Intrinsic::x86_avx512_psrli_w_512:
3738 case Intrinsic::x86_avx512_psrli_d_512:
3739 case Intrinsic::x86_avx512_psrli_q_512:
3740 case Intrinsic::x86_avx512_psrai_w_512:
3741 case Intrinsic::x86_avx512_psrai_d_512:
3742 case Intrinsic::x86_avx512_psrai_q_512:
3743 case Intrinsic::x86_avx512_psra_q_256:
3744 case Intrinsic::x86_avx512_psra_q_128:
3745 case Intrinsic::x86_avx512_psrai_q_256:
3746 case Intrinsic::x86_avx512_psrai_q_128:
3747 case Intrinsic::x86_avx2_psll_w:
3748 case Intrinsic::x86_avx2_psll_d:
3749 case Intrinsic::x86_avx2_psll_q:
3750 case Intrinsic::x86_avx2_pslli_w:
3751 case Intrinsic::x86_avx2_pslli_d:
3752 case Intrinsic::x86_avx2_pslli_q:
3753 case Intrinsic::x86_avx2_psrl_w:
3754 case Intrinsic::x86_avx2_psrl_d:
3755 case Intrinsic::x86_avx2_psrl_q:
3756 case Intrinsic::x86_avx2_psra_w:
3757 case Intrinsic::x86_avx2_psra_d:
3758 case Intrinsic::x86_avx2_psrli_w:
3759 case Intrinsic::x86_avx2_psrli_d:
3760 case Intrinsic::x86_avx2_psrli_q:
3761 case Intrinsic::x86_avx2_psrai_w:
3762 case Intrinsic::x86_avx2_psrai_d:
3763 case Intrinsic::x86_sse2_psll_w:
3764 case Intrinsic::x86_sse2_psll_d:
3765 case Intrinsic::x86_sse2_psll_q:
3766 case Intrinsic::x86_sse2_pslli_w:
3767 case Intrinsic::x86_sse2_pslli_d:
3768 case Intrinsic::x86_sse2_pslli_q:
3769 case Intrinsic::x86_sse2_psrl_w:
3770 case Intrinsic::x86_sse2_psrl_d:
3771 case Intrinsic::x86_sse2_psrl_q:
3772 case Intrinsic::x86_sse2_psra_w:
3773 case Intrinsic::x86_sse2_psra_d:
3774 case Intrinsic::x86_sse2_psrli_w:
3775 case Intrinsic::x86_sse2_psrli_d:
3776 case Intrinsic::x86_sse2_psrli_q:
3777 case Intrinsic::x86_sse2_psrai_w:
3778 case Intrinsic::x86_sse2_psrai_d:
3779 case Intrinsic::x86_mmx_psll_w:
3780 case Intrinsic::x86_mmx_psll_d:
3781 case Intrinsic::x86_mmx_psll_q:
3782 case Intrinsic::x86_mmx_pslli_w:
3783 case Intrinsic::x86_mmx_pslli_d:
3784 case Intrinsic::x86_mmx_pslli_q:
3785 case Intrinsic::x86_mmx_psrl_w:
3786 case Intrinsic::x86_mmx_psrl_d:
3787 case Intrinsic::x86_mmx_psrl_q:
3788 case Intrinsic::x86_mmx_psra_w:
3789 case Intrinsic::x86_mmx_psra_d:
3790 case Intrinsic::x86_mmx_psrli_w:
3791 case Intrinsic::x86_mmx_psrli_d:
3792 case Intrinsic::x86_mmx_psrli_q:
3793 case Intrinsic::x86_mmx_psrai_w:
3794 case Intrinsic::x86_mmx_psrai_d:
3795 handleVectorShiftIntrinsic(
I,
false);
3797 case Intrinsic::x86_avx2_psllv_d:
3798 case Intrinsic::x86_avx2_psllv_d_256:
3799 case Intrinsic::x86_avx512_psllv_d_512:
3800 case Intrinsic::x86_avx2_psllv_q:
3801 case Intrinsic::x86_avx2_psllv_q_256:
3802 case Intrinsic::x86_avx512_psllv_q_512:
3803 case Intrinsic::x86_avx2_psrlv_d:
3804 case Intrinsic::x86_avx2_psrlv_d_256:
3805 case Intrinsic::x86_avx512_psrlv_d_512:
3806 case Intrinsic::x86_avx2_psrlv_q:
3807 case Intrinsic::x86_avx2_psrlv_q_256:
3808 case Intrinsic::x86_avx512_psrlv_q_512:
3809 case Intrinsic::x86_avx2_psrav_d:
3810 case Intrinsic::x86_avx2_psrav_d_256:
3811 case Intrinsic::x86_avx512_psrav_d_512:
3812 case Intrinsic::x86_avx512_psrav_q_128:
3813 case Intrinsic::x86_avx512_psrav_q_256:
3814 case Intrinsic::x86_avx512_psrav_q_512:
3815 handleVectorShiftIntrinsic(
I,
true);
3818 case Intrinsic::x86_sse2_packsswb_128:
3819 case Intrinsic::x86_sse2_packssdw_128:
3820 case Intrinsic::x86_sse2_packuswb_128:
3821 case Intrinsic::x86_sse41_packusdw:
3822 case Intrinsic::x86_avx2_packsswb:
3823 case Intrinsic::x86_avx2_packssdw:
3824 case Intrinsic::x86_avx2_packuswb:
3825 case Intrinsic::x86_avx2_packusdw:
3826 handleVectorPackIntrinsic(
I);
3829 case Intrinsic::x86_mmx_packsswb:
3830 case Intrinsic::x86_mmx_packuswb:
3831 handleVectorPackIntrinsic(
I, 16);
3834 case Intrinsic::x86_mmx_packssdw:
3835 handleVectorPackIntrinsic(
I, 32);
3838 case Intrinsic::x86_mmx_psad_bw:
3839 case Intrinsic::x86_sse2_psad_bw:
3840 case Intrinsic::x86_avx2_psad_bw:
3841 handleVectorSadIntrinsic(
I);
3844 case Intrinsic::x86_sse2_pmadd_wd:
3845 case Intrinsic::x86_avx2_pmadd_wd:
3846 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3847 case Intrinsic::x86_avx2_pmadd_ub_sw:
3848 handleVectorPmaddIntrinsic(
I);
3851 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3852 handleVectorPmaddIntrinsic(
I, 8);
3855 case Intrinsic::x86_mmx_pmadd_wd:
3856 handleVectorPmaddIntrinsic(
I, 16);
3859 case Intrinsic::x86_sse_cmp_ss:
3860 case Intrinsic::x86_sse2_cmp_sd:
3861 case Intrinsic::x86_sse_comieq_ss:
3862 case Intrinsic::x86_sse_comilt_ss:
3863 case Intrinsic::x86_sse_comile_ss:
3864 case Intrinsic::x86_sse_comigt_ss:
3865 case Intrinsic::x86_sse_comige_ss:
3866 case Intrinsic::x86_sse_comineq_ss:
3867 case Intrinsic::x86_sse_ucomieq_ss:
3868 case Intrinsic::x86_sse_ucomilt_ss:
3869 case Intrinsic::x86_sse_ucomile_ss:
3870 case Intrinsic::x86_sse_ucomigt_ss:
3871 case Intrinsic::x86_sse_ucomige_ss:
3872 case Intrinsic::x86_sse_ucomineq_ss:
3873 case Intrinsic::x86_sse2_comieq_sd:
3874 case Intrinsic::x86_sse2_comilt_sd:
3875 case Intrinsic::x86_sse2_comile_sd:
3876 case Intrinsic::x86_sse2_comigt_sd:
3877 case Intrinsic::x86_sse2_comige_sd:
3878 case Intrinsic::x86_sse2_comineq_sd:
3879 case Intrinsic::x86_sse2_ucomieq_sd:
3880 case Intrinsic::x86_sse2_ucomilt_sd:
3881 case Intrinsic::x86_sse2_ucomile_sd:
3882 case Intrinsic::x86_sse2_ucomigt_sd:
3883 case Intrinsic::x86_sse2_ucomige_sd:
3884 case Intrinsic::x86_sse2_ucomineq_sd:
3885 handleVectorCompareScalarIntrinsic(
I);
3888 case Intrinsic::x86_avx_cmp_pd_256:
3889 case Intrinsic::x86_avx_cmp_ps_256:
3890 case Intrinsic::x86_sse2_cmp_pd:
3891 case Intrinsic::x86_sse_cmp_ps:
3892 handleVectorComparePackedIntrinsic(
I);
3895 case Intrinsic::x86_bmi_bextr_32:
3896 case Intrinsic::x86_bmi_bextr_64:
3897 case Intrinsic::x86_bmi_bzhi_32:
3898 case Intrinsic::x86_bmi_bzhi_64:
3899 case Intrinsic::x86_bmi_pdep_32:
3900 case Intrinsic::x86_bmi_pdep_64:
3901 case Intrinsic::x86_bmi_pext_32:
3902 case Intrinsic::x86_bmi_pext_64:
3903 handleBmiIntrinsic(
I);
3906 case Intrinsic::x86_pclmulqdq:
3907 case Intrinsic::x86_pclmulqdq_256:
3908 case Intrinsic::x86_pclmulqdq_512:
3909 handlePclmulIntrinsic(
I);
3912 case Intrinsic::x86_sse41_round_sd:
3913 case Intrinsic::x86_sse41_round_ss:
3914 handleUnarySdSsIntrinsic(
I);
3916 case Intrinsic::x86_sse2_max_sd:
3917 case Intrinsic::x86_sse_max_ss:
3918 case Intrinsic::x86_sse2_min_sd:
3919 case Intrinsic::x86_sse_min_ss:
3920 handleBinarySdSsIntrinsic(
I);
3923 case Intrinsic::x86_avx_vtestc_pd:
3924 case Intrinsic::x86_avx_vtestc_pd_256:
3925 case Intrinsic::x86_avx_vtestc_ps:
3926 case Intrinsic::x86_avx_vtestc_ps_256:
3927 case Intrinsic::x86_avx_vtestnzc_pd:
3928 case Intrinsic::x86_avx_vtestnzc_pd_256:
3929 case Intrinsic::x86_avx_vtestnzc_ps:
3930 case Intrinsic::x86_avx_vtestnzc_ps_256:
3931 case Intrinsic::x86_avx_vtestz_pd:
3932 case Intrinsic::x86_avx_vtestz_pd_256:
3933 case Intrinsic::x86_avx_vtestz_ps:
3934 case Intrinsic::x86_avx_vtestz_ps_256:
3935 case Intrinsic::x86_avx_ptestc_256:
3936 case Intrinsic::x86_avx_ptestnzc_256:
3937 case Intrinsic::x86_avx_ptestz_256:
3938 case Intrinsic::x86_sse41_ptestc:
3939 case Intrinsic::x86_sse41_ptestnzc:
3940 case Intrinsic::x86_sse41_ptestz:
3941 handleVtestIntrinsic(
I);
3944 case Intrinsic::fshl:
3945 case Intrinsic::fshr:
3946 handleFunnelShift(
I);
3949 case Intrinsic::is_constant:
3951 setShadow(&
I, getCleanShadow(&
I));
3952 setOrigin(&
I, getCleanOrigin());
3956 if (!handleUnknownIntrinsic(
I))
3957 visitInstruction(
I);
3962 void visitLibAtomicLoad(
CallBase &CB) {
3964 assert(isa<CallInst>(CB));
3973 Value *NewOrdering =
3977 NextNodeIRBuilder NextIRB(&CB);
3978 Value *SrcShadowPtr, *SrcOriginPtr;
3979 std::tie(SrcShadowPtr, SrcOriginPtr) =
3980 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
3982 Value *DstShadowPtr =
3983 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
3987 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
3988 if (MS.TrackOrigins) {
3989 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
3991 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
3992 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
3996 void visitLibAtomicStore(
CallBase &CB) {
4003 Value *NewOrdering =
4007 Value *DstShadowPtr =
4025 visitAsmInstruction(CB);
4027 visitInstruction(CB);
4036 case LibFunc_atomic_load:
4037 if (!isa<CallInst>(CB)) {
4038 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4042 visitLibAtomicLoad(CB);
4044 case LibFunc_atomic_store:
4045 visitLibAtomicStore(CB);
4052 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4053 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4061 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4063 Call->removeFnAttrs(
B);
4065 Func->removeFnAttrs(
B);
4071 bool MayCheckCall = MS.EagerChecks;
4075 MayCheckCall &= !
Func->getName().startswith(
"__sanitizer_unaligned_");
4078 unsigned ArgOffset = 0;
4081 if (!
A->getType()->isSized()) {
4082 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4090 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4093 insertShadowCheck(
A, &CB);
4094 Size =
DL.getTypeAllocSize(
A->getType());
4100 Value *ArgShadow = getShadow(
A);
4101 Value *ArgShadowBase = getShadowPtrForArgument(
A, IRB, ArgOffset);
4103 <<
" Shadow: " << *ArgShadow <<
"\n");
4107 assert(
A->getType()->isPointerTy() &&
4108 "ByVal argument is not a pointer!");
4116 Value *AShadowPtr, *AOriginPtr;
4117 std::tie(AShadowPtr, AOriginPtr) =
4118 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4120 if (!PropagateShadow) {
4127 if (MS.TrackOrigins) {
4128 Value *ArgOriginBase = getOriginPtrForArgument(
A, IRB, ArgOffset);
4142 Size =
DL.getTypeAllocSize(
A->getType());
4147 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4148 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4150 getOriginPtrForArgument(
A, IRB, ArgOffset));
4154 assert(Store !=
nullptr);
4163 if (FT->isVarArg()) {
4164 VAHelper->visitCallBase(CB, IRB);
4174 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4175 setShadow(&CB, getCleanShadow(&CB));
4176 setOrigin(&CB, getCleanOrigin());
4182 Value *
Base = getShadowPtrForRetval(&CB, IRBBefore);
4183 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4186 if (isa<CallInst>(CB)) {
4190 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4195 setShadow(&CB, getCleanShadow(&CB));
4196 setOrigin(&CB, getCleanOrigin());
4203 "Could not find insertion point for retval shadow load");
4206 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4207 getShadowTy(&CB), getShadowPtrForRetval(&CB, IRBAfter),
4209 setShadow(&CB, RetvalShadow);
4210 if (MS.TrackOrigins)
4211 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
4212 getOriginPtrForRetval(IRBAfter)));
4216 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4217 RetVal =
I->getOperand(0);
4219 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4220 return I->isMustTailCall();
4227 Value *RetVal =
I.getReturnValue();
4233 Value *ShadowPtr = getShadowPtrForRetval(RetVal, IRB);
4234 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4235 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4238 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4240 Value *Shadow = getShadow(RetVal);
4241 bool StoreOrigin =
true;
4243 insertShadowCheck(RetVal, &
I);
4244 Shadow = getCleanShadow(RetVal);
4245 StoreOrigin =
false;
4252 if (MS.TrackOrigins && StoreOrigin)
4253 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval(IRB));
4259 if (!PropagateShadow) {
4260 setShadow(&
I, getCleanShadow(&
I));
4261 setOrigin(&
I, getCleanOrigin());
4265 ShadowPHINodes.push_back(&
I);
4266 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4268 if (MS.TrackOrigins)
4270 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4288 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
4290 Value *ShadowBase, *OriginBase;
4291 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4298 if (PoisonStack && MS.TrackOrigins) {
4299 Value *Idptr = getLocalVarIdptr(
I);
4301 Value *Descr = getLocalVarDescription(
I);
4302 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4303 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
4304 IRB.CreatePointerCast(Idptr, IRB.getInt8PtrTy()),
4305 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
4307 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn,
4308 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
4309 IRB.CreatePointerCast(Idptr, IRB.getInt8PtrTy())});
4315 Value *Descr = getLocalVarDescription(
I);
4318 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len,
4319 IRB.CreatePointerCast(Descr, IRB.getInt8PtrTy())});
4322 {IRB.CreatePointerCast(&I, IRB.getInt8PtrTy()), Len});
4329 NextNodeIRBuilder IRB(InsPoint);
4333 if (
I.isArrayAllocation())
4337 if (MS.CompileKernel)
4338 poisonAllocaKmsan(
I, IRB, Len);
4340 poisonAllocaUserspace(
I, IRB, Len);
4344 setShadow(&
I, getCleanShadow(&
I));
4345 setOrigin(&
I, getCleanOrigin());
4357 Value *Sb = getShadow(
B);
4358 Value *Sc = getShadow(
C);
4359 Value *Sd = getShadow(
D);
4364 if (
I.getType()->isAggregateType()) {
4368 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4376 C = CreateAppToShadowCast(IRB,
C);
4377 D = CreateAppToShadowCast(IRB,
D);
4384 if (MS.TrackOrigins) {
4387 if (
B->getType()->isVectorTy()) {
4388 B = convertToBool(
B, IRB);
4389 Sb = convertToBool(Sb, IRB);
4396 getOrigin(
I.getFalseValue()))));
4403 setShadow(&
I, getCleanShadow(&
I));
4404 setOrigin(&
I, getCleanOrigin());
4408 setShadow(&
I, getCleanShadow(&
I));
4409 setOrigin(&
I, getCleanOrigin());
4413 setShadow(&
I, getCleanShadow(&
I));
4414 setOrigin(&
I, getCleanOrigin());
4421 Value *Agg =
I.getAggregateOperand();
4423 Value *AggShadow = getShadow(Agg);
4427 setShadow(&
I, ResShadow);
4428 setOriginForNaryOp(
I);
4434 Value *AggShadow = getShadow(
I.getAggregateOperand());
4435 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4441 setOriginForNaryOp(
I);
4445 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4446 errs() <<
"ZZZ call " << CI->getCalledFunction()->getName() <<
"\n";
4448 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4450 errs() <<
"QQQ " <<
I <<
"\n";
4477 insertShadowCheck(Operand, &
I);
4487 IRB.
CreateCall(MS.MsanInstrumentAsmStoreFn, {Ptr, SizeVal});
4492 int NumRetOutputs = 0;
4494 Type *
RetTy = cast<Value>(CB)->getType();
4495 if (!
RetTy->isVoidTy()) {
4497 auto *
ST = dyn_cast<StructType>(
RetTy);
4499 NumRetOutputs =
ST->getNumElements();
4505 switch (
Info.Type) {
4513 return NumOutputs - NumRetOutputs;
4536 int OutputArgs = getNumOutputArgs(IA, CB);
4542 for (
int i = OutputArgs; i < NumOperands; i++) {
4550 for (
int i = 0; i < OutputArgs; i++) {
4556 setShadow(&
I, getCleanShadow(&
I));
4557 setOrigin(&
I, getCleanOrigin());
4562 setShadow(&
I, getCleanShadow(&
I));
4563 setOrigin(&
I, getCleanOrigin());
4571 for (
size_t i = 0, n =
I.getNumOperands(); i < n; i++) {
4572 Value *Operand =
I.getOperand(i);
4574 insertShadowCheck(Operand, &
I);
4576 setShadow(&
I, getCleanShadow(&
I));
4577 setOrigin(&
I, getCleanOrigin());
4582struct VarArgAMD64Helper :
public VarArgHelper {
4585 static const unsigned AMD64GpEndOffset = 48;
4586 static const unsigned AMD64FpEndOffsetSSE = 176;
4588 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4590 unsigned AMD64FpEndOffset;
4592 MemorySanitizer &MS;
4593 MemorySanitizerVisitor &MSV;
4594 Value *VAArgTLSCopy =
nullptr;
4595 Value *VAArgTLSOriginCopy =
nullptr;
4596 Value *VAArgOverflowSize =
nullptr;
4600 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4602 VarArgAMD64Helper(
Function &
F, MemorySanitizer &MS,
4603 MemorySanitizerVisitor &MSV)
4604 :
F(
F), MS(MS), MSV(MSV) {
4605 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4606 for (
const auto &Attr :
F.getAttributes().getFnAttrs()) {
4607 if (Attr.isStringAttribute() &&
4608 (Attr.getKindAsString() ==
"target-features")) {
4609 if (Attr.getValueAsString().contains(
"-sse"))
4610 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4616 ArgKind classifyArgument(
Value *arg) {
4619 if (
T->isFPOrFPVectorTy() ||
T->isX86_MMXTy())
4620 return AK_FloatingPoint;
4621 if (
T->isIntegerTy() &&
T->getPrimitiveSizeInBits() <= 64)
4622 return AK_GeneralPurpose;
4623 if (
T->isPointerTy())
4624 return AK_GeneralPurpose;
4637 unsigned GpOffset = 0;
4638 unsigned FpOffset = AMD64GpEndOffset;
4639 unsigned OverflowOffset = AMD64FpEndOffset;
4643 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
4650 assert(
A->getType()->isPointerTy());
4652 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
4653 Value *ShadowBase = getShadowPtrForVAArgument(
4654 RealTy, IRB, OverflowOffset,
alignTo(ArgSize, 8));
4655 Value *OriginBase =
nullptr;
4656 if (MS.TrackOrigins)
4657 OriginBase = getOriginPtrForVAArgument(RealTy, IRB, OverflowOffset);
4658 OverflowOffset +=
alignTo(ArgSize, 8);
4661 Value *ShadowPtr, *OriginPtr;
4662 std::tie(ShadowPtr, OriginPtr) =
4668 if (MS.TrackOrigins)
4672 ArgKind AK = classifyArgument(
A);
4673 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4675 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4677 Value *ShadowBase, *OriginBase =
nullptr;
4679 case AK_GeneralPurpose:
4681 getShadowPtrForVAArgument(
A->getType(), IRB, GpOffset, 8);
4682 if (MS.TrackOrigins)
4683 OriginBase = getOriginPtrForVAArgument(
A->getType(), IRB, GpOffset);
4686 case AK_FloatingPoint:
4688 getShadowPtrForVAArgument(
A->getType(), IRB, FpOffset, 16);
4689 if (MS.TrackOrigins)
4690 OriginBase = getOriginPtrForVAArgument(
A->getType(), IRB, FpOffset);
4696 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
4698 getShadowPtrForVAArgument(
A->getType(), IRB, OverflowOffset, 8);
4699 if (MS.TrackOrigins)
4701 getOriginPtrForVAArgument(
A->getType(), IRB, OverflowOffset);
4702 OverflowOffset +=
alignTo(ArgSize, 8);
4711 Value *Shadow = MSV.getShadow(
A);
4713 if (MS.TrackOrigins) {
4714 Value *Origin = MSV.getOrigin(
A);
4716 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4723 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4728 unsigned ArgOffset,
unsigned ArgSize) {
4751 Value *VAListTag =
I.getArgOperand(0);
4752 Value *ShadowPtr, *OriginPtr;
4754 std::tie(ShadowPtr, OriginPtr) =
4755 MSV.getShadowOriginPtr(VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
4761 24, Alignment,
false);
4770 unpoisonVAListTagForInst(
I);
4776 unpoisonVAListTagForInst(
I);
4779 void finalizeInstrumentation()
override {
4780 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4781 "finalizeInstrumentation called twice");
4782 if (!VAStartInstrumentationList.
empty()) {
4792 if (MS.TrackOrigins) {
4795 Align(8), CopySize);
4801 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
4802 CallInst *OrigInst = VAStartInstrumentationList[i];
4803 NextNodeIRBuilder IRB(OrigInst);
4810 PointerType::get(RegSaveAreaPtrTy, 0));
4811 Value *RegSaveAreaPtr =
4812 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
4813 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
4815 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
4816 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
4818 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
4820 if (MS.TrackOrigins)
4821 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
4822 Alignment, AMD64FpEndOffset);
4827 PointerType::get(OverflowArgAreaPtrTy, 0));
4828 Value *OverflowArgAreaPtr =
4829 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
4830 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
4831 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
4832 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
4836 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
4838 if (MS.TrackOrigins) {
4841 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
4849struct VarArgMIPS64Helper :
public VarArgHelper {
4851 MemorySanitizer &MS;
4852 MemorySanitizerVisitor &MSV;
4853 Value *VAArgTLSCopy =
nullptr;
4854 Value *VAArgSize =
nullptr;
4858 VarArgMIPS64Helper(
Function &
F, MemorySanitizer &MS,
4859 MemorySanitizerVisitor &MSV)
4860 :
F(
F), MS(MS), MSV(MSV) {}
4863 unsigned VAArgOffset = 0;
4867 Triple TargetTriple(
F.getParent()->getTargetTriple());
4869 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
4874 VAArgOffset += (8 - ArgSize);
4876 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VAArgOffset, ArgSize);
4877 VAArgOffset += ArgSize;
4878 VAArgOffset =
alignTo(VAArgOffset, 8);
4887 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
4892 unsigned ArgOffset,
unsigned ArgSize) {
4905 Value *VAListTag =
I.getArgOperand(0);
4906 Value *ShadowPtr, *OriginPtr;
4908 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4909 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4911 8, Alignment,
false);
4917 Value *VAListTag =
I.getArgOperand(0);
4918 Value *ShadowPtr, *OriginPtr;
4920 std::tie(ShadowPtr, OriginPtr) = MSV.getShadowOriginPtr(
4921 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4923 8, Alignment,
false);
4926 void finalizeInstrumentation()
override {