182#include "llvm/IR/IntrinsicsX86.h"
211#define DEBUG_TYPE "msan"
214 "Controls which checks to insert");
217 "Controls which instruction to instrument");
235 "msan-track-origins",
240 cl::desc(
"keep going after reporting a UMR"),
249 "msan-poison-stack-with-call",
254 "msan-poison-stack-pattern",
255 cl::desc(
"poison uninitialized stack variables with the given pattern"),
260 cl::desc(
"Print name of local stack variable"),
269 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
274 cl::desc(
"exact handling of relational integer ICmp"),
278 "msan-handle-lifetime-intrinsics",
280 "when possible, poison scoped variables at the beginning of the scope "
281 "(slower, but more precise)"),
292 "msan-handle-asm-conservative",
303 "msan-check-access-address",
304 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
309 cl::desc(
"check arguments and return values at function call boundaries"),
313 "msan-dump-strict-instructions",
314 cl::desc(
"print out instructions with default strict semantics"),
318 "msan-instrumentation-with-call-threshold",
320 "If the function being instrumented requires more than "
321 "this number of checks and origin stores, use callbacks instead of "
322 "inline checks (-1 means never use callbacks)."),
327 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
337 cl::desc(
"Insert checks for constant shadow values"),
344 cl::desc(
"Place MSan constructors in comdat sections"),
350 cl::desc(
"Define custom MSan AndMask"),
354 cl::desc(
"Define custom MSan XorMask"),
358 cl::desc(
"Define custom MSan ShadowBase"),
362 cl::desc(
"Define custom MSan OriginBase"),
367 cl::desc(
"Define threshold for number of checks per "
368 "debug location to force origin update."),
380struct MemoryMapParams {
387struct PlatformMemoryMapParams {
388 const MemoryMapParams *bits32;
389 const MemoryMapParams *bits64;
535class MemorySanitizer {
544 MemorySanitizer(MemorySanitizer &&) =
delete;
545 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
546 MemorySanitizer(
const MemorySanitizer &) =
delete;
547 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
552 friend struct MemorySanitizerVisitor;
553 friend struct VarArgHelperBase;
554 friend struct VarArgAMD64Helper;
555 friend struct VarArgMIPS64Helper;
556 friend struct VarArgAArch64Helper;
557 friend struct VarArgPowerPC64Helper;
558 friend struct VarArgSystemZHelper;
560 void initializeModule(
Module &M);
565 template <
typename... ArgsTy>
592 Value *ParamOriginTLS;
598 Value *RetvalOriginTLS;
604 Value *VAArgOriginTLS;
607 Value *VAArgOverflowSizeTLS;
610 bool CallbacksInitialized =
false;
655 Value *MsanMetadataAlloca;
661 const MemoryMapParams *MapParams;
665 MemoryMapParams CustomMapParams;
670 MDNode *OriginStoreWeights;
673void insertModuleCtor(
Module &M) {
701 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
716 MemorySanitizer Msan(*
F.getParent(),
Options);
735 OS, MapClassName2PassName);
742 OS <<
"eager-checks;";
743 OS <<
"track-origins=" <<
Options.TrackOrigins;
759template <
typename... ArgsTy>
767 std::forward<ArgsTy>(Args)...);
770 return M.getOrInsertFunction(
Name, MsanMetadata,
771 std::forward<ArgsTy>(Args)...);
780 RetvalOriginTLS =
nullptr;
782 ParamOriginTLS =
nullptr;
784 VAArgOriginTLS =
nullptr;
785 VAArgOverflowSizeTLS =
nullptr;
787 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
789 IRB.getVoidTy(), IRB.getInt32Ty());
800 MsanGetContextStateFn =
M.getOrInsertFunction(
806 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
807 std::string name_load =
808 "__msan_metadata_ptr_for_load_" + std::to_string(size);
809 std::string name_store =
810 "__msan_metadata_ptr_for_store_" + std::to_string(size);
811 MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
813 MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
817 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
820 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
821 M,
"__msan_metadata_ptr_for_store_n",
825 MsanPoisonAllocaFn =
M.getOrInsertFunction(
826 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
827 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
828 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
832 return M.getOrInsertGlobal(
Name, Ty, [&] {
834 nullptr,
Name,
nullptr,
847 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
848 :
"__msan_warning_with_origin_noreturn";
849 WarningFn =
M.getOrInsertFunction(WarningFnName,
851 IRB.getVoidTy(), IRB.getInt32Ty());
854 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
855 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
881 VAArgOverflowSizeTLS =
886 unsigned AccessSize = 1 << AccessSizeIndex;
887 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
888 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
890 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
892 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
893 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
895 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
899 MsanSetAllocaOriginWithDescriptionFn =
900 M.getOrInsertFunction(
"__msan_set_alloca_origin_with_descr",
901 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
902 MsanSetAllocaOriginNoDescriptionFn =
903 M.getOrInsertFunction(
"__msan_set_alloca_origin_no_descr",
904 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
905 MsanPoisonStackFn =
M.getOrInsertFunction(
"__msan_poison_stack",
906 IRB.getVoidTy(), PtrTy, IntptrTy);
912 if (CallbacksInitialized)
918 MsanChainOriginFn =
M.getOrInsertFunction(
919 "__msan_chain_origin",
922 MsanSetOriginFn =
M.getOrInsertFunction(
924 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
926 M.getOrInsertFunction(
"__msan_memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
928 M.getOrInsertFunction(
"__msan_memcpy", PtrTy, PtrTy, PtrTy, IntptrTy);
929 MemsetFn =
M.getOrInsertFunction(
"__msan_memset",
931 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
933 MsanInstrumentAsmStoreFn =
934 M.getOrInsertFunction(
"__msan_instrument_asm_store", IRB.getVoidTy(),
938 createKernelApi(M, TLI);
940 createUserspaceApi(M, TLI);
942 CallbacksInitialized =
true;
948 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
966void MemorySanitizer::initializeModule(
Module &M) {
967 auto &
DL =
M.getDataLayout();
969 TargetTriple =
Triple(
M.getTargetTriple());
971 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
972 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
974 if (ShadowPassed || OriginPassed) {
979 MapParams = &CustomMapParams;
981 switch (TargetTriple.getOS()) {
983 switch (TargetTriple.getArch()) {
998 switch (TargetTriple.getArch()) {
1007 switch (TargetTriple.getArch()) {
1041 C = &(
M.getContext());
1043 IntptrTy = IRB.getIntPtrTy(
DL);
1044 OriginTy = IRB.getInt32Ty();
1045 PtrTy = IRB.getPtrTy();
1050 if (!CompileKernel) {
1052 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1053 return new GlobalVariable(
1054 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1055 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1059 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1060 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1061 GlobalValue::WeakODRLinkage,
1062 IRB.getInt32(Recover),
"__msan_keep_going");
1077struct VarArgHelper {
1078 virtual ~VarArgHelper() =
default;
1093 virtual void finalizeInstrumentation() = 0;
1096struct MemorySanitizerVisitor;
1101 MemorySanitizerVisitor &Visitor);
1108 if (TypeSizeFixed <= 8)
1117class NextNodeIRBuilder :
public IRBuilder<> {
1130struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1132 MemorySanitizer &MS;
1135 std::unique_ptr<VarArgHelper> VAHelper;
1143 bool PropagateShadow;
1147 struct ShadowOriginAndInsertPoint {
1153 : Shadow(S), Origin(
O), OrigIns(
I) {}
1161 int64_t SplittableBlocksCount = 0;
1163 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1166 bool SanitizeFunction =
1168 InsertChecks = SanitizeFunction;
1169 PropagateShadow = SanitizeFunction;
1179 MS.initializeCallbacks(*
F.getParent(), TLI);
1180 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1183 if (MS.CompileKernel) {
1185 insertKmsanPrologue(IRB);
1189 <<
"MemorySanitizer is not inserting checks into '"
1190 <<
F.getName() <<
"'\n");
1193 bool instrumentWithCalls(
Value *V) {
1195 if (isa<Constant>(V))
1198 ++SplittableBlocksCount;
1204 return I.getParent() == FnPrologueEnd->
getParent() &&
1205 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1213 if (MS.TrackOrigins <= 1)
1215 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1220 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1232 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1233 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1245 auto [InsertPt,
Index] =
1257 Align CurrentAlignment = Alignment;
1258 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1259 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1260 Value *IntptrOriginPtr =
1262 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1267 CurrentAlignment = IntptrAlignment;
1284 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1285 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1293 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1302 if (instrumentWithCalls(ConvertedShadow) &&
1305 Value *ConvertedShadow2 =
1311 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1315 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1320 void materializeStores() {
1323 Value *Val =
SI->getValueOperand();
1325 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1326 Value *ShadowPtr, *OriginPtr;
1328 const Align Alignment =
SI->getAlign();
1330 std::tie(ShadowPtr, OriginPtr) =
1331 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1340 if (MS.TrackOrigins && !
SI->isAtomic())
1341 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1348 if (MS.TrackOrigins < 2)
1351 if (LazyWarningDebugLocationCount.
empty())
1352 for (
const auto &
I : InstrumentationList)
1353 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1367 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1369 auto NewDebugLoc = OI->getDebugLoc();
1376 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1377 Origin = updateOrigin(Origin, IRBOrigin);
1382 if (MS.CompileKernel || MS.TrackOrigins)
1396 if (instrumentWithCalls(ConvertedShadow) &&
1399 Value *ConvertedShadow2 =
1402 Fn, {ConvertedShadow2,
1403 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1407 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1410 !MS.Recover, MS.ColdCallWeights);
1413 insertWarningFn(IRB, Origin);
1418 void materializeInstructionChecks(
1423 bool Combine = !MS.TrackOrigins;
1425 Value *Shadow =
nullptr;
1426 for (
const auto &ShadowData : InstructionChecks) {
1430 Value *ConvertedShadow = ShadowData.Shadow;
1432 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1439 insertWarningFn(IRB, ShadowData.Origin);
1449 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1454 Shadow = ConvertedShadow;
1458 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1459 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1460 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1466 materializeOneCheck(IRB, Shadow,
nullptr);
1470 void materializeChecks() {
1476 for (
auto I = InstrumentationList.begin();
1477 I != InstrumentationList.end();) {
1478 auto OrigIns =
I->OrigIns;
1482 auto J = std::find_if(
I + 1, InstrumentationList.end(),
1483 [OrigIns](
const ShadowOriginAndInsertPoint &R) {
1484 return OrigIns != R.OrigIns;
1498 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1499 {Zero, IRB.getInt32(0)},
"param_shadow");
1500 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1501 {Zero, IRB.getInt32(1)},
"retval_shadow");
1502 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1503 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1504 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1505 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1506 MS.VAArgOverflowSizeTLS =
1507 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1508 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1509 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1510 {Zero, IRB.getInt32(5)},
"param_origin");
1511 MS.RetvalOriginTLS =
1512 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1513 {Zero, IRB.getInt32(6)},
"retval_origin");
1515 MS.MsanMetadataAlloca = IRB.
CreateAlloca(MS.MsanMetadata, 0u);
1532 for (
PHINode *PN : ShadowPHINodes) {
1533 PHINode *PNS = cast<PHINode>(getShadow(PN));
1534 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1535 size_t NumValues = PN->getNumIncomingValues();
1536 for (
size_t v = 0;
v < NumValues;
v++) {
1537 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1539 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1543 VAHelper->finalizeInstrumentation();
1547 if (InstrumentLifetimeStart) {
1548 for (
auto Item : LifetimeStartList) {
1549 instrumentAlloca(*Item.second, Item.first);
1550 AllocaSet.
remove(Item.second);
1556 instrumentAlloca(*AI);
1559 materializeChecks();
1563 materializeStores();
1569 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1581 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1582 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1584 VT->getElementCount());
1586 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1587 return ArrayType::get(getShadowTy(AT->getElementType()),
1588 AT->getNumElements());
1590 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1592 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1593 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1595 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1611 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1613 if (Aggregator != FalseVal)
1614 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1616 Aggregator = ShadowBool;
1625 if (!
Array->getNumElements())
1629 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1633 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1634 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1644 return collapseStructShadow(
Struct, V, IRB);
1645 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1646 return collapseArrayShadow(Array, V, IRB);
1647 if (isa<VectorType>(
V->getType())) {
1648 if (isa<ScalableVectorType>(
V->getType()))
1651 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1659 Type *VTy =
V->getType();
1661 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1668 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1669 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1670 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1671 VectTy->getElementCount());
1677 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1678 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1679 return VectorType::get(
1680 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1681 VectTy->getElementCount());
1683 assert(IntPtrTy == MS.IntptrTy);
1684 return PointerType::get(*MS.C, 0);
1688 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1690 VectTy->getElementCount(), constToIntPtr(VectTy->getElementType(),
C));
1692 assert(IntPtrTy == MS.IntptrTy);
1693 return ConstantInt::get(MS.IntptrTy,
C);
1704 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1707 if (
uint64_t AndMask = MS.MapParams->AndMask)
1708 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1710 if (
uint64_t XorMask = MS.MapParams->XorMask)
1711 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1723 std::pair<Value *, Value *>
1730 assert(VectTy->getElementType()->isPointerTy());
1732 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1733 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1734 Value *ShadowLong = ShadowOffset;
1735 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1737 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1740 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1742 Value *OriginPtr =
nullptr;
1743 if (MS.TrackOrigins) {
1744 Value *OriginLong = ShadowOffset;
1745 uint64_t OriginBase = MS.MapParams->OriginBase;
1746 if (OriginBase != 0)
1748 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1751 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1754 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1756 return std::make_pair(ShadowPtr, OriginPtr);
1759 template <
typename... ArgsTy>
1764 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1765 return IRB.
CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1768 return IRB.
CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1771 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1775 Value *ShadowOriginPtrs;
1783 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1785 Value *SizeVal = ConstantInt::get(MS.IntptrTy,
Size);
1786 ShadowOriginPtrs = createMetadataCall(
1788 isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1795 return std::make_pair(ShadowPtr, OriginPtr);
1801 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1808 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1812 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1813 Value *ShadowPtrs = ConstantInt::getNullValue(
1815 Value *OriginPtrs =
nullptr;
1816 if (MS.TrackOrigins)
1817 OriginPtrs = ConstantInt::getNullValue(
1819 for (
unsigned i = 0; i < NumElements; ++i) {
1822 auto [ShadowPtr, OriginPtr] =
1823 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1826 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1827 if (MS.TrackOrigins)
1829 OriginPtrs, OriginPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1831 return {ShadowPtrs, OriginPtrs};
1838 if (MS.CompileKernel)
1839 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1840 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1855 if (!MS.TrackOrigins)
1869 Value *getOriginPtrForRetval() {
1871 return MS.RetvalOriginTLS;
1876 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1877 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1882 if (!MS.TrackOrigins)
1884 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1885 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1886 OriginMap[
V] = Origin;
1890 Type *ShadowTy = getShadowTy(OrigTy);
1900 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1905 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1907 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1909 getPoisonedShadow(AT->getElementType()));
1912 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1914 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1915 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1923 Type *ShadowTy = getShadowTy(V);
1926 return getPoisonedShadow(ShadowTy);
1938 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1939 return getCleanShadow(V);
1941 Value *Shadow = ShadowMap[
V];
1943 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1945 assert(Shadow &&
"No shadow for a value");
1949 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1950 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1951 : getCleanShadow(V);
1956 if (
Argument *
A = dyn_cast<Argument>(V)) {
1958 Value *&ShadowPtr = ShadowMap[
V];
1963 unsigned ArgOffset = 0;
1965 for (
auto &FArg :
F->args()) {
1966 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
1968 ?
"vscale not fully supported\n"
1969 :
"Arg is not sized\n"));
1971 ShadowPtr = getCleanShadow(V);
1972 setOrigin(
A, getCleanOrigin());
1978 unsigned Size = FArg.hasByValAttr()
1979 ?
DL.getTypeAllocSize(FArg.getParamByValType())
1980 :
DL.getTypeAllocSize(FArg.getType());
1984 if (FArg.hasByValAttr()) {
1988 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
1989 FArg.getParamAlign(), FArg.getParamByValType());
1990 Value *CpShadowPtr, *CpOriginPtr;
1991 std::tie(CpShadowPtr, CpOriginPtr) =
1992 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
1994 if (!PropagateShadow || Overflow) {
1996 EntryIRB.CreateMemSet(
2000 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2002 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
2007 if (MS.TrackOrigins) {
2009 getOriginPtrForArgument(EntryIRB, ArgOffset);
2013 EntryIRB.CreateMemCpy(
2022 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2023 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
2024 ShadowPtr = getCleanShadow(V);
2025 setOrigin(
A, getCleanOrigin());
2028 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2029 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
2031 if (MS.TrackOrigins) {
2033 getOriginPtrForArgument(EntryIRB, ArgOffset);
2034 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
2038 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
2044 assert(ShadowPtr &&
"Could not find shadow for an argument");
2048 return getCleanShadow(V);
2053 return getShadow(
I->getOperand(i));
2058 if (!MS.TrackOrigins)
2060 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
2061 return getCleanOrigin();
2062 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2063 "Unexpected value type in getOrigin()");
2065 if (
I->getMetadata(LLVMContext::MD_nosanitize))
2066 return getCleanOrigin();
2068 Value *Origin = OriginMap[
V];
2069 assert(Origin &&
"Missing origin");
2075 return getOrigin(
I->getOperand(i));
2088 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
2089 << *OrigIns <<
"\n");
2094 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2095 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2096 "Can only insert checks for integer, vector, and aggregate shadow "
2099 InstrumentationList.push_back(
2100 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2109 Value *Shadow, *Origin;
2111 Shadow = getShadow(Val);
2114 Origin = getOrigin(Val);
2116 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2119 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2121 insertShadowCheck(Shadow, Origin, OrigIns);
2126 case AtomicOrdering::NotAtomic:
2127 return AtomicOrdering::NotAtomic;
2128 case AtomicOrdering::Unordered:
2129 case AtomicOrdering::Monotonic:
2130 case AtomicOrdering::Release:
2131 return AtomicOrdering::Release;
2132 case AtomicOrdering::Acquire:
2133 case AtomicOrdering::AcquireRelease:
2134 return AtomicOrdering::AcquireRelease;
2135 case AtomicOrdering::SequentiallyConsistent:
2136 return AtomicOrdering::SequentiallyConsistent;
2142 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2143 uint32_t OrderingTable[NumOrderings] = {};
2145 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2146 OrderingTable[(
int)AtomicOrderingCABI::release] =
2147 (int)AtomicOrderingCABI::release;
2148 OrderingTable[(int)AtomicOrderingCABI::consume] =
2149 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2150 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2151 (
int)AtomicOrderingCABI::acq_rel;
2152 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2153 (
int)AtomicOrderingCABI::seq_cst;
2160 case AtomicOrdering::NotAtomic:
2161 return AtomicOrdering::NotAtomic;
2162 case AtomicOrdering::Unordered:
2163 case AtomicOrdering::Monotonic:
2164 case AtomicOrdering::Acquire:
2165 return AtomicOrdering::Acquire;
2166 case AtomicOrdering::Release:
2167 case AtomicOrdering::AcquireRelease:
2168 return AtomicOrdering::AcquireRelease;
2169 case AtomicOrdering::SequentiallyConsistent:
2170 return AtomicOrdering::SequentiallyConsistent;
2176 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2177 uint32_t OrderingTable[NumOrderings] = {};
2179 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2180 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2181 OrderingTable[(int)AtomicOrderingCABI::consume] =
2182 (
int)AtomicOrderingCABI::acquire;
2183 OrderingTable[(int)AtomicOrderingCABI::release] =
2184 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2185 (int)AtomicOrderingCABI::acq_rel;
2186 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2187 (
int)AtomicOrderingCABI::seq_cst;
2195 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2198 if (isInPrologue(
I))
2203 setShadow(&
I, getCleanShadow(&
I));
2204 setOrigin(&
I, getCleanOrigin());
2216 assert(
I.getType()->isSized() &&
"Load type must have size");
2217 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2218 NextNodeIRBuilder IRB(&
I);
2219 Type *ShadowTy = getShadowTy(&
I);
2221 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2222 const Align Alignment =
I.getAlign();
2223 if (PropagateShadow) {
2224 std::tie(ShadowPtr, OriginPtr) =
2225 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2229 setShadow(&
I, getCleanShadow(&
I));
2233 insertShadowCheck(
I.getPointerOperand(), &
I);
2238 if (MS.TrackOrigins) {
2239 if (PropagateShadow) {
2244 setOrigin(&
I, getCleanOrigin());
2254 StoreList.push_back(&
I);
2256 insertShadowCheck(
I.getPointerOperand(), &
I);
2260 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2264 Value *Val =
I.getOperand(1);
2265 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2270 insertShadowCheck(
Addr, &
I);
2275 if (isa<AtomicCmpXchgInst>(
I))
2276 insertShadowCheck(Val, &
I);
2280 setShadow(&
I, getCleanShadow(&
I));
2281 setOrigin(&
I, getCleanOrigin());
2296 insertShadowCheck(
I.getOperand(1), &
I);
2300 setOrigin(&
I, getOrigin(&
I, 0));
2304 insertShadowCheck(
I.getOperand(2), &
I);
2306 auto *Shadow0 = getShadow(&
I, 0);
2307 auto *Shadow1 = getShadow(&
I, 1);
2310 setOriginForNaryOp(
I);
2315 auto *Shadow0 = getShadow(&
I, 0);
2316 auto *Shadow1 = getShadow(&
I, 1);
2319 setOriginForNaryOp(
I);
2325 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2326 setOrigin(&
I, getOrigin(&
I, 0));
2331 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2332 setOrigin(&
I, getOrigin(&
I, 0));
2337 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2338 setOrigin(&
I, getOrigin(&
I, 0));
2345 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2346 if (CI->isMustTailCall())
2350 setOrigin(&
I, getOrigin(&
I, 0));
2356 "_msprop_ptrtoint"));
2357 setOrigin(&
I, getOrigin(&
I, 0));
2363 "_msprop_inttoptr"));
2364 setOrigin(&
I, getOrigin(&
I, 0));
2367 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2368 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2369 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2370 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2371 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2372 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2387 Value *S2 = getShadow(&
I, 1);
2388 Value *V1 =
I.getOperand(0);
2397 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2398 setOriginForNaryOp(
I);
2409 Value *S2 = getShadow(&
I, 1);
2419 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2420 setOriginForNaryOp(
I);
2438 template <
bool CombineShadow>
class Combiner {
2439 Value *Shadow =
nullptr;
2440 Value *Origin =
nullptr;
2442 MemorySanitizerVisitor *MSV;
2446 : IRB(IRB), MSV(MSV) {}
2450 if (CombineShadow) {
2455 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2456 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2460 if (MSV->MS.TrackOrigins) {
2465 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2467 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2468 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2478 Value *OpShadow = MSV->getShadow(V);
2479 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2480 return Add(OpShadow, OpOrigin);
2486 if (CombineShadow) {
2488 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2489 MSV->setShadow(
I, Shadow);
2491 if (MSV->MS.TrackOrigins) {
2493 MSV->setOrigin(
I, Origin);
2503 if (!MS.TrackOrigins)
2506 OriginCombiner
OC(
this, IRB);
2507 for (
Use &
Op :
I.operands())
2512 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2514 "Vector of pointers is not a valid shadow type");
2515 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2524 Type *srcTy =
V->getType();
2527 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2528 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2529 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2535 cast<VectorType>(dstTy)->getElementCount() ==
2536 cast<VectorType>(srcTy)->getElementCount())
2547 Type *ShadowTy = getShadowTy(V);
2548 if (
V->getType() == ShadowTy)
2550 if (
V->getType()->isPtrOrPtrVectorTy())
2559 ShadowAndOriginCombiner
SC(
this, IRB);
2560 for (
Use &
Op :
I.operands())
2580 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2581 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2582 Type *EltTy = VTy->getElementType();
2584 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2587 const APInt &
V = Elt->getValue();
2589 Elements.push_back(ConstantInt::get(EltTy, V2));
2591 Elements.push_back(ConstantInt::get(EltTy, 1));
2596 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2597 const APInt &
V = Elt->getValue();
2599 ShadowMul = ConstantInt::get(Ty, V2);
2601 ShadowMul = ConstantInt::get(Ty, 1);
2607 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2608 setOrigin(&
I, getOrigin(OtherArg));
2612 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2613 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2614 if (constOp0 && !constOp1)
2615 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2616 else if (constOp1 && !constOp0)
2617 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2632 insertShadowCheck(
I.getOperand(1), &
I);
2633 setShadow(&
I, getShadow(&
I, 0));
2634 setOrigin(&
I, getOrigin(&
I, 0));
2651 void handleEqualityComparison(
ICmpInst &
I) {
2655 Value *Sa = getShadow(
A);
2656 Value *Sb = getShadow(
B);
2682 setOriginForNaryOp(
I);
2724 void handleRelationalComparisonExact(
ICmpInst &
I) {
2728 Value *Sa = getShadow(
A);
2729 Value *Sb = getShadow(
B);
2740 bool IsSigned =
I.isSigned();
2742 getLowestPossibleValue(IRB,
A, Sa, IsSigned),
2743 getHighestPossibleValue(IRB,
B, Sb, IsSigned));
2745 getHighestPossibleValue(IRB,
A, Sa, IsSigned),
2746 getLowestPossibleValue(IRB,
B, Sb, IsSigned));
2749 setOriginForNaryOp(
I);
2756 void handleSignedRelationalComparison(
ICmpInst &
I) {
2760 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2761 op =
I.getOperand(0);
2762 pre =
I.getPredicate();
2763 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2764 op =
I.getOperand(1);
2765 pre =
I.getSwappedPredicate();
2778 setShadow(&
I, Shadow);
2779 setOrigin(&
I, getOrigin(
op));
2790 if (
I.isEquality()) {
2791 handleEqualityComparison(
I);
2797 handleRelationalComparisonExact(
I);
2801 handleSignedRelationalComparison(
I);
2806 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2807 handleRelationalComparisonExact(
I);
2814 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2821 Value *S2 = getShadow(&
I, 1);
2826 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2827 setOriginForNaryOp(
I);
2838 Value *S0 = getShadow(&
I, 0);
2840 Value *S2 = getShadow(&
I, 2);
2845 I.getModule(),
I.getIntrinsicID(), S2Conv->
getType());
2847 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2848 setOriginForNaryOp(
I);
2862 getShadow(
I.getArgOperand(1));
2865 {I.getArgOperand(0), I.getArgOperand(1),
2866 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2867 I.eraseFromParent();
2885 getShadow(
I.getArgOperand(1));
2888 {I.getArgOperand(0), I.getArgOperand(1),
2889 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2890 I.eraseFromParent();
2898 {I.getArgOperand(0),
2899 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2900 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2901 I.eraseFromParent();
2904 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2906 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2915 Value *Shadow = getShadow(&
I, 1);
2916 Value *ShadowPtr, *OriginPtr;
2920 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2925 insertShadowCheck(
Addr, &
I);
2928 if (MS.TrackOrigins)
2941 Type *ShadowTy = getShadowTy(&
I);
2942 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2943 if (PropagateShadow) {
2947 std::tie(ShadowPtr, OriginPtr) =
2948 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2952 setShadow(&
I, getCleanShadow(&
I));
2956 insertShadowCheck(
Addr, &
I);
2958 if (MS.TrackOrigins) {
2959 if (PropagateShadow)
2960 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2962 setOrigin(&
I, getCleanOrigin());
2975 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy() ||
2976 RetTy->isX86_MMXTy()))
2979 unsigned NumArgOperands =
I.arg_size();
2980 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2981 Type *Ty =
I.getArgOperand(i)->getType();
2987 ShadowAndOriginCombiner
SC(
this, IRB);
2988 for (
unsigned i = 0; i < NumArgOperands; ++i)
2989 SC.Add(
I.getArgOperand(i));
3006 unsigned NumArgOperands =
I.arg_size();
3007 if (NumArgOperands == 0)
3010 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3011 I.getArgOperand(1)->getType()->isVectorTy() &&
3012 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
3014 return handleVectorStoreIntrinsic(
I);
3017 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3018 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
3020 return handleVectorLoadIntrinsic(
I);
3023 if (
I.doesNotAccessMemory())
3024 if (maybeHandleSimpleNomemIntrinsic(
I))
3032 setShadow(&
I, getShadow(&
I, 0));
3033 setOrigin(&
I, getOrigin(&
I, 0));
3041 InstrumentLifetimeStart =
false;
3042 LifetimeStartList.push_back(std::make_pair(&
I, AI));
3048 Type *OpType =
Op->getType();
3050 F.getParent(), Intrinsic::bswap,
ArrayRef(&OpType, 1));
3052 setOrigin(&
I, getOrigin(
Op));
3057 Value *Src =
I.getArgOperand(0);
3063 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
3066 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
3069 Value *OutputShadow =
3070 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
3072 setShadow(&
I, OutputShadow);
3073 setOriginForNaryOp(
I);
3091 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
3092 bool HasRoundingMode =
false) {
3094 Value *CopyOp, *ConvertOp;
3096 assert((!HasRoundingMode ||
3097 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
3098 "Invalid rounding mode");
3100 switch (
I.arg_size() - HasRoundingMode) {
3102 CopyOp =
I.getArgOperand(0);
3103 ConvertOp =
I.getArgOperand(1);
3106 ConvertOp =
I.getArgOperand(0);
3120 Value *ConvertShadow = getShadow(ConvertOp);
3121 Value *AggShadow =
nullptr;
3124 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), 0));
3125 for (
int i = 1; i < NumUsedElements; ++i) {
3127 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), i));
3128 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3131 AggShadow = ConvertShadow;
3134 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3141 Value *ResultShadow = getShadow(CopyOp);
3142 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3143 for (
int i = 0; i < NumUsedElements; ++i) {
3145 ResultShadow, ConstantInt::getNullValue(EltTy),
3148 setShadow(&
I, ResultShadow);
3149 setOrigin(&
I, getOrigin(CopyOp));
3151 setShadow(&
I, getCleanShadow(&
I));
3152 setOrigin(&
I, getCleanOrigin());
3160 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3163 return CreateShadowCast(IRB, S2,
T,
true);
3171 return CreateShadowCast(IRB, S2,
T,
true);
3188 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3194 Value *S2 = getShadow(&
I, 1);
3195 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3196 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3197 Value *V1 =
I.getOperand(0);
3200 {IRB.CreateBitCast(S1, V1->getType()), V2});
3202 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3203 setOriginForNaryOp(
I);
3207 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3208 const unsigned X86_MMXSizeInBits = 64;
3209 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3210 "Illegal MMX vector element size");
3212 X86_MMXSizeInBits / EltSizeInBits);
3219 case Intrinsic::x86_sse2_packsswb_128:
3220 case Intrinsic::x86_sse2_packuswb_128:
3221 return Intrinsic::x86_sse2_packsswb_128;
3223 case Intrinsic::x86_sse2_packssdw_128:
3224 case Intrinsic::x86_sse41_packusdw:
3225 return Intrinsic::x86_sse2_packssdw_128;
3227 case Intrinsic::x86_avx2_packsswb:
3228 case Intrinsic::x86_avx2_packuswb:
3229 return Intrinsic::x86_avx2_packsswb;
3231 case Intrinsic::x86_avx2_packssdw:
3232 case Intrinsic::x86_avx2_packusdw:
3233 return Intrinsic::x86_avx2_packssdw;
3235 case Intrinsic::x86_mmx_packsswb:
3236 case Intrinsic::x86_mmx_packuswb:
3237 return Intrinsic::x86_mmx_packsswb;
3239 case Intrinsic::x86_mmx_packssdw:
3240 return Intrinsic::x86_mmx_packssdw;
3253 void handleVectorPackIntrinsic(
IntrinsicInst &
I,
unsigned EltSizeInBits = 0) {
3255 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3258 Value *S2 = getShadow(&
I, 1);
3259 assert(isX86_MMX ||
S1->getType()->isVectorTy());
3264 Type *
T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) :
S1->
getType();
3280 F.getParent(), getSignedPackIntrinsic(
I.getIntrinsicID()));
3283 IRB.
CreateCall(ShadowFn, {S1_ext, S2_ext},
"_msprop_vector_pack");
3287 setOriginForNaryOp(
I);
3292 const unsigned SignificantBitsPerResultElement = 16;
3293 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3295 unsigned ZeroBitsPerResultElement =
3299 auto *Shadow0 = getShadow(&
I, 0);
3300 auto *Shadow1 = getShadow(&
I, 1);
3305 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3308 setOriginForNaryOp(
I);
3313 unsigned EltSizeInBits = 0) {
3314 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3315 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) :
I.
getType();
3317 auto *Shadow0 = getShadow(&
I, 0);
3318 auto *Shadow1 = getShadow(&
I, 1);
3325 setOriginForNaryOp(
I);
3333 Type *ResTy = getShadowTy(&
I);
3334 auto *Shadow0 = getShadow(&
I, 0);
3335 auto *Shadow1 = getShadow(&
I, 1);
3340 setOriginForNaryOp(
I);
3348 auto *Shadow0 = getShadow(&
I, 0);
3349 auto *Shadow1 = getShadow(&
I, 1);
3351 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3353 setOriginForNaryOp(
I);
3362 setOrigin(&
I, getOrigin(&
I, 0));
3370 Value *OperandShadow = getShadow(&
I, 0);
3372 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3380 setOrigin(&
I, getOrigin(&
I, 0));
3388 Value *OperandShadow = getShadow(&
I, 0);
3389 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3397 setOrigin(&
I, getOrigin(&
I, 0));
3405 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3410 insertShadowCheck(
Addr, &
I);
3421 Value *ShadowPtr, *OriginPtr;
3422 std::tie(ShadowPtr, OriginPtr) =
3423 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3426 insertShadowCheck(
Addr, &
I);
3429 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3431 insertShadowCheck(Shadow, Origin, &
I);
3438 Value *PassThru =
I.getArgOperand(2);
3441 insertShadowCheck(
Ptr, &
I);
3442 insertShadowCheck(Mask, &
I);
3445 if (!PropagateShadow) {
3446 setShadow(&
I, getCleanShadow(&
I));
3447 setOrigin(&
I, getCleanOrigin());
3451 Type *ShadowTy = getShadowTy(&
I);
3452 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3453 auto [ShadowPtr, OriginPtr] =
3454 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3457 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3459 setShadow(&
I, Shadow);
3462 setOrigin(&
I, getCleanOrigin());
3467 Value *Values =
I.getArgOperand(0);
3472 insertShadowCheck(
Ptr, &
I);
3473 insertShadowCheck(Mask, &
I);
3476 Value *Shadow = getShadow(Values);
3477 Type *ElementShadowTy =
3478 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3479 auto [ShadowPtr, OriginPtrs] =
3480 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3489 Value *Ptrs =
I.getArgOperand(0);
3490 const Align Alignment(
3491 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3493 Value *PassThru =
I.getArgOperand(3);
3495 Type *PtrsShadowTy = getShadowTy(Ptrs);
3497 insertShadowCheck(Mask, &
I);
3501 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3504 if (!PropagateShadow) {
3505 setShadow(&
I, getCleanShadow(&
I));
3506 setOrigin(&
I, getCleanOrigin());
3510 Type *ShadowTy = getShadowTy(&
I);
3511 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3512 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3513 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3517 getShadow(PassThru),
"_msmaskedgather");
3519 setShadow(&
I, Shadow);
3522 setOrigin(&
I, getCleanOrigin());
3527 Value *Values =
I.getArgOperand(0);
3528 Value *Ptrs =
I.getArgOperand(1);
3529 const Align Alignment(
3530 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3533 Type *PtrsShadowTy = getShadowTy(Ptrs);
3535 insertShadowCheck(Mask, &
I);
3539 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3542 Value *Shadow = getShadow(Values);
3543 Type *ElementShadowTy =
3544 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3545 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3546 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3555 Value *
V =
I.getArgOperand(0);
3557 const Align Alignment(
3558 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3560 Value *Shadow = getShadow(V);
3563 insertShadowCheck(
Ptr, &
I);
3564 insertShadowCheck(Mask, &
I);
3569 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3570 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3574 if (!MS.TrackOrigins)
3577 auto &
DL =
F.getParent()->getDataLayout();
3578 paintOrigin(IRB, getOrigin(V), OriginPtr,
3586 const Align Alignment(
3587 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3589 Value *PassThru =
I.getArgOperand(3);
3592 insertShadowCheck(
Ptr, &
I);
3593 insertShadowCheck(Mask, &
I);
3596 if (!PropagateShadow) {
3597 setShadow(&
I, getCleanShadow(&
I));
3598 setOrigin(&
I, getCleanOrigin());
3602 Type *ShadowTy = getShadowTy(&
I);
3603 Value *ShadowPtr, *OriginPtr;
3604 std::tie(ShadowPtr, OriginPtr) =
3605 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3607 getShadow(PassThru),
"_msmaskedld"));
3609 if (!MS.TrackOrigins)
3616 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3621 setOrigin(&
I, Origin);
3631 Type *ShadowTy = getShadowTy(&
I);
3634 Value *SMask = getShadow(&
I, 1);
3639 {getShadow(&I, 0), I.getOperand(1)});
3642 setOriginForNaryOp(
I);
3647 for (
unsigned X = OddElements ? 1 : 0;
X < Width;
X += 2) {
3664 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3665 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3666 "pclmul 3rd operand must be a constant");
3667 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3669 getPclmulMask(Width, Imm & 0x01));
3671 getPclmulMask(Width, Imm & 0x10));
3672 ShadowAndOriginCombiner SOC(
this, IRB);
3673 SOC.Add(Shuf0, getOrigin(&
I, 0));
3674 SOC.Add(Shuf1, getOrigin(&
I, 1));
3682 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3684 Value *Second = getShadow(&
I, 1);
3687 Mask.push_back(Width);
3688 for (
unsigned i = 1; i < Width; i++)
3692 setShadow(&
I, Shadow);
3693 setOriginForNaryOp(
I);
3698 Value *Shadow0 = getShadow(&
I, 0);
3699 Value *Shadow1 = getShadow(&
I, 1);
3705 setShadow(&
I, Shadow);
3706 setOriginForNaryOp(
I);
3712 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3714 Value *Second = getShadow(&
I, 1);
3718 Mask.push_back(Width);
3719 for (
unsigned i = 1; i < Width; i++)
3723 setShadow(&
I, Shadow);
3724 setOriginForNaryOp(
I);
3731 assert(
I.getType()->isIntOrIntVectorTy());
3732 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3736 setShadow(&
I, getShadow(&
I, 0));
3737 setOrigin(&
I, getOrigin(&
I, 0));
3742 Value *Shadow = getShadow(&
I, 0);
3743 setShadow(&
I, IRB.
CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3744 setOrigin(&
I, getOrigin(&
I, 0));
3749 Value *Shadow0 = getShadow(&
I, 0);
3750 Value *Shadow1 = getShadow(&
I, 1);
3753 IRB.
CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
3759 setShadow(&
I, Shadow);
3760 setOriginForNaryOp(
I);
3764 switch (
I.getIntrinsicID()) {
3765 case Intrinsic::uadd_with_overflow:
3766 case Intrinsic::sadd_with_overflow:
3767 case Intrinsic::usub_with_overflow:
3768 case Intrinsic::ssub_with_overflow:
3769 case Intrinsic::umul_with_overflow:
3770 case Intrinsic::smul_with_overflow:
3771 handleArithmeticWithOverflow(
I);
3773 case Intrinsic::abs:
3774 handleAbsIntrinsic(
I);
3776 case Intrinsic::is_fpclass:
3779 case Intrinsic::lifetime_start:
3780 handleLifetimeStart(
I);
3782 case Intrinsic::launder_invariant_group:
3783 case Intrinsic::strip_invariant_group:
3784 handleInvariantGroup(
I);
3786 case Intrinsic::bswap:
3789 case Intrinsic::ctlz:
3790 case Intrinsic::cttz:
3791 handleCountZeroes(
I);
3793 case Intrinsic::masked_compressstore:
3794 handleMaskedCompressStore(
I);
3796 case Intrinsic::masked_expandload:
3797 handleMaskedExpandLoad(
I);
3799 case Intrinsic::masked_gather:
3800 handleMaskedGather(
I);
3802 case Intrinsic::masked_scatter:
3803 handleMaskedScatter(
I);
3805 case Intrinsic::masked_store:
3806 handleMaskedStore(
I);
3808 case Intrinsic::masked_load:
3809 handleMaskedLoad(
I);
3811 case Intrinsic::vector_reduce_and:
3812 handleVectorReduceAndIntrinsic(
I);
3814 case Intrinsic::vector_reduce_or:
3815 handleVectorReduceOrIntrinsic(
I);
3817 case Intrinsic::vector_reduce_add:
3818 case Intrinsic::vector_reduce_xor:
3819 case Intrinsic::vector_reduce_mul:
3820 handleVectorReduceIntrinsic(
I);
3822 case Intrinsic::x86_sse_stmxcsr:
3825 case Intrinsic::x86_sse_ldmxcsr:
3828 case Intrinsic::x86_avx512_vcvtsd2usi64:
3829 case Intrinsic::x86_avx512_vcvtsd2usi32:
3830 case Intrinsic::x86_avx512_vcvtss2usi64:
3831 case Intrinsic::x86_avx512_vcvtss2usi32:
3832 case Intrinsic::x86_avx512_cvttss2usi64:
3833 case Intrinsic::x86_avx512_cvttss2usi:
3834 case Intrinsic::x86_avx512_cvttsd2usi64:
3835 case Intrinsic::x86_avx512_cvttsd2usi:
3836 case Intrinsic::x86_avx512_cvtusi2ss:
3837 case Intrinsic::x86_avx512_cvtusi642sd:
3838 case Intrinsic::x86_avx512_cvtusi642ss:
3839 handleVectorConvertIntrinsic(
I, 1,
true);
3841 case Intrinsic::x86_sse2_cvtsd2si64:
3842 case Intrinsic::x86_sse2_cvtsd2si:
3843 case Intrinsic::x86_sse2_cvtsd2ss:
3844 case Intrinsic::x86_sse2_cvttsd2si64:
3845 case Intrinsic::x86_sse2_cvttsd2si:
3846 case Intrinsic::x86_sse_cvtss2si64:
3847 case Intrinsic::x86_sse_cvtss2si:
3848 case Intrinsic::x86_sse_cvttss2si64:
3849 case Intrinsic::x86_sse_cvttss2si:
3850 handleVectorConvertIntrinsic(
I, 1);
3852 case Intrinsic::x86_sse_cvtps2pi:
3853 case Intrinsic::x86_sse_cvttps2pi:
3854 handleVectorConvertIntrinsic(
I, 2);
3857 case Intrinsic::x86_avx512_psll_w_512:
3858 case Intrinsic::x86_avx512_psll_d_512:
3859 case Intrinsic::x86_avx512_psll_q_512:
3860 case Intrinsic::x86_avx512_pslli_w_512:
3861 case Intrinsic::x86_avx512_pslli_d_512:
3862 case Intrinsic::x86_avx512_pslli_q_512:
3863 case Intrinsic::x86_avx512_psrl_w_512:
3864 case Intrinsic::x86_avx512_psrl_d_512:
3865 case Intrinsic::x86_avx512_psrl_q_512:
3866 case Intrinsic::x86_avx512_psra_w_512:
3867 case Intrinsic::x86_avx512_psra_d_512:
3868 case Intrinsic::x86_avx512_psra_q_512:
3869 case Intrinsic::x86_avx512_psrli_w_512:
3870 case Intrinsic::x86_avx512_psrli_d_512:
3871 case Intrinsic::x86_avx512_psrli_q_512:
3872 case Intrinsic::x86_avx512_psrai_w_512:
3873 case Intrinsic::x86_avx512_psrai_d_512:
3874 case Intrinsic::x86_avx512_psrai_q_512:
3875 case Intrinsic::x86_avx512_psra_q_256:
3876 case Intrinsic::x86_avx512_psra_q_128:
3877 case Intrinsic::x86_avx512_psrai_q_256:
3878 case Intrinsic::x86_avx512_psrai_q_128:
3879 case Intrinsic::x86_avx2_psll_w:
3880 case Intrinsic::x86_avx2_psll_d:
3881 case Intrinsic::x86_avx2_psll_q:
3882 case Intrinsic::x86_avx2_pslli_w:
3883 case Intrinsic::x86_avx2_pslli_d:
3884 case Intrinsic::x86_avx2_pslli_q:
3885 case Intrinsic::x86_avx2_psrl_w:
3886 case Intrinsic::x86_avx2_psrl_d:
3887 case Intrinsic::x86_avx2_psrl_q:
3888 case Intrinsic::x86_avx2_psra_w:
3889 case Intrinsic::x86_avx2_psra_d:
3890 case Intrinsic::x86_avx2_psrli_w:
3891 case Intrinsic::x86_avx2_psrli_d:
3892 case Intrinsic::x86_avx2_psrli_q:
3893 case Intrinsic::x86_avx2_psrai_w:
3894 case Intrinsic::x86_avx2_psrai_d:
3895 case Intrinsic::x86_sse2_psll_w:
3896 case Intrinsic::x86_sse2_psll_d:
3897 case Intrinsic::x86_sse2_psll_q:
3898 case Intrinsic::x86_sse2_pslli_w:
3899 case Intrinsic::x86_sse2_pslli_d:
3900 case Intrinsic::x86_sse2_pslli_q:
3901 case Intrinsic::x86_sse2_psrl_w:
3902 case Intrinsic::x86_sse2_psrl_d:
3903 case Intrinsic::x86_sse2_psrl_q:
3904 case Intrinsic::x86_sse2_psra_w:
3905 case Intrinsic::x86_sse2_psra_d:
3906 case Intrinsic::x86_sse2_psrli_w:
3907 case Intrinsic::x86_sse2_psrli_d:
3908 case Intrinsic::x86_sse2_psrli_q:
3909 case Intrinsic::x86_sse2_psrai_w:
3910 case Intrinsic::x86_sse2_psrai_d:
3911 case Intrinsic::x86_mmx_psll_w:
3912 case Intrinsic::x86_mmx_psll_d:
3913 case Intrinsic::x86_mmx_psll_q:
3914 case Intrinsic::x86_mmx_pslli_w:
3915 case Intrinsic::x86_mmx_pslli_d:
3916 case Intrinsic::x86_mmx_pslli_q:
3917 case Intrinsic::x86_mmx_psrl_w:
3918 case Intrinsic::x86_mmx_psrl_d:
3919 case Intrinsic::x86_mmx_psrl_q:
3920 case Intrinsic::x86_mmx_psra_w:
3921 case Intrinsic::x86_mmx_psra_d:
3922 case Intrinsic::x86_mmx_psrli_w:
3923 case Intrinsic::x86_mmx_psrli_d:
3924 case Intrinsic::x86_mmx_psrli_q:
3925 case Intrinsic::x86_mmx_psrai_w:
3926 case Intrinsic::x86_mmx_psrai_d:
3927 handleVectorShiftIntrinsic(
I,
false);
3929 case Intrinsic::x86_avx2_psllv_d:
3930 case Intrinsic::x86_avx2_psllv_d_256:
3931 case Intrinsic::x86_avx512_psllv_d_512:
3932 case Intrinsic::x86_avx2_psllv_q:
3933 case Intrinsic::x86_avx2_psllv_q_256:
3934 case Intrinsic::x86_avx512_psllv_q_512:
3935 case Intrinsic::x86_avx2_psrlv_d:
3936 case Intrinsic::x86_avx2_psrlv_d_256:
3937 case Intrinsic::x86_avx512_psrlv_d_512:
3938 case Intrinsic::x86_avx2_psrlv_q:
3939 case Intrinsic::x86_avx2_psrlv_q_256:
3940 case Intrinsic::x86_avx512_psrlv_q_512:
3941 case Intrinsic::x86_avx2_psrav_d:
3942 case Intrinsic::x86_avx2_psrav_d_256:
3943 case Intrinsic::x86_avx512_psrav_d_512:
3944 case Intrinsic::x86_avx512_psrav_q_128:
3945 case Intrinsic::x86_avx512_psrav_q_256:
3946 case Intrinsic::x86_avx512_psrav_q_512:
3947 handleVectorShiftIntrinsic(
I,
true);
3950 case Intrinsic::x86_sse2_packsswb_128:
3951 case Intrinsic::x86_sse2_packssdw_128:
3952 case Intrinsic::x86_sse2_packuswb_128:
3953 case Intrinsic::x86_sse41_packusdw:
3954 case Intrinsic::x86_avx2_packsswb:
3955 case Intrinsic::x86_avx2_packssdw:
3956 case Intrinsic::x86_avx2_packuswb:
3957 case Intrinsic::x86_avx2_packusdw:
3958 handleVectorPackIntrinsic(
I);
3961 case Intrinsic::x86_mmx_packsswb:
3962 case Intrinsic::x86_mmx_packuswb:
3963 handleVectorPackIntrinsic(
I, 16);
3966 case Intrinsic::x86_mmx_packssdw:
3967 handleVectorPackIntrinsic(
I, 32);
3970 case Intrinsic::x86_mmx_psad_bw:
3971 case Intrinsic::x86_sse2_psad_bw:
3972 case Intrinsic::x86_avx2_psad_bw:
3973 handleVectorSadIntrinsic(
I);
3976 case Intrinsic::x86_sse2_pmadd_wd:
3977 case Intrinsic::x86_avx2_pmadd_wd:
3978 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
3979 case Intrinsic::x86_avx2_pmadd_ub_sw:
3980 handleVectorPmaddIntrinsic(
I);
3983 case Intrinsic::x86_ssse3_pmadd_ub_sw:
3984 handleVectorPmaddIntrinsic(
I, 8);
3987 case Intrinsic::x86_mmx_pmadd_wd:
3988 handleVectorPmaddIntrinsic(
I, 16);
3991 case Intrinsic::x86_sse_cmp_ss:
3992 case Intrinsic::x86_sse2_cmp_sd:
3993 case Intrinsic::x86_sse_comieq_ss:
3994 case Intrinsic::x86_sse_comilt_ss:
3995 case Intrinsic::x86_sse_comile_ss:
3996 case Intrinsic::x86_sse_comigt_ss:
3997 case Intrinsic::x86_sse_comige_ss:
3998 case Intrinsic::x86_sse_comineq_ss:
3999 case Intrinsic::x86_sse_ucomieq_ss:
4000 case Intrinsic::x86_sse_ucomilt_ss:
4001 case Intrinsic::x86_sse_ucomile_ss:
4002 case Intrinsic::x86_sse_ucomigt_ss:
4003 case Intrinsic::x86_sse_ucomige_ss:
4004 case Intrinsic::x86_sse_ucomineq_ss:
4005 case Intrinsic::x86_sse2_comieq_sd:
4006 case Intrinsic::x86_sse2_comilt_sd:
4007 case Intrinsic::x86_sse2_comile_sd:
4008 case Intrinsic::x86_sse2_comigt_sd:
4009 case Intrinsic::x86_sse2_comige_sd:
4010 case Intrinsic::x86_sse2_comineq_sd:
4011 case Intrinsic::x86_sse2_ucomieq_sd:
4012 case Intrinsic::x86_sse2_ucomilt_sd:
4013 case Intrinsic::x86_sse2_ucomile_sd:
4014 case Intrinsic::x86_sse2_ucomigt_sd:
4015 case Intrinsic::x86_sse2_ucomige_sd:
4016 case Intrinsic::x86_sse2_ucomineq_sd:
4017 handleVectorCompareScalarIntrinsic(
I);
4020 case Intrinsic::x86_avx_cmp_pd_256:
4021 case Intrinsic::x86_avx_cmp_ps_256:
4022 case Intrinsic::x86_sse2_cmp_pd:
4023 case Intrinsic::x86_sse_cmp_ps:
4024 handleVectorComparePackedIntrinsic(
I);
4027 case Intrinsic::x86_bmi_bextr_32:
4028 case Intrinsic::x86_bmi_bextr_64:
4029 case Intrinsic::x86_bmi_bzhi_32:
4030 case Intrinsic::x86_bmi_bzhi_64:
4031 case Intrinsic::x86_bmi_pdep_32:
4032 case Intrinsic::x86_bmi_pdep_64:
4033 case Intrinsic::x86_bmi_pext_32:
4034 case Intrinsic::x86_bmi_pext_64:
4035 handleBmiIntrinsic(
I);
4038 case Intrinsic::x86_pclmulqdq:
4039 case Intrinsic::x86_pclmulqdq_256:
4040 case Intrinsic::x86_pclmulqdq_512:
4041 handlePclmulIntrinsic(
I);
4044 case Intrinsic::x86_sse41_round_sd:
4045 case Intrinsic::x86_sse41_round_ss:
4046 handleUnarySdSsIntrinsic(
I);
4048 case Intrinsic::x86_sse2_max_sd:
4049 case Intrinsic::x86_sse_max_ss:
4050 case Intrinsic::x86_sse2_min_sd:
4051 case Intrinsic::x86_sse_min_ss:
4052 handleBinarySdSsIntrinsic(
I);
4055 case Intrinsic::x86_avx_vtestc_pd:
4056 case Intrinsic::x86_avx_vtestc_pd_256:
4057 case Intrinsic::x86_avx_vtestc_ps:
4058 case Intrinsic::x86_avx_vtestc_ps_256:
4059 case Intrinsic::x86_avx_vtestnzc_pd:
4060 case Intrinsic::x86_avx_vtestnzc_pd_256:
4061 case Intrinsic::x86_avx_vtestnzc_ps:
4062 case Intrinsic::x86_avx_vtestnzc_ps_256:
4063 case Intrinsic::x86_avx_vtestz_pd:
4064 case Intrinsic::x86_avx_vtestz_pd_256:
4065 case Intrinsic::x86_avx_vtestz_ps:
4066 case Intrinsic::x86_avx_vtestz_ps_256:
4067 case Intrinsic::x86_avx_ptestc_256:
4068 case Intrinsic::x86_avx_ptestnzc_256:
4069 case Intrinsic::x86_avx_ptestz_256:
4070 case Intrinsic::x86_sse41_ptestc:
4071 case Intrinsic::x86_sse41_ptestnzc:
4072 case Intrinsic::x86_sse41_ptestz:
4073 handleVtestIntrinsic(
I);
4076 case Intrinsic::fshl:
4077 case Intrinsic::fshr:
4078 handleFunnelShift(
I);
4081 case Intrinsic::is_constant:
4083 setShadow(&
I, getCleanShadow(&
I));
4084 setOrigin(&
I, getCleanOrigin());
4088 if (!handleUnknownIntrinsic(
I))
4089 visitInstruction(
I);
4094 void visitLibAtomicLoad(
CallBase &CB) {
4096 assert(isa<CallInst>(CB));
4105 Value *NewOrdering =
4109 NextNodeIRBuilder NextIRB(&CB);
4110 Value *SrcShadowPtr, *SrcOriginPtr;
4111 std::tie(SrcShadowPtr, SrcOriginPtr) =
4112 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4114 Value *DstShadowPtr =
4115 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4119 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
4120 if (MS.TrackOrigins) {
4121 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
4123 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
4124 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
4128 void visitLibAtomicStore(
CallBase &CB) {
4135 Value *NewOrdering =
4139 Value *DstShadowPtr =
4157 visitAsmInstruction(CB);
4159 visitInstruction(CB);
4168 case LibFunc_atomic_load:
4169 if (!isa<CallInst>(CB)) {
4170 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4174 visitLibAtomicLoad(CB);
4176 case LibFunc_atomic_store:
4177 visitLibAtomicStore(CB);
4184 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4185 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4193 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4195 Call->removeFnAttrs(
B);
4197 Func->removeFnAttrs(
B);
4203 bool MayCheckCall = MS.EagerChecks;
4207 MayCheckCall &= !
Func->getName().starts_with(
"__sanitizer_unaligned_");
4210 unsigned ArgOffset = 0;
4213 if (!
A->getType()->isSized()) {
4214 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4218 if (
A->getType()->isScalableTy()) {
4219 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is vscale: " << CB <<
"\n");
4221 insertShadowCheck(
A, &CB);
4230 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4233 insertShadowCheck(
A, &CB);
4234 Size =
DL.getTypeAllocSize(
A->getType());
4240 Value *ArgShadow = getShadow(
A);
4241 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4243 <<
" Shadow: " << *ArgShadow <<
"\n");
4247 assert(
A->getType()->isPointerTy() &&
4248 "ByVal argument is not a pointer!");
4256 Value *AShadowPtr, *AOriginPtr;
4257 std::tie(AShadowPtr, AOriginPtr) =
4258 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4260 if (!PropagateShadow) {
4267 if (MS.TrackOrigins) {
4268 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4282 Size =
DL.getTypeAllocSize(
A->getType());
4287 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4288 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4290 getOriginPtrForArgument(IRB, ArgOffset));
4294 assert(Store !=
nullptr);
4303 if (FT->isVarArg()) {
4304 VAHelper->visitCallBase(CB, IRB);
4311 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4314 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4315 setShadow(&CB, getCleanShadow(&CB));
4316 setOrigin(&CB, getCleanOrigin());
4322 Value *
Base = getShadowPtrForRetval(IRBBefore);
4323 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4326 if (isa<CallInst>(CB)) {
4330 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4335 setShadow(&CB, getCleanShadow(&CB));
4336 setOrigin(&CB, getCleanOrigin());
4343 "Could not find insertion point for retval shadow load");
4346 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4347 getShadowTy(&CB), getShadowPtrForRetval(IRBAfter),
4349 setShadow(&CB, RetvalShadow);
4350 if (MS.TrackOrigins)
4351 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
4352 getOriginPtrForRetval()));
4356 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4357 RetVal =
I->getOperand(0);
4359 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4360 return I->isMustTailCall();
4367 Value *RetVal =
I.getReturnValue();
4373 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4374 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4375 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4378 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4380 Value *Shadow = getShadow(RetVal);
4381 bool StoreOrigin =
true;
4383 insertShadowCheck(RetVal, &
I);
4384 Shadow = getCleanShadow(RetVal);
4385 StoreOrigin =
false;
4392 if (MS.TrackOrigins && StoreOrigin)
4393 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4399 if (!PropagateShadow) {
4400 setShadow(&
I, getCleanShadow(&
I));
4401 setOrigin(&
I, getCleanOrigin());
4405 ShadowPHINodes.push_back(&
I);
4406 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4408 if (MS.TrackOrigins)
4410 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4427 IRB.
CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4429 Value *ShadowBase, *OriginBase;
4430 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4437 if (PoisonStack && MS.TrackOrigins) {
4438 Value *Idptr = getLocalVarIdptr(
I);
4440 Value *Descr = getLocalVarDescription(
I);
4441 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4442 {&I, Len, Idptr, Descr});
4444 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4450 Value *Descr = getLocalVarDescription(
I);
4452 IRB.
CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4454 IRB.
CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4461 NextNodeIRBuilder IRB(InsPoint);
4463 TypeSize TS =
DL.getTypeAllocSize(
I.getAllocatedType());
4465 if (
I.isArrayAllocation())
4469 if (MS.CompileKernel)
4470 poisonAllocaKmsan(
I, IRB, Len);
4472 poisonAllocaUserspace(
I, IRB, Len);
4476 setShadow(&
I, getCleanShadow(&
I));
4477 setOrigin(&
I, getCleanOrigin());
4489 Value *Sb = getShadow(
B);
4490 Value *Sc = getShadow(
C);
4491 Value *Sd = getShadow(
D);
4496 if (
I.getType()->isAggregateType()) {
4500 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4508 C = CreateAppToShadowCast(IRB,
C);
4509 D = CreateAppToShadowCast(IRB,
D);
4516 if (MS.TrackOrigins) {
4519 if (
B->getType()->isVectorTy()) {
4520 B = convertToBool(
B, IRB);
4521 Sb = convertToBool(Sb, IRB);
4528 getOrigin(
I.getFalseValue()))));
4535 setShadow(&
I, getCleanShadow(&
I));
4536 setOrigin(&
I, getCleanOrigin());
4540 setShadow(&
I, getCleanShadow(&
I));
4541 setOrigin(&
I, getCleanOrigin());
4545 setShadow(&
I, getCleanShadow(&
I));
4546 setOrigin(&
I, getCleanOrigin());
4553 Value *Agg =
I.getAggregateOperand();
4555 Value *AggShadow = getShadow(Agg);
4559 setShadow(&
I, ResShadow);
4560 setOriginForNaryOp(
I);
4566 Value *AggShadow = getShadow(
I.getAggregateOperand());
4567 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4573 setOriginForNaryOp(
I);
4577 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4578 errs() <<
"ZZZ call " << CI->getCalledFunction()->getName() <<
"\n";
4580 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4582 errs() <<
"QQQ " <<
I <<
"\n";
4609 insertShadowCheck(Operand, &
I);
4616 auto Size =
DL.getTypeStoreSize(ElemTy);
4618 if (MS.CompileKernel) {
4619 IRB.
CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
4625 auto [ShadowPtr,
_] =
4626 getShadowOriginPtrUserspace(Operand, IRB, IRB.
getInt8Ty(),
Align(1));
4637 int NumRetOutputs = 0;
4639 Type *
RetTy = cast<Value>(CB)->getType();
4640 if (!
RetTy->isVoidTy()) {
4642 auto *
ST = dyn_cast<StructType>(
RetTy);
4644 NumRetOutputs =
ST->getNumElements();
4650 switch (
Info.Type) {
4658 return NumOutputs - NumRetOutputs;
4681 int OutputArgs = getNumOutputArgs(IA, CB);
4687 for (
int i = OutputArgs; i < NumOperands; i++) {
4695 for (
int i = 0; i < OutputArgs; i++) {
4701 setShadow(&
I, getCleanShadow(&
I));
4702 setOrigin(&
I, getCleanOrigin());
4707 setShadow(&
I, getCleanShadow(&
I));
4708 setOrigin(&
I, getCleanOrigin());
4716 for (
size_t i = 0, n =
I.getNumOperands(); i < n; i++) {
4717 Value *Operand =
I.getOperand(i);
4719 insertShadowCheck(Operand, &
I);
4721 setShadow(&
I, getCleanShadow(&
I));
4722 setOrigin(&
I, getCleanOrigin());
4726struct VarArgHelperBase :
public VarArgHelper {
4728 MemorySanitizer &MS;
4729 MemorySanitizerVisitor &MSV;
4731 const unsigned VAListTagSize;
4733 VarArgHelperBase(
Function &
F, MemorySanitizer &MS,
4734 MemorySanitizerVisitor &MSV,
unsigned VAListTagSize)
4735 :
F(
F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
4739 return IRB.
CreateAdd(
Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4744 unsigned ArgOffset) {
4753 unsigned ArgOffset,
unsigned ArgSize) {
4757 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4772 unsigned BaseOffset) {
4781 TailSize,
Align(8));
4786 Value *VAListTag =
I.getArgOperand(0);
4788 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
4789 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4792 VAListTagSize, Alignment,
false);
4799 unpoisonVAListTagForInst(
I);
4805 unpoisonVAListTagForInst(
I);
4810struct VarArgAMD64Helper :
public VarArgHelperBase {
4813 static const unsigned AMD64GpEndOffset = 48;
4814 static const unsigned AMD64FpEndOffsetSSE = 176;
4816 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
4818 unsigned AMD64FpEndOffset;
4821 Value *VAArgOverflowSize =
nullptr;
4823 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
4825 VarArgAMD64Helper(
Function &
F, MemorySanitizer &MS,
4826 MemorySanitizerVisitor &MSV)
4827 : VarArgHelperBase(
F, MS, MSV, 24) {
4828 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
4829 for (
const auto &Attr :
F.getAttributes().getFnAttrs()) {
4830 if (Attr.isStringAttribute() &&
4831 (Attr.getKindAsString() ==
"target-features")) {
4832 if (Attr.getValueAsString().contains(
"-sse"))
4833 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
4839 ArgKind classifyArgument(
Value *arg) {
4842 if (
T->isX86_FP80Ty())
4844 if (
T->isFPOrFPVectorTy() ||
T->isX86_MMXTy())
4845 return AK_FloatingPoint;
4846 if (
T->isIntegerTy() &&
T->getPrimitiveSizeInBits() <= 64)
4847 return AK_GeneralPurpose;
4848 if (
T->isPointerTy())
4849 return AK_GeneralPurpose;
4862 unsigned GpOffset = 0;
4863 unsigned FpOffset = AMD64GpEndOffset;
4864 unsigned OverflowOffset = AMD64FpEndOffset;
4869 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
4876 assert(
A->getType()->isPointerTy());
4878 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
4880 unsigned BaseOffset = OverflowOffset;
4882 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
4883 Value *OriginBase =
nullptr;
4884 if (MS.TrackOrigins)
4885 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4886 OverflowOffset += AlignedSize;
4889 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4893 Value *ShadowPtr, *OriginPtr;
4894 std::tie(ShadowPtr, OriginPtr) =
4899 if (MS.TrackOrigins)
4903 ArgKind AK = classifyArgument(
A);
4904 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
4906 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
4908 Value *ShadowBase, *OriginBase =
nullptr;
4910 case AK_GeneralPurpose:
4911 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, GpOffset);
4912 if (MS.TrackOrigins)
4913 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
4917 case AK_FloatingPoint:
4918 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, FpOffset);
4919 if (MS.TrackOrigins)
4920 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
4927 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
4929 unsigned BaseOffset = OverflowOffset;
4931 getShadowPtrForVAArgument(
A->getType(), IRB, OverflowOffset);
4932 if (MS.TrackOrigins) {
4933 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
4935 OverflowOffset += AlignedSize;
4938 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
4947 Value *Shadow = MSV.getShadow(
A);
4949 if (MS.TrackOrigins) {
4950 Value *Origin = MSV.getOrigin(
A);
4952 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
4958 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
4959 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
4962 void finalizeInstrumentation()
override {
4963 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
4964 "finalizeInstrumentation called twice");
4965 if (!VAStartInstrumentationList.
empty()) {
4972 ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
4979 Intrinsic::umin, CopySize,
4983 if (MS.TrackOrigins) {
4993 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
4994 CallInst *OrigInst = VAStartInstrumentationList[i];
4995 NextNodeIRBuilder IRB(OrigInst);
4998 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5001 ConstantInt::get(MS.IntptrTy, 16)),
5002 PointerType::get(RegSaveAreaPtrTy, 0));
5003 Value *RegSaveAreaPtr =
5004 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5005 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5007 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5008 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5010 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5012 if (MS.TrackOrigins)
5013 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5014 Alignment, AMD64FpEndOffset);
5015 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5018 ConstantInt::get(MS.IntptrTy, 8)),
5019 PointerType::get(OverflowArgAreaPtrTy, 0));
5020 Value *OverflowArgAreaPtr =
5021 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5022 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5023 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5024 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5028 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5030 if (MS.TrackOrigins) {
5033 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5042struct VarArgMIPS64Helper :
public VarArgHelperBase {
5044 Value *VAArgSize =
nullptr;
5046 VarArgMIPS64Helper(
Function &
F, MemorySanitizer &MS,
5047 MemorySanitizerVisitor &MSV)
5048 : VarArgHelperBase(
F, MS, MSV, 8) {}
5051 unsigned VAArgOffset = 0;
5055 Triple TargetTriple(
F.getParent()->getTargetTriple());
5057 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5062 VAArgOffset += (8 - ArgSize);
5064 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VAArgOffset, ArgSize);
5065 VAArgOffset += ArgSize;
5066 VAArgOffset =
alignTo(VAArgOffset, 8);
5075 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5078 void finalizeInstrumentation()
override {
5079 assert(!VAArgSize && !VAArgTLSCopy &&
5080 "finalizeInstrumentation called twice");
5084 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5086 if (!VAStartInstrumentationList.
empty()) {
5095 Intrinsic::umin, CopySize,
5103 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5104 CallInst *OrigInst = VAStartInstrumentationList[i];
5105 NextNodeIRBuilder IRB(OrigInst);
5107 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5108 Value *RegSaveAreaPtrPtr =
5110 PointerType::get(RegSaveAreaPtrTy, 0));
5111 Value *RegSaveAreaPtr =
5112 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5113 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5115 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5116 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5118 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5125struct VarArgAArch64Helper :
public VarArgHelperBase {
5126 static const unsigned kAArch64GrArgSize = 64;
5127 static const unsigned kAArch64VrArgSize = 128;
5129 static const unsigned AArch64GrBegOffset = 0;
5130 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
5132 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
5133 static const unsigned AArch64VrEndOffset =
5134 AArch64VrBegOffset + kAArch64VrArgSize;
5135 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
5138 Value *VAArgOverflowSize =
nullptr;
5140 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5142 VarArgAArch64Helper(
Function &
F, MemorySanitizer &MS,
5143 MemorySanitizerVisitor &MSV)
5144 : VarArgHelperBase(
F, MS, MSV, 32) {}
5147 std::pair<ArgKind, uint64_t> classifyArgument(
Type *
T) {
5148 if (
T->isIntOrPtrTy() &&
T->getPrimitiveSizeInBits() <= 64)
5149 return {AK_GeneralPurpose, 1};
5150 if (
T->isFloatingPointTy() &&
T->getPrimitiveSizeInBits() <= 128)
5151 return {AK_FloatingPoint, 1};
5153 if (
T->isArrayTy()) {
5154 auto R = classifyArgument(
T->getArrayElementType());
5155 R.second *=
T->getScalarType()->getArrayNumElements();
5160 auto R = classifyArgument(FV->getScalarType());
5161 R.second *= FV->getNumElements();
5166 return {AK_Memory, 0};
5179 unsigned GrOffset = AArch64GrBegOffset;
5180 unsigned VrOffset = AArch64VrBegOffset;
5181 unsigned OverflowOffset = AArch64VAEndOffset;
5186 auto [AK, RegNum] = classifyArgument(
A->getType());
5187 if (AK == AK_GeneralPurpose &&
5188 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
5190 if (AK == AK_FloatingPoint &&
5191 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
5195 case AK_GeneralPurpose:
5196 Base = getShadowPtrForVAArgument(
A->getType(), IRB, GrOffset);
5197 GrOffset += 8 * RegNum;
5199 case AK_FloatingPoint:
5200 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VrOffset);
5201 VrOffset += 16 * RegNum;
5208 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5210 unsigned BaseOffset = OverflowOffset;
5211 Base = getShadowPtrForVAArgument(
A->getType(), IRB, BaseOffset);
5212 OverflowOffset += AlignedSize;
5215 CleanUnusedTLS(IRB,
Base, BaseOffset);
5227 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5228 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5235 ConstantInt::get(MS.IntptrTy, offset)),
5236 PointerType::get(*MS.C, 0));
5244 ConstantInt::get(MS.IntptrTy, offset)),
5245 PointerType::get(*MS.C, 0));
5247 return IRB.
CreateSExt(SaveArea32, MS.IntptrTy);
5250 void finalizeInstrumentation()
override {
5251 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5252 "finalizeInstrumentation called twice");
5253 if (!VAStartInstrumentationList.
empty()) {
5260 ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
5267 Intrinsic::umin, CopySize,
5273 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
5274 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
5278 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5279 CallInst *OrigInst = VAStartInstrumentationList[i];
5280 NextNodeIRBuilder IRB(OrigInst);
5299 Value *StackSaveAreaPtr =
5300 IRB.
CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5303 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5304 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5307 IRB.
CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5310 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5311 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5314 IRB.
CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5320 Value *GrRegSaveAreaShadowPtrOff =
5321 IRB.
CreateAdd(GrArgSize, GrOffSaveArea);
5323 Value *GrRegSaveAreaShadowPtr =
5324 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5330 Value *GrCopySize = IRB.
CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5336 Value *VrRegSaveAreaShadowPtrOff =
5337 IRB.
CreateAdd(VrArgSize, VrOffSaveArea);
5339 Value *VrRegSaveAreaShadowPtr =
5340 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5347 VrRegSaveAreaShadowPtrOff);
5348 Value *VrCopySize = IRB.
CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5354 Value *StackSaveAreaShadowPtr =
5355 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5360 VAArgTLSCopy, IRB.
getInt32(AArch64VAEndOffset));
5363 Align(16), VAArgOverflowSize);
5369struct VarArgPowerPC64Helper :
public VarArgHelperBase {
5371 Value *VAArgSize =
nullptr;
5373 VarArgPowerPC64Helper(
Function &
F, MemorySanitizer &MS,
5374 MemorySanitizerVisitor &MSV)
5375 : VarArgHelperBase(
F, MS, MSV, 8) {}
5385 Triple TargetTriple(
F.getParent()->getTargetTriple());
5393 unsigned VAArgOffset = VAArgBase;
5397 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5399 assert(
A->getType()->isPointerTy());
5401 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5404 ArgAlign =
Align(8);
5405 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5407 Value *
Base = getShadowPtrForVAArgument(
5408 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
5410 Value *AShadowPtr, *AOriginPtr;
5411 std::tie(AShadowPtr, AOriginPtr) =
5412 MSV.getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(),
5422 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5424 if (
A->getType()->isArrayTy()) {
5427 Type *ElementTy =
A->getType()->getArrayElementType();
5429 ArgAlign =
Align(
DL.getTypeAllocSize(ElementTy));
5430 }
else if (
A->getType()->isVectorTy()) {
5432 ArgAlign =
Align(ArgSize);
5435 ArgAlign =
Align(8);
5436 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5437 if (
DL.isBigEndian()) {
5441 VAArgOffset += (8 - ArgSize);
5444 Base = getShadowPtrForVAArgument(
A->getType(), IRB,
5445 VAArgOffset - VAArgBase, ArgSize);
5449 VAArgOffset += ArgSize;
5453 VAArgBase = VAArgOffset;
5457 ConstantInt::get(IRB.
getInt64Ty(), VAArgOffset - VAArgBase);
5460 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5463 void finalizeInstrumentation()
override {
5464 assert(!VAArgSize && !VAArgTLSCopy &&
5465 "finalizeInstrumentation called twice");
5469 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5471 if (!VAStartInstrumentationList.
empty()) {
5481 Intrinsic::umin, CopySize,
5489 for (
size_t i = 0, n = VAStartInstrumentationList.
size(); i < n; i++) {
5490 CallInst *OrigInst = VAStartInstrumentationList[i];
5491 NextNodeIRBuilder IRB(OrigInst);
5493 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5494 Value *RegSaveAreaPtrPtr =
5496 PointerType::get(RegSaveAreaPtrTy, 0));
5497 Value *RegSaveAreaPtr =
5498 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5499 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5501 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5502 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5504 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5511struct VarArgSystemZHelper :
public VarArgHelperBase {
5512 static const unsigned SystemZGpOffset = 16;
5513 static const unsigned SystemZGpEndOffset = 56;
5514 static const unsigned SystemZFpOffset = 128;
5515 static const unsigned SystemZFpEndOffset = 160;
5516 static const unsigned SystemZMaxVrArgs = 8;
5517 static const unsigned SystemZRegSaveAreaSize = 160;
5518 static const unsigned SystemZOverflowOffset = 160;
5519 static const unsigned SystemZVAListTagSize = 32;
5520 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5521 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5523 bool IsSoftFloatABI;
5526 Value *VAArgOverflowSize =
nullptr;
5528 enum class ArgKind {
5536 enum class ShadowExtension {
None,
Zero, Sign };
5538 VarArgSystemZHelper(
Function &
F, MemorySanitizer &MS,
5539 MemorySanitizerVisitor &MSV)
5540 : VarArgHelperBase(
F, MS, MSV, SystemZVAListTagSize),
5541 IsSoftFloatABI(
F.getFnAttribute(
"use-soft-float").getValueAsBool()) {}
5543 ArgKind classifyArgument(
Type *
T) {
5550 if (
T->isIntegerTy(128) ||
T->isFP128Ty())
5551 return ArgKind::Indirect;
5552 if (
T->isFloatingPointTy())
5553 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5554 if (
T->isIntegerTy() ||
T->isPointerTy())
5555 return ArgKind::GeneralPurpose;
5556 if (
T->isVectorTy())
5557 return ArgKind::Vector;
5558 return ArgKind::Memory;
5561 ShadowExtension getShadowExtension(
const CallBase &CB,
unsigned ArgNo) {
5571 return ShadowExtension::Zero;
5575 return ShadowExtension::Sign;
5577 return ShadowExtension::None;
5581 unsigned GpOffset = SystemZGpOffset;
5582 unsigned FpOffset = SystemZFpOffset;
5583 unsigned VrIndex = 0;
5584 unsigned OverflowOffset = SystemZOverflowOffset;
5591 ArgKind AK = classifyArgument(
T);
5592 if (AK == ArgKind::Indirect) {
5593 T = PointerType::get(
T, 0);
5594 AK = ArgKind::GeneralPurpose;
5596 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5597 AK = ArgKind::Memory;
5598 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5599 AK = ArgKind::Memory;
5600 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5601 AK = ArgKind::Memory;
5602 Value *ShadowBase =
nullptr;
5603 Value *OriginBase =
nullptr;
5604 ShadowExtension SE = ShadowExtension::None;
5606 case ArgKind::GeneralPurpose: {
5611 SE = getShadowExtension(CB, ArgNo);
5613 if (SE == ShadowExtension::None) {
5615 assert(ArgAllocSize <= ArgSize);
5616 GapSize = ArgSize - ArgAllocSize;
5618 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5619 if (MS.TrackOrigins)
5620 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5622 GpOffset += ArgSize;
5628 case ArgKind::FloatingPoint: {
5637 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5638 if (MS.TrackOrigins)
5639 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5641 FpOffset += ArgSize;
5647 case ArgKind::Vector: {
5654 case ArgKind::Memory: {
5662 SE = getShadowExtension(CB, ArgNo);
5664 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5666 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5667 if (MS.TrackOrigins)
5669 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5670 OverflowOffset += ArgSize;
5677 case ArgKind::Indirect:
5680 if (ShadowBase ==
nullptr)
5682 Value *Shadow = MSV.getShadow(
A);
5683 if (SE != ShadowExtension::None)
5684 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.
getInt64Ty(),
5685 SE == ShadowExtension::Sign);
5687 ShadowBase, PointerType::get(Shadow->
getType(), 0),
"_msarg_va_s");
5689 if (MS.TrackOrigins) {
5690 Value *Origin = MSV.getOrigin(
A);
5692 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5696 Constant *OverflowSize = ConstantInt::get(
5697 IRB.
getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5698 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5702 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5706 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5707 PointerType::get(RegSaveAreaPtrTy, 0));
5708 Value *RegSaveAreaPtr = IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5709 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5711 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5712 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(), Alignment,
5717 unsigned RegSaveAreaSize =
5718 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
5719 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5721 if (MS.TrackOrigins)
5722 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5723 Alignment, RegSaveAreaSize);
5729 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5733 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5734 PointerType::get(OverflowArgAreaPtrTy, 0));
5735 Value *OverflowArgAreaPtr =
5736 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5737 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5739 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5740 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5743 SystemZOverflowOffset);
5744 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5746 if (MS.TrackOrigins) {
5748 SystemZOverflowOffset);
5749 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5754 void finalizeInstrumentation()
override {
5755 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5756 "finalizeInstrumentation called twice");
5757 if (!VAStartInstrumentationList.
empty()) {
5764 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5772 Intrinsic::umin, CopySize,
5776 if (MS.TrackOrigins) {
5786 for (
size_t VaStartNo = 0, VaStartNum = VAStartInstrumentationList.
size();
5787 VaStartNo < VaStartNum; VaStartNo++) {
5788 CallInst *OrigInst = VAStartInstrumentationList[VaStartNo];
5789 NextNodeIRBuilder IRB(OrigInst);
5791 copyRegSaveArea(IRB, VAListTag);
5792 copyOverflowArea(IRB, VAListTag);
5799using VarArgLoongArch64Helper = VarArgMIPS64Helper;
5802struct VarArgNoOpHelper :
public VarArgHelper {
5803 VarArgNoOpHelper(
Function &
F, MemorySanitizer &MS,
5804 MemorySanitizerVisitor &MSV) {}
5812 void finalizeInstrumentation()
override {}
5818 MemorySanitizerVisitor &Visitor) {
5821 Triple TargetTriple(Func.getParent()->getTargetTriple());
5823 return new VarArgAMD64Helper(Func, Msan, Visitor);
5825 return new VarArgMIPS64Helper(Func, Msan, Visitor);
5827 return new VarArgAArch64Helper(Func, Msan, Visitor);
5830 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
5832 return new VarArgSystemZHelper(Func, Msan, Visitor);
5834 return new VarArgLoongArch64Helper(Func, Msan, Visitor);
5836 return new VarArgNoOpHelper(Func, Msan, Visitor);
5843 if (
F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
5846 MemorySanitizerVisitor Visitor(
F, *
this, TLI);
5850 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
5853 return Visitor.runOnFunction();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
static bool isStore(int Opcode)
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
static const size_t kNumberOfAccessSizes
VarLocInsertPt getNextNode(const DbgRecord *DVR)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
static bool isAMustTailRetVal(Value *RetVal)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool runOnFunction(Function &F, bool PostInlining)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
static size_t TypeSizeToSizeIndex(uint32_t TypeSize)
static const MemoryMapParams Linux_LoongArch64_MemoryMapParams
static const PlatformMemoryMapParams Linux_S390_MemoryMapParams
static const Align kMinOriginAlignment
static const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< uint64_t > ClShadowBase("msan-shadow-base", cl::desc("Define custom MSan ShadowBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClDumpStrictInstructions("msan-dump-strict-instructions", cl::desc("print out instructions with default strict semantics"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClHandleICmpExact("msan-handle-icmp-exact", cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams Linux_X86_MemoryMapParams
static cl::opt< uint64_t > ClOriginBase("msan-origin-base", cl::desc("Define custom MSan OriginBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClCheckConstantShadow("msan-check-constant-shadow", cl::desc("Insert checks for constant shadow values"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams
static const MemoryMapParams NetBSD_X86_64_MemoryMapParams
static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams
static const unsigned kOriginSize
static cl::opt< bool > ClWithComdat("msan-with-comdat", cl::desc("Place MSan constructors in comdat sections"), cl::Hidden, cl::init(false))
static cl::opt< int > ClTrackOrigins("msan-track-origins", cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden, cl::init(0))
Track origins of uninitialized values.
static cl::opt< int > ClInstrumentationWithCallThreshold("msan-instrumentation-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of checks and origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< int > ClPoisonStackPattern("msan-poison-stack-pattern", cl::desc("poison uninitialized stack variables with the given pattern"), cl::Hidden, cl::init(0xff))
static const Align kShadowTLSAlignment
static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams
static Constant * getOrInsertGlobal(Module &M, StringRef Name, Type *Ty)
static const MemoryMapParams Linux_S390X_MemoryMapParams
static cl::opt< bool > ClPoisonUndef("msan-poison-undef", cl::desc("poison undef temps"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClPoisonStack("msan-poison-stack", cl::desc("poison uninitialized stack variables"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_I386_MemoryMapParams
const char kMsanInitName[]
static cl::opt< bool > ClPrintStackNames("msan-print-stack-names", cl::desc("Print name of local stack variable"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< uint64_t > ClAndMask("msan-and-mask", cl::desc("Define custom MSan AndMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleLifetimeIntrinsics("msan-handle-lifetime-intrinsics", cl::desc("when possible, poison scoped variables at the beginning of the scope " "(slower, but more precise)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClKeepGoing("msan-keep-going", cl::desc("keep going after reporting a UMR"), cl::Hidden, cl::init(false))
static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams
static GlobalVariable * createPrivateConstGlobalForString(Module &M, StringRef Str)
Create a non-const global initialized with the given string.
static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams
static const size_t kNumberOfAccessSizes
static cl::opt< bool > ClEagerChecks("msan-eager-checks", cl::desc("check arguments and return values at function call boundaries"), cl::Hidden, cl::init(false))
static cl::opt< int > ClDisambiguateWarning("msan-disambiguate-warning-threshold", cl::desc("Define threshold for number of checks per " "debug location to force origin update."), cl::Hidden, cl::init(3))
static VarArgHelper * CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, MemorySanitizerVisitor &Visitor)
static const MemoryMapParams Linux_MIPS64_MemoryMapParams
static const MemoryMapParams Linux_PowerPC64_MemoryMapParams
static cl::opt< uint64_t > ClXorMask("msan-xor-mask", cl::desc("Define custom MSan XorMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleAsmConservative("msan-handle-asm-conservative", cl::desc("conservative handling of inline assembly"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams
static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams
static const unsigned kParamTLSSize
static cl::opt< bool > ClHandleICmp("msan-handle-icmp", cl::desc("propagate shadow through ICmpEQ and ICmpNE"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClEnableKmsan("msan-kernel", cl::desc("Enable KernelMemorySanitizer instrumentation"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClPoisonStackWithCall("msan-poison-stack-with-call", cl::desc("poison uninitialized stack variables with a call"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams
static const unsigned kRetvalTLSSize
static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams
const char kMsanModuleCtorName[]
static const MemoryMapParams FreeBSD_I386_MemoryMapParams
static cl::opt< bool > ClCheckAccessAddress("msan-check-access-address", cl::desc("report accesses through a pointer which has poisoned shadow"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClDisableChecks("msan-disable-checks", cl::desc("Apply no_sanitize to the whole file"), cl::Hidden, cl::init(false))
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
Value * getCalledOperand() const
Type * getParamElementType(unsigned ArgNo) const
Extract the elementtype type for a parameter.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_SGT
signed greater than
@ ICMP_SGE
signed greater or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
bool isZeroValue() const
Return true if the value is negative zero or null value.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
static bool shouldExecute(unsigned CounterName)
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void setComdat(Comdat *C)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ ExternalLinkage
Externally visible function.
Analysis pass providing a never-invalidated alias analysis result.
This instruction compares its operands according to the predicate given to the constructor.
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::vector< ConstraintInfo > ConstraintInfoVector
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
This class represents a cast from an integer to a pointer.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
This class wraps the llvm.memcpy intrinsic.
This class wraps the llvm.memmove intrinsic.
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
This class represents a cast from a pointer to an integer.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This instruction constructs a fixed permutation of two input vectors.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
AttributeList getAttrList(LLVMContext *C, ArrayRef< unsigned > ArgNos, bool Signed, bool Ret=false, AttributeList AL=AttributeList()) const
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Triple - Helper class for working with autoconf configuration names.
bool isMIPS64() const
Tests whether the target is MIPS 64-bit (little and big endian).
ArchType getArch() const
Get the parsed architecture type of this triple.
bool isLoongArch64() const
Tests whether the target is 64-bit LoongArch.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
static Type * getX86_MMXTy(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
'undef' values are things that do not have specified contents.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This represents the llvm.va_copy intrinsic.
This represents the llvm.va_start intrinsic.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
This class represents zero extension of integer types.
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
This class provides various memory handling functions that manipulate MemoryBlock instances.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
initializer< Ty > init(const Ty &Val)
Function * Kernel
Summary of a kernel (=entry point for target offloading).
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
AllocaInst * findAllocaForValue(Value *V, bool OffsetZero=false)
Returns unique alloca where the value comes from, or nullptr.
std::pair< Function *, FunctionCallee > getOrCreateSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, function_ref< void(Function *, FunctionCallee)> FunctionsCreatedCallback, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function lazily.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Or
Bitwise or logical OR of integers.
std::pair< Instruction *, Value * > SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore)
Insert a for (int i = 0; i < End; i++) loop structure (with the exception that End is assumed > 0,...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, const TargetLibraryInfo *TLI)
Given a CallInst, check if it calls a string function known to CodeGen, and mark it with NoBuiltin if...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A CRTP mix-in to automatically provide informational APIs needed for passes.