184#include "llvm/IR/IntrinsicsAArch64.h"
185#include "llvm/IR/IntrinsicsX86.h"
214#define DEBUG_TYPE "msan"
217 "Controls which checks to insert");
220 "Controls which instruction to instrument");
238 "msan-track-origins",
243 cl::desc(
"keep going after reporting a UMR"),
252 "msan-poison-stack-with-call",
257 "msan-poison-stack-pattern",
258 cl::desc(
"poison uninitialized stack variables with the given pattern"),
263 cl::desc(
"Print name of local stack variable"),
272 cl::desc(
"propagate shadow through ICmpEQ and ICmpNE"),
277 cl::desc(
"exact handling of relational integer ICmp"),
281 "msan-handle-lifetime-intrinsics",
283 "when possible, poison scoped variables at the beginning of the scope "
284 "(slower, but more precise)"),
295 "msan-handle-asm-conservative",
306 "msan-check-access-address",
307 cl::desc(
"report accesses through a pointer which has poisoned shadow"),
312 cl::desc(
"check arguments and return values at function call boundaries"),
316 "msan-dump-strict-instructions",
317 cl::desc(
"print out instructions with default strict semantics"),
321 "msan-instrumentation-with-call-threshold",
323 "If the function being instrumented requires more than "
324 "this number of checks and origin stores, use callbacks instead of "
325 "inline checks (-1 means never use callbacks)."),
330 cl::desc(
"Enable KernelMemorySanitizer instrumentation"),
340 cl::desc(
"Insert checks for constant shadow values"),
347 cl::desc(
"Place MSan constructors in comdat sections"),
353 cl::desc(
"Define custom MSan AndMask"),
357 cl::desc(
"Define custom MSan XorMask"),
361 cl::desc(
"Define custom MSan ShadowBase"),
365 cl::desc(
"Define custom MSan OriginBase"),
370 cl::desc(
"Define threshold for number of checks per "
371 "debug location to force origin update."),
383struct MemoryMapParams {
390struct PlatformMemoryMapParams {
391 const MemoryMapParams *bits32;
392 const MemoryMapParams *bits64;
538class MemorySanitizer {
547 MemorySanitizer(MemorySanitizer &&) =
delete;
548 MemorySanitizer &operator=(MemorySanitizer &&) =
delete;
549 MemorySanitizer(
const MemorySanitizer &) =
delete;
550 MemorySanitizer &operator=(
const MemorySanitizer &) =
delete;
555 friend struct MemorySanitizerVisitor;
556 friend struct VarArgHelperBase;
557 friend struct VarArgAMD64Helper;
558 friend struct VarArgMIPS64Helper;
559 friend struct VarArgAArch64Helper;
560 friend struct VarArgPowerPC64Helper;
561 friend struct VarArgSystemZHelper;
563 void initializeModule(
Module &M);
568 template <
typename... ArgsTy>
595 Value *ParamOriginTLS;
601 Value *RetvalOriginTLS;
607 Value *VAArgOriginTLS;
610 Value *VAArgOverflowSizeTLS;
613 bool CallbacksInitialized =
false;
658 Value *MsanMetadataAlloca;
664 const MemoryMapParams *MapParams;
668 MemoryMapParams CustomMapParams;
673 MDNode *OriginStoreWeights;
676void insertModuleCtor(
Module &M) {
704 Recover(getOptOrDefault(
ClKeepGoing, Kernel || R)),
719 MemorySanitizer Msan(*
F.getParent(),
Options);
738 OS, MapClassName2PassName);
745 OS <<
"eager-checks;";
746 OS <<
"track-origins=" <<
Options.TrackOrigins;
762template <
typename... ArgsTy>
770 std::forward<ArgsTy>(Args)...);
773 return M.getOrInsertFunction(
Name, MsanMetadata,
774 std::forward<ArgsTy>(Args)...);
783 RetvalOriginTLS =
nullptr;
785 ParamOriginTLS =
nullptr;
787 VAArgOriginTLS =
nullptr;
788 VAArgOverflowSizeTLS =
nullptr;
790 WarningFn =
M.getOrInsertFunction(
"__msan_warning",
792 IRB.getVoidTy(), IRB.getInt32Ty());
803 MsanGetContextStateFn =
M.getOrInsertFunction(
809 for (
int ind = 0, size = 1; ind < 4; ind++,
size <<= 1) {
810 std::string name_load =
811 "__msan_metadata_ptr_for_load_" + std::to_string(size);
812 std::string name_store =
813 "__msan_metadata_ptr_for_store_" + std::to_string(size);
814 MsanMetadataPtrForLoad_1_8[ind] = getOrInsertMsanMetadataFunction(
816 MsanMetadataPtrForStore_1_8[ind] = getOrInsertMsanMetadataFunction(
820 MsanMetadataPtrForLoadN = getOrInsertMsanMetadataFunction(
823 MsanMetadataPtrForStoreN = getOrInsertMsanMetadataFunction(
824 M,
"__msan_metadata_ptr_for_store_n",
828 MsanPoisonAllocaFn =
M.getOrInsertFunction(
829 "__msan_poison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
830 MsanUnpoisonAllocaFn =
M.getOrInsertFunction(
831 "__msan_unpoison_alloca", IRB.getVoidTy(), PtrTy, IntptrTy);
835 return M.getOrInsertGlobal(
Name, Ty, [&] {
837 nullptr,
Name,
nullptr,
850 StringRef WarningFnName = Recover ?
"__msan_warning_with_origin"
851 :
"__msan_warning_with_origin_noreturn";
852 WarningFn =
M.getOrInsertFunction(WarningFnName,
854 IRB.getVoidTy(), IRB.getInt32Ty());
857 Recover ?
"__msan_warning" :
"__msan_warning_noreturn";
858 WarningFn =
M.getOrInsertFunction(WarningFnName, IRB.getVoidTy());
884 VAArgOverflowSizeTLS =
889 unsigned AccessSize = 1 << AccessSizeIndex;
890 std::string FunctionName =
"__msan_maybe_warning_" + itostr(AccessSize);
891 MaybeWarningFn[AccessSizeIndex] =
M.getOrInsertFunction(
893 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), IRB.getInt32Ty());
895 FunctionName =
"__msan_maybe_store_origin_" + itostr(AccessSize);
896 MaybeStoreOriginFn[AccessSizeIndex] =
M.getOrInsertFunction(
898 IRB.getVoidTy(), IRB.getIntNTy(AccessSize * 8), PtrTy,
902 MsanSetAllocaOriginWithDescriptionFn =
903 M.getOrInsertFunction(
"__msan_set_alloca_origin_with_descr",
904 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy, PtrTy);
905 MsanSetAllocaOriginNoDescriptionFn =
906 M.getOrInsertFunction(
"__msan_set_alloca_origin_no_descr",
907 IRB.getVoidTy(), PtrTy, IntptrTy, PtrTy);
908 MsanPoisonStackFn =
M.getOrInsertFunction(
"__msan_poison_stack",
909 IRB.getVoidTy(), PtrTy, IntptrTy);
915 if (CallbacksInitialized)
921 MsanChainOriginFn =
M.getOrInsertFunction(
922 "__msan_chain_origin",
925 MsanSetOriginFn =
M.getOrInsertFunction(
927 IRB.getVoidTy(), PtrTy, IntptrTy, IRB.getInt32Ty());
929 M.getOrInsertFunction(
"__msan_memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
931 M.getOrInsertFunction(
"__msan_memcpy", PtrTy, PtrTy, PtrTy, IntptrTy);
932 MemsetFn =
M.getOrInsertFunction(
"__msan_memset",
934 PtrTy, PtrTy, IRB.getInt32Ty(), IntptrTy);
936 MsanInstrumentAsmStoreFn =
937 M.getOrInsertFunction(
"__msan_instrument_asm_store", IRB.getVoidTy(),
941 createKernelApi(M, TLI);
943 createUserspaceApi(M, TLI);
945 CallbacksInitialized =
true;
951 isStore ? MsanMetadataPtrForStore_1_8 : MsanMetadataPtrForLoad_1_8;
969void MemorySanitizer::initializeModule(
Module &M) {
970 auto &
DL =
M.getDataLayout();
972 TargetTriple =
Triple(
M.getTargetTriple());
974 bool ShadowPassed =
ClShadowBase.getNumOccurrences() > 0;
975 bool OriginPassed =
ClOriginBase.getNumOccurrences() > 0;
977 if (ShadowPassed || OriginPassed) {
982 MapParams = &CustomMapParams;
984 switch (TargetTriple.getOS()) {
986 switch (TargetTriple.getArch()) {
1001 switch (TargetTriple.getArch()) {
1010 switch (TargetTriple.getArch()) {
1044 C = &(
M.getContext());
1046 IntptrTy = IRB.getIntPtrTy(
DL);
1047 OriginTy = IRB.getInt32Ty();
1048 PtrTy = IRB.getPtrTy();
1053 if (!CompileKernel) {
1055 M.getOrInsertGlobal(
"__msan_track_origins", IRB.getInt32Ty(), [&] {
1056 return new GlobalVariable(
1057 M, IRB.getInt32Ty(), true, GlobalValue::WeakODRLinkage,
1058 IRB.getInt32(TrackOrigins),
"__msan_track_origins");
1062 M.getOrInsertGlobal(
"__msan_keep_going", IRB.getInt32Ty(), [&] {
1063 return new GlobalVariable(M, IRB.getInt32Ty(), true,
1064 GlobalValue::WeakODRLinkage,
1065 IRB.getInt32(Recover),
"__msan_keep_going");
1080struct VarArgHelper {
1081 virtual ~VarArgHelper() =
default;
1096 virtual void finalizeInstrumentation() = 0;
1099struct MemorySanitizerVisitor;
1104 MemorySanitizerVisitor &Visitor);
1111 if (TypeSizeFixed <= 8)
1120class NextNodeIRBuilder :
public IRBuilder<> {
1133struct MemorySanitizerVisitor :
public InstVisitor<MemorySanitizerVisitor> {
1135 MemorySanitizer &MS;
1138 std::unique_ptr<VarArgHelper> VAHelper;
1146 bool PropagateShadow;
1150 struct ShadowOriginAndInsertPoint {
1156 : Shadow(S), Origin(
O), OrigIns(
I) {}
1164 int64_t SplittableBlocksCount = 0;
1166 MemorySanitizerVisitor(
Function &
F, MemorySanitizer &MS,
1169 bool SanitizeFunction =
1171 InsertChecks = SanitizeFunction;
1172 PropagateShadow = SanitizeFunction;
1182 MS.initializeCallbacks(*
F.getParent(), TLI);
1183 FnPrologueEnd =
IRBuilder<>(
F.getEntryBlock().getFirstNonPHI())
1186 if (MS.CompileKernel) {
1188 insertKmsanPrologue(IRB);
1192 <<
"MemorySanitizer is not inserting checks into '"
1193 <<
F.getName() <<
"'\n");
1196 bool instrumentWithCalls(
Value *V) {
1198 if (isa<Constant>(V))
1201 ++SplittableBlocksCount;
1207 return I.getParent() == FnPrologueEnd->
getParent() &&
1208 (&
I == FnPrologueEnd ||
I.comesBefore(FnPrologueEnd));
1216 if (MS.TrackOrigins <= 1)
1218 return IRB.
CreateCall(MS.MsanChainOriginFn, V);
1223 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1235 const Align IntptrAlignment =
DL.getABITypeAlign(MS.IntptrTy);
1236 unsigned IntptrSize =
DL.getTypeStoreSize(MS.IntptrTy);
1248 auto [InsertPt,
Index] =
1260 Align CurrentAlignment = Alignment;
1261 if (Alignment >= IntptrAlignment && IntptrSize >
kOriginSize) {
1262 Value *IntptrOrigin = originToIntptr(IRB, Origin);
1263 Value *IntptrOriginPtr =
1265 for (
unsigned i = 0; i <
Size / IntptrSize; ++i) {
1270 CurrentAlignment = IntptrAlignment;
1288 Value *ConvertedShadow = convertShadowToScalar(Shadow, IRB);
1289 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1297 paintOrigin(IRB, updateOrigin(Origin, IRB), OriginPtr, StoreSize,
1306 if (instrumentWithCalls(ConvertedShadow) &&
1309 Value *ConvertedShadow2 =
1315 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1319 paintOrigin(IRBNew, updateOrigin(Origin, IRBNew), OriginPtr, StoreSize,
1324 void materializeStores() {
1327 Value *Val =
SI->getValueOperand();
1329 Value *Shadow =
SI->isAtomic() ? getCleanShadow(Val) : getShadow(Val);
1330 Value *ShadowPtr, *OriginPtr;
1332 const Align Alignment =
SI->getAlign();
1334 std::tie(ShadowPtr, OriginPtr) =
1335 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
true);
1344 if (MS.TrackOrigins && !
SI->isAtomic())
1345 storeOrigin(IRB,
Addr, Shadow, getOrigin(Val), OriginPtr,
1352 if (MS.TrackOrigins < 2)
1355 if (LazyWarningDebugLocationCount.
empty())
1356 for (
const auto &
I : InstrumentationList)
1357 ++LazyWarningDebugLocationCount[
I.OrigIns->getDebugLoc()];
1371 if (
Instruction *OI = dyn_cast_or_null<Instruction>(Origin)) {
1373 auto NewDebugLoc = OI->getDebugLoc();
1380 IRBOrigin.SetCurrentDebugLocation(NewDebugLoc);
1381 Origin = updateOrigin(Origin, IRBOrigin);
1386 if (MS.CompileKernel || MS.TrackOrigins)
1400 if (instrumentWithCalls(ConvertedShadow) &&
1404 ConvertedShadow = convertShadowToScalar(ConvertedShadow, IRB);
1405 Value *ConvertedShadow2 =
1408 Fn, {ConvertedShadow2,
1409 MS.TrackOrigins && Origin ? Origin : (
Value *)IRB.
getInt32(0)});
1413 Value *
Cmp = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1416 !MS.Recover, MS.ColdCallWeights);
1419 insertWarningFn(IRB, Origin);
1424 void materializeInstructionChecks(
1429 bool Combine = !MS.TrackOrigins;
1431 Value *Shadow =
nullptr;
1432 for (
const auto &ShadowData : InstructionChecks) {
1436 Value *ConvertedShadow = ShadowData.Shadow;
1438 if (
auto *ConstantShadow = dyn_cast<Constant>(ConvertedShadow)) {
1445 insertWarningFn(IRB, ShadowData.Origin);
1455 materializeOneCheck(IRB, ConvertedShadow, ShadowData.Origin);
1460 Shadow = ConvertedShadow;
1464 Shadow = convertToBool(Shadow, IRB,
"_mscmp");
1465 ConvertedShadow = convertToBool(ConvertedShadow, IRB,
"_mscmp");
1466 Shadow = IRB.
CreateOr(Shadow, ConvertedShadow,
"_msor");
1472 materializeOneCheck(IRB, Shadow,
nullptr);
1476 void materializeChecks() {
1482 for (
auto I = InstrumentationList.begin();
1483 I != InstrumentationList.end();) {
1484 auto OrigIns =
I->OrigIns;
1488 auto J = std::find_if(
I + 1, InstrumentationList.end(),
1489 [OrigIns](
const ShadowOriginAndInsertPoint &R) {
1490 return OrigIns != R.OrigIns;
1504 MS.ParamTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1505 {Zero, IRB.getInt32(0)},
"param_shadow");
1506 MS.RetvalTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1507 {Zero, IRB.getInt32(1)},
"retval_shadow");
1508 MS.VAArgTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1509 {Zero, IRB.getInt32(2)},
"va_arg_shadow");
1510 MS.VAArgOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1511 {Zero, IRB.getInt32(3)},
"va_arg_origin");
1512 MS.VAArgOverflowSizeTLS =
1513 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1514 {Zero, IRB.getInt32(4)},
"va_arg_overflow_size");
1515 MS.ParamOriginTLS = IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1516 {Zero, IRB.getInt32(5)},
"param_origin");
1517 MS.RetvalOriginTLS =
1518 IRB.
CreateGEP(MS.MsanContextStateTy, ContextState,
1519 {Zero, IRB.getInt32(6)},
"retval_origin");
1521 MS.MsanMetadataAlloca = IRB.
CreateAlloca(MS.MsanMetadata, 0u);
1538 for (
PHINode *PN : ShadowPHINodes) {
1539 PHINode *PNS = cast<PHINode>(getShadow(PN));
1540 PHINode *PNO = MS.TrackOrigins ? cast<PHINode>(getOrigin(PN)) : nullptr;
1541 size_t NumValues = PN->getNumIncomingValues();
1542 for (
size_t v = 0;
v < NumValues;
v++) {
1543 PNS->
addIncoming(getShadow(PN, v), PN->getIncomingBlock(v));
1545 PNO->
addIncoming(getOrigin(PN, v), PN->getIncomingBlock(v));
1549 VAHelper->finalizeInstrumentation();
1553 if (InstrumentLifetimeStart) {
1554 for (
auto Item : LifetimeStartList) {
1555 instrumentAlloca(*Item.second, Item.first);
1556 AllocaSet.
remove(Item.second);
1562 instrumentAlloca(*AI);
1565 materializeChecks();
1569 materializeStores();
1575 Type *getShadowTy(
Value *V) {
return getShadowTy(
V->getType()); }
1587 if (
VectorType *VT = dyn_cast<VectorType>(OrigTy)) {
1588 uint32_t EltSize =
DL.getTypeSizeInBits(VT->getElementType());
1590 VT->getElementCount());
1592 if (
ArrayType *AT = dyn_cast<ArrayType>(OrigTy)) {
1593 return ArrayType::get(getShadowTy(AT->getElementType()),
1594 AT->getNumElements());
1596 if (
StructType *ST = dyn_cast<StructType>(OrigTy)) {
1598 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1599 Elements.push_back(getShadowTy(
ST->getElementType(i)));
1601 LLVM_DEBUG(
dbgs() <<
"getShadowTy: " << *ST <<
" ===> " << *Res <<
"\n");
1617 Value *ShadowBool = convertToBool(ShadowItem, IRB);
1619 if (Aggregator != FalseVal)
1620 Aggregator = IRB.
CreateOr(Aggregator, ShadowBool);
1622 Aggregator = ShadowBool;
1631 if (!
Array->getNumElements())
1635 Value *Aggregator = convertShadowToScalar(FirstItem, IRB);
1639 Value *ShadowInner = convertShadowToScalar(ShadowItem, IRB);
1640 Aggregator = IRB.
CreateOr(Aggregator, ShadowInner);
1650 return collapseStructShadow(
Struct, V, IRB);
1651 if (
ArrayType *Array = dyn_cast<ArrayType>(
V->getType()))
1652 return collapseArrayShadow(Array, V, IRB);
1653 if (isa<VectorType>(
V->getType())) {
1654 if (isa<ScalableVectorType>(
V->getType()))
1657 V->getType()->getPrimitiveSizeInBits().getFixedValue();
1665 Type *VTy =
V->getType();
1667 return convertToBool(convertShadowToScalar(V, IRB), IRB,
name);
1674 Type *ptrToIntPtrType(
Type *PtrTy)
const {
1675 if (
VectorType *VectTy = dyn_cast<VectorType>(PtrTy)) {
1676 return VectorType::get(ptrToIntPtrType(VectTy->getElementType()),
1677 VectTy->getElementCount());
1683 Type *getPtrToShadowPtrType(
Type *IntPtrTy,
Type *ShadowTy)
const {
1684 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1685 return VectorType::get(
1686 getPtrToShadowPtrType(VectTy->getElementType(), ShadowTy),
1687 VectTy->getElementCount());
1689 assert(IntPtrTy == MS.IntptrTy);
1690 return PointerType::get(*MS.C, 0);
1694 if (
VectorType *VectTy = dyn_cast<VectorType>(IntPtrTy)) {
1696 VectTy->getElementCount(), constToIntPtr(VectTy->getElementType(),
C));
1698 assert(IntPtrTy == MS.IntptrTy);
1699 return ConstantInt::get(MS.IntptrTy,
C);
1710 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1713 if (
uint64_t AndMask = MS.MapParams->AndMask)
1714 OffsetLong = IRB.
CreateAnd(OffsetLong, constToIntPtr(IntptrTy, ~AndMask));
1716 if (
uint64_t XorMask = MS.MapParams->XorMask)
1717 OffsetLong = IRB.
CreateXor(OffsetLong, constToIntPtr(IntptrTy, XorMask));
1729 std::pair<Value *, Value *>
1736 assert(VectTy->getElementType()->isPointerTy());
1738 Type *IntptrTy = ptrToIntPtrType(
Addr->getType());
1739 Value *ShadowOffset = getShadowPtrOffset(
Addr, IRB);
1740 Value *ShadowLong = ShadowOffset;
1741 if (
uint64_t ShadowBase = MS.MapParams->ShadowBase) {
1743 IRB.
CreateAdd(ShadowLong, constToIntPtr(IntptrTy, ShadowBase));
1746 ShadowLong, getPtrToShadowPtrType(IntptrTy, ShadowTy));
1748 Value *OriginPtr =
nullptr;
1749 if (MS.TrackOrigins) {
1750 Value *OriginLong = ShadowOffset;
1751 uint64_t OriginBase = MS.MapParams->OriginBase;
1752 if (OriginBase != 0)
1754 IRB.
CreateAdd(OriginLong, constToIntPtr(IntptrTy, OriginBase));
1757 OriginLong = IRB.
CreateAnd(OriginLong, constToIntPtr(IntptrTy, ~Mask));
1760 OriginLong, getPtrToShadowPtrType(IntptrTy, MS.OriginTy));
1762 return std::make_pair(ShadowPtr, OriginPtr);
1765 template <
typename... ArgsTy>
1770 {MS.MsanMetadataAlloca, std::forward<ArgsTy>(Args)...});
1771 return IRB.
CreateLoad(MS.MsanMetadata, MS.MsanMetadataAlloca);
1774 return IRB.
CreateCall(Callee, {std::forward<ArgsTy>(Args)...});
1777 std::pair<Value *, Value *> getShadowOriginPtrKernelNoVec(
Value *
Addr,
1781 Value *ShadowOriginPtrs;
1789 ShadowOriginPtrs = createMetadataCall(IRB, Getter, AddrCast);
1791 Value *SizeVal = ConstantInt::get(MS.IntptrTy,
Size);
1792 ShadowOriginPtrs = createMetadataCall(
1794 isStore ? MS.MsanMetadataPtrForStoreN : MS.MsanMetadataPtrForLoadN,
1801 return std::make_pair(ShadowPtr, OriginPtr);
1807 std::pair<Value *, Value *> getShadowOriginPtrKernel(
Value *
Addr,
1814 return getShadowOriginPtrKernelNoVec(
Addr, IRB, ShadowTy,
isStore);
1818 unsigned NumElements = cast<FixedVectorType>(VectTy)->getNumElements();
1819 Value *ShadowPtrs = ConstantInt::getNullValue(
1821 Value *OriginPtrs =
nullptr;
1822 if (MS.TrackOrigins)
1823 OriginPtrs = ConstantInt::getNullValue(
1825 for (
unsigned i = 0; i < NumElements; ++i) {
1828 auto [ShadowPtr, OriginPtr] =
1829 getShadowOriginPtrKernelNoVec(OneAddr, IRB, ShadowTy,
isStore);
1832 ShadowPtrs, ShadowPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1833 if (MS.TrackOrigins)
1835 OriginPtrs, OriginPtr, ConstantInt::get(IRB.
getInt32Ty(), i));
1837 return {ShadowPtrs, OriginPtrs};
1844 if (MS.CompileKernel)
1845 return getShadowOriginPtrKernel(
Addr, IRB, ShadowTy,
isStore);
1846 return getShadowOriginPtrUserspace(
Addr, IRB, ShadowTy, Alignment);
1861 if (!MS.TrackOrigins)
1875 Value *getOriginPtrForRetval() {
1877 return MS.RetvalOriginTLS;
1882 assert(!ShadowMap.
count(V) &&
"Values may only have one shadow");
1883 ShadowMap[
V] = PropagateShadow ? SV : getCleanShadow(V);
1888 if (!MS.TrackOrigins)
1890 assert(!OriginMap.
count(V) &&
"Values may only have one origin");
1891 LLVM_DEBUG(
dbgs() <<
"ORIGIN: " << *V <<
" ==> " << *Origin <<
"\n");
1892 OriginMap[
V] = Origin;
1896 Type *ShadowTy = getShadowTy(OrigTy);
1906 Constant *getCleanShadow(
Value *V) {
return getCleanShadow(
V->getType()); }
1911 if (isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy))
1913 if (
ArrayType *AT = dyn_cast<ArrayType>(ShadowTy)) {
1915 getPoisonedShadow(AT->getElementType()));
1918 if (
StructType *ST = dyn_cast<StructType>(ShadowTy)) {
1920 for (
unsigned i = 0, n =
ST->getNumElements(); i < n; i++)
1921 Vals.
push_back(getPoisonedShadow(
ST->getElementType(i)));
1929 Type *ShadowTy = getShadowTy(V);
1932 return getPoisonedShadow(ShadowTy);
1944 if (!PropagateShadow ||
I->getMetadata(LLVMContext::MD_nosanitize))
1945 return getCleanShadow(V);
1947 Value *Shadow = ShadowMap[
V];
1949 LLVM_DEBUG(
dbgs() <<
"No shadow: " << *V <<
"\n" << *(
I->getParent()));
1951 assert(Shadow &&
"No shadow for a value");
1955 if (
UndefValue *U = dyn_cast<UndefValue>(V)) {
1956 Value *
AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
1957 : getCleanShadow(V);
1962 if (
Argument *
A = dyn_cast<Argument>(V)) {
1964 Value *&ShadowPtr = ShadowMap[
V];
1969 unsigned ArgOffset = 0;
1971 for (
auto &FArg :
F->args()) {
1972 if (!FArg.getType()->isSized() || FArg.getType()->isScalableTy()) {
1974 ?
"vscale not fully supported\n"
1975 :
"Arg is not sized\n"));
1977 ShadowPtr = getCleanShadow(V);
1978 setOrigin(
A, getCleanOrigin());
1984 unsigned Size = FArg.hasByValAttr()
1985 ?
DL.getTypeAllocSize(FArg.getParamByValType())
1986 :
DL.getTypeAllocSize(FArg.getType());
1990 if (FArg.hasByValAttr()) {
1994 const Align ArgAlign =
DL.getValueOrABITypeAlignment(
1995 FArg.getParamAlign(), FArg.getParamByValType());
1996 Value *CpShadowPtr, *CpOriginPtr;
1997 std::tie(CpShadowPtr, CpOriginPtr) =
1998 getShadowOriginPtr(V, EntryIRB, EntryIRB.getInt8Ty(), ArgAlign,
2000 if (!PropagateShadow || Overflow) {
2002 EntryIRB.CreateMemSet(
2006 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2008 Value *Cpy = EntryIRB.CreateMemCpy(CpShadowPtr, CopyAlign,
Base,
2013 if (MS.TrackOrigins) {
2015 getOriginPtrForArgument(EntryIRB, ArgOffset);
2019 EntryIRB.CreateMemCpy(
2028 if (!PropagateShadow || Overflow || FArg.hasByValAttr() ||
2029 (MS.EagerChecks && FArg.hasAttribute(Attribute::NoUndef))) {
2030 ShadowPtr = getCleanShadow(V);
2031 setOrigin(
A, getCleanOrigin());
2034 Value *
Base = getShadowPtrForArgument(EntryIRB, ArgOffset);
2035 ShadowPtr = EntryIRB.CreateAlignedLoad(getShadowTy(&FArg),
Base,
2037 if (MS.TrackOrigins) {
2039 getOriginPtrForArgument(EntryIRB, ArgOffset);
2040 setOrigin(
A, EntryIRB.CreateLoad(MS.OriginTy, OriginPtr));
2044 <<
" ARG: " << FArg <<
" ==> " << *ShadowPtr <<
"\n");
2050 assert(ShadowPtr &&
"Could not find shadow for an argument");
2054 return getCleanShadow(V);
2059 return getShadow(
I->getOperand(i));
2064 if (!MS.TrackOrigins)
2066 if (!PropagateShadow || isa<Constant>(V) || isa<InlineAsm>(V))
2067 return getCleanOrigin();
2068 assert((isa<Instruction>(V) || isa<Argument>(V)) &&
2069 "Unexpected value type in getOrigin()");
2071 if (
I->getMetadata(LLVMContext::MD_nosanitize))
2072 return getCleanOrigin();
2074 Value *Origin = OriginMap[
V];
2075 assert(Origin &&
"Missing origin");
2081 return getOrigin(
I->getOperand(i));
2094 LLVM_DEBUG(
dbgs() <<
"Skipping check of " << *Shadow <<
" before "
2095 << *OrigIns <<
"\n");
2100 assert((isa<IntegerType>(ShadowTy) || isa<VectorType>(ShadowTy) ||
2101 isa<StructType>(ShadowTy) || isa<ArrayType>(ShadowTy)) &&
2102 "Can only insert checks for integer, vector, and aggregate shadow "
2105 InstrumentationList.push_back(
2106 ShadowOriginAndInsertPoint(Shadow, Origin, OrigIns));
2115 Value *Shadow, *Origin;
2117 Shadow = getShadow(Val);
2120 Origin = getOrigin(Val);
2122 Shadow = dyn_cast_or_null<Instruction>(getShadow(Val));
2125 Origin = dyn_cast_or_null<Instruction>(getOrigin(Val));
2127 insertShadowCheck(Shadow, Origin, OrigIns);
2132 case AtomicOrdering::NotAtomic:
2133 return AtomicOrdering::NotAtomic;
2134 case AtomicOrdering::Unordered:
2135 case AtomicOrdering::Monotonic:
2136 case AtomicOrdering::Release:
2137 return AtomicOrdering::Release;
2138 case AtomicOrdering::Acquire:
2139 case AtomicOrdering::AcquireRelease:
2140 return AtomicOrdering::AcquireRelease;
2141 case AtomicOrdering::SequentiallyConsistent:
2142 return AtomicOrdering::SequentiallyConsistent;
2148 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2149 uint32_t OrderingTable[NumOrderings] = {};
2151 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2152 OrderingTable[(
int)AtomicOrderingCABI::release] =
2153 (int)AtomicOrderingCABI::release;
2154 OrderingTable[(int)AtomicOrderingCABI::consume] =
2155 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2156 OrderingTable[(int)AtomicOrderingCABI::acq_rel] =
2157 (
int)AtomicOrderingCABI::acq_rel;
2158 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2159 (
int)AtomicOrderingCABI::seq_cst;
2166 case AtomicOrdering::NotAtomic:
2167 return AtomicOrdering::NotAtomic;
2168 case AtomicOrdering::Unordered:
2169 case AtomicOrdering::Monotonic:
2170 case AtomicOrdering::Acquire:
2171 return AtomicOrdering::Acquire;
2172 case AtomicOrdering::Release:
2173 case AtomicOrdering::AcquireRelease:
2174 return AtomicOrdering::AcquireRelease;
2175 case AtomicOrdering::SequentiallyConsistent:
2176 return AtomicOrdering::SequentiallyConsistent;
2182 constexpr int NumOrderings = (int)AtomicOrderingCABI::seq_cst + 1;
2183 uint32_t OrderingTable[NumOrderings] = {};
2185 OrderingTable[(int)AtomicOrderingCABI::relaxed] =
2186 OrderingTable[(
int)AtomicOrderingCABI::acquire] =
2187 OrderingTable[(int)AtomicOrderingCABI::consume] =
2188 (
int)AtomicOrderingCABI::acquire;
2189 OrderingTable[(int)AtomicOrderingCABI::release] =
2190 OrderingTable[(
int)AtomicOrderingCABI::acq_rel] =
2191 (int)AtomicOrderingCABI::acq_rel;
2192 OrderingTable[(int)AtomicOrderingCABI::seq_cst] =
2193 (
int)AtomicOrderingCABI::seq_cst;
2201 if (
I.getMetadata(LLVMContext::MD_nosanitize))
2204 if (isInPrologue(
I))
2209 setShadow(&
I, getCleanShadow(&
I));
2210 setOrigin(&
I, getCleanOrigin());
2222 assert(
I.getType()->isSized() &&
"Load type must have size");
2223 assert(!
I.getMetadata(LLVMContext::MD_nosanitize));
2224 NextNodeIRBuilder IRB(&
I);
2225 Type *ShadowTy = getShadowTy(&
I);
2227 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2228 const Align Alignment =
I.getAlign();
2229 if (PropagateShadow) {
2230 std::tie(ShadowPtr, OriginPtr) =
2231 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2235 setShadow(&
I, getCleanShadow(&
I));
2239 insertShadowCheck(
I.getPointerOperand(), &
I);
2244 if (MS.TrackOrigins) {
2245 if (PropagateShadow) {
2250 setOrigin(&
I, getCleanOrigin());
2260 StoreList.push_back(&
I);
2262 insertShadowCheck(
I.getPointerOperand(), &
I);
2266 assert(isa<AtomicRMWInst>(
I) || isa<AtomicCmpXchgInst>(
I));
2270 Value *Val =
I.getOperand(1);
2271 Value *ShadowPtr = getShadowOriginPtr(
Addr, IRB, getShadowTy(Val),
Align(1),
2276 insertShadowCheck(
Addr, &
I);
2281 if (isa<AtomicCmpXchgInst>(
I))
2282 insertShadowCheck(Val, &
I);
2286 setShadow(&
I, getCleanShadow(&
I));
2287 setOrigin(&
I, getCleanOrigin());
2302 insertShadowCheck(
I.getOperand(1), &
I);
2306 setOrigin(&
I, getOrigin(&
I, 0));
2310 insertShadowCheck(
I.getOperand(2), &
I);
2312 auto *Shadow0 = getShadow(&
I, 0);
2313 auto *Shadow1 = getShadow(&
I, 1);
2316 setOriginForNaryOp(
I);
2321 auto *Shadow0 = getShadow(&
I, 0);
2322 auto *Shadow1 = getShadow(&
I, 1);
2325 setOriginForNaryOp(
I);
2331 setShadow(&
I, IRB.
CreateSExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2332 setOrigin(&
I, getOrigin(&
I, 0));
2337 setShadow(&
I, IRB.
CreateZExt(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2338 setOrigin(&
I, getOrigin(&
I, 0));
2343 setShadow(&
I, IRB.
CreateTrunc(getShadow(&
I, 0),
I.getType(),
"_msprop"));
2344 setOrigin(&
I, getOrigin(&
I, 0));
2351 if (
auto *CI = dyn_cast<CallInst>(
I.getOperand(0)))
2352 if (CI->isMustTailCall())
2356 setOrigin(&
I, getOrigin(&
I, 0));
2362 "_msprop_ptrtoint"));
2363 setOrigin(&
I, getOrigin(&
I, 0));
2369 "_msprop_inttoptr"));
2370 setOrigin(&
I, getOrigin(&
I, 0));
2373 void visitFPToSIInst(
CastInst &
I) { handleShadowOr(
I); }
2374 void visitFPToUIInst(
CastInst &
I) { handleShadowOr(
I); }
2375 void visitSIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2376 void visitUIToFPInst(
CastInst &
I) { handleShadowOr(
I); }
2377 void visitFPExtInst(
CastInst &
I) { handleShadowOr(
I); }
2378 void visitFPTruncInst(
CastInst &
I) { handleShadowOr(
I); }
2393 Value *S2 = getShadow(&
I, 1);
2394 Value *V1 =
I.getOperand(0);
2403 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2404 setOriginForNaryOp(
I);
2415 Value *S2 = getShadow(&
I, 1);
2425 setShadow(&
I, IRB.
CreateOr({S1S2, V1S2, S1V2}));
2426 setOriginForNaryOp(
I);
2444 template <
bool CombineShadow>
class Combiner {
2445 Value *Shadow =
nullptr;
2446 Value *Origin =
nullptr;
2448 MemorySanitizerVisitor *MSV;
2452 : IRB(IRB), MSV(MSV) {}
2456 if (CombineShadow) {
2461 OpShadow = MSV->CreateShadowCast(IRB, OpShadow, Shadow->getType());
2462 Shadow = IRB.
CreateOr(Shadow, OpShadow,
"_msprop");
2466 if (MSV->MS.TrackOrigins) {
2471 Constant *ConstOrigin = dyn_cast<Constant>(OpOrigin);
2473 if (!ConstOrigin || !ConstOrigin->
isNullValue()) {
2474 Value *
Cond = MSV->convertToBool(OpShadow, IRB);
2484 Value *OpShadow = MSV->getShadow(V);
2485 Value *OpOrigin = MSV->MS.TrackOrigins ? MSV->getOrigin(V) :
nullptr;
2486 return Add(OpShadow, OpOrigin);
2492 if (CombineShadow) {
2494 Shadow = MSV->CreateShadowCast(IRB, Shadow, MSV->getShadowTy(
I));
2495 MSV->setShadow(
I, Shadow);
2497 if (MSV->MS.TrackOrigins) {
2499 MSV->setOrigin(
I, Origin);
2506 if (MSV->MS.TrackOrigins) {
2518 if (!MS.TrackOrigins)
2521 OriginCombiner
OC(
this, IRB);
2522 for (
Use &
Op :
I.operands())
2527 size_t VectorOrPrimitiveTypeSizeInBits(
Type *Ty) {
2529 "Vector of pointers is not a valid shadow type");
2530 return Ty->
isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() *
2539 Type *srcTy =
V->getType();
2542 size_t srcSizeInBits = VectorOrPrimitiveTypeSizeInBits(srcTy);
2543 size_t dstSizeInBits = VectorOrPrimitiveTypeSizeInBits(dstTy);
2544 if (srcSizeInBits > 1 && dstSizeInBits == 1)
2550 cast<VectorType>(dstTy)->getElementCount() ==
2551 cast<VectorType>(srcTy)->getElementCount())
2562 Type *ShadowTy = getShadowTy(V);
2563 if (
V->getType() == ShadowTy)
2565 if (
V->getType()->isPtrOrPtrVectorTy())
2574 ShadowAndOriginCombiner
SC(
this, IRB);
2575 for (
Use &
Op :
I.operands())
2595 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
2596 unsigned NumElements = cast<FixedVectorType>(VTy)->getNumElements();
2597 Type *EltTy = VTy->getElementType();
2599 for (
unsigned Idx = 0;
Idx < NumElements; ++
Idx) {
2602 const APInt &
V = Elt->getValue();
2604 Elements.push_back(ConstantInt::get(EltTy, V2));
2606 Elements.push_back(ConstantInt::get(EltTy, 1));
2611 if (
ConstantInt *Elt = dyn_cast<ConstantInt>(ConstArg)) {
2612 const APInt &
V = Elt->getValue();
2614 ShadowMul = ConstantInt::get(Ty, V2);
2616 ShadowMul = ConstantInt::get(Ty, 1);
2622 IRB.
CreateMul(getShadow(OtherArg), ShadowMul,
"msprop_mul_cst"));
2623 setOrigin(&
I, getOrigin(OtherArg));
2627 Constant *constOp0 = dyn_cast<Constant>(
I.getOperand(0));
2628 Constant *constOp1 = dyn_cast<Constant>(
I.getOperand(1));
2629 if (constOp0 && !constOp1)
2630 handleMulByConstant(
I, constOp0,
I.getOperand(1));
2631 else if (constOp1 && !constOp0)
2632 handleMulByConstant(
I, constOp1,
I.getOperand(0));
2647 insertShadowCheck(
I.getOperand(1), &
I);
2648 setShadow(&
I, getShadow(&
I, 0));
2649 setOrigin(&
I, getOrigin(&
I, 0));
2666 void handleEqualityComparison(
ICmpInst &
I) {
2670 Value *Sa = getShadow(
A);
2671 Value *Sb = getShadow(
B);
2697 setOriginForNaryOp(
I);
2739 void handleRelationalComparisonExact(
ICmpInst &
I) {
2743 Value *Sa = getShadow(
A);
2744 Value *Sb = getShadow(
B);
2755 bool IsSigned =
I.isSigned();
2757 getLowestPossibleValue(IRB,
A, Sa, IsSigned),
2758 getHighestPossibleValue(IRB,
B, Sb, IsSigned));
2760 getHighestPossibleValue(IRB,
A, Sa, IsSigned),
2761 getLowestPossibleValue(IRB,
B, Sb, IsSigned));
2764 setOriginForNaryOp(
I);
2771 void handleSignedRelationalComparison(
ICmpInst &
I) {
2775 if ((constOp = dyn_cast<Constant>(
I.getOperand(1)))) {
2776 op =
I.getOperand(0);
2777 pre =
I.getPredicate();
2778 }
else if ((constOp = dyn_cast<Constant>(
I.getOperand(0)))) {
2779 op =
I.getOperand(1);
2780 pre =
I.getSwappedPredicate();
2793 setShadow(&
I, Shadow);
2794 setOrigin(&
I, getOrigin(
op));
2805 if (
I.isEquality()) {
2806 handleEqualityComparison(
I);
2812 handleRelationalComparisonExact(
I);
2816 handleSignedRelationalComparison(
I);
2821 if ((isa<Constant>(
I.getOperand(0)) || isa<Constant>(
I.getOperand(1)))) {
2822 handleRelationalComparisonExact(
I);
2829 void visitFCmpInst(
FCmpInst &
I) { handleShadowOr(
I); }
2836 Value *S2 = getShadow(&
I, 1);
2841 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2842 setOriginForNaryOp(
I);
2853 Value *S0 = getShadow(&
I, 0);
2855 Value *S2 = getShadow(&
I, 2);
2860 I.getModule(),
I.getIntrinsicID(), S2Conv->
getType());
2862 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
2863 setOriginForNaryOp(
I);
2877 getShadow(
I.getArgOperand(1));
2880 {I.getArgOperand(0), I.getArgOperand(1),
2881 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2882 I.eraseFromParent();
2900 getShadow(
I.getArgOperand(1));
2903 {I.getArgOperand(0), I.getArgOperand(1),
2904 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2905 I.eraseFromParent();
2913 {I.getArgOperand(0),
2914 IRB.CreateIntCast(I.getArgOperand(1), IRB.getInt32Ty(), false),
2915 IRB.CreateIntCast(I.getArgOperand(2), MS.IntptrTy, false)});
2916 I.eraseFromParent();
2919 void visitVAStartInst(
VAStartInst &
I) { VAHelper->visitVAStartInst(
I); }
2921 void visitVACopyInst(
VACopyInst &
I) { VAHelper->visitVACopyInst(
I); }
2930 Value *Shadow = getShadow(&
I, 1);
2931 Value *ShadowPtr, *OriginPtr;
2935 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
2940 insertShadowCheck(
Addr, &
I);
2943 if (MS.TrackOrigins)
2956 Type *ShadowTy = getShadowTy(&
I);
2957 Value *ShadowPtr =
nullptr, *OriginPtr =
nullptr;
2958 if (PropagateShadow) {
2962 std::tie(ShadowPtr, OriginPtr) =
2963 getShadowOriginPtr(
Addr, IRB, ShadowTy, Alignment,
false);
2967 setShadow(&
I, getCleanShadow(&
I));
2971 insertShadowCheck(
Addr, &
I);
2973 if (MS.TrackOrigins) {
2974 if (PropagateShadow)
2975 setOrigin(&
I, IRB.
CreateLoad(MS.OriginTy, OriginPtr));
2977 setOrigin(&
I, getCleanOrigin());
2990 if (!(
RetTy->isIntOrIntVectorTy() ||
RetTy->isFPOrFPVectorTy() ||
2991 RetTy->isX86_MMXTy()))
2994 unsigned NumArgOperands =
I.arg_size();
2995 for (
unsigned i = 0; i < NumArgOperands; ++i) {
2996 Type *Ty =
I.getArgOperand(i)->getType();
3002 ShadowAndOriginCombiner
SC(
this, IRB);
3003 for (
unsigned i = 0; i < NumArgOperands; ++i)
3004 SC.Add(
I.getArgOperand(i));
3021 unsigned NumArgOperands =
I.arg_size();
3022 if (NumArgOperands == 0)
3025 if (NumArgOperands == 2 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3026 I.getArgOperand(1)->getType()->isVectorTy() &&
3027 I.getType()->isVoidTy() && !
I.onlyReadsMemory()) {
3029 return handleVectorStoreIntrinsic(
I);
3032 if (NumArgOperands == 1 &&
I.getArgOperand(0)->getType()->isPointerTy() &&
3033 I.getType()->isVectorTy() &&
I.onlyReadsMemory()) {
3035 return handleVectorLoadIntrinsic(
I);
3038 if (
I.doesNotAccessMemory())
3039 if (maybeHandleSimpleNomemIntrinsic(
I))
3047 setShadow(&
I, getShadow(&
I, 0));
3048 setOrigin(&
I, getOrigin(&
I, 0));
3056 InstrumentLifetimeStart =
false;
3057 LifetimeStartList.push_back(std::make_pair(&
I, AI));
3063 Type *OpType =
Op->getType();
3065 F.getParent(), Intrinsic::bswap,
ArrayRef(&OpType, 1));
3067 setOrigin(&
I, getOrigin(
Op));
3072 Value *Src =
I.getArgOperand(0);
3078 Constant *IsZeroPoison = cast<Constant>(
I.getOperand(1));
3081 BoolShadow = IRB.
CreateOr(BoolShadow, BoolZeroPoison,
"_mscz_bs");
3084 Value *OutputShadow =
3085 IRB.
CreateSExt(BoolShadow, getShadowTy(Src),
"_mscz_os");
3087 setShadow(&
I, OutputShadow);
3088 setOriginForNaryOp(
I);
3106 void handleVectorConvertIntrinsic(
IntrinsicInst &
I,
int NumUsedElements,
3107 bool HasRoundingMode =
false) {
3109 Value *CopyOp, *ConvertOp;
3111 assert((!HasRoundingMode ||
3112 isa<ConstantInt>(
I.getArgOperand(
I.arg_size() - 1))) &&
3113 "Invalid rounding mode");
3115 switch (
I.arg_size() - HasRoundingMode) {
3117 CopyOp =
I.getArgOperand(0);
3118 ConvertOp =
I.getArgOperand(1);
3121 ConvertOp =
I.getArgOperand(0);
3135 Value *ConvertShadow = getShadow(ConvertOp);
3136 Value *AggShadow =
nullptr;
3139 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), 0));
3140 for (
int i = 1; i < NumUsedElements; ++i) {
3142 ConvertShadow, ConstantInt::get(IRB.
getInt32Ty(), i));
3143 AggShadow = IRB.
CreateOr(AggShadow, MoreShadow);
3146 AggShadow = ConvertShadow;
3149 insertShadowCheck(AggShadow, getOrigin(ConvertOp), &
I);
3156 Value *ResultShadow = getShadow(CopyOp);
3157 Type *EltTy = cast<VectorType>(ResultShadow->
getType())->getElementType();
3158 for (
int i = 0; i < NumUsedElements; ++i) {
3160 ResultShadow, ConstantInt::getNullValue(EltTy),
3163 setShadow(&
I, ResultShadow);
3164 setOrigin(&
I, getOrigin(CopyOp));
3166 setShadow(&
I, getCleanShadow(&
I));
3167 setOrigin(&
I, getCleanOrigin());
3175 S = CreateShadowCast(IRB, S, IRB.
getInt64Ty(),
true);
3178 return CreateShadowCast(IRB, S2,
T,
true);
3186 return CreateShadowCast(IRB, S2,
T,
true);
3203 void handleVectorShiftIntrinsic(
IntrinsicInst &
I,
bool Variable) {
3209 Value *S2 = getShadow(&
I, 1);
3210 Value *S2Conv = Variable ? VariableShadowExtend(IRB, S2)
3211 : Lower64ShadowExtend(IRB, S2, getShadowTy(&
I));
3212 Value *V1 =
I.getOperand(0);
3215 {IRB.CreateBitCast(S1, V1->getType()), V2});
3217 setShadow(&
I, IRB.
CreateOr(Shift, S2Conv));
3218 setOriginForNaryOp(
I);
3222 Type *getMMXVectorTy(
unsigned EltSizeInBits) {
3223 const unsigned X86_MMXSizeInBits = 64;
3224 assert(EltSizeInBits != 0 && (X86_MMXSizeInBits % EltSizeInBits) == 0 &&
3225 "Illegal MMX vector element size");
3227 X86_MMXSizeInBits / EltSizeInBits);
3234 case Intrinsic::x86_sse2_packsswb_128:
3235 case Intrinsic::x86_sse2_packuswb_128:
3236 return Intrinsic::x86_sse2_packsswb_128;
3238 case Intrinsic::x86_sse2_packssdw_128:
3239 case Intrinsic::x86_sse41_packusdw:
3240 return Intrinsic::x86_sse2_packssdw_128;
3242 case Intrinsic::x86_avx2_packsswb:
3243 case Intrinsic::x86_avx2_packuswb:
3244 return Intrinsic::x86_avx2_packsswb;
3246 case Intrinsic::x86_avx2_packssdw:
3247 case Intrinsic::x86_avx2_packusdw:
3248 return Intrinsic::x86_avx2_packssdw;
3250 case Intrinsic::x86_mmx_packsswb:
3251 case Intrinsic::x86_mmx_packuswb:
3252 return Intrinsic::x86_mmx_packsswb;
3254 case Intrinsic::x86_mmx_packssdw:
3255 return Intrinsic::x86_mmx_packssdw;
3268 void handleVectorPackIntrinsic(
IntrinsicInst &
I,
unsigned EltSizeInBits = 0) {
3270 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3273 Value *S2 = getShadow(&
I, 1);
3274 assert(isX86_MMX ||
S1->getType()->isVectorTy());
3279 Type *
T = isX86_MMX ? getMMXVectorTy(EltSizeInBits) :
S1->
getType();
3295 F.getParent(), getSignedPackIntrinsic(
I.getIntrinsicID()));
3298 IRB.
CreateCall(ShadowFn, {S1_ext, S2_ext},
"_msprop_vector_pack");
3302 setOriginForNaryOp(
I);
3306 Constant *createDppMask(
unsigned Width,
unsigned Mask) {
3319 const unsigned Width =
3320 cast<FixedVectorType>(S->
getType())->getNumElements();
3326 Value *DstMaskV = createDppMask(Width, DstMask);
3346 Value *S0 = getShadow(&
I, 0);
3350 const unsigned Width =
3351 cast<FixedVectorType>(S->
getType())->getNumElements();
3352 assert(Width == 2 || Width == 4 || Width == 8);
3354 const unsigned Mask = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3355 const unsigned SrcMask =
Mask >> 4;
3356 const unsigned DstMask =
Mask & 0xf;
3359 Value *SI1 = findDppPoisonedOutput(IRB, S, SrcMask, DstMask);
3364 SI1, findDppPoisonedOutput(IRB, S, SrcMask << 4, DstMask << 4));
3371 setOriginForNaryOp(
I);
3375 C = CreateAppToShadowCast(IRB,
C);
3389 Value *Sc = getShadow(&
I, 2);
3390 Value *Oc = MS.TrackOrigins ? getOrigin(
C) : nullptr;
3395 C = convertBlendvToSelectMask(IRB,
C);
3396 Sc = convertBlendvToSelectMask(IRB, Sc);
3402 handleSelectLikeInst(
I,
C,
T,
F);
3407 const unsigned SignificantBitsPerResultElement = 16;
3408 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3410 unsigned ZeroBitsPerResultElement =
3414 auto *Shadow0 = getShadow(&
I, 0);
3415 auto *Shadow1 = getShadow(&
I, 1);
3420 S = IRB.
CreateLShr(S, ZeroBitsPerResultElement);
3423 setOriginForNaryOp(
I);
3428 unsigned EltSizeInBits = 0) {
3429 bool isX86_MMX =
I.getOperand(0)->getType()->isX86_MMXTy();
3430 Type *ResTy = isX86_MMX ? getMMXVectorTy(EltSizeInBits * 2) :
I.
getType();
3432 auto *Shadow0 = getShadow(&
I, 0);
3433 auto *Shadow1 = getShadow(&
I, 1);
3440 setOriginForNaryOp(
I);
3448 Type *ResTy = getShadowTy(&
I);
3449 auto *Shadow0 = getShadow(&
I, 0);
3450 auto *Shadow1 = getShadow(&
I, 1);
3455 setOriginForNaryOp(
I);
3463 auto *Shadow0 = getShadow(&
I, 0);
3464 auto *Shadow1 = getShadow(&
I, 1);
3466 Value *S = LowerElementShadowExtend(IRB, S0, getShadowTy(&
I));
3468 setOriginForNaryOp(
I);
3477 setOrigin(&
I, getOrigin(&
I, 0));
3485 Value *OperandShadow = getShadow(&
I, 0);
3487 Value *OperandUnsetOrPoison = IRB.
CreateOr(OperandUnsetBits, OperandShadow);
3495 setOrigin(&
I, getOrigin(&
I, 0));
3503 Value *OperandShadow = getShadow(&
I, 0);
3504 Value *OperandSetOrPoison = IRB.
CreateOr(
I.getOperand(0), OperandShadow);
3512 setOrigin(&
I, getOrigin(&
I, 0));
3520 getShadowOriginPtr(
Addr, IRB, Ty,
Align(1),
true).first;
3525 insertShadowCheck(
Addr, &
I);
3536 Value *ShadowPtr, *OriginPtr;
3537 std::tie(ShadowPtr, OriginPtr) =
3538 getShadowOriginPtr(
Addr, IRB, Ty, Alignment,
false);
3541 insertShadowCheck(
Addr, &
I);
3544 Value *Origin = MS.TrackOrigins ? IRB.
CreateLoad(MS.OriginTy, OriginPtr)
3546 insertShadowCheck(Shadow, Origin, &
I);
3553 Value *PassThru =
I.getArgOperand(2);
3556 insertShadowCheck(
Ptr, &
I);
3557 insertShadowCheck(Mask, &
I);
3560 if (!PropagateShadow) {
3561 setShadow(&
I, getCleanShadow(&
I));
3562 setOrigin(&
I, getCleanOrigin());
3566 Type *ShadowTy = getShadowTy(&
I);
3567 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3568 auto [ShadowPtr, OriginPtr] =
3569 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
false);
3572 ShadowTy, ShadowPtr, Mask, getShadow(PassThru),
"_msmaskedexpload");
3574 setShadow(&
I, Shadow);
3577 setOrigin(&
I, getCleanOrigin());
3582 Value *Values =
I.getArgOperand(0);
3587 insertShadowCheck(
Ptr, &
I);
3588 insertShadowCheck(Mask, &
I);
3591 Value *Shadow = getShadow(Values);
3592 Type *ElementShadowTy =
3593 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3594 auto [ShadowPtr, OriginPtrs] =
3595 getShadowOriginPtr(
Ptr, IRB, ElementShadowTy, {},
true);
3604 Value *Ptrs =
I.getArgOperand(0);
3605 const Align Alignment(
3606 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3608 Value *PassThru =
I.getArgOperand(3);
3610 Type *PtrsShadowTy = getShadowTy(Ptrs);
3612 insertShadowCheck(Mask, &
I);
3616 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3619 if (!PropagateShadow) {
3620 setShadow(&
I, getCleanShadow(&
I));
3621 setOrigin(&
I, getCleanOrigin());
3625 Type *ShadowTy = getShadowTy(&
I);
3626 Type *ElementShadowTy = cast<VectorType>(ShadowTy)->getElementType();
3627 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3628 Ptrs, IRB, ElementShadowTy, Alignment,
false);
3632 getShadow(PassThru),
"_msmaskedgather");
3634 setShadow(&
I, Shadow);
3637 setOrigin(&
I, getCleanOrigin());
3642 Value *Values =
I.getArgOperand(0);
3643 Value *Ptrs =
I.getArgOperand(1);
3644 const Align Alignment(
3645 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3648 Type *PtrsShadowTy = getShadowTy(Ptrs);
3650 insertShadowCheck(Mask, &
I);
3654 insertShadowCheck(MaskedPtrShadow, getOrigin(Ptrs), &
I);
3657 Value *Shadow = getShadow(Values);
3658 Type *ElementShadowTy =
3659 getShadowTy(cast<VectorType>(Values->
getType())->getElementType());
3660 auto [ShadowPtrs, OriginPtrs] = getShadowOriginPtr(
3661 Ptrs, IRB, ElementShadowTy, Alignment,
true);
3670 Value *
V =
I.getArgOperand(0);
3672 const Align Alignment(
3673 cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue());
3675 Value *Shadow = getShadow(V);
3678 insertShadowCheck(
Ptr, &
I);
3679 insertShadowCheck(Mask, &
I);
3684 std::tie(ShadowPtr, OriginPtr) = getShadowOriginPtr(
3685 Ptr, IRB, Shadow->
getType(), Alignment,
true);
3689 if (!MS.TrackOrigins)
3692 auto &
DL =
F.getDataLayout();
3693 paintOrigin(IRB, getOrigin(V), OriginPtr,
3701 const Align Alignment(
3702 cast<ConstantInt>(
I.getArgOperand(1))->getZExtValue());
3704 Value *PassThru =
I.getArgOperand(3);
3707 insertShadowCheck(
Ptr, &
I);
3708 insertShadowCheck(Mask, &
I);
3711 if (!PropagateShadow) {
3712 setShadow(&
I, getCleanShadow(&
I));
3713 setOrigin(&
I, getCleanOrigin());
3717 Type *ShadowTy = getShadowTy(&
I);
3718 Value *ShadowPtr, *OriginPtr;
3719 std::tie(ShadowPtr, OriginPtr) =
3720 getShadowOriginPtr(
Ptr, IRB, ShadowTy, Alignment,
false);
3722 getShadow(PassThru),
"_msmaskedld"));
3724 if (!MS.TrackOrigins)
3731 Value *NotNull = convertToBool(MaskedPassThruShadow, IRB,
"_mscmp");
3736 setOrigin(&
I, Origin);
3746 Type *ShadowTy = getShadowTy(&
I);
3749 Value *SMask = getShadow(&
I, 1);
3754 {getShadow(&I, 0), I.getOperand(1)});
3757 setOriginForNaryOp(
I);
3762 for (
unsigned X = OddElements ? 1 : 0;
X < Width;
X += 2) {
3779 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3780 assert(isa<ConstantInt>(
I.getArgOperand(2)) &&
3781 "pclmul 3rd operand must be a constant");
3782 unsigned Imm = cast<ConstantInt>(
I.getArgOperand(2))->getZExtValue();
3784 getPclmulMask(Width, Imm & 0x01));
3786 getPclmulMask(Width, Imm & 0x10));
3787 ShadowAndOriginCombiner SOC(
this, IRB);
3788 SOC.Add(Shuf0, getOrigin(&
I, 0));
3789 SOC.Add(Shuf1, getOrigin(&
I, 1));
3797 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3799 Value *Second = getShadow(&
I, 1);
3802 Mask.push_back(Width);
3803 for (
unsigned i = 1; i < Width; i++)
3807 setShadow(&
I, Shadow);
3808 setOriginForNaryOp(
I);
3813 Value *Shadow0 = getShadow(&
I, 0);
3814 Value *Shadow1 = getShadow(&
I, 1);
3820 setShadow(&
I, Shadow);
3821 setOriginForNaryOp(
I);
3827 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements();
3829 Value *Second = getShadow(&
I, 1);
3833 Mask.push_back(Width);
3834 for (
unsigned i = 1; i < Width; i++)
3838 setShadow(&
I, Shadow);
3839 setOriginForNaryOp(
I);
3846 assert(
I.getType()->isIntOrIntVectorTy());
3847 assert(
I.getArgOperand(0)->getType() ==
I.getType());
3851 setShadow(&
I, getShadow(&
I, 0));
3852 setOrigin(&
I, getOrigin(&
I, 0));
3857 Value *Shadow = getShadow(&
I, 0);
3858 setShadow(&
I, IRB.
CreateICmpNE(Shadow, getCleanShadow(Shadow)));
3859 setOrigin(&
I, getOrigin(&
I, 0));
3864 Value *Shadow0 = getShadow(&
I, 0);
3865 Value *Shadow1 = getShadow(&
I, 1);
3868 IRB.
CreateICmpNE(ShadowElt0, getCleanShadow(ShadowElt0));
3874 setShadow(&
I, Shadow);
3875 setOriginForNaryOp(
I);
3887 int numArgOperands =
I.arg_size();
3888 assert(numArgOperands >= 1);
3891 Value *
Addr =
I.getArgOperand(numArgOperands - 1);
3895 insertShadowCheck(
Addr, &
I);
3899 for (
int i = 0; i < numArgOperands - 1; i++) {
3900 assert(isa<FixedVectorType>(
I.getArgOperand(i)->getType()));
3916 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getElementType(),
3917 cast<FixedVectorType>(
I.getArgOperand(0)->getType())->getNumElements() *
3918 (numArgOperands - 1));
3919 Type *ShadowTy = getShadowTy(OutputVectorTy);
3920 Value *ShadowPtr, *OriginPtr;
3922 std::tie(ShadowPtr, OriginPtr) =
3923 getShadowOriginPtr(
Addr, IRB, ShadowTy,
Align(1),
true);
3927 if (MS.TrackOrigins) {
3932 OriginCombiner
OC(
this, IRB);
3933 for (
int i = 0; i < numArgOperands - 1; i++)
3934 OC.Add(
I.getArgOperand(i));
3937 OC.DoneAndStoreOrigin(
DL.getTypeStoreSize(OutputVectorTy), OriginPtr);
3942 switch (
I.getIntrinsicID()) {
3943 case Intrinsic::uadd_with_overflow:
3944 case Intrinsic::sadd_with_overflow:
3945 case Intrinsic::usub_with_overflow:
3946 case Intrinsic::ssub_with_overflow:
3947 case Intrinsic::umul_with_overflow:
3948 case Intrinsic::smul_with_overflow:
3949 handleArithmeticWithOverflow(
I);
3951 case Intrinsic::abs:
3952 handleAbsIntrinsic(
I);
3954 case Intrinsic::is_fpclass:
3957 case Intrinsic::lifetime_start:
3958 handleLifetimeStart(
I);
3960 case Intrinsic::launder_invariant_group:
3961 case Intrinsic::strip_invariant_group:
3962 handleInvariantGroup(
I);
3964 case Intrinsic::bswap:
3967 case Intrinsic::ctlz:
3968 case Intrinsic::cttz:
3969 handleCountZeroes(
I);
3971 case Intrinsic::masked_compressstore:
3972 handleMaskedCompressStore(
I);
3974 case Intrinsic::masked_expandload:
3975 handleMaskedExpandLoad(
I);
3977 case Intrinsic::masked_gather:
3978 handleMaskedGather(
I);
3980 case Intrinsic::masked_scatter:
3981 handleMaskedScatter(
I);
3983 case Intrinsic::masked_store:
3984 handleMaskedStore(
I);
3986 case Intrinsic::masked_load:
3987 handleMaskedLoad(
I);
3989 case Intrinsic::vector_reduce_and:
3990 handleVectorReduceAndIntrinsic(
I);
3992 case Intrinsic::vector_reduce_or:
3993 handleVectorReduceOrIntrinsic(
I);
3995 case Intrinsic::vector_reduce_add:
3996 case Intrinsic::vector_reduce_xor:
3997 case Intrinsic::vector_reduce_mul:
3998 handleVectorReduceIntrinsic(
I);
4000 case Intrinsic::x86_sse_stmxcsr:
4003 case Intrinsic::x86_sse_ldmxcsr:
4006 case Intrinsic::x86_avx512_vcvtsd2usi64:
4007 case Intrinsic::x86_avx512_vcvtsd2usi32:
4008 case Intrinsic::x86_avx512_vcvtss2usi64:
4009 case Intrinsic::x86_avx512_vcvtss2usi32:
4010 case Intrinsic::x86_avx512_cvttss2usi64:
4011 case Intrinsic::x86_avx512_cvttss2usi:
4012 case Intrinsic::x86_avx512_cvttsd2usi64:
4013 case Intrinsic::x86_avx512_cvttsd2usi:
4014 case Intrinsic::x86_avx512_cvtusi2ss:
4015 case Intrinsic::x86_avx512_cvtusi642sd:
4016 case Intrinsic::x86_avx512_cvtusi642ss:
4017 handleVectorConvertIntrinsic(
I, 1,
true);
4019 case Intrinsic::x86_sse2_cvtsd2si64:
4020 case Intrinsic::x86_sse2_cvtsd2si:
4021 case Intrinsic::x86_sse2_cvtsd2ss:
4022 case Intrinsic::x86_sse2_cvttsd2si64:
4023 case Intrinsic::x86_sse2_cvttsd2si:
4024 case Intrinsic::x86_sse_cvtss2si64:
4025 case Intrinsic::x86_sse_cvtss2si:
4026 case Intrinsic::x86_sse_cvttss2si64:
4027 case Intrinsic::x86_sse_cvttss2si:
4028 handleVectorConvertIntrinsic(
I, 1);
4030 case Intrinsic::x86_sse_cvtps2pi:
4031 case Intrinsic::x86_sse_cvttps2pi:
4032 handleVectorConvertIntrinsic(
I, 2);
4035 case Intrinsic::x86_avx512_psll_w_512:
4036 case Intrinsic::x86_avx512_psll_d_512:
4037 case Intrinsic::x86_avx512_psll_q_512:
4038 case Intrinsic::x86_avx512_pslli_w_512:
4039 case Intrinsic::x86_avx512_pslli_d_512:
4040 case Intrinsic::x86_avx512_pslli_q_512:
4041 case Intrinsic::x86_avx512_psrl_w_512:
4042 case Intrinsic::x86_avx512_psrl_d_512:
4043 case Intrinsic::x86_avx512_psrl_q_512:
4044 case Intrinsic::x86_avx512_psra_w_512:
4045 case Intrinsic::x86_avx512_psra_d_512:
4046 case Intrinsic::x86_avx512_psra_q_512:
4047 case Intrinsic::x86_avx512_psrli_w_512:
4048 case Intrinsic::x86_avx512_psrli_d_512:
4049 case Intrinsic::x86_avx512_psrli_q_512:
4050 case Intrinsic::x86_avx512_psrai_w_512:
4051 case Intrinsic::x86_avx512_psrai_d_512:
4052 case Intrinsic::x86_avx512_psrai_q_512:
4053 case Intrinsic::x86_avx512_psra_q_256:
4054 case Intrinsic::x86_avx512_psra_q_128:
4055 case Intrinsic::x86_avx512_psrai_q_256:
4056 case Intrinsic::x86_avx512_psrai_q_128:
4057 case Intrinsic::x86_avx2_psll_w:
4058 case Intrinsic::x86_avx2_psll_d:
4059 case Intrinsic::x86_avx2_psll_q:
4060 case Intrinsic::x86_avx2_pslli_w:
4061 case Intrinsic::x86_avx2_pslli_d:
4062 case Intrinsic::x86_avx2_pslli_q:
4063 case Intrinsic::x86_avx2_psrl_w:
4064 case Intrinsic::x86_avx2_psrl_d:
4065 case Intrinsic::x86_avx2_psrl_q:
4066 case Intrinsic::x86_avx2_psra_w:
4067 case Intrinsic::x86_avx2_psra_d:
4068 case Intrinsic::x86_avx2_psrli_w:
4069 case Intrinsic::x86_avx2_psrli_d:
4070 case Intrinsic::x86_avx2_psrli_q:
4071 case Intrinsic::x86_avx2_psrai_w:
4072 case Intrinsic::x86_avx2_psrai_d:
4073 case Intrinsic::x86_sse2_psll_w:
4074 case Intrinsic::x86_sse2_psll_d:
4075 case Intrinsic::x86_sse2_psll_q:
4076 case Intrinsic::x86_sse2_pslli_w:
4077 case Intrinsic::x86_sse2_pslli_d:
4078 case Intrinsic::x86_sse2_pslli_q:
4079 case Intrinsic::x86_sse2_psrl_w:
4080 case Intrinsic::x86_sse2_psrl_d:
4081 case Intrinsic::x86_sse2_psrl_q:
4082 case Intrinsic::x86_sse2_psra_w:
4083 case Intrinsic::x86_sse2_psra_d:
4084 case Intrinsic::x86_sse2_psrli_w:
4085 case Intrinsic::x86_sse2_psrli_d:
4086 case Intrinsic::x86_sse2_psrli_q:
4087 case Intrinsic::x86_sse2_psrai_w:
4088 case Intrinsic::x86_sse2_psrai_d:
4089 case Intrinsic::x86_mmx_psll_w:
4090 case Intrinsic::x86_mmx_psll_d:
4091 case Intrinsic::x86_mmx_psll_q:
4092 case Intrinsic::x86_mmx_pslli_w:
4093 case Intrinsic::x86_mmx_pslli_d:
4094 case Intrinsic::x86_mmx_pslli_q:
4095 case Intrinsic::x86_mmx_psrl_w:
4096 case Intrinsic::x86_mmx_psrl_d:
4097 case Intrinsic::x86_mmx_psrl_q:
4098 case Intrinsic::x86_mmx_psra_w:
4099 case Intrinsic::x86_mmx_psra_d:
4100 case Intrinsic::x86_mmx_psrli_w:
4101 case Intrinsic::x86_mmx_psrli_d:
4102 case Intrinsic::x86_mmx_psrli_q:
4103 case Intrinsic::x86_mmx_psrai_w:
4104 case Intrinsic::x86_mmx_psrai_d:
4105 handleVectorShiftIntrinsic(
I,
false);
4107 case Intrinsic::x86_avx2_psllv_d:
4108 case Intrinsic::x86_avx2_psllv_d_256:
4109 case Intrinsic::x86_avx512_psllv_d_512:
4110 case Intrinsic::x86_avx2_psllv_q:
4111 case Intrinsic::x86_avx2_psllv_q_256:
4112 case Intrinsic::x86_avx512_psllv_q_512:
4113 case Intrinsic::x86_avx2_psrlv_d:
4114 case Intrinsic::x86_avx2_psrlv_d_256:
4115 case Intrinsic::x86_avx512_psrlv_d_512:
4116 case Intrinsic::x86_avx2_psrlv_q:
4117 case Intrinsic::x86_avx2_psrlv_q_256:
4118 case Intrinsic::x86_avx512_psrlv_q_512:
4119 case Intrinsic::x86_avx2_psrav_d:
4120 case Intrinsic::x86_avx2_psrav_d_256:
4121 case Intrinsic::x86_avx512_psrav_d_512:
4122 case Intrinsic::x86_avx512_psrav_q_128:
4123 case Intrinsic::x86_avx512_psrav_q_256:
4124 case Intrinsic::x86_avx512_psrav_q_512:
4125 handleVectorShiftIntrinsic(
I,
true);
4128 case Intrinsic::x86_sse2_packsswb_128:
4129 case Intrinsic::x86_sse2_packssdw_128:
4130 case Intrinsic::x86_sse2_packuswb_128:
4131 case Intrinsic::x86_sse41_packusdw:
4132 case Intrinsic::x86_avx2_packsswb:
4133 case Intrinsic::x86_avx2_packssdw:
4134 case Intrinsic::x86_avx2_packuswb:
4135 case Intrinsic::x86_avx2_packusdw:
4136 handleVectorPackIntrinsic(
I);
4139 case Intrinsic::x86_sse41_pblendvb:
4140 case Intrinsic::x86_sse41_blendvpd:
4141 case Intrinsic::x86_sse41_blendvps:
4142 case Intrinsic::x86_avx_blendv_pd_256:
4143 case Intrinsic::x86_avx_blendv_ps_256:
4144 case Intrinsic::x86_avx2_pblendvb:
4145 handleBlendvIntrinsic(
I);
4148 case Intrinsic::x86_avx_dp_ps_256:
4149 case Intrinsic::x86_sse41_dppd:
4150 case Intrinsic::x86_sse41_dpps:
4151 handleDppIntrinsic(
I);
4154 case Intrinsic::x86_mmx_packsswb:
4155 case Intrinsic::x86_mmx_packuswb:
4156 handleVectorPackIntrinsic(
I, 16);
4159 case Intrinsic::x86_mmx_packssdw:
4160 handleVectorPackIntrinsic(
I, 32);
4163 case Intrinsic::x86_mmx_psad_bw:
4164 case Intrinsic::x86_sse2_psad_bw:
4165 case Intrinsic::x86_avx2_psad_bw:
4166 handleVectorSadIntrinsic(
I);
4169 case Intrinsic::x86_sse2_pmadd_wd:
4170 case Intrinsic::x86_avx2_pmadd_wd:
4171 case Intrinsic::x86_ssse3_pmadd_ub_sw_128:
4172 case Intrinsic::x86_avx2_pmadd_ub_sw:
4173 handleVectorPmaddIntrinsic(
I);
4176 case Intrinsic::x86_ssse3_pmadd_ub_sw:
4177 handleVectorPmaddIntrinsic(
I, 8);
4180 case Intrinsic::x86_mmx_pmadd_wd:
4181 handleVectorPmaddIntrinsic(
I, 16);
4184 case Intrinsic::x86_sse_cmp_ss:
4185 case Intrinsic::x86_sse2_cmp_sd:
4186 case Intrinsic::x86_sse_comieq_ss:
4187 case Intrinsic::x86_sse_comilt_ss:
4188 case Intrinsic::x86_sse_comile_ss:
4189 case Intrinsic::x86_sse_comigt_ss:
4190 case Intrinsic::x86_sse_comige_ss:
4191 case Intrinsic::x86_sse_comineq_ss:
4192 case Intrinsic::x86_sse_ucomieq_ss:
4193 case Intrinsic::x86_sse_ucomilt_ss:
4194 case Intrinsic::x86_sse_ucomile_ss:
4195 case Intrinsic::x86_sse_ucomigt_ss:
4196 case Intrinsic::x86_sse_ucomige_ss:
4197 case Intrinsic::x86_sse_ucomineq_ss:
4198 case Intrinsic::x86_sse2_comieq_sd:
4199 case Intrinsic::x86_sse2_comilt_sd:
4200 case Intrinsic::x86_sse2_comile_sd:
4201 case Intrinsic::x86_sse2_comigt_sd:
4202 case Intrinsic::x86_sse2_comige_sd:
4203 case Intrinsic::x86_sse2_comineq_sd:
4204 case Intrinsic::x86_sse2_ucomieq_sd:
4205 case Intrinsic::x86_sse2_ucomilt_sd:
4206 case Intrinsic::x86_sse2_ucomile_sd:
4207 case Intrinsic::x86_sse2_ucomigt_sd:
4208 case Intrinsic::x86_sse2_ucomige_sd:
4209 case Intrinsic::x86_sse2_ucomineq_sd:
4210 handleVectorCompareScalarIntrinsic(
I);
4213 case Intrinsic::x86_avx_cmp_pd_256:
4214 case Intrinsic::x86_avx_cmp_ps_256:
4215 case Intrinsic::x86_sse2_cmp_pd:
4216 case Intrinsic::x86_sse_cmp_ps:
4217 handleVectorComparePackedIntrinsic(
I);
4220 case Intrinsic::x86_bmi_bextr_32:
4221 case Intrinsic::x86_bmi_bextr_64:
4222 case Intrinsic::x86_bmi_bzhi_32:
4223 case Intrinsic::x86_bmi_bzhi_64:
4224 case Intrinsic::x86_bmi_pdep_32:
4225 case Intrinsic::x86_bmi_pdep_64:
4226 case Intrinsic::x86_bmi_pext_32:
4227 case Intrinsic::x86_bmi_pext_64:
4228 handleBmiIntrinsic(
I);
4231 case Intrinsic::x86_pclmulqdq:
4232 case Intrinsic::x86_pclmulqdq_256:
4233 case Intrinsic::x86_pclmulqdq_512:
4234 handlePclmulIntrinsic(
I);
4237 case Intrinsic::x86_sse41_round_sd:
4238 case Intrinsic::x86_sse41_round_ss:
4239 handleUnarySdSsIntrinsic(
I);
4241 case Intrinsic::x86_sse2_max_sd:
4242 case Intrinsic::x86_sse_max_ss:
4243 case Intrinsic::x86_sse2_min_sd:
4244 case Intrinsic::x86_sse_min_ss:
4245 handleBinarySdSsIntrinsic(
I);
4248 case Intrinsic::x86_avx_vtestc_pd:
4249 case Intrinsic::x86_avx_vtestc_pd_256:
4250 case Intrinsic::x86_avx_vtestc_ps:
4251 case Intrinsic::x86_avx_vtestc_ps_256:
4252 case Intrinsic::x86_avx_vtestnzc_pd:
4253 case Intrinsic::x86_avx_vtestnzc_pd_256:
4254 case Intrinsic::x86_avx_vtestnzc_ps:
4255 case Intrinsic::x86_avx_vtestnzc_ps_256:
4256 case Intrinsic::x86_avx_vtestz_pd:
4257 case Intrinsic::x86_avx_vtestz_pd_256:
4258 case Intrinsic::x86_avx_vtestz_ps:
4259 case Intrinsic::x86_avx_vtestz_ps_256:
4260 case Intrinsic::x86_avx_ptestc_256:
4261 case Intrinsic::x86_avx_ptestnzc_256:
4262 case Intrinsic::x86_avx_ptestz_256:
4263 case Intrinsic::x86_sse41_ptestc:
4264 case Intrinsic::x86_sse41_ptestnzc:
4265 case Intrinsic::x86_sse41_ptestz:
4266 handleVtestIntrinsic(
I);
4269 case Intrinsic::fshl:
4270 case Intrinsic::fshr:
4271 handleFunnelShift(
I);
4274 case Intrinsic::is_constant:
4276 setShadow(&
I, getCleanShadow(&
I));
4277 setOrigin(&
I, getCleanOrigin());
4280 case Intrinsic::aarch64_neon_st2:
4281 case Intrinsic::aarch64_neon_st3:
4282 case Intrinsic::aarch64_neon_st4: {
4283 handleNEONVectorStoreIntrinsic(
I);
4288 if (!handleUnknownIntrinsic(
I))
4289 visitInstruction(
I);
4294 void visitLibAtomicLoad(
CallBase &CB) {
4296 assert(isa<CallInst>(CB));
4305 Value *NewOrdering =
4309 NextNodeIRBuilder NextIRB(&CB);
4310 Value *SrcShadowPtr, *SrcOriginPtr;
4311 std::tie(SrcShadowPtr, SrcOriginPtr) =
4312 getShadowOriginPtr(SrcPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4314 Value *DstShadowPtr =
4315 getShadowOriginPtr(DstPtr, NextIRB, NextIRB.getInt8Ty(),
Align(1),
4319 NextIRB.CreateMemCpy(DstShadowPtr,
Align(1), SrcShadowPtr,
Align(1),
Size);
4320 if (MS.TrackOrigins) {
4321 Value *SrcOrigin = NextIRB.CreateAlignedLoad(MS.OriginTy, SrcOriginPtr,
4323 Value *NewOrigin = updateOrigin(SrcOrigin, NextIRB);
4324 NextIRB.CreateCall(MS.MsanSetOriginFn, {DstPtr, Size, NewOrigin});
4328 void visitLibAtomicStore(
CallBase &CB) {
4335 Value *NewOrdering =
4339 Value *DstShadowPtr =
4357 visitAsmInstruction(CB);
4359 visitInstruction(CB);
4368 case LibFunc_atomic_load:
4369 if (!isa<CallInst>(CB)) {
4370 llvm::errs() <<
"MSAN -- cannot instrument invoke of libatomic load."
4374 visitLibAtomicLoad(CB);
4376 case LibFunc_atomic_store:
4377 visitLibAtomicStore(CB);
4384 if (
auto *Call = dyn_cast<CallInst>(&CB)) {
4385 assert(!isa<IntrinsicInst>(Call) &&
"intrinsics are handled elsewhere");
4393 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
4395 Call->removeFnAttrs(
B);
4397 Func->removeFnAttrs(
B);
4403 bool MayCheckCall = MS.EagerChecks;
4407 MayCheckCall &= !
Func->getName().starts_with(
"__sanitizer_unaligned_");
4410 unsigned ArgOffset = 0;
4413 if (!
A->getType()->isSized()) {
4414 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is not sized: " << CB <<
"\n");
4418 if (
A->getType()->isScalableTy()) {
4419 LLVM_DEBUG(
dbgs() <<
"Arg " << i <<
" is vscale: " << CB <<
"\n");
4421 insertShadowCheck(
A, &CB);
4430 bool EagerCheck = MayCheckCall && !ByVal && NoUndef;
4433 insertShadowCheck(
A, &CB);
4434 Size =
DL.getTypeAllocSize(
A->getType());
4440 Value *ArgShadow = getShadow(
A);
4441 Value *ArgShadowBase = getShadowPtrForArgument(IRB, ArgOffset);
4443 <<
" Shadow: " << *ArgShadow <<
"\n");
4447 assert(
A->getType()->isPointerTy() &&
4448 "ByVal argument is not a pointer!");
4456 Value *AShadowPtr, *AOriginPtr;
4457 std::tie(AShadowPtr, AOriginPtr) =
4458 getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(), Alignment,
4460 if (!PropagateShadow) {
4467 if (MS.TrackOrigins) {
4468 Value *ArgOriginBase = getOriginPtrForArgument(IRB, ArgOffset);
4482 Size =
DL.getTypeAllocSize(
A->getType());
4487 Constant *Cst = dyn_cast<Constant>(ArgShadow);
4488 if (MS.TrackOrigins && !(Cst && Cst->
isNullValue())) {
4490 getOriginPtrForArgument(IRB, ArgOffset));
4494 assert(Store !=
nullptr);
4503 if (FT->isVarArg()) {
4504 VAHelper->visitCallBase(CB, IRB);
4511 if (isa<CallInst>(CB) && cast<CallInst>(CB).isMustTailCall())
4514 if (MayCheckCall && CB.
hasRetAttr(Attribute::NoUndef)) {
4515 setShadow(&CB, getCleanShadow(&CB));
4516 setOrigin(&CB, getCleanOrigin());
4522 Value *
Base = getShadowPtrForRetval(IRBBefore);
4523 IRBBefore.CreateAlignedStore(getCleanShadow(&CB),
Base,
4526 if (isa<CallInst>(CB)) {
4530 BasicBlock *NormalDest = cast<InvokeInst>(CB).getNormalDest();
4535 setShadow(&CB, getCleanShadow(&CB));
4536 setOrigin(&CB, getCleanOrigin());
4543 "Could not find insertion point for retval shadow load");
4546 Value *RetvalShadow = IRBAfter.CreateAlignedLoad(
4547 getShadowTy(&CB), getShadowPtrForRetval(IRBAfter),
4549 setShadow(&CB, RetvalShadow);
4550 if (MS.TrackOrigins)
4551 setOrigin(&CB, IRBAfter.CreateLoad(MS.OriginTy,
4552 getOriginPtrForRetval()));
4556 if (
auto *
I = dyn_cast<BitCastInst>(RetVal)) {
4557 RetVal =
I->getOperand(0);
4559 if (
auto *
I = dyn_cast<CallInst>(RetVal)) {
4560 return I->isMustTailCall();
4567 Value *RetVal =
I.getReturnValue();
4573 Value *ShadowPtr = getShadowPtrForRetval(IRB);
4574 bool HasNoUndef =
F.hasRetAttribute(Attribute::NoUndef);
4575 bool StoreShadow = !(MS.EagerChecks && HasNoUndef);
4578 bool EagerCheck = (MS.EagerChecks && HasNoUndef) || (
F.getName() ==
"main");
4580 Value *Shadow = getShadow(RetVal);
4581 bool StoreOrigin =
true;
4583 insertShadowCheck(RetVal, &
I);
4584 Shadow = getCleanShadow(RetVal);
4585 StoreOrigin =
false;
4592 if (MS.TrackOrigins && StoreOrigin)
4593 IRB.
CreateStore(getOrigin(RetVal), getOriginPtrForRetval());
4599 if (!PropagateShadow) {
4600 setShadow(&
I, getCleanShadow(&
I));
4601 setOrigin(&
I, getCleanOrigin());
4605 ShadowPHINodes.push_back(&
I);
4606 setShadow(&
I, IRB.
CreatePHI(getShadowTy(&
I),
I.getNumIncomingValues(),
4608 if (MS.TrackOrigins)
4610 &
I, IRB.
CreatePHI(MS.OriginTy,
I.getNumIncomingValues(),
"_msphi_o"));
4627 IRB.
CreateCall(MS.MsanPoisonStackFn, {&I, Len});
4629 Value *ShadowBase, *OriginBase;
4630 std::tie(ShadowBase, OriginBase) = getShadowOriginPtr(
4637 if (PoisonStack && MS.TrackOrigins) {
4638 Value *Idptr = getLocalVarIdptr(
I);
4640 Value *Descr = getLocalVarDescription(
I);
4641 IRB.
CreateCall(MS.MsanSetAllocaOriginWithDescriptionFn,
4642 {&I, Len, Idptr, Descr});
4644 IRB.
CreateCall(MS.MsanSetAllocaOriginNoDescriptionFn, {&I, Len, Idptr});
4650 Value *Descr = getLocalVarDescription(
I);
4652 IRB.
CreateCall(MS.MsanPoisonAllocaFn, {&I, Len, Descr});
4654 IRB.
CreateCall(MS.MsanUnpoisonAllocaFn, {&I, Len});
4661 NextNodeIRBuilder IRB(InsPoint);
4663 TypeSize TS =
DL.getTypeAllocSize(
I.getAllocatedType());
4665 if (
I.isArrayAllocation())
4669 if (MS.CompileKernel)
4670 poisonAllocaKmsan(
I, IRB, Len);
4672 poisonAllocaUserspace(
I, IRB, Len);
4676 setShadow(&
I, getCleanShadow(&
I));
4677 setOrigin(&
I, getCleanOrigin());
4689 handleSelectLikeInst(
I,
B,
C,
D);
4695 Value *Sb = getShadow(
B);
4696 Value *Sc = getShadow(
C);
4697 Value *Sd = getShadow(
D);
4699 Value *Ob = MS.TrackOrigins ? getOrigin(
B) : nullptr;
4700 Value *Oc = MS.TrackOrigins ? getOrigin(
C) : nullptr;
4701 Value *Od = MS.TrackOrigins ? getOrigin(
D) : nullptr;
4706 if (
I.getType()->isAggregateType()) {
4710 Sa1 = getPoisonedShadow(getShadowTy(
I.getType()));
4718 C = CreateAppToShadowCast(IRB,
C);
4719 D = CreateAppToShadowCast(IRB,
D);
4726 if (MS.TrackOrigins) {
4729 if (
B->getType()->isVectorTy()) {
4730 B = convertToBool(
B, IRB);
4731 Sb = convertToBool(Sb, IRB);
4742 setShadow(&
I, getCleanShadow(&
I));
4743 setOrigin(&
I, getCleanOrigin());
4747 setShadow(&
I, getCleanShadow(&
I));
4748 setOrigin(&
I, getCleanOrigin());
4752 setShadow(&
I, getCleanShadow(&
I));
4753 setOrigin(&
I, getCleanOrigin());
4760 Value *Agg =
I.getAggregateOperand();
4762 Value *AggShadow = getShadow(Agg);
4766 setShadow(&
I, ResShadow);
4767 setOriginForNaryOp(
I);
4773 Value *AggShadow = getShadow(
I.getAggregateOperand());
4774 Value *InsShadow = getShadow(
I.getInsertedValueOperand());
4780 setOriginForNaryOp(
I);
4784 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
4785 errs() <<
"ZZZ call " << CI->getCalledFunction()->getName() <<
"\n";
4787 errs() <<
"ZZZ " <<
I.getOpcodeName() <<
"\n";
4789 errs() <<
"QQQ " <<
I <<
"\n";
4816 insertShadowCheck(Operand, &
I);
4823 auto Size =
DL.getTypeStoreSize(ElemTy);
4825 if (MS.CompileKernel) {
4826 IRB.
CreateCall(MS.MsanInstrumentAsmStoreFn, {Operand, SizeVal});
4832 auto [ShadowPtr,
_] =
4833 getShadowOriginPtrUserspace(Operand, IRB, IRB.
getInt8Ty(),
Align(1));
4844 int NumRetOutputs = 0;
4846 Type *
RetTy = cast<Value>(CB)->getType();
4847 if (!
RetTy->isVoidTy()) {
4849 auto *
ST = dyn_cast<StructType>(
RetTy);
4851 NumRetOutputs =
ST->getNumElements();
4857 switch (
Info.Type) {
4865 return NumOutputs - NumRetOutputs;
4888 int OutputArgs = getNumOutputArgs(IA, CB);
4894 for (
int i = OutputArgs; i < NumOperands; i++) {
4902 for (
int i = 0; i < OutputArgs; i++) {
4908 setShadow(&
I, getCleanShadow(&
I));
4909 setOrigin(&
I, getCleanOrigin());
4914 setShadow(&
I, getCleanShadow(&
I));
4915 setOrigin(&
I, getCleanOrigin());
4923 for (
size_t i = 0, n =
I.getNumOperands(); i < n; i++) {
4924 Value *Operand =
I.getOperand(i);
4926 insertShadowCheck(Operand, &
I);
4928 setShadow(&
I, getCleanShadow(&
I));
4929 setOrigin(&
I, getCleanOrigin());
4933struct VarArgHelperBase :
public VarArgHelper {
4935 MemorySanitizer &MS;
4936 MemorySanitizerVisitor &MSV;
4938 const unsigned VAListTagSize;
4940 VarArgHelperBase(
Function &
F, MemorySanitizer &MS,
4941 MemorySanitizerVisitor &MSV,
unsigned VAListTagSize)
4942 :
F(
F), MS(MS), MSV(MSV), VAListTagSize(VAListTagSize) {}
4946 return IRB.
CreateAdd(
Base, ConstantInt::get(MS.IntptrTy, ArgOffset));
4951 unsigned ArgOffset) {
4960 unsigned ArgOffset,
unsigned ArgSize) {
4964 return getShadowPtrForVAArgument(Ty, IRB, ArgOffset);
4979 unsigned BaseOffset) {
4988 TailSize,
Align(8));
4993 Value *VAListTag =
I.getArgOperand(0);
4995 auto [ShadowPtr, OriginPtr] = MSV.getShadowOriginPtr(
4996 VAListTag, IRB, IRB.
getInt8Ty(), Alignment,
true);
4999 VAListTagSize, Alignment,
false);
5006 unpoisonVAListTagForInst(
I);
5012 unpoisonVAListTagForInst(
I);
5017struct VarArgAMD64Helper :
public VarArgHelperBase {
5020 static const unsigned AMD64GpEndOffset = 48;
5021 static const unsigned AMD64FpEndOffsetSSE = 176;
5023 static const unsigned AMD64FpEndOffsetNoSSE = AMD64GpEndOffset;
5025 unsigned AMD64FpEndOffset;
5028 Value *VAArgOverflowSize =
nullptr;
5030 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5032 VarArgAMD64Helper(
Function &
F, MemorySanitizer &MS,
5033 MemorySanitizerVisitor &MSV)
5034 : VarArgHelperBase(
F, MS, MSV, 24) {
5035 AMD64FpEndOffset = AMD64FpEndOffsetSSE;
5036 for (
const auto &Attr :
F.getAttributes().getFnAttrs()) {
5037 if (Attr.isStringAttribute() &&
5038 (Attr.getKindAsString() ==
"target-features")) {
5039 if (Attr.getValueAsString().contains(
"-sse"))
5040 AMD64FpEndOffset = AMD64FpEndOffsetNoSSE;
5046 ArgKind classifyArgument(
Value *arg) {
5049 if (
T->isX86_FP80Ty())
5051 if (
T->isFPOrFPVectorTy() ||
T->isX86_MMXTy())
5052 return AK_FloatingPoint;
5053 if (
T->isIntegerTy() &&
T->getPrimitiveSizeInBits() <= 64)
5054 return AK_GeneralPurpose;
5055 if (
T->isPointerTy())
5056 return AK_GeneralPurpose;
5069 unsigned GpOffset = 0;
5070 unsigned FpOffset = AMD64GpEndOffset;
5071 unsigned OverflowOffset = AMD64FpEndOffset;
5076 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5083 assert(
A->getType()->isPointerTy());
5085 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5087 unsigned BaseOffset = OverflowOffset;
5089 getShadowPtrForVAArgument(RealTy, IRB, OverflowOffset);
5090 Value *OriginBase =
nullptr;
5091 if (MS.TrackOrigins)
5092 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
5093 OverflowOffset += AlignedSize;
5096 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
5100 Value *ShadowPtr, *OriginPtr;
5101 std::tie(ShadowPtr, OriginPtr) =
5106 if (MS.TrackOrigins)
5110 ArgKind AK = classifyArgument(
A);
5111 if (AK == AK_GeneralPurpose && GpOffset >= AMD64GpEndOffset)
5113 if (AK == AK_FloatingPoint && FpOffset >= AMD64FpEndOffset)
5115 Value *ShadowBase, *OriginBase =
nullptr;
5117 case AK_GeneralPurpose:
5118 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, GpOffset);
5119 if (MS.TrackOrigins)
5120 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset);
5124 case AK_FloatingPoint:
5125 ShadowBase = getShadowPtrForVAArgument(
A->getType(), IRB, FpOffset);
5126 if (MS.TrackOrigins)
5127 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5134 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5136 unsigned BaseOffset = OverflowOffset;
5138 getShadowPtrForVAArgument(
A->getType(), IRB, OverflowOffset);
5139 if (MS.TrackOrigins) {
5140 OriginBase = getOriginPtrForVAArgument(IRB, OverflowOffset);
5142 OverflowOffset += AlignedSize;
5145 CleanUnusedTLS(IRB, ShadowBase, BaseOffset);
5154 Value *Shadow = MSV.getShadow(
A);
5156 if (MS.TrackOrigins) {
5157 Value *Origin = MSV.getOrigin(
A);
5159 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5165 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AMD64FpEndOffset);
5166 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5169 void finalizeInstrumentation()
override {
5170 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5171 "finalizeInstrumentation called twice");
5172 if (!VAStartInstrumentationList.
empty()) {
5179 ConstantInt::get(MS.IntptrTy, AMD64FpEndOffset), VAArgOverflowSize);
5186 Intrinsic::umin, CopySize,
5190 if (MS.TrackOrigins) {
5200 for (
CallInst *OrigInst : VAStartInstrumentationList) {
5201 NextNodeIRBuilder IRB(OrigInst);
5202 Value *VAListTag = OrigInst->getArgOperand(0);
5204 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5207 ConstantInt::get(MS.IntptrTy, 16)),
5208 PointerType::get(RegSaveAreaPtrTy, 0));
5209 Value *RegSaveAreaPtr =
5210 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5211 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5213 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5214 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5216 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5218 if (MS.TrackOrigins)
5219 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5220 Alignment, AMD64FpEndOffset);
5221 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5224 ConstantInt::get(MS.IntptrTy, 8)),
5225 PointerType::get(OverflowArgAreaPtrTy, 0));
5226 Value *OverflowArgAreaPtr =
5227 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5228 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5229 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5230 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5234 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5236 if (MS.TrackOrigins) {
5239 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5248struct VarArgMIPS64Helper :
public VarArgHelperBase {
5250 Value *VAArgSize =
nullptr;
5252 VarArgMIPS64Helper(
Function &
F, MemorySanitizer &MS,
5253 MemorySanitizerVisitor &MSV)
5254 : VarArgHelperBase(
F, MS, MSV, 8) {}
5257 unsigned VAArgOffset = 0;
5261 Triple TargetTriple(
F.getParent()->getTargetTriple());
5263 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5268 VAArgOffset += (8 - ArgSize);
5270 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VAArgOffset, ArgSize);
5271 VAArgOffset += ArgSize;
5272 VAArgOffset =
alignTo(VAArgOffset, 8);
5281 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5284 void finalizeInstrumentation()
override {
5285 assert(!VAArgSize && !VAArgTLSCopy &&
5286 "finalizeInstrumentation called twice");
5290 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5292 if (!VAStartInstrumentationList.empty()) {
5301 Intrinsic::umin, CopySize,
5309 for (
CallInst *OrigInst : VAStartInstrumentationList) {
5310 NextNodeIRBuilder IRB(OrigInst);
5311 Value *VAListTag = OrigInst->getArgOperand(0);
5312 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5313 Value *RegSaveAreaPtrPtr =
5315 PointerType::get(RegSaveAreaPtrTy, 0));
5316 Value *RegSaveAreaPtr =
5317 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5318 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5320 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5321 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5323 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5330struct VarArgAArch64Helper :
public VarArgHelperBase {
5331 static const unsigned kAArch64GrArgSize = 64;
5332 static const unsigned kAArch64VrArgSize = 128;
5334 static const unsigned AArch64GrBegOffset = 0;
5335 static const unsigned AArch64GrEndOffset = kAArch64GrArgSize;
5337 static const unsigned AArch64VrBegOffset = AArch64GrEndOffset;
5338 static const unsigned AArch64VrEndOffset =
5339 AArch64VrBegOffset + kAArch64VrArgSize;
5340 static const unsigned AArch64VAEndOffset = AArch64VrEndOffset;
5343 Value *VAArgOverflowSize =
nullptr;
5345 enum ArgKind { AK_GeneralPurpose, AK_FloatingPoint, AK_Memory };
5347 VarArgAArch64Helper(
Function &
F, MemorySanitizer &MS,
5348 MemorySanitizerVisitor &MSV)
5349 : VarArgHelperBase(
F, MS, MSV, 32) {}
5352 std::pair<ArgKind, uint64_t> classifyArgument(
Type *
T) {
5353 if (
T->isIntOrPtrTy() &&
T->getPrimitiveSizeInBits() <= 64)
5354 return {AK_GeneralPurpose, 1};
5355 if (
T->isFloatingPointTy() &&
T->getPrimitiveSizeInBits() <= 128)
5356 return {AK_FloatingPoint, 1};
5358 if (
T->isArrayTy()) {
5359 auto R = classifyArgument(
T->getArrayElementType());
5360 R.second *=
T->getScalarType()->getArrayNumElements();
5365 auto R = classifyArgument(FV->getScalarType());
5366 R.second *= FV->getNumElements();
5371 return {AK_Memory, 0};
5384 unsigned GrOffset = AArch64GrBegOffset;
5385 unsigned VrOffset = AArch64VrBegOffset;
5386 unsigned OverflowOffset = AArch64VAEndOffset;
5391 auto [AK, RegNum] = classifyArgument(
A->getType());
5392 if (AK == AK_GeneralPurpose &&
5393 (GrOffset + RegNum * 8) > AArch64GrEndOffset)
5395 if (AK == AK_FloatingPoint &&
5396 (VrOffset + RegNum * 16) > AArch64VrEndOffset)
5400 case AK_GeneralPurpose:
5401 Base = getShadowPtrForVAArgument(
A->getType(), IRB, GrOffset);
5402 GrOffset += 8 * RegNum;
5404 case AK_FloatingPoint:
5405 Base = getShadowPtrForVAArgument(
A->getType(), IRB, VrOffset);
5406 VrOffset += 16 * RegNum;
5413 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5415 unsigned BaseOffset = OverflowOffset;
5416 Base = getShadowPtrForVAArgument(
A->getType(), IRB, BaseOffset);
5417 OverflowOffset += AlignedSize;
5420 CleanUnusedTLS(IRB,
Base, BaseOffset);
5432 ConstantInt::get(IRB.
getInt64Ty(), OverflowOffset - AArch64VAEndOffset);
5433 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5440 ConstantInt::get(MS.IntptrTy, offset)),
5441 PointerType::get(*MS.C, 0));
5449 ConstantInt::get(MS.IntptrTy, offset)),
5450 PointerType::get(*MS.C, 0));
5452 return IRB.
CreateSExt(SaveArea32, MS.IntptrTy);
5455 void finalizeInstrumentation()
override {
5456 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5457 "finalizeInstrumentation called twice");
5458 if (!VAStartInstrumentationList.empty()) {
5465 ConstantInt::get(MS.IntptrTy, AArch64VAEndOffset), VAArgOverflowSize);
5472 Intrinsic::umin, CopySize,
5478 Value *GrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64GrArgSize);
5479 Value *VrArgSize = ConstantInt::get(MS.IntptrTy, kAArch64VrArgSize);
5483 for (
CallInst *OrigInst : VAStartInstrumentationList) {
5484 NextNodeIRBuilder IRB(OrigInst);
5486 Value *VAListTag = OrigInst->getArgOperand(0);
5503 Value *StackSaveAreaPtr =
5504 IRB.
CreateIntToPtr(getVAField64(IRB, VAListTag, 0), RegSaveAreaPtrTy);
5507 Value *GrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 8);
5508 Value *GrOffSaveArea = getVAField32(IRB, VAListTag, 24);
5511 IRB.
CreateAdd(GrTopSaveAreaPtr, GrOffSaveArea), RegSaveAreaPtrTy);
5514 Value *VrTopSaveAreaPtr = getVAField64(IRB, VAListTag, 16);
5515 Value *VrOffSaveArea = getVAField32(IRB, VAListTag, 28);
5518 IRB.
CreateAdd(VrTopSaveAreaPtr, VrOffSaveArea), RegSaveAreaPtrTy);
5524 Value *GrRegSaveAreaShadowPtrOff =
5525 IRB.
CreateAdd(GrArgSize, GrOffSaveArea);
5527 Value *GrRegSaveAreaShadowPtr =
5528 MSV.getShadowOriginPtr(GrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5534 Value *GrCopySize = IRB.
CreateSub(GrArgSize, GrRegSaveAreaShadowPtrOff);
5540 Value *VrRegSaveAreaShadowPtrOff =
5541 IRB.
CreateAdd(VrArgSize, VrOffSaveArea);
5543 Value *VrRegSaveAreaShadowPtr =
5544 MSV.getShadowOriginPtr(VrRegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5551 VrRegSaveAreaShadowPtrOff);
5552 Value *VrCopySize = IRB.
CreateSub(VrArgSize, VrRegSaveAreaShadowPtrOff);
5558 Value *StackSaveAreaShadowPtr =
5559 MSV.getShadowOriginPtr(StackSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5564 VAArgTLSCopy, IRB.
getInt32(AArch64VAEndOffset));
5567 Align(16), VAArgOverflowSize);
5573struct VarArgPowerPC64Helper :
public VarArgHelperBase {
5575 Value *VAArgSize =
nullptr;
5577 VarArgPowerPC64Helper(
Function &
F, MemorySanitizer &MS,
5578 MemorySanitizerVisitor &MSV)
5579 : VarArgHelperBase(
F, MS, MSV, 8) {}
5589 Triple TargetTriple(
F.getParent()->getTargetTriple());
5597 unsigned VAArgOffset = VAArgBase;
5601 bool IsByVal = CB.
paramHasAttr(ArgNo, Attribute::ByVal);
5603 assert(
A->getType()->isPointerTy());
5605 uint64_t ArgSize =
DL.getTypeAllocSize(RealTy);
5608 ArgAlign =
Align(8);
5609 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5611 Value *
Base = getShadowPtrForVAArgument(
5612 RealTy, IRB, VAArgOffset - VAArgBase, ArgSize);
5614 Value *AShadowPtr, *AOriginPtr;
5615 std::tie(AShadowPtr, AOriginPtr) =
5616 MSV.getShadowOriginPtr(
A, IRB, IRB.
getInt8Ty(),
5626 uint64_t ArgSize =
DL.getTypeAllocSize(
A->getType());
5628 if (
A->getType()->isArrayTy()) {
5631 Type *ElementTy =
A->getType()->getArrayElementType();
5633 ArgAlign =
Align(
DL.getTypeAllocSize(ElementTy));
5634 }
else if (
A->getType()->isVectorTy()) {
5636 ArgAlign =
Align(ArgSize);
5639 ArgAlign =
Align(8);
5640 VAArgOffset =
alignTo(VAArgOffset, ArgAlign);
5641 if (
DL.isBigEndian()) {
5645 VAArgOffset += (8 - ArgSize);
5648 Base = getShadowPtrForVAArgument(
A->getType(), IRB,
5649 VAArgOffset - VAArgBase, ArgSize);
5653 VAArgOffset += ArgSize;
5657 VAArgBase = VAArgOffset;
5661 ConstantInt::get(IRB.
getInt64Ty(), VAArgOffset - VAArgBase);
5664 IRB.
CreateStore(TotalVAArgSize, MS.VAArgOverflowSizeTLS);
5667 void finalizeInstrumentation()
override {
5668 assert(!VAArgSize && !VAArgTLSCopy &&
5669 "finalizeInstrumentation called twice");
5673 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, 0), VAArgSize);
5675 if (!VAStartInstrumentationList.empty()) {
5685 Intrinsic::umin, CopySize,
5693 for (
CallInst *OrigInst : VAStartInstrumentationList) {
5694 NextNodeIRBuilder IRB(OrigInst);
5695 Value *VAListTag = OrigInst->getArgOperand(0);
5696 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5697 Value *RegSaveAreaPtrPtr =
5699 PointerType::get(RegSaveAreaPtrTy, 0));
5700 Value *RegSaveAreaPtr =
5701 IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5702 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5704 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5705 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(),
5707 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5714struct VarArgSystemZHelper :
public VarArgHelperBase {
5715 static const unsigned SystemZGpOffset = 16;
5716 static const unsigned SystemZGpEndOffset = 56;
5717 static const unsigned SystemZFpOffset = 128;
5718 static const unsigned SystemZFpEndOffset = 160;
5719 static const unsigned SystemZMaxVrArgs = 8;
5720 static const unsigned SystemZRegSaveAreaSize = 160;
5721 static const unsigned SystemZOverflowOffset = 160;
5722 static const unsigned SystemZVAListTagSize = 32;
5723 static const unsigned SystemZOverflowArgAreaPtrOffset = 16;
5724 static const unsigned SystemZRegSaveAreaPtrOffset = 24;
5726 bool IsSoftFloatABI;
5729 Value *VAArgOverflowSize =
nullptr;
5731 enum class ArgKind {
5739 enum class ShadowExtension {
None,
Zero, Sign };
5741 VarArgSystemZHelper(
Function &
F, MemorySanitizer &MS,
5742 MemorySanitizerVisitor &MSV)
5743 : VarArgHelperBase(
F, MS, MSV, SystemZVAListTagSize),
5744 IsSoftFloatABI(
F.getFnAttribute(
"use-soft-float").getValueAsBool()) {}
5746 ArgKind classifyArgument(
Type *
T) {
5753 if (
T->isIntegerTy(128) ||
T->isFP128Ty())
5754 return ArgKind::Indirect;
5755 if (
T->isFloatingPointTy())
5756 return IsSoftFloatABI ? ArgKind::GeneralPurpose : ArgKind::FloatingPoint;
5757 if (
T->isIntegerTy() ||
T->isPointerTy())
5758 return ArgKind::GeneralPurpose;
5759 if (
T->isVectorTy())
5760 return ArgKind::Vector;
5761 return ArgKind::Memory;
5764 ShadowExtension getShadowExtension(
const CallBase &CB,
unsigned ArgNo) {
5774 return ShadowExtension::Zero;
5778 return ShadowExtension::Sign;
5780 return ShadowExtension::None;
5784 unsigned GpOffset = SystemZGpOffset;
5785 unsigned FpOffset = SystemZFpOffset;
5786 unsigned VrIndex = 0;
5787 unsigned OverflowOffset = SystemZOverflowOffset;
5794 ArgKind AK = classifyArgument(
T);
5795 if (AK == ArgKind::Indirect) {
5796 T = PointerType::get(
T, 0);
5797 AK = ArgKind::GeneralPurpose;
5799 if (AK == ArgKind::GeneralPurpose && GpOffset >= SystemZGpEndOffset)
5800 AK = ArgKind::Memory;
5801 if (AK == ArgKind::FloatingPoint && FpOffset >= SystemZFpEndOffset)
5802 AK = ArgKind::Memory;
5803 if (AK == ArgKind::Vector && (VrIndex >= SystemZMaxVrArgs || !IsFixed))
5804 AK = ArgKind::Memory;
5805 Value *ShadowBase =
nullptr;
5806 Value *OriginBase =
nullptr;
5807 ShadowExtension SE = ShadowExtension::None;
5809 case ArgKind::GeneralPurpose: {
5814 SE = getShadowExtension(CB, ArgNo);
5816 if (SE == ShadowExtension::None) {
5818 assert(ArgAllocSize <= ArgSize);
5819 GapSize = ArgSize - ArgAllocSize;
5821 ShadowBase = getShadowAddrForVAArgument(IRB, GpOffset + GapSize);
5822 if (MS.TrackOrigins)
5823 OriginBase = getOriginPtrForVAArgument(IRB, GpOffset + GapSize);
5825 GpOffset += ArgSize;
5831 case ArgKind::FloatingPoint: {
5840 ShadowBase = getShadowAddrForVAArgument(IRB, FpOffset);
5841 if (MS.TrackOrigins)
5842 OriginBase = getOriginPtrForVAArgument(IRB, FpOffset);
5844 FpOffset += ArgSize;
5850 case ArgKind::Vector: {
5857 case ArgKind::Memory: {
5865 SE = getShadowExtension(CB, ArgNo);
5867 SE == ShadowExtension::None ? ArgSize - ArgAllocSize : 0;
5869 getShadowAddrForVAArgument(IRB, OverflowOffset + GapSize);
5870 if (MS.TrackOrigins)
5872 getOriginPtrForVAArgument(IRB, OverflowOffset + GapSize);
5873 OverflowOffset += ArgSize;
5880 case ArgKind::Indirect:
5883 if (ShadowBase ==
nullptr)
5885 Value *Shadow = MSV.getShadow(
A);
5886 if (SE != ShadowExtension::None)
5887 Shadow = MSV.CreateShadowCast(IRB, Shadow, IRB.
getInt64Ty(),
5888 SE == ShadowExtension::Sign);
5890 ShadowBase, PointerType::get(Shadow->
getType(), 0),
"_msarg_va_s");
5892 if (MS.TrackOrigins) {
5893 Value *Origin = MSV.getOrigin(
A);
5895 MSV.paintOrigin(IRB, Origin, OriginBase, StoreSize,
5899 Constant *OverflowSize = ConstantInt::get(
5900 IRB.
getInt64Ty(), OverflowOffset - SystemZOverflowOffset);
5901 IRB.
CreateStore(OverflowSize, MS.VAArgOverflowSizeTLS);
5905 Type *RegSaveAreaPtrTy = PointerType::getUnqual(*MS.C);
5909 ConstantInt::get(MS.IntptrTy, SystemZRegSaveAreaPtrOffset)),
5910 PointerType::get(RegSaveAreaPtrTy, 0));
5911 Value *RegSaveAreaPtr = IRB.
CreateLoad(RegSaveAreaPtrTy, RegSaveAreaPtrPtr);
5912 Value *RegSaveAreaShadowPtr, *RegSaveAreaOriginPtr;
5914 std::tie(RegSaveAreaShadowPtr, RegSaveAreaOriginPtr) =
5915 MSV.getShadowOriginPtr(RegSaveAreaPtr, IRB, IRB.
getInt8Ty(), Alignment,
5920 unsigned RegSaveAreaSize =
5921 IsSoftFloatABI ? SystemZGpEndOffset : SystemZRegSaveAreaSize;
5922 IRB.
CreateMemCpy(RegSaveAreaShadowPtr, Alignment, VAArgTLSCopy, Alignment,
5924 if (MS.TrackOrigins)
5925 IRB.
CreateMemCpy(RegSaveAreaOriginPtr, Alignment, VAArgTLSOriginCopy,
5926 Alignment, RegSaveAreaSize);
5932 Type *OverflowArgAreaPtrTy = PointerType::getUnqual(*MS.C);
5936 ConstantInt::get(MS.IntptrTy, SystemZOverflowArgAreaPtrOffset)),
5937 PointerType::get(OverflowArgAreaPtrTy, 0));
5938 Value *OverflowArgAreaPtr =
5939 IRB.
CreateLoad(OverflowArgAreaPtrTy, OverflowArgAreaPtrPtr);
5940 Value *OverflowArgAreaShadowPtr, *OverflowArgAreaOriginPtr;
5942 std::tie(OverflowArgAreaShadowPtr, OverflowArgAreaOriginPtr) =
5943 MSV.getShadowOriginPtr(OverflowArgAreaPtr, IRB, IRB.
getInt8Ty(),
5946 SystemZOverflowOffset);
5947 IRB.
CreateMemCpy(OverflowArgAreaShadowPtr, Alignment, SrcPtr, Alignment,
5949 if (MS.TrackOrigins) {
5951 SystemZOverflowOffset);
5952 IRB.
CreateMemCpy(OverflowArgAreaOriginPtr, Alignment, SrcPtr, Alignment,
5957 void finalizeInstrumentation()
override {
5958 assert(!VAArgOverflowSize && !VAArgTLSCopy &&
5959 "finalizeInstrumentation called twice");
5960 if (!VAStartInstrumentationList.empty()) {
5967 IRB.
CreateAdd(ConstantInt::get(MS.IntptrTy, SystemZOverflowOffset),
5975 Intrinsic::umin, CopySize,
5979 if (MS.TrackOrigins) {
5989 for (
CallInst *OrigInst : VAStartInstrumentationList) {
5990 NextNodeIRBuilder IRB(OrigInst);
5991 Value *VAListTag = OrigInst->getArgOperand(0);
5992 copyRegSaveArea(IRB, VAListTag);
5993 copyOverflowArea(IRB, VAListTag);
6000using VarArgLoongArch64Helper = VarArgMIPS64Helper;
6003struct VarArgNoOpHelper :
public VarArgHelper {
6004 VarArgNoOpHelper(
Function &
F, MemorySanitizer &MS,
6005 MemorySanitizerVisitor &MSV) {}
6013 void finalizeInstrumentation()
override {}
6019 MemorySanitizerVisitor &Visitor) {
6022 Triple TargetTriple(Func.getParent()->getTargetTriple());
6024 return new VarArgAMD64Helper(Func, Msan, Visitor);
6026 return new VarArgMIPS64Helper(Func, Msan, Visitor);
6028 return new VarArgAArch64Helper(Func, Msan, Visitor);
6031 return new VarArgPowerPC64Helper(Func, Msan, Visitor);
6033 return new VarArgSystemZHelper(Func, Msan, Visitor);
6035 return new VarArgLoongArch64Helper(Func, Msan, Visitor);
6037 return new VarArgNoOpHelper(Func, Msan, Visitor);
6044 if (
F.hasFnAttribute(Attribute::DisableSanitizerInstrumentation))
6047 MemorySanitizerVisitor Visitor(
F, *
this, TLI);
6051 B.addAttribute(Attribute::Memory).addAttribute(Attribute::Speculatable);
6054 return Visitor.runOnFunction();
This file implements a class to represent arbitrary precision integral constant values and operations...
static bool isStore(int Opcode)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
static const size_t kNumberOfAccessSizes
VarLocInsertPt getNextNode(const DbgRecord *DVR)
Atomic ordering constants.
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static AtomicOrdering addReleaseOrdering(AtomicOrdering AO)
static AtomicOrdering addAcquireOrdering(AtomicOrdering AO)
static bool isAMustTailRetVal(Value *RetVal)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
static bool runOnFunction(Function &F, bool PostInlining)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
This is the interface for a simple mod/ref and alias analysis over globals.
static size_t TypeSizeToSizeIndex(uint32_t TypeSize)
static const MemoryMapParams Linux_LoongArch64_MemoryMapParams
static const PlatformMemoryMapParams Linux_S390_MemoryMapParams
static const Align kMinOriginAlignment
static const MemoryMapParams Linux_X86_64_MemoryMapParams
static cl::opt< uint64_t > ClShadowBase("msan-shadow-base", cl::desc("Define custom MSan ShadowBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClDumpStrictInstructions("msan-dump-strict-instructions", cl::desc("print out instructions with default strict semantics"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClHandleICmpExact("msan-handle-icmp-exact", cl::desc("exact handling of relational integer ICmp"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams Linux_X86_MemoryMapParams
static cl::opt< uint64_t > ClOriginBase("msan-origin-base", cl::desc("Define custom MSan OriginBase"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClCheckConstantShadow("msan-check-constant-shadow", cl::desc("Insert checks for constant shadow values"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams Linux_LoongArch_MemoryMapParams
static const MemoryMapParams NetBSD_X86_64_MemoryMapParams
static const PlatformMemoryMapParams Linux_MIPS_MemoryMapParams
static const unsigned kOriginSize
static cl::opt< bool > ClWithComdat("msan-with-comdat", cl::desc("Place MSan constructors in comdat sections"), cl::Hidden, cl::init(false))
static cl::opt< int > ClTrackOrigins("msan-track-origins", cl::desc("Track origins (allocation sites) of poisoned memory"), cl::Hidden, cl::init(0))
Track origins of uninitialized values.
static cl::opt< int > ClInstrumentationWithCallThreshold("msan-instrumentation-with-call-threshold", cl::desc("If the function being instrumented requires more than " "this number of checks and origin stores, use callbacks instead of " "inline checks (-1 means never use callbacks)."), cl::Hidden, cl::init(3500))
static cl::opt< int > ClPoisonStackPattern("msan-poison-stack-pattern", cl::desc("poison uninitialized stack variables with the given pattern"), cl::Hidden, cl::init(0xff))
static const Align kShadowTLSAlignment
static const PlatformMemoryMapParams Linux_ARM_MemoryMapParams
static Constant * getOrInsertGlobal(Module &M, StringRef Name, Type *Ty)
static const MemoryMapParams Linux_S390X_MemoryMapParams
static cl::opt< bool > ClPoisonUndef("msan-poison-undef", cl::desc("poison undef temps"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClPoisonStack("msan-poison-stack", cl::desc("poison uninitialized stack variables"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_I386_MemoryMapParams
const char kMsanInitName[]
static cl::opt< bool > ClPrintStackNames("msan-print-stack-names", cl::desc("Print name of local stack variable"), cl::Hidden, cl::init(true))
static const MemoryMapParams Linux_AArch64_MemoryMapParams
static cl::opt< uint64_t > ClAndMask("msan-and-mask", cl::desc("Define custom MSan AndMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleLifetimeIntrinsics("msan-handle-lifetime-intrinsics", cl::desc("when possible, poison scoped variables at the beginning of the scope " "(slower, but more precise)"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClKeepGoing("msan-keep-going", cl::desc("keep going after reporting a UMR"), cl::Hidden, cl::init(false))
static const MemoryMapParams FreeBSD_X86_64_MemoryMapParams
static GlobalVariable * createPrivateConstGlobalForString(Module &M, StringRef Str)
Create a non-const global initialized with the given string.
static const PlatformMemoryMapParams Linux_PowerPC_MemoryMapParams
static const size_t kNumberOfAccessSizes
static cl::opt< bool > ClEagerChecks("msan-eager-checks", cl::desc("check arguments and return values at function call boundaries"), cl::Hidden, cl::init(false))
static cl::opt< int > ClDisambiguateWarning("msan-disambiguate-warning-threshold", cl::desc("Define threshold for number of checks per " "debug location to force origin update."), cl::Hidden, cl::init(3))
static VarArgHelper * CreateVarArgHelper(Function &Func, MemorySanitizer &Msan, MemorySanitizerVisitor &Visitor)
static const MemoryMapParams Linux_MIPS64_MemoryMapParams
static const MemoryMapParams Linux_PowerPC64_MemoryMapParams
static cl::opt< uint64_t > ClXorMask("msan-xor-mask", cl::desc("Define custom MSan XorMask"), cl::Hidden, cl::init(0))
static cl::opt< bool > ClHandleAsmConservative("msan-handle-asm-conservative", cl::desc("conservative handling of inline assembly"), cl::Hidden, cl::init(true))
static const PlatformMemoryMapParams FreeBSD_X86_MemoryMapParams
static const PlatformMemoryMapParams FreeBSD_ARM_MemoryMapParams
static const unsigned kParamTLSSize
static cl::opt< bool > ClHandleICmp("msan-handle-icmp", cl::desc("propagate shadow through ICmpEQ and ICmpNE"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClEnableKmsan("msan-kernel", cl::desc("Enable KernelMemorySanitizer instrumentation"), cl::Hidden, cl::init(false))
static cl::opt< bool > ClPoisonStackWithCall("msan-poison-stack-with-call", cl::desc("poison uninitialized stack variables with a call"), cl::Hidden, cl::init(false))
static const PlatformMemoryMapParams NetBSD_X86_MemoryMapParams
static const unsigned kRetvalTLSSize
static const MemoryMapParams FreeBSD_AArch64_MemoryMapParams
const char kMsanModuleCtorName[]
static const MemoryMapParams FreeBSD_I386_MemoryMapParams
static cl::opt< bool > ClCheckAccessAddress("msan-check-access-address", cl::desc("report accesses through a pointer which has poisoned shadow"), cl::Hidden, cl::init(true))
static cl::opt< bool > ClDisableChecks("msan-disable-checks", cl::desc("Apply no_sanitize to the whole file"), cl::Hidden, cl::init(false))
Module.h This file contains the declarations for the Module class.
FunctionAnalysisManager FAM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
Class for arbitrary precision integers.
an instruction to allocate memory on the stack
void setAlignment(Align Align)
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
An instruction that atomically checks whether a specified value is in a memory location,...
an instruction that atomically reads a memory location, combines it with another value,...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::iterator iterator
Instruction iterators...
This class represents a no-op cast from one type to another.
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
bool isInlineAsm() const
Check if this call is an inline asm statement.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
bool paramHasAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Determine whether the argument or parameter has the given attribute.
MaybeAlign getParamAlign(unsigned ArgNo) const
Extract the alignment for a call or parameter (0=unknown).
Type * getParamByValType(unsigned ArgNo) const
Extract the byval type for a call or parameter.
Value * getCalledOperand() const
Type * getParamElementType(unsigned ArgNo) const
Extract the elementtype type for a parameter.
Value * getArgOperand(unsigned i) const
void setArgOperand(unsigned i, Value *v)
FunctionType * getFunctionType() const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
This class represents a function call, abstracting a target machine's calling convention.
This is the base class for all instructions that perform data casts.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ ICMP_SGT
signed greater than
@ ICMP_SGE
signed greater or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< uint8_t > Elts)
get() constructors - Return a constant with vector type with an element count and element type matchi...
This is the shared class of boolean and integer constants.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static ConstantInt * getBool(LLVMContext &Context, bool V)
static Constant * get(StructType *T, ArrayRef< Constant * > V)
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
bool isAllOnesValue() const
Return true if this is the value that would be returned by getAllOnesValue.
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
bool isZeroValue() const
Return true if the value is negative zero or null value.
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
static bool shouldExecute(unsigned CounterName)
This instruction compares its operands according to the predicate given to the constructor.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
This class represents a freeze function that returns random concrete value if an operand is either a ...
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void setComdat(Comdat *C)
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ ExternalLinkage
Externally visible function.
Analysis pass providing a never-invalidated alias analysis result.
This instruction compares its operands according to the predicate given to the constructor.
CallInst * CreateMaskedCompressStore(Value *Val, Value *Ptr, Value *Mask=nullptr)
Create a call to Masked Compress Store intrinsic.
CallInst * CreateMaskedExpandLoad(Type *Ty, Value *Ptr, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Expand Load intrinsic.
Value * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
IntegerType * getInt1Ty()
Fetch the type representing a single bit.
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
CallInst * CreateAndReduce(Value *Src)
Create a vector int AND reduction intrinsic of the source vector.
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
CallInst * CreateMaskedLoad(Type *Ty, Value *Ptr, Align Alignment, Value *Mask, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Load intrinsic.
CallInst * CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, MaybeAlign Align, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memset to the specified pointer and the specified value.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateTypeSize(Type *DstType, TypeSize Size)
Create an expression which evaluates to the number of units in Size at runtime.
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
CallInst * CreateOrReduce(Value *Src)
Create a vector int OR reduction intrinsic of the source vector.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
CallInst * CreateMaskedStore(Value *Val, Value *Ptr, Align Alignment, Value *Mask)
Create a call to Masked Store intrinsic.
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
CallInst * CreateMaskedScatter(Value *Val, Value *Ptrs, Align Alignment, Value *Mask=nullptr)
Create a call to Masked Scatter intrinsic.
CallInst * CreateMaskedGather(Type *Ty, Value *Ptrs, Align Alignment, Value *Mask=nullptr, Value *PassThru=nullptr, const Twine &Name="")
Create a call to Masked Gather intrinsic.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
std::vector< ConstraintInfo > ConstraintInfoVector
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
This instruction inserts a single (scalar) element into a VectorType value.
This instruction inserts a struct field of array element value into an aggregate value.
Base class for instruction visitors.
void visit(Iterator Start, Iterator End)
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
This class represents a cast from an integer to a pointer.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
This is an important class for using LLVM in a threaded context.
The landingpad instruction holds all of the information necessary to generate correct exception handl...
An instruction for reading from memory.
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
This class wraps the llvm.memcpy intrinsic.
This class wraps the llvm.memmove intrinsic.
This class wraps the llvm.memset and llvm.memset.inline intrinsics.
A Module instance is used to store all the information related to an LLVM module.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void abandon()
Mark an analysis as abandoned.
This class represents a cast from a pointer to an integer.
Resume the propagation of an exception.
Return a value (possibly void), from a function.
This class represents a sign extension of integer types.
This class represents the LLVM 'select' instruction.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This instruction constructs a fixed permutation of two input vectors.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
StringRef - Represent a constant reference to a string, i.e.
Class to represent struct types.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
AttributeList getAttrList(LLVMContext *C, ArrayRef< unsigned > ArgNos, bool Signed, bool Ret=false, AttributeList AL=AttributeList()) const
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
Triple - Helper class for working with autoconf configuration names.
bool isMIPS64() const
Tests whether the target is MIPS 64-bit (little and big endian).
ArchType getArch() const
Get the parsed architecture type of this triple.
bool isLoongArch64() const
Tests whether the target is 64-bit LoongArch.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isPPC_FP128Ty() const
Return true if this is powerpc long double.
static Type * getX86_MMXTy(LLVMContext &C)
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
static IntegerType * getInt8Ty(LLVMContext &C)
bool isIntOrPtrTy() const
Return true if this is an integer type or a pointer type.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
'undef' values are things that do not have specified contents.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
This represents the llvm.va_copy intrinsic.
This represents the llvm.va_start intrinsic.
size_type count(const KeyT &Val) const
Return 1 if the specified key is in the map, 0 otherwise.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
Type * getElementType() const
This class represents zero extension of integer types.
int getNumOccurrences() const
constexpr ScalarTy getFixedValue() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
This class provides various memory handling functions that manipulate MemoryBlock instances.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Win64
The C convention as implemented on Windows/x86-64 and AArch64.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
initializer< Ty > init(const Ty &Val)
Function * Kernel
Summary of a kernel (=entry point for target offloading).
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
AllocaInst * findAllocaForValue(Value *V, bool OffsetZero=false)
Returns unique alloca where the value comes from, or nullptr.
std::pair< Function *, FunctionCallee > getOrCreateSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, function_ref< void(Function *, FunctionCallee)> FunctionsCreatedCallback, StringRef VersionCheckName=StringRef(), bool Weak=false)
Creates sanitizer constructor function lazily.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
bool isKnownNonZero(const Value *V, const SimplifyQuery &Q, unsigned Depth=0)
Return true if the given value is known to be non-zero when defined.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Or
Bitwise or logical OR of integers.
std::pair< Instruction *, Value * > SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore)
Insert a for (int i = 0; i < End; i++) loop structure (with the exception that End is assumed > 0,...
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
iterator_range< df_iterator< T > > depth_first(const T &G)
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, const TargetLibraryInfo *TLI)
Given a CallInst, check if it calls a string function known to CodeGen, and mark it with NoBuiltin if...
bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Remove all blocks that can not be reached from the function's entry.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
void printPipeline(raw_ostream &OS, function_ref< StringRef(StringRef)> MapClassName2PassName)
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
A CRTP mix-in to automatically provide informational APIs needed for passes.