36#define DEBUG_TYPE "vector-combine"
42STATISTIC(NumVecLoad,
"Number of vector loads formed");
43STATISTIC(NumVecCmp,
"Number of vector compares formed");
44STATISTIC(NumVecBO,
"Number of vector binops formed");
45STATISTIC(NumVecCmpBO,
"Number of vector compare + binop formed");
46STATISTIC(NumShufOfBitcast,
"Number of shuffles moved after bitcast");
47STATISTIC(NumScalarBO,
"Number of scalar binops formed");
48STATISTIC(NumScalarCmp,
"Number of scalar compares formed");
52 cl::desc(
"Disable all vector combine transforms"));
56 cl::desc(
"Disable binop extract to shuffle transforms"));
60 cl::desc(
"Max number of instructions to scan for vector combining."));
62static const unsigned InvalidIndex = std::numeric_limits<unsigned>::max();
70 :
F(
F), Builder(
F.getContext()),
TTI(
TTI), DT(DT), AA(AA), AC(AC),
DL(
DL),
71 TryEarlyFoldsOnly(TryEarlyFoldsOnly) {}
86 bool TryEarlyFoldsOnly;
97 unsigned PreferredExtractIndex)
const;
101 unsigned PreferredExtractIndex);
117 bool foldSelectShuffle(
Instruction &
I,
bool FromReduction =
false);
121 if (
auto *NewI = dyn_cast<Instruction>(&New)) {
142 if (!Load || !Load->isSimple() || !Load->hasOneUse() ||
143 Load->getFunction()->hasFnAttribute(Attribute::SanitizeMemTag) ||
149 Type *ScalarTy = Load->getType()->getScalarType();
152 if (!ScalarSize || !MinVectorSize || MinVectorSize % ScalarSize != 0 ||
159bool VectorCombine::vectorizeLoadInsert(
Instruction &
I) {
173 auto *
Load = dyn_cast<LoadInst>(
X);
185 Value *SrcPtr =
Load->getPointerOperand()->stripPointerCasts();
186 assert(isa<PointerType>(SrcPtr->
getType()) &&
"Expected a pointer type");
188 unsigned MinVecNumElts = MinVectorSize / ScalarSize;
189 auto *MinVecTy = VectorType::get(ScalarTy, MinVecNumElts,
false);
190 unsigned OffsetEltIndex = 0;
198 unsigned OffsetBitWidth =
DL->getIndexTypeSizeInBits(SrcPtr->
getType());
209 uint64_t ScalarSizeInBytes = ScalarSize / 8;
210 if (
Offset.urem(ScalarSizeInBytes) != 0)
214 OffsetEltIndex =
Offset.udiv(ScalarSizeInBytes).getZExtValue();
215 if (OffsetEltIndex >= MinVecNumElts)
232 unsigned AS =
Load->getPointerAddressSpace();
251 auto *Ty = cast<FixedVectorType>(
I.getType());
252 unsigned OutputNumElts = Ty->getNumElements();
254 assert(OffsetEltIndex < MinVecNumElts &&
"Address offset too big");
255 Mask[0] = OffsetEltIndex;
261 if (OldCost < NewCost || !NewCost.
isValid())
272 replaceValue(
I, *VecLd);
282 auto *Shuf = cast<ShuffleVectorInst>(&
I);
283 if (!Shuf->isIdentityWithPadding())
288 cast<FixedVectorType>(Shuf->getOperand(0)->getType())->getNumElements();
289 unsigned OpIndex =
any_of(Shuf->getShuffleMask(), [&NumOpElts](
int M) {
290 return M >= (int)(NumOpElts);
293 auto *
Load = dyn_cast<LoadInst>(Shuf->getOperand(
OpIndex));
300 auto *Ty = cast<FixedVectorType>(
I.getType());
301 Value *SrcPtr =
Load->getPointerOperand()->stripPointerCasts();
302 assert(isa<PointerType>(SrcPtr->
getType()) &&
"Expected a pointer type");
309 unsigned AS =
Load->getPointerAddressSpace();
324 if (OldCost < NewCost || !NewCost.
isValid())
331 replaceValue(
I, *VecLd);
343 assert(Index0C && Index1C &&
"Expected constant extract indexes");
345 unsigned Index0 = Index0C->getZExtValue();
346 unsigned Index1 = Index1C->getZExtValue();
349 if (Index0 == Index1)
374 if (PreferredExtractIndex == Index0)
376 if (PreferredExtractIndex == Index1)
380 return Index0 > Index1 ? Ext0 : Ext1;
392 unsigned PreferredExtractIndex) {
393 auto *Ext0IndexC = dyn_cast<ConstantInt>(Ext0->
getOperand(1));
394 auto *Ext1IndexC = dyn_cast<ConstantInt>(Ext1->
getOperand(1));
395 assert(Ext0IndexC && Ext1IndexC &&
"Expected constant extract indexes");
397 unsigned Opcode =
I.getOpcode();
408 assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
409 "Expected a compare");
419 unsigned Ext0Index = Ext0IndexC->getZExtValue();
420 unsigned Ext1Index = Ext1IndexC->getZExtValue();
435 InstructionCost CheapExtractCost = std::min(Extract0Cost, Extract1Cost);
445 bool HasUseTax = Ext0 == Ext1 ? !Ext0->
hasNUses(2)
447 OldCost = CheapExtractCost + ScalarOpCost;
448 NewCost = VectorOpCost + CheapExtractCost + HasUseTax * CheapExtractCost;
452 OldCost = Extract0Cost + Extract1Cost + ScalarOpCost;
453 NewCost = VectorOpCost + CheapExtractCost +
458 ConvertToShuffle = getShuffleExtract(Ext0, Ext1, PreferredExtractIndex);
459 if (ConvertToShuffle) {
477 return OldCost < NewCost;
487 auto *VecTy = cast<FixedVectorType>(Vec->
getType());
489 ShufMask[NewIndex] = OldIndex;
508 assert(isa<ConstantInt>(
C) &&
"Expected a constant index operand");
509 if (isa<Constant>(
X))
522 assert(isa<CmpInst>(&
I) &&
"Expected a compare");
525 "Expected matching constant extract indexes");
533 replaceValue(
I, *NewExt);
541 assert(isa<BinaryOperator>(&
I) &&
"Expected a binary operator");
544 "Expected matching constant extract indexes");
554 if (
auto *VecBOInst = dyn_cast<Instruction>(VecBO))
555 VecBOInst->copyIRFlags(&
I);
558 replaceValue(
I, *NewExt);
586 auto *Ext0 = cast<ExtractElementInst>(I0);
587 auto *Ext1 = cast<ExtractElementInst>(I1);
594 if (isExtractExtractCheap(Ext0, Ext1,
I, ExtractToChange, InsertIndex))
597 if (ExtractToChange) {
598 unsigned CheapExtractIdx = ExtractToChange == Ext0 ? C1 : C0;
603 if (ExtractToChange == Ext0)
610 foldExtExtCmp(Ext0, Ext1,
I);
612 foldExtExtBinop(Ext0, Ext1,
I);
639 auto *VecTy = cast<FixedVectorType>(
I.getType());
640 if (SrcVec->
getType() != VecTy)
644 unsigned NumElts = VecTy->getNumElements();
645 if (
Index >= NumElts)
652 std::iota(
Mask.begin(),
Mask.end(), 0);
671 if (NewCost > OldCost)
678 replaceValue(
I, *Shuf);
697 auto *DestTy = dyn_cast<FixedVectorType>(
I.getType());
698 auto *SrcTy = dyn_cast<FixedVectorType>(V0->
getType());
699 if (!DestTy || !SrcTy)
702 unsigned DestEltSize = DestTy->getScalarSizeInBits();
703 unsigned SrcEltSize = SrcTy->getScalarSizeInBits();
704 if (SrcTy->getPrimitiveSizeInBits() % DestEltSize != 0)
708 if (DestEltSize <= SrcEltSize) {
711 assert(SrcEltSize % DestEltSize == 0 &&
"Unexpected shuffle mask");
712 unsigned ScaleFactor = SrcEltSize / DestEltSize;
717 assert(DestEltSize % SrcEltSize == 0 &&
"Unexpected shuffle mask");
718 unsigned ScaleFactor = DestEltSize / SrcEltSize;
725 unsigned NumSrcElts = SrcTy->getPrimitiveSizeInBits() / DestEltSize;
730 bool IsUnary = isa<UndefValue>(V1);
731 unsigned NumOps = IsUnary ? 1 : 2;
743 TargetTransformInfo::CastContextHint::None,
748 TargetTransformInfo::CastContextHint::None, CK);
749 if (DestCost > SrcCost || !DestCost.
isValid())
757 replaceValue(
I, *Shuf);
764bool VectorCombine::scalarizeVPIntrinsic(
Instruction &
I) {
765 if (!isa<VPIntrinsic>(
I))
778 if (!ScalarOp0 || !ScalarOp1)
786 auto IsAllTrueMask = [](
Value *MaskVal) {
788 if (
auto *ConstValue = dyn_cast<Constant>(SplattedVal))
789 return ConstValue->isAllOnesValue();
805 if (
auto *FVTy = dyn_cast<FixedVectorType>(VecTy))
806 Mask.resize(FVTy->getNumElements(), 0);
814 Args.push_back(
V->getType());
820 std::optional<unsigned> FunctionalOpcode =
822 std::optional<Intrinsic::ID> ScalarIntrID = std::nullopt;
823 if (!FunctionalOpcode) {
847 <<
", Cost of scalarizing:" << NewCost <<
"\n");
850 if (OldCost < NewCost || !NewCost.
isValid())
861 bool SafeToSpeculate;
864 .
hasFnAttr(Attribute::AttrKind::Speculatable);
867 *FunctionalOpcode, &VPI,
nullptr, &AC, &DT);
874 {ScalarOp0, ScalarOp1})
876 ScalarOp0, ScalarOp1);
884bool VectorCombine::scalarizeBinopOrCmp(
Instruction &
I) {
895 bool IsCmp = Pred != CmpInst::Predicate::BAD_ICMP_PREDICATE;
897 for (
User *U :
I.users())
907 Constant *VecC0 =
nullptr, *VecC1 =
nullptr;
908 Value *V0 =
nullptr, *V1 =
nullptr;
921 if (IsConst0 && IsConst1)
923 if (!IsConst0 && !IsConst1 && Index0 != Index1)
928 auto *I0 = dyn_cast_or_null<Instruction>(V0);
929 auto *
I1 = dyn_cast_or_null<Instruction>(V1);
930 if ((IsConst0 && I1 &&
I1->mayReadFromMemory()) ||
936 Type *VecTy =
I.getType();
941 "Unexpected types for insert element into binop or cmp");
943 unsigned Opcode =
I.getOpcode();
962 (IsConst0 ? 0 : InsertCost) + (IsConst1 ? 0 : InsertCost) + VectorOpCost;
964 (IsConst0 ? 0 : !Ins0->
hasOneUse() * InsertCost) +
965 (IsConst1 ? 0 : !Ins1->
hasOneUse() * InsertCost);
968 if (OldCost < NewCost || !NewCost.
isValid())
988 Scalar->setName(
I.getName() +
".scalar");
992 if (
auto *ScalarInst = dyn_cast<Instruction>(Scalar))
993 ScalarInst->copyIRFlags(&
I);
997 IsCmp ? Builder.
CreateCmp(Pred, VecC0, VecC1)
1000 replaceValue(
I, *Insert);
1010 if (!
I.isBinaryOp() || !
I.getType()->isIntegerTy(1))
1016 Value *B0 =
I.getOperand(0), *B1 =
I.getOperand(1);
1034 auto *Ext0 = cast<ExtractElementInst>(I0);
1035 auto *Ext1 = cast<ExtractElementInst>(I1);
1044 : Instruction::ICmp;
1045 auto *VecTy = dyn_cast<FixedVectorType>(
X->getType());
1062 int CheapIndex = ConvertToShuf == Ext0 ? Index1 : Index0;
1063 int ExpensiveIndex = ConvertToShuf == Ext0 ? Index0 : Index1;
1068 ShufMask[CheapIndex] = ExpensiveIndex;
1077 if (OldCost < NewCost || !NewCost.
isValid())
1091 replaceValue(
I, *NewExt);
1100 unsigned NumScanned = 0;
1110class ScalarizationResult {
1111 enum class StatusTy { Unsafe, Safe, SafeWithFreeze };
1116 ScalarizationResult(StatusTy
Status,
Value *ToFreeze =
nullptr)
1120 ScalarizationResult(
const ScalarizationResult &
Other) =
default;
1121 ~ScalarizationResult() {
1122 assert(!ToFreeze &&
"freeze() not called with ToFreeze being set");
1125 static ScalarizationResult unsafe() {
return {StatusTy::Unsafe}; }
1126 static ScalarizationResult safe() {
return {StatusTy::Safe}; }
1127 static ScalarizationResult safeWithFreeze(
Value *ToFreeze) {
1128 return {StatusTy::SafeWithFreeze, ToFreeze};
1132 bool isSafe()
const {
return Status == StatusTy::Safe; }
1134 bool isUnsafe()
const {
return Status == StatusTy::Unsafe; }
1137 bool isSafeWithFreeze()
const {
return Status == StatusTy::SafeWithFreeze; }
1142 Status = StatusTy::Unsafe;
1147 assert(isSafeWithFreeze() &&
1148 "should only be used when freezing is required");
1150 "UserI must be a user of ToFreeze");
1156 if (
U.get() == ToFreeze)
1173 uint64_t NumElements = VecTy->getElementCount().getKnownMinValue();
1175 if (
auto *
C = dyn_cast<ConstantInt>(
Idx)) {
1176 if (
C->getValue().ult(NumElements))
1177 return ScalarizationResult::safe();
1178 return ScalarizationResult::unsafe();
1181 unsigned IntWidth =
Idx->getType()->getScalarSizeInBits();
1182 APInt Zero(IntWidth, 0);
1183 APInt MaxElts(IntWidth, NumElements);
1189 true, &AC, CtxI, &DT)))
1190 return ScalarizationResult::safe();
1191 return ScalarizationResult::unsafe();
1204 if (ValidIndices.
contains(IdxRange))
1205 return ScalarizationResult::safeWithFreeze(IdxBase);
1206 return ScalarizationResult::unsafe();
1216 if (
auto *
C = dyn_cast<ConstantInt>(
Idx))
1218 C->getZExtValue() *
DL.getTypeStoreSize(ScalarType));
1230bool VectorCombine::foldSingleElementStore(
Instruction &
I) {
1231 auto *
SI = cast<StoreInst>(&
I);
1232 if (!
SI->isSimple() || !isa<VectorType>(
SI->getValueOperand()->getType()))
1240 if (!
match(
SI->getValueOperand(),
1245 if (
auto *Load = dyn_cast<LoadInst>(Source)) {
1246 auto VecTy = cast<VectorType>(
SI->getValueOperand()->getType());
1247 Value *SrcAddr =
Load->getPointerOperand()->stripPointerCasts();
1250 if (!
Load->isSimple() ||
Load->getParent() !=
SI->getParent() ||
1251 !
DL->typeSizeEqualsStoreSize(
Load->getType()->getScalarType()) ||
1252 SrcAddr !=
SI->getPointerOperand()->stripPointerCasts())
1256 if (ScalarizableIdx.isUnsafe() ||
1261 if (ScalarizableIdx.isSafeWithFreeze())
1262 ScalarizableIdx.freeze(Builder, *cast<Instruction>(
Idx));
1264 SI->getValueOperand()->getType(),
SI->getPointerOperand(),
1265 {ConstantInt::get(Idx->getType(), 0), Idx});
1272 replaceValue(
I, *NSI);
1281bool VectorCombine::scalarizeLoadExtract(
Instruction &
I) {
1286 auto *VecTy = cast<VectorType>(
I.getType());
1287 auto *LI = cast<LoadInst>(&
I);
1288 if (LI->isVolatile() || !
DL->typeSizeEqualsStoreSize(VecTy->
getScalarType()))
1293 LI->getPointerAddressSpace());
1297 unsigned NumInstChecked = 0;
1301 for (
auto &Pair : NeedFreeze)
1302 Pair.second.discard();
1309 auto *UI = dyn_cast<ExtractElementInst>(U);
1310 if (!UI || UI->getParent() != LI->getParent())
1317 make_range(std::next(LI->getIterator()), UI->getIterator())) {
1324 LastCheckedInst = UI;
1328 if (ScalarIdx.isUnsafe())
1330 if (ScalarIdx.isSafeWithFreeze()) {
1332 ScalarIdx.discard();
1335 auto *
Index = dyn_cast<ConstantInt>(UI->getOperand(1));
1342 Align(1), LI->getPointerAddressSpace());
1346 if (ScalarizedCost >= OriginalCost)
1351 auto *EI = cast<ExtractElementInst>(U);
1355 auto It = NeedFreeze.
find(EI);
1356 if (It != NeedFreeze.
end())
1357 It->second.freeze(Builder, *cast<Instruction>(
Idx));
1362 auto *NewLoad = cast<LoadInst>(Builder.
CreateLoad(
1363 VecTy->getElementType(),
GEP, EI->getName() +
".scalar"));
1366 LI->getAlign(), VecTy->getElementType(),
Idx, *
DL);
1367 NewLoad->setAlignment(ScalarOpAlignment);
1369 replaceValue(*EI, *NewLoad);
1372 FailureGuard.release();
1378bool VectorCombine::foldShuffleOfBinops(
Instruction &
I) {
1379 auto *VecTy = cast<FixedVectorType>(
I.getType());
1395 if (ShufCost > BinopCost)
1401 if (BinaryOperator::isCommutative(Opcode) &&
X != Z &&
Y != W)
1404 Value *Shuf0, *Shuf1;
1409 }
else if (
Y == W) {
1419 if (
auto *NewInst = dyn_cast<Instruction>(NewBO)) {
1420 NewInst->copyIRFlags(B0);
1421 NewInst->andIRFlags(B1);
1423 replaceValue(
I, *NewBO);
1430bool VectorCombine::foldShuffleFromReductions(
Instruction &
I) {
1431 auto *II = dyn_cast<IntrinsicInst>(&
I);
1434 switch (II->getIntrinsicID()) {
1435 case Intrinsic::vector_reduce_add:
1436 case Intrinsic::vector_reduce_mul:
1437 case Intrinsic::vector_reduce_and:
1438 case Intrinsic::vector_reduce_or:
1439 case Intrinsic::vector_reduce_xor:
1440 case Intrinsic::vector_reduce_smin:
1441 case Intrinsic::vector_reduce_smax:
1442 case Intrinsic::vector_reduce_umin:
1443 case Intrinsic::vector_reduce_umax:
1452 std::queue<Value *> Worklist;
1455 if (
auto *
Op = dyn_cast<Instruction>(
I.getOperand(0)))
1458 while (!Worklist.empty()) {
1459 Value *CV = Worklist.front();
1470 if (
auto *CI = dyn_cast<Instruction>(CV)) {
1471 if (CI->isBinaryOp()) {
1472 for (
auto *
Op : CI->operand_values())
1475 }
else if (
auto *SV = dyn_cast<ShuffleVectorInst>(CI)) {
1476 if (Shuffle && Shuffle != SV)
1493 for (
auto *V : Visited)
1494 for (
auto *U :
V->users())
1495 if (!Visited.contains(U) && U != &
I)
1499 dyn_cast<FixedVectorType>(II->getOperand(0)->getType());
1504 if (!ShuffleInputType)
1512 sort(ConcatMask, [](
int X,
int Y) {
return (
unsigned)
X < (
unsigned)
Y; });
1516 bool IsTruncatingShuffle =
VecType->getNumElements() < NumInputElts;
1517 bool UsesSecondVec =
1518 any_of(ConcatMask, [&](
int M) {
return M >= (int)NumInputElts; });
1521 (UsesSecondVec && !IsTruncatingShuffle) ? VecType : ShuffleInputType;
1527 VecTyForCost, ConcatMask);
1529 LLVM_DEBUG(
dbgs() <<
"Found a reduction feeding from a shuffle: " << *Shuffle
1531 LLVM_DEBUG(
dbgs() <<
" OldCost: " << OldCost <<
" vs NewCost: " << NewCost
1533 if (NewCost < OldCost) {
1537 LLVM_DEBUG(
dbgs() <<
"Created new shuffle: " << *NewShuffle <<
"\n");
1538 replaceValue(*Shuffle, *NewShuffle);
1543 return foldSelectShuffle(*Shuffle,
true);
1548bool VectorCombine::foldTruncFromReductions(
Instruction &
I) {
1549 auto *II = dyn_cast<IntrinsicInst>(&
I);
1555 case Intrinsic::vector_reduce_add:
1556 case Intrinsic::vector_reduce_mul:
1557 case Intrinsic::vector_reduce_and:
1558 case Intrinsic::vector_reduce_or:
1559 case Intrinsic::vector_reduce_xor:
1566 Value *ReductionSrc =
I.getOperand(0);
1572 auto *Trunc = cast<CastInst>(ReductionSrc);
1573 auto *TruncSrcTy = cast<VectorType>(TruncSrc->
getType());
1574 auto *ReductionSrcTy = cast<VectorType>(ReductionSrc->
getType());
1575 Type *ResultTy =
I.getType();
1587 ReductionSrcTy->getScalarType(),
1590 if (OldCost <= NewCost || !NewCost.
isValid())
1594 TruncSrcTy->getScalarType(), II->getIntrinsicID(), {TruncSrc});
1596 replaceValue(
I, *NewTruncation);
1610bool VectorCombine::foldSelectShuffle(
Instruction &
I,
bool FromReduction) {
1611 auto *SVI = cast<ShuffleVectorInst>(&
I);
1612 auto *VT = cast<FixedVectorType>(
I.getType());
1613 auto *Op0 = dyn_cast<Instruction>(SVI->getOperand(0));
1614 auto *Op1 = dyn_cast<Instruction>(SVI->getOperand(1));
1615 if (!Op0 || !Op1 || Op0 == Op1 || !Op0->isBinaryOp() || !Op1->isBinaryOp() ||
1619 auto *SVI0A = dyn_cast<Instruction>(Op0->getOperand(0));
1620 auto *SVI0B = dyn_cast<Instruction>(Op0->getOperand(1));
1621 auto *SVI1A = dyn_cast<Instruction>(Op1->getOperand(0));
1622 auto *SVI1B = dyn_cast<Instruction>(Op1->getOperand(1));
1625 if (!
I ||
I->getOperand(0)->getType() != VT)
1628 return U != Op0 && U != Op1 &&
1629 !(isa<ShuffleVectorInst>(U) &&
1630 (InputShuffles.contains(cast<Instruction>(U)) ||
1631 isInstructionTriviallyDead(cast<Instruction>(U))));
1634 if (checkSVNonOpUses(SVI0A) || checkSVNonOpUses(SVI0B) ||
1635 checkSVNonOpUses(SVI1A) || checkSVNonOpUses(SVI1B))
1643 for (
auto *U :
I->users()) {
1644 auto *SV = dyn_cast<ShuffleVectorInst>(U);
1645 if (!SV || SV->getType() != VT)
1647 if ((SV->getOperand(0) != Op0 && SV->getOperand(0) != Op1) ||
1648 (SV->getOperand(1) != Op0 && SV->getOperand(1) != Op1))
1655 if (!collectShuffles(Op0) || !collectShuffles(Op1))
1659 if (FromReduction && Shuffles.
size() > 1)
1664 if (!FromReduction) {
1666 for (
auto *U : SV->users()) {
1669 Shuffles.push_back(SSV);
1681 int MaxV1Elt = 0, MaxV2Elt = 0;
1682 unsigned NumElts = VT->getNumElements();
1685 SVN->getShuffleMask(Mask);
1689 Value *SVOp0 = SVN->getOperand(0);
1690 Value *SVOp1 = SVN->getOperand(1);
1691 if (isa<UndefValue>(SVOp1)) {
1692 auto *SSV = cast<ShuffleVectorInst>(SVOp0);
1695 for (
unsigned I = 0, E =
Mask.size();
I != E;
I++) {
1701 if (SVOp0 == Op1 && SVOp1 == Op0) {
1705 if (SVOp0 != Op0 || SVOp1 != Op1)
1712 for (
unsigned I = 0;
I <
Mask.size();
I++) {
1715 }
else if (Mask[
I] <
static_cast<int>(NumElts)) {
1716 MaxV1Elt = std::max(MaxV1Elt, Mask[
I]);
1717 auto It =
find_if(V1, [&](
const std::pair<int, int> &
A) {
1718 return Mask[
I] ==
A.first;
1727 MaxV2Elt = std::max<int>(MaxV2Elt, Mask[
I] - NumElts);
1728 auto It =
find_if(V2, [&](
const std::pair<int, int> &
A) {
1729 return Mask[
I] -
static_cast<int>(NumElts) ==
A.first;
1732 ReconstructMask.
push_back(NumElts + It -
V2.begin());
1735 V2.emplace_back(Mask[
I] - NumElts, NumElts +
V2.size());
1743 sort(ReconstructMask);
1744 OrigReconstructMasks.
push_back(std::move(ReconstructMask));
1751 if (V1.
empty() ||
V2.empty() ||
1752 (MaxV1Elt ==
static_cast<int>(V1.
size()) - 1 &&
1753 MaxV2Elt ==
static_cast<int>(
V2.size()) - 1))
1760 auto *SV = dyn_cast<ShuffleVectorInst>(
I);
1763 if (isa<UndefValue>(SV->getOperand(1)))
1764 if (
auto *SSV = dyn_cast<ShuffleVectorInst>(SV->getOperand(0)))
1765 if (InputShuffles.contains(SSV))
1767 return SV->getMaskValue(M);
1775 std::pair<int, int>
Y) {
1776 int MXA = GetBaseMaskValue(
A,
X.first);
1777 int MYA = GetBaseMaskValue(
A,
Y.first);
1780 stable_sort(V1, [&](std::pair<int, int>
A, std::pair<int, int>
B) {
1781 return SortBase(SVI0A,
A,
B);
1783 stable_sort(V2, [&](std::pair<int, int>
A, std::pair<int, int>
B) {
1784 return SortBase(SVI1A,
A,
B);
1789 for (
const auto &Mask : OrigReconstructMasks) {
1791 for (
int M : Mask) {
1793 auto It =
find_if(V, [M](
auto A) {
return A.second ==
M; });
1794 assert(It !=
V.end() &&
"Expected all entries in Mask");
1795 return std::distance(
V.begin(), It);
1799 else if (M <
static_cast<int>(NumElts)) {
1800 ReconstructMask.
push_back(FindIndex(V1, M));
1802 ReconstructMask.
push_back(NumElts + FindIndex(V2, M));
1805 ReconstructMasks.push_back(std::move(ReconstructMask));
1811 for (
unsigned I = 0;
I < V1.
size();
I++) {
1812 V1A.
push_back(GetBaseMaskValue(SVI0A, V1[
I].first));
1813 V1B.
push_back(GetBaseMaskValue(SVI0B, V1[
I].first));
1815 for (
unsigned I = 0;
I <
V2.size();
I++) {
1816 V2A.
push_back(GetBaseMaskValue(SVI1A, V2[
I].first));
1817 V2B.
push_back(GetBaseMaskValue(SVI1B, V2[
I].first));
1819 while (V1A.
size() < NumElts) {
1823 while (V2A.
size() < NumElts) {
1829 auto *SV = dyn_cast<ShuffleVectorInst>(
I);
1835 VT, SV->getShuffleMask());
1846 CostBefore += std::accumulate(Shuffles.begin(), Shuffles.end(),
1848 CostBefore += std::accumulate(InputShuffles.begin(), InputShuffles.end(),
1860 CostAfter += std::accumulate(ReconstructMasks.begin(), ReconstructMasks.end(),
1862 std::set<SmallVector<int>> OutputShuffleMasks({V1A, V1B, V2A, V2B});
1864 std::accumulate(OutputShuffleMasks.begin(), OutputShuffleMasks.end(),
1867 LLVM_DEBUG(
dbgs() <<
"Found a binop select shuffle pattern: " <<
I <<
"\n");
1869 <<
" vs CostAfter: " << CostAfter <<
"\n");
1870 if (CostBefore <= CostAfter)
1875 auto *SV = dyn_cast<ShuffleVectorInst>(
I);
1878 if (isa<UndefValue>(SV->getOperand(1)))
1879 if (
auto *SSV = dyn_cast<ShuffleVectorInst>(SV->getOperand(0)))
1880 if (InputShuffles.contains(SSV))
1882 return SV->getOperand(
Op);
1886 GetShuffleOperand(SVI0A, 1), V1A);
1889 GetShuffleOperand(SVI0B, 1), V1B);
1892 GetShuffleOperand(SVI1A, 1), V2A);
1895 GetShuffleOperand(SVI1B, 1), V2B);
1899 if (
auto *
I = dyn_cast<Instruction>(NOp0))
1900 I->copyIRFlags(Op0,
true);
1904 if (
auto *
I = dyn_cast<Instruction>(NOp1))
1905 I->copyIRFlags(Op1,
true);
1907 for (
int S = 0, E = ReconstructMasks.size(); S != E; S++) {
1910 replaceValue(*Shuffles[S], *NSV);
1913 Worklist.pushValue(NSV0A);
1914 Worklist.pushValue(NSV0B);
1915 Worklist.pushValue(NSV1A);
1916 Worklist.pushValue(NSV1B);
1917 for (
auto *S : Shuffles)
1924bool VectorCombine::run() {
1932 bool MadeChange =
false;
1935 bool IsFixedVectorType = isa<FixedVectorType>(
I.getType());
1936 auto Opcode =
I.getOpcode();
1942 if (IsFixedVectorType) {
1944 case Instruction::InsertElement:
1945 MadeChange |= vectorizeLoadInsert(
I);
1947 case Instruction::ShuffleVector:
1948 MadeChange |= widenSubvectorLoad(
I);
1957 if (isa<VectorType>(
I.getType())) {
1958 MadeChange |= scalarizeBinopOrCmp(
I);
1959 MadeChange |= scalarizeLoadExtract(
I);
1960 MadeChange |= scalarizeVPIntrinsic(
I);
1963 if (Opcode == Instruction::Store)
1964 MadeChange |= foldSingleElementStore(
I);
1967 if (TryEarlyFoldsOnly)
1974 if (IsFixedVectorType) {
1976 case Instruction::InsertElement:
1977 MadeChange |= foldInsExtFNeg(
I);
1979 case Instruction::ShuffleVector:
1980 MadeChange |= foldShuffleOfBinops(
I);
1981 MadeChange |= foldSelectShuffle(
I);
1983 case Instruction::BitCast:
1984 MadeChange |= foldBitcastShuffle(
I);
1989 case Instruction::Call:
1990 MadeChange |= foldShuffleFromReductions(
I);
1991 MadeChange |= foldTruncFromReductions(
I);
1993 case Instruction::ICmp:
1994 case Instruction::FCmp:
1995 MadeChange |= foldExtractExtract(
I);
1999 MadeChange |= foldExtractExtract(
I);
2000 MadeChange |= foldExtractedCmps(
I);
2013 if (
I.isDebugOrPseudoInst())
2019 while (!Worklist.isEmpty()) {
2042 VectorCombine
Combiner(
F,
TTI, DT, AA, AC,
DL, TryEarlyFoldsOnly);
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This is the interface for LLVM's primary stateless and local alias analysis.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static cl::opt< TargetTransformInfo::TargetCostKind > CostKind("cost-kind", cl::desc("Target cost kind"), cl::init(TargetTransformInfo::TCK_RecipThroughput), cl::values(clEnumValN(TargetTransformInfo::TCK_RecipThroughput, "throughput", "Reciprocal throughput"), clEnumValN(TargetTransformInfo::TCK_Latency, "latency", "Instruction latency"), clEnumValN(TargetTransformInfo::TCK_CodeSize, "code-size", "Code size"), clEnumValN(TargetTransformInfo::TCK_SizeAndLatency, "size-latency", "Code size and latency")))
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
std::optional< std::vector< StOtherPiece > > Other
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This is the interface for a simple mod/ref and alias analysis over globals.
static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo, MemorySSAUpdater &MSSAU)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static Value * createShiftShuffle(Value *Vec, unsigned OldIndex, unsigned NewIndex, IRBuilder<> &Builder)
Create a shuffle that translates (shifts) 1 element from the input vector to a new element location.
static Align computeAlignmentAfterScalarization(Align VectorAlignment, Type *ScalarType, Value *Idx, const DataLayout &DL)
The memory operation on a vector of ScalarType had alignment of VectorAlignment.
static ScalarizationResult canScalarizeAccess(VectorType *VecTy, Value *Idx, Instruction *CtxI, AssumptionCache &AC, const DominatorTree &DT)
Check if it is legal to scalarize a memory access to VecTy at index Idx.
static cl::opt< bool > DisableVectorCombine("disable-vector-combine", cl::init(false), cl::Hidden, cl::desc("Disable all vector combine transforms"))
static bool canWidenLoad(LoadInst *Load, const TargetTransformInfo &TTI)
static const unsigned InvalidIndex
static cl::opt< unsigned > MaxInstrsToScan("vector-combine-max-scan-instrs", cl::init(30), cl::Hidden, cl::desc("Max number of instructions to scan for vector combining."))
static cl::opt< bool > DisableBinopExtractShuffle("disable-binop-extract-shuffle", cl::init(false), cl::Hidden, cl::desc("Disable binop extract to shuffle transforms"))
static bool isMemModifiedBetween(BasicBlock::iterator Begin, BasicBlock::iterator End, const MemoryLocation &Loc, AAResults &AA)
static ExtractElementInst * translateExtract(ExtractElementInst *ExtElt, unsigned NewIndex, IRBuilder<> &Builder)
Given an extract element instruction with constant index operand, shuffle the source vector (shift th...
A manager for alias analyses.
ModRefInfo getModRefInfo(const Instruction *I, const std::optional< MemoryLocation > &OptLoc)
Check whether or not an instruction may read or write the optionally specified memory location.
Class for arbitrary precision integers.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
A container for analyses that lazily runs them and caches their results.
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
A function analysis which provides an AssumptionCache.
A cache of @llvm.assume calls within a function.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
LLVM Basic Block Representation.
InstListType::iterator iterator
Instruction iterators...
BinaryOps getOpcode() const
Represents analyses that only rely on functions' control flow.
Value * getArgOperand(unsigned i) const
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
bool isFPPredicate() const
static Constant * getExtractElement(Constant *Vec, Constant *Idx, Type *OnlyIfReducedTy=nullptr)
This is the shared class of boolean and integer constants.
const APInt & getValue() const
Return the constant as an APInt value reference.
This class represents a range of values.
ConstantRange urem(const ConstantRange &Other) const
Return a new range representing the possible values resulting from an unsigned remainder operation of...
ConstantRange binaryAnd(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a binary-and of a value in this ra...
bool contains(const APInt &Val) const
Return true if the specified value is in the set.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Analysis pass which computes a DominatorTree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Class to represent fixed width SIMD vectors.
unsigned getNumElements() const
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateFNegFMF(Value *V, Instruction *FMFSource, const Twine &Name="")
Copy fast-math-flags from an instruction rather than using the builder's default FMF.
Value * CreateFreeze(Value *V, const Twine &Name="")
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Value * CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
InstructionWorklist - This is the worklist management logic for InstCombine and other simplification ...
void pushUsersToWorkList(Instruction &I)
When an instruction is simplified, add all users of the instruction to the work lists because they mi...
void push(Instruction *I)
Push the instruction onto the worklist stack.
void remove(Instruction *I)
Remove I from the worklist if it exists.
bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
bool mayReadFromMemory() const LLVM_READONLY
Return true if this instruction may read memory.
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
An instruction for reading from memory.
Representation for a specific memory location.
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
void preserveSet()
Mark an analysis set as preserved.
This instruction constructs a fixed permutation of two input vectors.
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static void commuteShuffleMask(MutableArrayRef< int > Mask, unsigned InVecNumElts)
Change values in a shuffle permute mask assuming the two vector operands of length InVecNumElts have ...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
void setAlignment(Align Align)
Analysis pass providing the TargetTransformInfo.
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
static bool isVPBinOp(Intrinsic::ID ID)
This is the common base class for vector predication intrinsics.
std::optional< unsigned > getFunctionalIntrinsicID() const
std::optional< unsigned > getFunctionalOpcode() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const Value * stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) const
This is a wrapper around stripAndAccumulateConstantOffsets with the in-bounds requirement set to fals...
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
bool hasNUses(unsigned N) const
Return true if this Value has exactly N uses.
StringRef getName() const
Return a constant reference to the value's name.
PreservedAnalyses run(Function &F, FunctionAnalysisManager &)
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr char Attrs[]
Key for Kernel::Metadata::mAttrs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
AttributeList getAttributes(LLVMContext &C, ID id)
Return the attributes for an intrinsic.
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
BinaryOp_match< LHS, RHS, Instruction::URem > m_URem(const LHS &L, const RHS &R)
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
CastOperator_match< OpTy, Instruction::Trunc > m_Trunc(const OpTy &Op)
Matches Trunc.
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
OneUse_match< T > m_OneUse(const T &SubPattern)
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
initializer< Ty > init(const Ty &Val)
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
bool isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to be non-zero when defined.
void stable_sort(R &&Range)
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
unsigned getArithmeticReductionInstruction(Intrinsic::ID RdxID)
Returns the arithmetic instruction opcode used when expanding a reduction.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool mustSuppressSpeculation(const LoadInst &LI)
Return true if speculation of the given load must be suppressed to avoid ordering or interfering with...
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
ConstantRange computeConstantRange(const Value *V, bool ForSigned, bool UseInstrInfo=true, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Determine the possible constant range of an integer or vector of integer value.
bool isSafeToSpeculativelyExecuteWithOpcode(unsigned Opcode, const Instruction *Inst, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
This returns the same result as isSafeToSpeculativelyExecute if Opcode is the actual opcode of Inst.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool isInstructionTriviallyDead(Instruction *I, const TargetLibraryInfo *TLI=nullptr)
Return true if the result produced by the instruction is not used, and the instruction will return.
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
bool isModSet(const ModRefInfo MRI)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr int PoisonMaskElem
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
DWARFExpression::Operation Op
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isSafeToLoadUnconditionally(Value *V, Align Alignment, APInt &Size, const DataLayout &DL, Instruction *ScanFrom=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if we know that executing a load from this value cannot trap.
bool isGuaranteedNotToBePoison(const Value *V, AssumptionCache *AC=nullptr, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, unsigned Depth=0)
Returns true if V cannot be poison, but may be undef.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.