Go to the documentation of this file.
44 #define DEBUG_TYPE "instcombine"
47 using namespace PatternMatch;
49 STATISTIC(NumAggregateReconstructionsSimplified,
50 "Number of aggregate reconstructions turned into reuse of the "
51 "original aggregate");
62 if (
auto *
C = dyn_cast<Constant>(V))
65 if (CEI &&
match(V, m_Intrinsic<Intrinsic::experimental_stepvector>())) {
108 for (
auto U : PN->
users()) {
111 Extracts.push_back(EU);
114 }
else if (!PHIUser) {
115 PHIUser = cast<Instruction>(U);
128 !(isa<BinaryOperator>(PHIUser)) ||
134 PHINode *scalarPHI = cast<PHINode>(InsertNewInstWith(
142 if (PHIInVal == PHIUser) {
147 unsigned opId = (B0->
getOperand(0) == PN) ? 1 : 0;
152 Value *newPHIUser = InsertNewInstWith(
154 scalarPHI,
Op, B0), *B0);
160 Instruction *pos = dyn_cast<Instruction>(PHIInVal);
162 if (pos && !isa<PHINode>(pos)) {
168 InsertNewInstWith(newEI, *InsertPos);
174 for (
auto E : Extracts)
175 replaceInstUsesWith(*
E, scalarPHI);
188 cast<VectorType>(
Ext.getVectorOperandType())->getElementCount();
190 bool IsBigEndian =
DL.isBigEndian();
195 if (
X->getType()->isIntegerTy() && DestTy->
isIntegerTy() &&
196 isDesirableIntType(
X->getType()->getPrimitiveSizeInBits())) {
197 assert(isa<FixedVectorType>(
Ext.getVectorOperand()->getType()) &&
198 "Expected fixed vector type for bitcast from scalar integer");
206 if (!ShiftAmountC ||
Ext.getVectorOperand()->hasOneUse()) {
207 Value *Lshr =
Builder.CreateLShr(
X, ShiftAmountC,
"extelt.offset");
212 if (!
X->getType()->isVectorTy())
218 auto *SrcTy = cast<VectorType>(
X->getType());
220 if (NumSrcElts == NumElts)
225 "Src and Dst must be the same sort of vector type");
240 unsigned NarrowingRatio =
242 if (ExtIndexC / NarrowingRatio != InsIndexC)
256 unsigned Chunk = ExtIndexC % NarrowingRatio;
258 Chunk = NarrowingRatio - 1 - Chunk;
263 bool NeedSrcBitcast = SrcTy->getScalarType()->isFloatingPointTy();
265 if (NeedSrcBitcast && NeedDestBitcast)
268 unsigned SrcWidth = SrcTy->getScalarSizeInBits();
270 unsigned ShAmt = Chunk * DestWidth;
275 if (!
X->hasOneUse() || !
Ext.getVectorOperand()->hasOneUse())
276 if (NeedSrcBitcast || NeedDestBitcast)
279 if (NeedSrcBitcast) {
286 if (!
Ext.getVectorOperand()->hasOneUse())
291 if (NeedDestBitcast) {
303 unsigned VWidth = cast<FixedVectorType>(V->
getType())->getNumElements();
309 case Instruction::ExtractElement: {
313 if (EEIIndexC && EEIIndexC->
getValue().
ult(VWidth)) {
318 case Instruction::ShuffleVector: {
320 unsigned MaskNumElts =
321 cast<FixedVectorType>(UserInstr->
getType())->getNumElements();
323 UsedElts =
APInt(VWidth, 0);
324 for (
unsigned i = 0;
i < MaskNumElts;
i++) {
326 if (MaskVal == -1u || MaskVal >= 2 * VWidth)
328 if (Shuffle->
getOperand(0) == V && (MaskVal < VWidth))
331 ((MaskVal >= VWidth) && (MaskVal < 2 * VWidth)))
332 UsedElts.
setBit(MaskVal - VWidth);
347 unsigned VWidth = cast<FixedVectorType>(V->
getType())->getNumElements();
349 APInt UnionUsedElts(VWidth, 0);
350 for (
const Use &U : V->
uses()) {
351 if (
Instruction *
I = dyn_cast<Instruction>(U.getUser())) {
362 return UnionUsedElts;
381 SQ.getWithInstruction(&EI)))
382 return replaceInstUsesWith(EI, V);
386 auto *IndexC = dyn_cast<ConstantInt>(
Index);
390 return replaceOperand(EI, 1, NewIdx);
393 unsigned NumElts = EC.getKnownMinValue();
399 if (IID == Intrinsic::experimental_stepvector &&
400 IndexC->getValue().ult(NumElts)) {
406 if (IndexC->getValue().getActiveBits() <=
BitWidth)
410 return replaceInstUsesWith(EI, Idx);
416 if (!EC.isScalable() && IndexC->getValue().uge(NumElts))
424 if (
auto *Phi = dyn_cast<PHINode>(SrcVec))
425 if (
Instruction *ScalarPHI = scalarizePHI(EI, Phi))
458 if (
auto *
I = dyn_cast<Instruction>(SrcVec)) {
459 if (
auto *
IE = dyn_cast<InsertElementInst>(
I)) {
463 if (isa<Constant>(
IE->getOperand(2)) && IndexC)
464 return replaceOperand(EI, 0,
IE->getOperand(0));
465 }
else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(
I)) {
466 auto *
VecType = cast<VectorType>(
GEP->getType());
468 uint64_t IdxVal = IndexC ? IndexC->getZExtValue() : 0;
469 if (IndexC && IdxVal < EC.getKnownMinValue() &&
GEP->hasOneUse()) {
480 return isa<VectorType>(V->getType());
482 if (VectorOps == 1) {
483 Value *NewPtr =
GEP->getPointerOperand();
484 if (isa<VectorType>(NewPtr->
getType()))
485 NewPtr =
Builder.CreateExtractElement(NewPtr, IndexC);
488 for (
unsigned I = 1;
I !=
GEP->getNumOperands(); ++
I) {
490 if (isa<VectorType>(
Op->getType()))
491 NewOps.push_back(
Builder.CreateExtractElement(
Op, IndexC));
493 NewOps.push_back(
Op);
497 GEP->getSourceElementType(), NewPtr, NewOps);
502 }
else if (
auto *SVI = dyn_cast<ShuffleVectorInst>(
I)) {
506 if (isa<FixedVectorType>(SVI->getType()) && isa<ConstantInt>(
Index)) {
508 SVI->getMaskValue(cast<ConstantInt>(
Index)->getZExtValue());
510 unsigned LHSWidth = cast<FixedVectorType>(SVI->getOperand(0)->getType())
515 if (SrcIdx < (
int)LHSWidth)
516 Src = SVI->getOperand(0);
519 Src = SVI->getOperand(1);
525 }
else if (
auto *CI = dyn_cast<CastInst>(
I)) {
529 if (CI->hasOneUse() && (CI->getOpcode() != Instruction::BitCast)) {
541 unsigned NumElts = EC.getKnownMinValue();
545 if (!EC.isScalable() && NumElts != 1) {
549 APInt UndefElts(NumElts, 0);
550 APInt DemandedElts(NumElts, 0);
551 DemandedElts.
setBit(IndexC->getZExtValue());
553 SimplifyDemandedVectorElts(SrcVec, DemandedElts, UndefElts))
554 return replaceOperand(EI, 0, V);
560 APInt UndefElts(NumElts, 0);
561 if (
Value *V = SimplifyDemandedVectorElts(
562 SrcVec, DemandedElts, UndefElts, 0 ,
581 "Invalid CollectSingleShuffleElements");
582 unsigned NumElts = cast<FixedVectorType>(V->
getType())->getNumElements();
585 Mask.assign(NumElts, -1);
590 for (
unsigned i = 0;
i != NumElts; ++
i)
596 for (
unsigned i = 0;
i != NumElts; ++
i)
597 Mask.push_back(
i + NumElts);
603 Value *VecOp = IEI->getOperand(0);
604 Value *ScalarOp = IEI->getOperand(1);
605 Value *IdxOp = IEI->getOperand(2);
607 if (!isa<ConstantInt>(IdxOp))
609 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
611 if (isa<UndefValue>(ScalarOp)) {
616 Mask[InsertedIdx] = -1;
621 unsigned ExtractedIdx =
622 cast<ConstantInt>(EI->
getOperand(1))->getZExtValue();
623 unsigned NumLHSElts =
624 cast<FixedVectorType>(
LHS->
getType())->getNumElements();
633 Mask[InsertedIdx % NumElts] = ExtractedIdx;
636 Mask[InsertedIdx % NumElts] = ExtractedIdx + NumLHSElts;
654 auto *InsVecType = cast<FixedVectorType>(InsElt->
getType());
656 unsigned NumInsElts = InsVecType->getNumElements();
657 unsigned NumExtElts = ExtVecType->getNumElements();
660 if (InsVecType->getElementType() != ExtVecType->getElementType() ||
661 NumExtElts >= NumInsElts)
669 for (
unsigned i = 0;
i < NumExtElts; ++
i)
670 ExtendMask.push_back(
i);
671 for (
unsigned i = NumExtElts;
i < NumInsElts; ++
i)
672 ExtendMask.push_back(-1);
675 auto *ExtVecOpInst = dyn_cast<Instruction>(ExtVecOp);
676 BasicBlock *InsertionBlock = (ExtVecOpInst && !isa<PHINode>(ExtVecOpInst))
689 if (InsertionBlock != InsElt->
getParent())
706 if (ExtVecOpInst && !isa<PHINode>(ExtVecOpInst))
707 WideVec->insertAfter(ExtVecOpInst);
718 NewExt->insertAfter(OldExt);
737 unsigned NumElts = cast<FixedVectorType>(V->
getType())->getNumElements();
740 Mask.assign(NumElts, -1);
741 return std::make_pair(
745 if (isa<ConstantAggregateZero>(V)) {
746 Mask.assign(NumElts, 0);
747 return std::make_pair(V,
nullptr);
752 Value *VecOp = IEI->getOperand(0);
753 Value *ScalarOp = IEI->getOperand(1);
754 Value *IdxOp = IEI->getOperand(2);
757 if (isa<ConstantInt>(EI->
getOperand(1)) && isa<ConstantInt>(IdxOp)) {
758 unsigned ExtractedIdx =
759 cast<ConstantInt>(EI->
getOperand(1))->getZExtValue();
760 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
764 if (EI->
getOperand(0) == PermittedRHS || PermittedRHS ==
nullptr) {
767 assert(LR.second ==
nullptr || LR.second ==
RHS);
776 for (
unsigned i = 0;
i < NumElts; ++
i)
778 return std::make_pair(V,
nullptr);
781 unsigned NumLHSElts =
782 cast<FixedVectorType>(
RHS->
getType())->getNumElements();
783 Mask[InsertedIdx % NumElts] = NumLHSElts + ExtractedIdx;
784 return std::make_pair(LR.first,
RHS);
787 if (VecOp == PermittedRHS) {
790 unsigned NumLHSElts =
793 for (
unsigned i = 0;
i != NumElts; ++
i)
794 Mask.push_back(
i == InsertedIdx ? ExtractedIdx : NumLHSElts +
i);
795 return std::make_pair(EI->
getOperand(0), PermittedRHS);
803 return std::make_pair(EI->
getOperand(0), PermittedRHS);
809 for (
unsigned i = 0;
i != NumElts; ++
i)
811 return std::make_pair(V,
nullptr);
837 assert(NumAggElts > 0 &&
"Aggregate should have elements.");
842 static constexpr
auto FoundMismatch =
nullptr;
849 auto KnowAllElts = [&AggElts]() {
858 static const int DepthLimit = 2 * NumAggElts;
863 Depth < DepthLimit && CurrIVI && !KnowAllElts();
864 CurrIVI = dyn_cast<InsertValueInst>(CurrIVI->getAggregateOperand()),
866 auto *InsertedValue =
867 dyn_cast<Instruction>(CurrIVI->getInsertedValueOperand());
874 if (Indices.
size() != 1)
894 enum class AggregateDescription {
913 if (*SourceAggregate == FoundMismatch)
914 return AggregateDescription::FoundMismatch;
915 return AggregateDescription::Found;
923 auto FindSourceAggregate =
932 auto *EVI = dyn_cast_or_null<ExtractValueInst>(Elt);
936 Value *SourceAggregate = EVI->getAggregateOperand();
939 if (SourceAggregate->
getType() != AggTy)
940 return FoundMismatch;
942 if (EVI->getNumIndices() != 1 || EltIdx != EVI->getIndices().front())
943 return FoundMismatch;
945 return SourceAggregate;
951 auto FindCommonSourceAggregate =
957 assert(Describe(SourceAggregate) != AggregateDescription::FoundMismatch &&
958 "We don't store nullptr in SourceAggregate!");
959 assert((Describe(SourceAggregate) == AggregateDescription::Found) ==
961 "SourceAggregate should be valid after the first element,");
967 FindSourceAggregate(*
I.value(),
I.index(), UseBB, PredBB);
974 if (Describe(SourceAggregateForElement) != AggregateDescription::Found)
975 return SourceAggregateForElement;
979 switch (Describe(SourceAggregate)) {
982 SourceAggregate = SourceAggregateForElement;
984 case AggregateDescription::Found:
987 if (*SourceAggregateForElement != *SourceAggregate)
988 return FoundMismatch;
990 case AggregateDescription::FoundMismatch:
995 assert(Describe(SourceAggregate) == AggregateDescription::Found &&
996 "Must be a valid Value");
997 return *SourceAggregate;
1003 SourceAggregate = FindCommonSourceAggregate(
None,
None);
1005 if (Describe(SourceAggregate) == AggregateDescription::FoundMismatch)
1007 ++NumAggregateReconstructionsSimplified;
1008 return replaceInstUsesWith(OrigIVI, *SourceAggregate);
1044 static const int PredCountLimit = 64;
1051 if (Preds.size() >= PredCountLimit)
1061 std::pair<decltype(SourceAggregates)::iterator,
bool>
IV =
1062 SourceAggregates.
insert({Pred,
nullptr});
1070 SourceAggregate = FindCommonSourceAggregate(UseBB, Pred);
1071 if (Describe(SourceAggregate) != AggregateDescription::Found)
1073 IV.first->second = *SourceAggregate;
1084 Builder.CreatePHI(AggTy, Preds.size(), OrigIVI.
getName() +
".merged");
1086 PHI->addIncoming(SourceAggregates[Pred], Pred);
1088 ++NumAggregateReconstructionsSimplified;
1089 return replaceInstUsesWith(OrigIVI, PHI);
1100 bool IsRedundant =
false;
1111 auto UserInsInst = dyn_cast<InsertValueInst>(U);
1114 if (UserInsInst->getIndices() == FirstIndices) {
1123 return replaceInstUsesWith(
I,
I.getOperand(0));
1125 if (
Instruction *NewI = foldAggregateConstructionIntoAggregateReuse(
I))
1147 for (
int i = 0;
i != MaskSize; ++
i) {
1149 if (Elt != -1 && Elt !=
i && Elt !=
i +
VecSize)
1168 if (isa<ScalableVectorType>(VecTy))
1170 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1174 if (NumElements == 1)
1185 auto *Idx = dyn_cast<ConstantInt>(CurrIE->
getOperand(2));
1186 if (!Idx || CurrIE->
getOperand(1) != SplatVal)
1189 auto *NextIE = dyn_cast<InsertElementInst>(CurrIE->
getOperand(0));
1193 if (CurrIE != &InsElt &&
1194 (!CurrIE->
hasOneUse() && (NextIE !=
nullptr || !Idx->isZero())))
1197 ElementPresent[Idx->getZExtValue()] =
true;
1203 if (FirstIE == &InsElt)
1211 if (!ElementPresent.
all())
1218 if (!cast<ConstantInt>(FirstIE->
getOperand(2))->isZero())
1223 for (
unsigned i = 0;
i != NumElements; ++
i)
1224 if (!ElementPresent[
i])
1234 auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0));
1235 if (!Shuf || !Shuf->isZeroEltSplat())
1240 if (isa<ScalableVectorType>(Shuf->getType()))
1250 Value *Op0 = Shuf->getOperand(0);
1258 unsigned NumMaskElts =
1259 cast<FixedVectorType>(Shuf->getType())->getNumElements();
1261 for (
unsigned i = 0;
i != NumMaskElts; ++
i)
1262 NewMask[
i] =
i == IdxC ? 0 : Shuf->getMaskValue(
i);
1271 auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0));
1273 !(Shuf->isIdentityWithExtract() || Shuf->isIdentityWithPadding()))
1278 if (isa<ScalableVectorType>(Shuf->getType()))
1289 Value *
X = Shuf->getOperand(0);
1297 unsigned NumMaskElts =
1298 cast<FixedVectorType>(Shuf->getType())->getNumElements();
1301 for (
unsigned i = 0;
i != NumMaskElts; ++
i) {
1304 NewMask[
i] = OldMask[
i];
1305 }
else if (OldMask[
i] == (
int)IdxC) {
1311 "Unexpected shuffle mask element for identity shuffle");
1330 auto *InsElt1 = dyn_cast<InsertElementInst>(InsElt2.
getOperand(0));
1331 if (!InsElt1 || !InsElt1->hasOneUse())
1338 match(InsElt1->getOperand(1),
m_Value(
Y)) && !isa<Constant>(
Y) &&
1342 Value *NewInsElt1 =
Builder.CreateInsertElement(
X, ScalarC, IdxC2);
1352 auto *Inst = dyn_cast<Instruction>(InsElt.
getOperand(0));
1355 if (!Inst || !Inst->hasOneUse())
1357 if (
auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0))) {
1360 Constant *ShufConstVec, *InsEltScalar;
1384 unsigned NumElts =
Mask.size();
1387 for (
unsigned I = 0;
I != NumElts; ++
I) {
1388 if (
I == InsEltIndex) {
1389 NewShufElts[
I] = InsEltScalar;
1390 NewMaskElts[
I] = InsEltIndex + NumElts;
1394 NewMaskElts[
I] =
Mask[
I];
1398 if (!NewShufElts[
I])
1406 }
else if (
auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
1411 if (isa<ScalableVectorType>(InsElt.
getType()))
1414 cast<FixedVectorType>(InsElt.
getType())->getNumElements();
1437 for (
unsigned I = 0;
I < NumElts; ++
I) {
1469 CastOpcode = Instruction::FPExt;
1471 CastOpcode = Instruction::SExt;
1473 CastOpcode = Instruction::ZExt;
1478 if (
X->getType()->getScalarType() !=
Y->getType())
1487 Value *VecOp =
IE.getOperand(0);
1488 Value *ScalarOp =
IE.getOperand(1);
1489 Value *IdxOp =
IE.getOperand(2);
1492 VecOp, ScalarOp, IdxOp, SQ.getWithInstruction(&
IE)))
1493 return replaceInstUsesWith(
IE, V);
1496 if (
auto *IndexC = dyn_cast<ConstantInt>(IdxOp))
1498 return replaceOperand(
IE, 2, NewIdx);
1513 Value *NewInsElt =
Builder.CreateInsertElement(NewUndef, ScalarSrc, IdxOp);
1524 cast<VectorType>(VecSrc->
getType())->getElementType() ==
1528 Value *NewInsElt =
Builder.CreateInsertElement(VecSrc, ScalarSrc, IdxOp);
1536 uint64_t InsertedIdx, ExtractedIdx;
1538 if (isa<FixedVectorType>(
IE.getType()) &&
1542 isa<FixedVectorType>(ExtVecOp->
getType()) &&
1544 cast<FixedVectorType>(ExtVecOp->
getType())->getNumElements()) {
1562 auto *InsertUser = dyn_cast<InsertElementInst>(
Insert.user_back());
1569 if (isShuffleRootCandidate(
IE)) {
1575 if (LR.first != &
IE && LR.second != &
IE) {
1577 if (LR.second ==
nullptr)
1584 if (
auto VecTy = dyn_cast<FixedVectorType>(VecOp->
getType())) {
1585 unsigned VWidth = VecTy->getNumElements();
1586 APInt UndefElts(VWidth, 0);
1588 if (
Value *V = SimplifyDemandedVectorElts(&
IE, AllOnesEltMask, UndefElts)) {
1590 return replaceInstUsesWith(
IE, V);
1608 return IdentityShuf;
1619 unsigned Depth = 5) {
1621 if (isa<Constant>(V))
1626 if (!
I)
return false;
1629 if (!
I->hasOneUse())
1632 if (
Depth == 0)
return false;
1634 switch (
I->getOpcode()) {
1635 case Instruction::UDiv:
1636 case Instruction::SDiv:
1637 case Instruction::URem:
1638 case Instruction::SRem:
1646 case Instruction::FAdd:
1647 case Instruction::Sub:
1648 case Instruction::FSub:
1650 case Instruction::FMul:
1651 case Instruction::FDiv:
1652 case Instruction::FRem:
1653 case Instruction::Shl:
1654 case Instruction::LShr:
1655 case Instruction::AShr:
1656 case Instruction::And:
1657 case Instruction::Or:
1658 case Instruction::Xor:
1659 case Instruction::ICmp:
1660 case Instruction::FCmp:
1661 case Instruction::Trunc:
1662 case Instruction::ZExt:
1663 case Instruction::SExt:
1664 case Instruction::FPToUI:
1665 case Instruction::FPToSI:
1666 case Instruction::UIToFP:
1667 case Instruction::SIToFP:
1668 case Instruction::FPTrunc:
1669 case Instruction::FPExt:
1670 case Instruction::GetElementPtr: {
1673 Type *ITy =
I->getType();
1675 Mask.size() > cast<FixedVectorType>(ITy)->getNumElements())
1677 for (
Value *Operand :
I->operands()) {
1683 case Instruction::InsertElement: {
1684 ConstantInt *CI = dyn_cast<ConstantInt>(
I->getOperand(2));
1685 if (!CI)
return false;
1690 bool SeenOnce =
false;
1691 for (
int i = 0,
e =
Mask.size();
i !=
e; ++
i) {
1692 if (
Mask[
i] == ElementNumber) {
1709 switch (
I->getOpcode()) {
1711 case Instruction::FAdd:
1712 case Instruction::Sub:
1713 case Instruction::FSub:
1715 case Instruction::FMul:
1716 case Instruction::UDiv:
1717 case Instruction::SDiv:
1718 case Instruction::FDiv:
1719 case Instruction::URem:
1720 case Instruction::SRem:
1721 case Instruction::FRem:
1722 case Instruction::Shl:
1723 case Instruction::LShr:
1724 case Instruction::AShr:
1725 case Instruction::And:
1726 case Instruction::Or:
1727 case Instruction::Xor: {
1729 assert(NewOps.
size() == 2 &&
"binary operator with #ops != 2");
1732 NewOps[0], NewOps[1],
"", BO);
1733 if (isa<OverflowingBinaryOperator>(BO)) {
1737 if (isa<PossiblyExactOperator>(BO)) {
1738 New->setIsExact(BO->
isExact());
1740 if (isa<FPMathOperator>(BO))
1741 New->copyFastMathFlags(
I);
1744 case Instruction::ICmp:
1745 assert(NewOps.
size() == 2 &&
"icmp with #ops != 2");
1747 NewOps[0], NewOps[1]);
1748 case Instruction::FCmp:
1749 assert(NewOps.
size() == 2 &&
"fcmp with #ops != 2");
1751 NewOps[0], NewOps[1]);
1752 case Instruction::Trunc:
1753 case Instruction::ZExt:
1754 case Instruction::SExt:
1755 case Instruction::FPToUI:
1756 case Instruction::FPToSI:
1757 case Instruction::UIToFP:
1758 case Instruction::SIToFP:
1759 case Instruction::FPTrunc:
1760 case Instruction::FPExt: {
1764 I->getType()->getScalarType(),
1765 cast<VectorType>(NewOps[0]->getType())->getElementCount());
1766 assert(NewOps.
size() == 1 &&
"cast with #ops != 1");
1770 case Instruction::GetElementPtr: {
1771 Value *Ptr = NewOps[0];
1774 cast<GetElementPtrInst>(
I)->getSourceElementType(), Ptr, Idx,
"",
I);
1775 GEP->setIsInBounds(cast<GetElementPtrInst>(
I)->isInBounds());
1791 if (isa<ConstantAggregateZero>(V))
1794 if (
Constant *
C = dyn_cast<Constant>(V))
1799 switch (
I->getOpcode()) {
1801 case Instruction::FAdd:
1802 case Instruction::Sub:
1803 case Instruction::FSub:
1805 case Instruction::FMul:
1806 case Instruction::UDiv:
1807 case Instruction::SDiv:
1808 case Instruction::FDiv:
1809 case Instruction::URem:
1810 case Instruction::SRem:
1811 case Instruction::FRem:
1812 case Instruction::Shl:
1813 case Instruction::LShr:
1814 case Instruction::AShr:
1815 case Instruction::And:
1816 case Instruction::Or:
1817 case Instruction::Xor:
1818 case Instruction::ICmp:
1819 case Instruction::FCmp:
1820 case Instruction::Trunc:
1821 case Instruction::ZExt:
1822 case Instruction::SExt:
1823 case Instruction::FPToUI:
1824 case Instruction::FPToSI:
1825 case Instruction::UIToFP:
1826 case Instruction::SIToFP:
1827 case Instruction::FPTrunc:
1828 case Instruction::FPExt:
1830 case Instruction::GetElementPtr: {
1834 cast<FixedVectorType>(
I->getType())->getNumElements());
1835 for (
int i = 0,
e =
I->getNumOperands();
i !=
e; ++
i) {
1840 if (
I->getOperand(
i)->getType()->isVectorTy())
1843 V =
I->getOperand(
i);
1844 NewOps.push_back(V);
1845 NeedsRebuild |= (V !=
I->getOperand(
i));
1852 case Instruction::InsertElement: {
1853 int Element = cast<ConstantInt>(
I->getOperand(2))->getLimitedValue();
1890 unsigned MaskElems =
Mask.size();
1891 unsigned BegIdx =
Mask.front();
1892 unsigned EndIdx =
Mask.back();
1893 if (BegIdx > EndIdx || EndIdx >= LHSElems || EndIdx - BegIdx != MaskElems - 1)
1895 for (
unsigned I = 0;
I != MaskElems; ++
I)
1896 if (
static_cast<unsigned>(
Mask[
I]) != BegIdx +
I)
1909 Opcode(Opc), Op0(V0), Op1(V1) {}
1910 operator bool()
const {
return Opcode != 0; }
1921 case Instruction::Shl: {
1930 case Instruction::Or: {
1937 case Instruction::Sub:
1949 assert(Shuf.
isSelect() &&
"Must have select-equivalent shuffle");
1966 auto *BO = cast<BinaryOperator>(Op0IsBinop ? Op0 : Op1);
1980 bool MightCreatePoisonOrUB =
1983 if (MightCreatePoisonOrUB)
1988 Value *
X = Op0IsBinop ? Op1 : Op0;
2020 Value *NewIns =
Builder.CreateInsertElement(UndefVec,
X, Zero);
2026 unsigned NumMaskElts =
2027 cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2029 for (
unsigned i = 0;
i != NumMaskElts; ++
i)
2043 unsigned NumElts = cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2065 bool ConstantsAreOp1;
2068 ConstantsAreOp1 =
false;
2073 ConstantsAreOp1 =
true;
2080 bool DropNSW =
false;
2081 if (ConstantsAreOp1 && Opc0 != Opc1) {
2085 if (Opc0 == Instruction::Shl || Opc1 == Instruction::Shl)
2088 assert(isa<Constant>(AltB0.Op1) &&
"Expecting constant with alt binop");
2089 Opc0 = AltB0.Opcode;
2090 C0 = cast<Constant>(AltB0.Op1);
2092 assert(isa<Constant>(AltB1.Op1) &&
"Expecting constant with alt binop");
2093 Opc1 = AltB1.Opcode;
2094 C1 = cast<Constant>(AltB1.Op1);
2098 if (Opc0 != Opc1 || !C0 || !
C1)
2111 bool MightCreatePoisonOrUB =
2114 if (MightCreatePoisonOrUB)
2137 if (MightCreatePoisonOrUB && !ConstantsAreOp1)
2150 Value *NewBO = ConstantsAreOp1 ?
Builder.CreateBinOp(BOpc, V, NewC) :
2151 Builder.CreateBinOp(BOpc, NewC, V);
2158 if (
auto *NewI = dyn_cast<Instruction>(NewBO)) {
2159 NewI->copyIRFlags(B0);
2160 NewI->andIRFlags(
B1);
2162 NewI->setHasNoSignedWrap(
false);
2164 NewI->dropPoisonGeneratingFlags();
2166 return replaceInstUsesWith(Shuf, NewBO);
2183 Type *SrcType =
X->getType();
2185 cast<FixedVectorType>(SrcType)->getNumElements() !=
2186 cast<FixedVectorType>(DestType)->getNumElements() ||
2191 "Expected a shuffle that decreases length");
2198 for (
unsigned i = 0,
e =
Mask.size();
i !=
e; ++
i) {
2201 uint64_t LSBIndex = IsBigEndian ? (
i + 1) * TruncRatio - 1 :
i * TruncRatio;
2202 assert(LSBIndex <= INT32_MAX &&
"Overflowed 32-bits");
2203 if (
Mask[
i] != (
int)LSBIndex)
2229 unsigned NarrowNumElts =
2230 cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2233 cast<FixedVectorType>(NarrowCond->
getType())->getNumElements() !=
2235 !cast<ShuffleVectorInst>(
Cond)->isIdentityWithPadding())
2269 Instruction *NewFNeg = UnaryOperator::CreateFNeg(NewShuf);
2282 auto *Cast0 = dyn_cast<CastInst>(Shuf.
getOperand(0));
2283 auto *Cast1 = dyn_cast<CastInst>(Shuf.
getOperand(1));
2284 if (!Cast0 || !Cast1 || Cast0->getOpcode() != Cast1->getOpcode() ||
2285 Cast0->getSrcTy() != Cast1->getSrcTy())
2291 switch (CastOpcode) {
2292 case Instruction::FPToSI:
2293 case Instruction::FPToUI:
2294 case Instruction::SIToFP:
2295 case Instruction::UIToFP:
2303 VectorType *CastSrcTy = cast<VectorType>(Cast0->getSrcTy());
2311 assert(isa<FixedVectorType>(CastSrcTy) && isa<FixedVectorType>(ShufOpTy) &&
2312 "Expected fixed vector operands for casts and binary shuffle");
2317 if (!Cast0->hasOneUse() && !Cast1->hasOneUse())
2321 Value *
X = Cast0->getOperand(0);
2322 Value *
Y = Cast1->getOperand(0);
2337 X->getType()->getPrimitiveSizeInBits() ==
2363 unsigned NumElts = cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2366 "Identity with extract must have less elements than its inputs");
2368 for (
unsigned i = 0;
i != NumElts; ++
i) {
2370 int MaskElt =
Mask[
i];
2371 NewMask[
i] = ExtractMaskElt ==
UndefMaskElem ? ExtractMaskElt : MaskElt;
2384 int NumElts =
Mask.size();
2385 int InpNumElts = cast<FixedVectorType>(V0->
getType())->getNumElements();
2410 if (NumElts != InpNumElts)
2414 auto isShufflingScalarIntoOp1 = [&](
Value *&Scalar,
ConstantInt *&IndexC) {
2422 int NewInsIndex = -1;
2423 for (
int i = 0;
i != NumElts; ++
i) {
2429 if (
Mask[
i] == NumElts +
i)
2433 if (NewInsIndex != -1 ||
Mask[
i] != IndexC->getSExtValue())
2440 assert(NewInsIndex != -1 &&
"Did not fold shuffle with unused operand?");
2452 if (isShufflingScalarIntoOp1(Scalar, IndexC))
2460 if (isShufflingScalarIntoOp1(Scalar, IndexC))
2470 auto *Shuffle0 = dyn_cast<ShuffleVectorInst>(Shuf.
getOperand(0));
2471 auto *Shuffle1 = dyn_cast<ShuffleVectorInst>(Shuf.
getOperand(1));
2472 if (!Shuffle0 || !Shuffle0->isIdentityWithPadding() ||
2473 !Shuffle1 || !Shuffle1->isIdentityWithPadding())
2481 Value *
X = Shuffle0->getOperand(0);
2482 Value *
Y = Shuffle1->getOperand(0);
2483 if (
X->getType() !=
Y->getType() ||
2486 cast<FixedVectorType>(Shuffle0->getType())->getNumElements()) ||
2487 !
isPowerOf2_32(cast<FixedVectorType>(
X->getType())->getNumElements()) ||
2492 "Unexpected operand for identity shuffle");
2498 int NarrowElts = cast<FixedVectorType>(
X->getType())->getNumElements();
2499 int WideElts = cast<FixedVectorType>(Shuffle0->getType())->getNumElements();
2500 assert(WideElts > NarrowElts &&
"Unexpected types for identity with padding");
2504 for (
int i = 0,
e =
Mask.size();
i !=
e; ++
i) {
2510 if (
Mask[
i] < WideElts) {
2511 if (Shuffle0->getMaskValue(
Mask[
i]) == -1)
2514 if (Shuffle1->getMaskValue(
Mask[
i] - WideElts) == -1)
2521 if (
Mask[
i] < WideElts) {
2522 assert(
Mask[
i] < NarrowElts &&
"Unexpected shuffle mask");
2525 assert(
Mask[
i] < (WideElts + NarrowElts) &&
"Unexpected shuffle mask");
2526 NewMask[
i] =
Mask[
i] - (WideElts - NarrowElts);
2538 return replaceInstUsesWith(SVI, V);
2544 unsigned VWidth = cast<FixedVectorType>(SVI.
getType())->getNumElements();
2545 unsigned LHSWidth = cast<FixedVectorType>(
LHS->
getType())->getNumElements();
2556 X->getType()->isVectorTy() &&
X->getType() ==
Y->getType() &&
2557 X->getType()->getScalarSizeInBits() ==
2575 X->getType()->isVectorTy() && VWidth == LHSWidth) {
2577 auto *XType = cast<FixedVectorType>(
X->getType());
2578 unsigned XNumElts = XType->getNumElements();
2580 if (XNumElts >= VWidth) {
2581 assert(XNumElts % VWidth == 0 &&
"Unexpected vector bitcast");
2584 assert(VWidth % XNumElts == 0 &&
"Unexpected vector bitcast");
2588 if (!ScaledMask.empty()) {
2592 ScaledMask, XType, ShufQuery))
2600 "Shuffle with 2 undef ops not simplified?");
2628 APInt UndefElts(VWidth, 0);
2630 if (
Value *V = SimplifyDemandedVectorElts(&SVI, AllOnesEltMask, UndefElts)) {
2632 return replaceInstUsesWith(SVI, V);
2648 return replaceInstUsesWith(SVI, V);
2680 bool MadeChange =
false;
2683 unsigned MaskElems =
Mask.size();
2684 auto *SrcTy = cast<FixedVectorType>(V->
getType());
2685 unsigned VecBitWidth = SrcTy->getPrimitiveSizeInBits().getFixedSize();
2686 unsigned SrcElemBitWidth =
DL.getTypeSizeInBits(SrcTy->getElementType());
2687 assert(SrcElemBitWidth &&
"vector elements must have a bitwidth");
2688 unsigned SrcNumElems = SrcTy->getNumElements();
2693 if (!BC->use_empty())
2697 unsigned BegIdx =
Mask.front();
2698 Type *TgtTy = BC->getDestTy();
2699 unsigned TgtElemBitWidth =
DL.getTypeSizeInBits(TgtTy);
2700 if (!TgtElemBitWidth)
2702 unsigned TgtNumElems = VecBitWidth / TgtElemBitWidth;
2703 bool VecBitWidthsEqual = VecBitWidth == TgtNumElems * TgtElemBitWidth;
2704 bool BegIsAligned = 0 == ((SrcElemBitWidth * BegIdx) % TgtElemBitWidth);
2705 if (!VecBitWidthsEqual)
2710 if (!BegIsAligned) {
2714 for (
unsigned I = 0,
E = MaskElems, Idx = BegIdx;
I !=
E; ++Idx, ++
I)
2715 ShuffleMask[
I] = Idx;
2716 V =
Builder.CreateShuffleVector(V, ShuffleMask,
2720 unsigned SrcElemsPerTgtElem = TgtElemBitWidth / SrcElemBitWidth;
2721 assert(SrcElemsPerTgtElem);
2722 BegIdx /= SrcElemsPerTgtElem;
2723 bool BCAlreadyExists = NewBCs.
find(CastSrcTy) != NewBCs.
end();
2728 if (!BCAlreadyExists)
2729 NewBCs[CastSrcTy] = NewBC;
2734 replaceInstUsesWith(*BC,
Ext);
2786 LHSShuffle =
nullptr;
2789 RHSShuffle =
nullptr;
2790 if (!LHSShuffle && !RHSShuffle)
2791 return MadeChange ? &SVI :
nullptr;
2793 Value* LHSOp0 =
nullptr;
2794 Value* LHSOp1 =
nullptr;
2795 Value* RHSOp0 =
nullptr;
2796 unsigned LHSOp0Width = 0;
2797 unsigned RHSOp0Width = 0;
2801 LHSOp0Width = cast<FixedVectorType>(LHSOp0->
getType())->getNumElements();
2805 RHSOp0Width = cast<FixedVectorType>(RHSOp0->
getType())->getNumElements();
2816 else if (LHSOp0Width == LHSWidth) {
2821 if (RHSShuffle && RHSOp0Width == LHSWidth) {
2825 if (LHSOp0 == RHSOp0) {
2830 if (newLHS ==
LHS && newRHS ==
RHS)
2831 return MadeChange ? &SVI :
nullptr;
2837 if (RHSShuffle && newRHS !=
RHS)
2840 unsigned newLHSWidth = (newLHS !=
LHS) ? LHSOp0Width : LHSWidth;
2846 for (
unsigned i = 0;
i < VWidth; ++
i) {
2851 }
else if (
Mask[
i] < (
int)LHSWidth) {
2856 if (newLHS !=
LHS) {
2857 eltMask = LHSMask[
Mask[
i]];
2860 if (eltMask >= (
int)LHSOp0Width && isa<UndefValue>(LHSOp1))
2873 else if (newRHS !=
RHS) {
2874 eltMask = RHSMask[
Mask[
i]-LHSWidth];
2877 if (eltMask >= (
int)RHSOp0Width) {
2879 "should have been check above");
2883 eltMask =
Mask[
i]-LHSWidth;
2891 if (eltMask >= 0 && newRHS !=
nullptr && newLHS != newRHS)
2892 eltMask += newLHSWidth;
2897 if (SplatElt >= 0 && SplatElt != eltMask)
2902 newMask.push_back(eltMask);
2907 if (
isSplat || newMask == LHSMask || newMask == RHSMask || newMask ==
Mask) {
2913 return MadeChange ? &SVI :
nullptr;
Value * simplifyInsertElementInst(Value *Vec, Value *Elt, Value *Idx, const SimplifyQuery &Q)
Given operands for an InsertElement, fold the result or return null.
BinaryOperator::BinaryOps Opcode
void setIsInBounds(bool b=true)
Set or clear the inbounds flag on this GEP instruction.
bool isIdentityWithExtract() const
Return true if this shuffle extracts the first N elements of exactly one source vector.
This is an optimization pass for GlobalISel generic memory operations.
bool MaskedValueIsZero(const Value *V, const APInt &Mask, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if 'V & Mask' is known to be zero.
A parsed version of the target data layout string in and methods for querying it.
bool hasOneUse() const
Return true if there is exactly one use of this value.
Vector Rotate Left Mask Mask Insert
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
InstListType::iterator iterator
Instruction iterators...
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
const Function * getParent() const
Return the enclosing method, or null if none.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static Instruction * foldInsSequenceIntoSplat(InsertElementInst &InsElt)
Turn a chain of inserts that splats a value into an insert + shuffle: insertelt(insertelt(insertelt(i...
static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Instruction *CopyO, const Twine &Name="", Instruction *InsertBefore=nullptr)
This class represents a no-op cast from one type to another.
instcombine should handle this C2 when C1
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
const APInt & getValue() const
Return the constant as an APInt value reference.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
bool hasNoUnsignedWrap() const
Determine whether the no unsigned wrap flag is set.
static Instruction * narrowInsElt(InsertElementInst &InsElt, InstCombiner::BuilderTy &Builder)
If both the base vector and the inserted element are extended from the same type, do the insert eleme...
TypeID getTypeID() const
Return the type id for the type.
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
static InsertElementInst * Create(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
static Value * buildNew(Instruction *I, ArrayRef< Value * > NewOps)
Rebuild a new instruction just like 'I' but with the new operands given.
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
The instances of the Type class are immutable: once they are created, they are never changed.
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Instruction * visitExtractElementInst(ExtractElementInst &EI)
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
Instruction * visitShuffleVectorInst(ShuffleVectorInst &SVI)
Type * getElementType() const
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
CastClass_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
static IntegerType * getInt32Ty(LLVMContext &C)
LLVM Basic Block Representation.
static Instruction * foldConstantInsEltIntoShuffle(InsertElementInst &InsElt)
insertelt (shufflevector X, CVec, Mask|insertelt X, C1, CIndex1), C, CIndex --> shufflevector X,...
constexpr int UndefMaskElem
OneUse_match< T > m_OneUse(const T &SubPattern)
Value * simplifyExtractElementInst(Value *Vec, Value *Idx, const SimplifyQuery &Q)
Given operands for an ExtractElementInst, fold the result or return null.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
static Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)
Return the identity constant for a binary opcode.
UnaryOps getOpcode() const
This is the shared class of boolean and integer constants.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isSelect() const
Return true if this shuffle chooses elements from its source vectors without lane crossings and all o...
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
bool match(Val *V, const Pattern &P)
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
class_match< UnaryOperator > m_UnOp()
Match an arbitrary unary operation and ignore it.
(vector float) vec_cmpeq(*A, *B) C
static bool isSplat(ArrayRef< Value * > VL)
This instruction compares its operands according to the predicate given to the constructor.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static Instruction * foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt)
Try to fold an extract+insert element into an existing identity shuffle by changing the shuffle's mas...
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
bool isVectorTy() const
True if this is an instance of VectorType.
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
This instruction inserts a single (scalar) element into a VectorType value.
iterator_range< use_iterator > uses()
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
CastClass_match< OpTy, Instruction::ZExt > m_ZExt(const OpTy &Op)
Matches ZExt.
bool all() const
Returns true if all bits are set.
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
static Constant * getAllOnesValue(Type *Ty)
BinaryOps getOpcode() const
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
STATISTIC(NumFunctions, "Total number of functions")
auto predecessors(MachineBasicBlock *BB)
static UnaryOperator * CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO, const Twine &Name="", Instruction *InsertBefore=nullptr)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl< int > &Mask, Value *PermittedRHS, InstCombinerImpl &IC)
Instruction * foldAggregateConstructionIntoAggregateReuse(InsertValueInst &OrigIVI)
Look for chain of insertvalue's that fully define an aggregate, and trace back the values inserted,...
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
unsigned getIntegerBitWidth() const
VectorType * getType() const
Overload to return most specific vector type.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
unsigned getStructNumElements() const
bool isIntegerTy() const
True if this is an instance of IntegerType.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
static Constant * getShuffleVector(Constant *V1, Constant *V2, ArrayRef< int > Mask, Type *OnlyIfReducedTy=nullptr)
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
static Instruction * foldShuffleWithInsert(ShuffleVectorInst &Shuf, InstCombinerImpl &IC)
Try to replace a shuffle with an insertelement or try to replace a shuffle operand with the operand o...
Base class of all SIMD vector types.
void andIRFlags(const Value *V)
Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction.
static Constant * getSafeVectorConstantForBinop(BinaryOperator::BinaryOps Opcode, Constant *In, bool IsRHSConstant)
Some binary operators require special handling to avoid poison and undefined behavior.
bool increasesLength() const
Return true if this shuffle returns a vector with a greater number of elements than its source vector...
static void commuteShuffleMask(MutableArrayRef< int > Mask, unsigned InVecNumElts)
Change values in a shuffle permute mask assuming the two vector operands of length InVecNumElts have ...
Instruction * foldSelectShuffle(ShuffleVectorInst &Shuf)
Try to fold shuffles that are the equivalent of a vector select.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
This is an important base class in LLVM.
static UnaryOperator * CreateFNegFMF(Value *Op, Instruction *FMFSource, const Twine &Name="", Instruction *InsertBefore=nullptr)
This instruction compares its operands according to the predicate given to the constructor.
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
This class represents a truncation of integer types.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static void replaceExtractElements(InsertElementInst *InsElt, ExtractElementInst *ExtElt, InstCombinerImpl &IC)
If we have insertion into a vector that is wider than the vector that we are extracting from,...
ConstantInt * getPreferredVectorIndex(ConstantInt *IndexC)
Given a constant index for a extractelement or insertelement instruction, return it with the canonica...
'undef' values are things that do not have specified contents.
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static Constant * getShl(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.
static Instruction * canonicalizeInsertSplat(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
If we have an insert of a scalar to a non-zero element of an undefined vector and then shuffle that v...
iterator find(const_arg_type_t< KeyT > Val)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
uint64_t getArrayNumElements() const
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
These are the ingredients in an alternate form binary operator as described below.
SimplifyQuery getWithInstruction(Instruction *I) const
static bool collectSingleShuffleElements(Value *V, Value *LHS, Value *RHS, SmallVectorImpl< int > &Mask)
If V is a shuffle of values that ONLY returns elements from either LHS or RHS, return the shuffle mas...
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
static Instruction * foldIdentityExtractShuffle(ShuffleVectorInst &Shuf)
Try to fold an extract subvector operation.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Class for arbitrary precision integers.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
CastClass_match< OpTy, Instruction::SExt > m_SExt(const OpTy &Op)
Matches SExt.
specific_intval< false > m_SpecificInt(APInt V)
Match a specific integer value or vector with all elements equal to the value.
match_combine_and< LTy, RTy > m_CombineAnd(const LTy &L, const RTy &R)
Combine two pattern matchers matching L && R.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
SmallVector< MachineOperand, 4 > Cond
static Value * evaluateInDifferentElementOrder(Value *V, ArrayRef< int > Mask)
auto m_Undef()
Match an arbitrary undef constant.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Type * getType() const
All values are typed, get the type of this value.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
self_iterator getIterator()
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Instruction * visitInsertValueInst(InsertValueInst &IV)
Try to find redundant insertvalue instructions, like the following ones: %0 = insertvalue { i8,...
bool isExact() const
Determine whether the exact flag is set.
bool pred_empty(const BasicBlock *BB)
static Constant * get(ArrayRef< Constant * > V)
static Instruction * foldCastShuffle(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
Canonicalize casts after shuffle.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
StringRef getName() const
Return a constant reference to the value's name.
llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, 4, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
const T & front() const
front - Get the first element.
static APInt findDemandedEltsBySingleUser(Value *V, Instruction *UserInstr)
Find elements of V demanded by UserInstr.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Instruction * visitInsertElementInst(InsertElementInst &IE)
CastClass_match< OpTy, Instruction::FPExt > m_FPExt(const OpTy &Op)
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Predicate getPredicate(unsigned Condition, unsigned Hint)
Return predicate consisting of specified condition and hint bits.
static Instruction * foldTruncShuffle(ShuffleVectorInst &Shuf, bool IsBigEndian)
Convert a narrowing shuffle of a bitcasted vector into a vector truncate.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Instruction * InsertNewInstWith(Instruction *New, Instruction &Old)
Same as InsertNewInstBefore, but also sets the debug loc.
static Instruction * narrowVectorSelect(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
Match a shuffle-select-shuffle pattern where the shuffles are widening and narrowing (concatenating w...
static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI, ArrayRef< int > Mask)
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static Instruction * hoistInsEltConst(InsertElementInst &InsElt2, InstCombiner::BuilderTy &Builder)
If we have an insertelement instruction feeding into another insertelement and the 2nd is inserting a...
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
constexpr unsigned BitWidth
static Instruction * foldInsEltIntoSplat(InsertElementInst &InsElt)
Try to fold an insert element into an existing splat shuffle by changing the shuffle's mask to includ...
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
A wrapper class for inspecting calls to intrinsic functions.
static CmpInst * Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name="", Instruction *InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate and the two operands.
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
VectorType * getType() const
Overload to return most specific vector type.
static bool canEvaluateShuffled(Value *V, ArrayRef< int > Mask, unsigned Depth=5)
Return true if we can evaluate the specified expression tree if the vector elements were shuffled in ...
This instruction constructs a fixed permutation of two input vectors.
static bool cheapToScalarize(Value *V, Value *EI)
Return true if the value is cheaper to scalarize than it is to leave as a vector operation.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
static APInt findDemandedEltsByAllUsers(Value *V)
Find union of elements of V demanded by all its users.
const Value * DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) const
Translate PHI node to its predecessor from the given basic block.
bool hasNoSignedWrap() const
Determine whether the no signed wrap flag is set.
const BasicBlock * getParent() const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
size_t size() const
size - Get the array size.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
unsigned getActiveBits() const
Compute the number of active bits in the value.
static Instruction * foldFNegShuffle(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
Canonicalize FP negate after shuffle.
Constant * getSplatValue(bool AllowUndefs=false) const
If all elements of the vector constant have the same value, return that value.
void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
static const uint32_t IV[8]
static Instruction * foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf)
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
static BinopElts getAlternateBinop(BinaryOperator *BO, const DataLayout &DL)
Binops may be transformed into binops with different opcodes and operands.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Value * getOperand(unsigned i) const
static Instruction * foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf)
Value * simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef< int > Mask, Type *RetTy, const SimplifyQuery &Q)
Given operands for a ShuffleVectorInst, fold the result or return null.
static ConstantAggregateZero * get(Type *Ty)
This instruction inserts a struct field of array element value into an aggregate value.
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
constexpr T value_or(U &&alt) const &
@ NotFound
Sentinel value for when no action was found in the specified table.
LLVM Value Representation.
BinopElts(BinaryOperator::BinaryOps Opc=(BinaryOperator::BinaryOps) 0, Value *V0=nullptr, Value *V1=nullptr)
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
iterator_range< user_iterator > users()
void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
bool changesLength() const
Return true if this shuffle returns a vector with a different number of elements than its source vect...
A Use represents the edge between a Value definition and its users.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
reference emplace_back(ArgTypes &&... Args)
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
void commute()
Swap the operands and adjust the mask to preserve the semantics of the instruction.