44#define DEBUG_TYPE "instcombine"
47using namespace PatternMatch;
50 "Number of aggregate reconstructions turned into reuse of the "
51 "original aggregate");
62 if (
auto *
C = dyn_cast<Constant>(V))
63 return CEI ||
C->getSplatValue();
65 if (CEI &&
match(V, m_Intrinsic<Intrinsic::experimental_stepvector>())) {
66 ElementCount EC = cast<VectorType>(V->getType())->getElementCount();
108 for (
auto *U : PN->
users()) {
114 }
else if (!PHIUser) {
115 PHIUser = cast<Instruction>(U);
128 !(isa<BinaryOperator>(PHIUser)) ||
142 if (PHIInVal == PHIUser) {
147 unsigned opId = (B0->
getOperand(0) == PN) ? 1 : 0;
160 Instruction *pos = dyn_cast<Instruction>(PHIInVal);
162 if (pos && !isa<PHINode>(pos)) {
174 for (
auto *
E : Extracts) {
191 cast<VectorType>(
Ext.getVectorOperandType())->getElementCount();
198 if (
X->getType()->isIntegerTy()) {
199 assert(isa<FixedVectorType>(
Ext.getVectorOperand()->getType()) &&
200 "Expected fixed vector type for bitcast from scalar integer");
207 unsigned ShiftAmountC = ExtIndexC * DestWidth;
209 (isDesirableIntType(
X->getType()->getPrimitiveSizeInBits()) &&
210 Ext.getVectorOperand()->hasOneUse())) {
222 if (!
X->getType()->isVectorTy())
228 auto *SrcTy = cast<VectorType>(
X->getType());
230 if (NumSrcElts == NumElts)
235 "Src and Dst must be the same sort of vector type");
251 unsigned NarrowingRatio =
254 if (ExtIndexC / NarrowingRatio != InsIndexC) {
260 if (
X->hasOneUse() &&
Ext.getVectorOperand()->hasOneUse()) {
278 unsigned Chunk = ExtIndexC % NarrowingRatio;
280 Chunk = NarrowingRatio - 1 - Chunk;
285 bool NeedSrcBitcast = SrcTy->getScalarType()->isFloatingPointTy();
287 if (NeedSrcBitcast && NeedDestBitcast)
290 unsigned SrcWidth = SrcTy->getScalarSizeInBits();
291 unsigned ShAmt = Chunk * DestWidth;
296 if (!
X->hasOneUse() || !
Ext.getVectorOperand()->hasOneUse())
297 if (NeedSrcBitcast || NeedDestBitcast)
300 if (NeedSrcBitcast) {
307 if (!
Ext.getVectorOperand()->hasOneUse())
312 if (NeedDestBitcast) {
324 unsigned VWidth = cast<FixedVectorType>(V->getType())->getNumElements();
330 case Instruction::ExtractElement: {
334 if (EEIIndexC && EEIIndexC->
getValue().
ult(VWidth)) {
339 case Instruction::ShuffleVector: {
341 unsigned MaskNumElts =
342 cast<FixedVectorType>(UserInstr->
getType())->getNumElements();
344 UsedElts =
APInt(VWidth, 0);
345 for (
unsigned i = 0; i < MaskNumElts; i++) {
347 if (MaskVal == -1u || MaskVal >= 2 * VWidth)
349 if (Shuffle->
getOperand(0) == V && (MaskVal < VWidth))
352 ((MaskVal >= VWidth) && (MaskVal < 2 * VWidth)))
353 UsedElts.
setBit(MaskVal - VWidth);
368 unsigned VWidth = cast<FixedVectorType>(V->getType())->getNumElements();
370 APInt UnionUsedElts(VWidth, 0);
371 for (
const Use &U : V->uses()) {
372 if (
Instruction *
I = dyn_cast<Instruction>(U.getUser())) {
383 return UnionUsedElts;
414 if (SI->getCondition()->getType()->isIntegerTy() &&
421 auto *IndexC = dyn_cast<ConstantInt>(
Index);
428 unsigned NumElts = EC.getKnownMinValue();
434 if (IID == Intrinsic::experimental_stepvector &&
435 IndexC->getValue().ult(NumElts)) {
441 if (IndexC->getValue().getActiveBits() <=
BitWidth)
451 if (!EC.isScalable() && IndexC->getValue().uge(NumElts))
459 if (
auto *Phi = dyn_cast<PHINode>(SrcVec))
460 if (
Instruction *ScalarPHI = scalarizePHI(EI, Phi))
493 if (
auto *
I = dyn_cast<Instruction>(SrcVec)) {
494 if (
auto *IE = dyn_cast<InsertElementInst>(
I)) {
498 if (isa<Constant>(IE->getOperand(2)) && IndexC)
500 }
else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(
I)) {
501 auto *VecType = cast<VectorType>(
GEP->getType());
503 uint64_t IdxVal = IndexC ? IndexC->getZExtValue() : 0;
504 if (IndexC && IdxVal < EC.getKnownMinValue() &&
GEP->hasOneUse()) {
515 return isa<VectorType>(V->getType());
517 if (VectorOps == 1) {
518 Value *NewPtr =
GEP->getPointerOperand();
519 if (isa<VectorType>(NewPtr->
getType()))
523 for (
unsigned I = 1;
I !=
GEP->getNumOperands(); ++
I) {
525 if (isa<VectorType>(
Op->getType()))
532 GEP->getSourceElementType(), NewPtr, NewOps);
537 }
else if (
auto *SVI = dyn_cast<ShuffleVectorInst>(
I)) {
541 if (isa<FixedVectorType>(SVI->getType()) && isa<ConstantInt>(
Index)) {
543 SVI->getMaskValue(cast<ConstantInt>(
Index)->getZExtValue());
545 unsigned LHSWidth = cast<FixedVectorType>(SVI->getOperand(0)->getType())
550 if (SrcIdx < (
int)LHSWidth)
551 Src = SVI->getOperand(0);
554 Src = SVI->getOperand(1);
560 }
else if (
auto *CI = dyn_cast<CastInst>(
I)) {
564 if (CI->hasOneUse() && (CI->getOpcode() != Instruction::BitCast)) {
576 unsigned NumElts = EC.getKnownMinValue();
580 if (!EC.isScalable() && NumElts != 1) {
584 APInt UndefElts(NumElts, 0);
585 APInt DemandedElts(NumElts, 0);
586 DemandedElts.
setBit(IndexC->getZExtValue());
595 APInt UndefElts(NumElts, 0);
597 SrcVec, DemandedElts, UndefElts, 0 ,
617 "Invalid CollectSingleShuffleElements");
618 unsigned NumElts = cast<FixedVectorType>(V->getType())->getNumElements();
621 Mask.assign(NumElts, -1);
626 for (
unsigned i = 0; i != NumElts; ++i)
632 for (
unsigned i = 0; i != NumElts; ++i)
633 Mask.push_back(i + NumElts);
639 Value *VecOp = IEI->getOperand(0);
640 Value *ScalarOp = IEI->getOperand(1);
641 Value *IdxOp = IEI->getOperand(2);
643 if (!isa<ConstantInt>(IdxOp))
645 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
647 if (isa<PoisonValue>(ScalarOp)) {
652 Mask[InsertedIdx] = -1;
657 unsigned ExtractedIdx =
658 cast<ConstantInt>(EI->
getOperand(1))->getZExtValue();
659 unsigned NumLHSElts =
660 cast<FixedVectorType>(
LHS->
getType())->getNumElements();
669 Mask[InsertedIdx % NumElts] = ExtractedIdx;
672 Mask[InsertedIdx % NumElts] = ExtractedIdx + NumLHSElts;
690 auto *InsVecType = cast<FixedVectorType>(InsElt->
getType());
692 unsigned NumInsElts = InsVecType->getNumElements();
693 unsigned NumExtElts = ExtVecType->getNumElements();
696 if (InsVecType->getElementType() != ExtVecType->getElementType() ||
697 NumExtElts >= NumInsElts)
705 for (
unsigned i = 0; i < NumExtElts; ++i)
707 for (
unsigned i = NumExtElts; i < NumInsElts; ++i)
711 auto *ExtVecOpInst = dyn_cast<Instruction>(ExtVecOp);
712 BasicBlock *InsertionBlock = (ExtVecOpInst && !isa<PHINode>(ExtVecOpInst))
725 if (InsertionBlock != InsElt->
getParent())
742 if (ExtVecOpInst && !isa<PHINode>(ExtVecOpInst))
743 WideVec->insertAfter(ExtVecOpInst);
777 assert(V->getType()->isVectorTy() &&
"Invalid shuffle!");
778 unsigned NumElts = cast<FixedVectorType>(V->getType())->getNumElements();
781 Mask.assign(NumElts, -1);
782 return std::make_pair(
786 if (isa<ConstantAggregateZero>(V)) {
787 Mask.assign(NumElts, 0);
788 return std::make_pair(V,
nullptr);
793 Value *VecOp = IEI->getOperand(0);
794 Value *ScalarOp = IEI->getOperand(1);
795 Value *IdxOp = IEI->getOperand(2);
798 if (isa<ConstantInt>(EI->
getOperand(1)) && isa<ConstantInt>(IdxOp)) {
799 unsigned ExtractedIdx =
800 cast<ConstantInt>(EI->
getOperand(1))->getZExtValue();
801 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
805 if (EI->
getOperand(0) == PermittedRHS || PermittedRHS ==
nullptr) {
808 assert(LR.second ==
nullptr || LR.second ==
RHS);
818 for (
unsigned i = 0; i < NumElts; ++i)
820 return std::make_pair(V,
nullptr);
823 unsigned NumLHSElts =
824 cast<FixedVectorType>(
RHS->
getType())->getNumElements();
825 Mask[InsertedIdx % NumElts] = NumLHSElts + ExtractedIdx;
826 return std::make_pair(LR.first,
RHS);
829 if (VecOp == PermittedRHS) {
832 unsigned NumLHSElts =
835 for (
unsigned i = 0; i != NumElts; ++i)
836 Mask.push_back(i == InsertedIdx ? ExtractedIdx : NumLHSElts + i);
837 return std::make_pair(EI->
getOperand(0), PermittedRHS);
845 return std::make_pair(EI->
getOperand(0), PermittedRHS);
851 for (
unsigned i = 0; i != NumElts; ++i)
853 return std::make_pair(V,
nullptr);
879 assert(NumAggElts > 0 &&
"Aggregate should have elements.");
883 static constexpr auto NotFound = std::nullopt;
884 static constexpr auto FoundMismatch =
nullptr;
891 auto KnowAllElts = [&AggElts]() {
899 static const int DepthLimit = 2 * NumAggElts;
904 Depth < DepthLimit && CurrIVI && !KnowAllElts();
905 CurrIVI = dyn_cast<InsertValueInst>(CurrIVI->getAggregateOperand()),
907 auto *InsertedValue =
908 dyn_cast<Instruction>(CurrIVI->getInsertedValueOperand());
915 if (Indices.
size() != 1)
921 std::optional<Instruction *> &Elt = AggElts[Indices.
front()];
922 Elt = Elt.value_or(InsertedValue);
935 enum class AggregateDescription {
951 auto Describe = [](std::optional<Value *> SourceAggregate) {
952 if (SourceAggregate == NotFound)
953 return AggregateDescription::NotFound;
954 if (*SourceAggregate == FoundMismatch)
955 return AggregateDescription::FoundMismatch;
956 return AggregateDescription::Found;
964 auto FindSourceAggregate =
965 [&](
Instruction *Elt,
unsigned EltIdx, std::optional<BasicBlock *> UseBB,
966 std::optional<BasicBlock *> PredBB) -> std::optional<Value *> {
973 auto *EVI = dyn_cast_or_null<ExtractValueInst>(Elt);
977 Value *SourceAggregate = EVI->getAggregateOperand();
980 if (SourceAggregate->
getType() != AggTy)
981 return FoundMismatch;
983 if (EVI->getNumIndices() != 1 || EltIdx != EVI->getIndices().front())
984 return FoundMismatch;
986 return SourceAggregate;
992 auto FindCommonSourceAggregate =
993 [&](std::optional<BasicBlock *> UseBB,
994 std::optional<BasicBlock *> PredBB) -> std::optional<Value *> {
995 std::optional<Value *> SourceAggregate;
998 assert(Describe(SourceAggregate) != AggregateDescription::FoundMismatch &&
999 "We don't store nullptr in SourceAggregate!");
1000 assert((Describe(SourceAggregate) == AggregateDescription::Found) ==
1002 "SourceAggregate should be valid after the first element,");
1007 std::optional<Value *> SourceAggregateForElement =
1008 FindSourceAggregate(*
I.value(),
I.index(), UseBB, PredBB);
1015 if (Describe(SourceAggregateForElement) != AggregateDescription::Found)
1016 return SourceAggregateForElement;
1020 switch (Describe(SourceAggregate)) {
1021 case AggregateDescription::NotFound:
1023 SourceAggregate = SourceAggregateForElement;
1025 case AggregateDescription::Found:
1028 if (*SourceAggregateForElement != *SourceAggregate)
1029 return FoundMismatch;
1031 case AggregateDescription::FoundMismatch:
1036 assert(Describe(SourceAggregate) == AggregateDescription::Found &&
1037 "Must be a valid Value");
1038 return *SourceAggregate;
1041 std::optional<Value *> SourceAggregate;
1044 SourceAggregate = FindCommonSourceAggregate(std::nullopt,
1046 if (Describe(SourceAggregate) != AggregateDescription::NotFound) {
1047 if (Describe(SourceAggregate) == AggregateDescription::FoundMismatch)
1049 ++NumAggregateReconstructionsSimplified;
1062 for (
const std::optional<Instruction *> &
I : AggElts) {
1086 static const int PredCountLimit = 64;
1093 if (Preds.
size() >= PredCountLimit)
1103 std::pair<
decltype(SourceAggregates)::iterator,
bool>
IV =
1104 SourceAggregates.
insert({Pred,
nullptr});
1112 SourceAggregate = FindCommonSourceAggregate(UseBB, Pred);
1113 if (Describe(SourceAggregate) != AggregateDescription::Found)
1115 IV.first->second = *SourceAggregate;
1124 Builder.SetInsertPoint(UseBB, UseBB->getFirstNonPHIIt());
1126 Builder.CreatePHI(AggTy, Preds.size(), OrigIVI.getName() +
".merged");
1128 PHI->addIncoming(SourceAggregates[Pred], Pred);
1130 ++NumAggregateReconstructionsSimplified;
1131 return replaceInstUsesWith(OrigIVI,
PHI);
1143 I.getAggregateOperand(),
I.getInsertedValueOperand(),
I.getIndices(),
1147 bool IsRedundant =
false;
1156 while (V->hasOneUse() &&
Depth < 10) {
1157 User *U = V->user_back();
1158 auto UserInsInst = dyn_cast<InsertValueInst>(U);
1159 if (!UserInsInst || U->getOperand(0) != V)
1161 if (UserInsInst->getIndices() == FirstIndices) {
1189 if (MaskSize != VecSize)
1194 for (
int i = 0; i != MaskSize; ++i) {
1196 if (Elt != -1 && Elt != i && Elt != i + VecSize)
1215 if (isa<ScalableVectorType>(VecTy))
1217 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1221 if (NumElements == 1)
1236 auto *NextIE = dyn_cast<InsertElementInst>(CurrIE->
getOperand(0));
1240 if (CurrIE != &InsElt &&
1241 (!CurrIE->
hasOneUse() && (NextIE !=
nullptr || !
Idx->isZero())))
1244 ElementPresent[
Idx->getZExtValue()] =
true;
1250 if (FirstIE == &InsElt)
1258 if (!ElementPresent.
all())
1265 if (!cast<ConstantInt>(FirstIE->
getOperand(2))->isZero())
1270 for (
unsigned i = 0; i != NumElements; ++i)
1271 if (!ElementPresent[i])
1281 auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0));
1282 if (!Shuf || !Shuf->isZeroEltSplat())
1287 if (isa<ScalableVectorType>(Shuf->getType()))
1297 Value *Op0 = Shuf->getOperand(0);
1305 unsigned NumMaskElts =
1306 cast<FixedVectorType>(Shuf->getType())->getNumElements();
1308 for (
unsigned i = 0; i != NumMaskElts; ++i)
1309 NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);
1318 auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0));
1320 !(Shuf->isIdentityWithExtract() || Shuf->isIdentityWithPadding()))
1325 if (isa<ScalableVectorType>(Shuf->getType()))
1336 Value *
X = Shuf->getOperand(0);
1344 unsigned NumMaskElts =
1345 cast<FixedVectorType>(Shuf->getType())->getNumElements();
1348 for (
unsigned i = 0; i != NumMaskElts; ++i) {
1351 NewMask[i] = OldMask[i];
1352 }
else if (OldMask[i] == (
int)IdxC) {
1358 "Unexpected shuffle mask element for identity shuffle");
1377 auto *InsElt1 = dyn_cast<InsertElementInst>(InsElt2.
getOperand(0));
1378 if (!InsElt1 || !InsElt1->hasOneUse())
1385 match(InsElt1->getOperand(1),
m_Value(
Y)) && !isa<Constant>(
Y) &&
1389 Value *NewInsElt1 =
Builder.CreateInsertElement(
X, ScalarC, IdxC2);
1399 auto *Inst = dyn_cast<Instruction>(InsElt.
getOperand(0));
1402 if (!Inst || !Inst->hasOneUse())
1404 if (
auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0))) {
1407 Constant *ShufConstVec, *InsEltScalar;
1431 unsigned NumElts = Mask.size();
1434 for (
unsigned I = 0;
I != NumElts; ++
I) {
1435 if (
I == InsEltIndex) {
1436 NewShufElts[
I] = InsEltScalar;
1437 NewMaskElts[
I] = InsEltIndex + NumElts;
1441 NewMaskElts[
I] = Mask[
I];
1445 if (!NewShufElts[
I])
1453 }
else if (
auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
1458 if (isa<ScalableVectorType>(InsElt.
getType()))
1461 cast<FixedVectorType>(InsElt.
getType())->getNumElements();
1472 auto ValI = std::begin(Val);
1479 Mask[
I] = NumElts +
I;
1484 for (
unsigned I = 0;
I < NumElts; ++
I) {
1516 CastOpcode = Instruction::FPExt;
1518 CastOpcode = Instruction::SExt;
1520 CastOpcode = Instruction::ZExt;
1525 if (
X->getType()->getScalarType() !=
Y->getType())
1552 auto *VTy = dyn_cast<FixedVectorType>(InsElt.
getType());
1553 Value *Scalar0, *BaseVec;
1555 if (!VTy || (VTy->getNumElements() & 1) ||
1564 if (Index0 + 1 != Index1 || Index0 & 1)
1581 Type *SrcTy =
X->getType();
1583 unsigned VecEltWidth = VTy->getScalarSizeInBits();
1584 if (ScalarWidth != VecEltWidth * 2 || ShAmt != VecEltWidth)
1589 Value *CastBaseVec =
Builder.CreateBitCast(BaseVec, CastTy);
1593 uint64_t NewIndex = IsBigEndian ? Index1 / 2 : Index0 / 2;
1594 Value *NewInsert =
Builder.CreateInsertElement(CastBaseVec,
X, NewIndex);
1599 Value *VecOp = IE.getOperand(0);
1600 Value *ScalarOp = IE.getOperand(1);
1601 Value *IdxOp = IE.getOperand(2);
1608 if (
auto *IndexC = dyn_cast<ConstantInt>(IdxOp)) {
1612 Value *BaseVec, *OtherScalar;
1617 !isa<Constant>(OtherScalar) && OtherIndexVal > IndexC->getZExtValue()) {
1648 cast<VectorType>(VecSrc->
getType())->getElementType() ==
1660 uint64_t InsertedIdx, ExtractedIdx;
1662 if (isa<FixedVectorType>(IE.getType()) &&
1666 isa<FixedVectorType>(ExtVecOp->
getType()) &&
1668 cast<FixedVectorType>(ExtVecOp->
getType())->getNumElements()) {
1684 if (!Insert.hasOneUse())
1686 auto *InsertUser = dyn_cast<InsertElementInst>(Insert.user_back());
1693 if (isShuffleRootCandidate(IE)) {
1704 if (LR.first != &IE && LR.second != &IE) {
1706 if (LR.second ==
nullptr)
1714 if (
auto VecTy = dyn_cast<FixedVectorType>(VecOp->
getType())) {
1715 unsigned VWidth = VecTy->getNumElements();
1716 APInt UndefElts(VWidth, 0);
1738 return IdentityShuf;
1752 unsigned Depth = 5) {
1754 if (isa<Constant>(V))
1759 if (!
I)
return false;
1762 if (!
I->hasOneUse())
1765 if (
Depth == 0)
return false;
1767 switch (
I->getOpcode()) {
1768 case Instruction::UDiv:
1769 case Instruction::SDiv:
1770 case Instruction::URem:
1771 case Instruction::SRem:
1778 case Instruction::Add:
1779 case Instruction::FAdd:
1780 case Instruction::Sub:
1781 case Instruction::FSub:
1782 case Instruction::Mul:
1783 case Instruction::FMul:
1784 case Instruction::FDiv:
1785 case Instruction::FRem:
1786 case Instruction::Shl:
1787 case Instruction::LShr:
1788 case Instruction::AShr:
1789 case Instruction::And:
1790 case Instruction::Or:
1791 case Instruction::Xor:
1792 case Instruction::ICmp:
1793 case Instruction::FCmp:
1794 case Instruction::Trunc:
1795 case Instruction::ZExt:
1796 case Instruction::SExt:
1797 case Instruction::FPToUI:
1798 case Instruction::FPToSI:
1799 case Instruction::UIToFP:
1800 case Instruction::SIToFP:
1801 case Instruction::FPTrunc:
1802 case Instruction::FPExt:
1803 case Instruction::GetElementPtr: {
1806 Type *ITy =
I->getType();
1808 Mask.size() > cast<FixedVectorType>(ITy)->getNumElements())
1810 for (
Value *Operand :
I->operands()) {
1816 case Instruction::InsertElement: {
1817 ConstantInt *CI = dyn_cast<ConstantInt>(
I->getOperand(2));
1818 if (!CI)
return false;
1823 bool SeenOnce =
false;
1824 for (
int I : Mask) {
1825 if (
I == ElementNumber) {
1842 switch (
I->getOpcode()) {
1843 case Instruction::Add:
1844 case Instruction::FAdd:
1845 case Instruction::Sub:
1846 case Instruction::FSub:
1847 case Instruction::Mul:
1848 case Instruction::FMul:
1849 case Instruction::UDiv:
1850 case Instruction::SDiv:
1851 case Instruction::FDiv:
1852 case Instruction::URem:
1853 case Instruction::SRem:
1854 case Instruction::FRem:
1855 case Instruction::Shl:
1856 case Instruction::LShr:
1857 case Instruction::AShr:
1858 case Instruction::And:
1859 case Instruction::Or:
1860 case Instruction::Xor: {
1862 assert(NewOps.
size() == 2 &&
"binary operator with #ops != 2");
1864 NewOps[0], NewOps[1]);
1865 if (
auto *NewI = dyn_cast<Instruction>(New)) {
1866 if (isa<OverflowingBinaryOperator>(BO)) {
1870 if (isa<PossiblyExactOperator>(BO)) {
1871 NewI->setIsExact(BO->
isExact());
1873 if (isa<FPMathOperator>(BO))
1874 NewI->copyFastMathFlags(
I);
1878 case Instruction::ICmp:
1879 assert(NewOps.
size() == 2 &&
"icmp with #ops != 2");
1880 return Builder.CreateICmp(cast<ICmpInst>(
I)->getPredicate(), NewOps[0],
1882 case Instruction::FCmp:
1883 assert(NewOps.
size() == 2 &&
"fcmp with #ops != 2");
1884 return Builder.CreateFCmp(cast<FCmpInst>(
I)->getPredicate(), NewOps[0],
1886 case Instruction::Trunc:
1887 case Instruction::ZExt:
1888 case Instruction::SExt:
1889 case Instruction::FPToUI:
1890 case Instruction::FPToSI:
1891 case Instruction::UIToFP:
1892 case Instruction::SIToFP:
1893 case Instruction::FPTrunc:
1894 case Instruction::FPExt: {
1898 I->getType()->getScalarType(),
1899 cast<VectorType>(NewOps[0]->getType())->getElementCount());
1900 assert(NewOps.
size() == 1 &&
"cast with #ops != 1");
1904 case Instruction::GetElementPtr: {
1907 return Builder.CreateGEP(cast<GEPOperator>(
I)->getSourceElementType(),
1909 cast<GEPOperator>(
I)->isInBounds());
1919 assert(V->getType()->isVectorTy() &&
"can't reorder non-vector elements");
1920 Type *EltTy = V->getType()->getScalarType();
1924 if (isa<ConstantAggregateZero>(V))
1927 if (
Constant *
C = dyn_cast<Constant>(V))
1932 switch (
I->getOpcode()) {
1933 case Instruction::Add:
1934 case Instruction::FAdd:
1935 case Instruction::Sub:
1936 case Instruction::FSub:
1937 case Instruction::Mul:
1938 case Instruction::FMul:
1939 case Instruction::UDiv:
1940 case Instruction::SDiv:
1941 case Instruction::FDiv:
1942 case Instruction::URem:
1943 case Instruction::SRem:
1944 case Instruction::FRem:
1945 case Instruction::Shl:
1946 case Instruction::LShr:
1947 case Instruction::AShr:
1948 case Instruction::And:
1949 case Instruction::Or:
1950 case Instruction::Xor:
1951 case Instruction::ICmp:
1952 case Instruction::FCmp:
1953 case Instruction::Trunc:
1954 case Instruction::ZExt:
1955 case Instruction::SExt:
1956 case Instruction::FPToUI:
1957 case Instruction::FPToSI:
1958 case Instruction::UIToFP:
1959 case Instruction::SIToFP:
1960 case Instruction::FPTrunc:
1961 case Instruction::FPExt:
1962 case Instruction::Select:
1963 case Instruction::GetElementPtr: {
1967 cast<FixedVectorType>(
I->getType())->getNumElements());
1968 for (
int i = 0, e =
I->getNumOperands(); i != e; ++i) {
1973 if (
I->getOperand(i)->getType()->isVectorTy())
1976 V =
I->getOperand(i);
1978 NeedsRebuild |= (V !=
I->getOperand(i));
1984 case Instruction::InsertElement: {
1985 int Element = cast<ConstantInt>(
I->getOperand(2))->getLimitedValue();
1992 for (
int e = Mask.size();
Index != e; ++
Index) {
1993 if (Mask[
Index] == Element) {
2007 return Builder.CreateInsertElement(V,
I->getOperand(1),
Index);
2023 unsigned MaskElems = Mask.size();
2024 unsigned BegIdx = Mask.front();
2025 unsigned EndIdx = Mask.back();
2026 if (BegIdx > EndIdx || EndIdx >= LHSElems || EndIdx - BegIdx != MaskElems - 1)
2028 for (
unsigned I = 0;
I != MaskElems; ++
I)
2029 if (
static_cast<unsigned>(Mask[
I]) != BegIdx +
I)
2042 Opcode(Opc), Op0(V0), Op1(V1) {}
2043 operator bool()
const {
return Opcode != 0; }
2054 case Instruction::Shl: {
2059 return {Instruction::Mul, BO0, ShlOne};
2063 case Instruction::Or: {
2067 return {Instruction::Add, BO0, BO1};
2070 case Instruction::Sub:
2085 assert(Shuf.
isSelect() &&
"Must have select-equivalent shuffle");
2090 unsigned NumElts = Mask.size();
2093 auto *ShufOp = dyn_cast<ShuffleVectorInst>(Op0);
2094 if (ShufOp && ShufOp->isSelect() &&
2095 (ShufOp->getOperand(0) == Op1 || ShufOp->getOperand(1) == Op1)) {
2100 ShufOp = dyn_cast<ShuffleVectorInst>(Op1);
2101 if (!ShufOp || !ShufOp->isSelect() ||
2102 (ShufOp->getOperand(0) != Op0 && ShufOp->getOperand(1) != Op0))
2105 Value *
X = ShufOp->getOperand(0), *
Y = ShufOp->getOperand(1);
2107 ShufOp->getShuffleMask(Mask1);
2108 assert(Mask1.
size() == NumElts &&
"Vector size changed with select shuffle");
2121 for (
unsigned i = 0; i != NumElts; ++i)
2122 NewMask[i] = Mask[i] < (
signed)NumElts ? Mask[i] : Mask1[i];
2127 "Unexpected shuffle mask");
2132 assert(Shuf.
isSelect() &&
"Must have select-equivalent shuffle");
2149 auto *BO = cast<BinaryOperator>(Op0IsBinop ? Op0 : Op1);
2163 bool MightCreatePoisonOrUB =
2166 if (MightCreatePoisonOrUB)
2171 Value *
X = Op0IsBinop ? Op1 : Op0;
2208 unsigned NumMaskElts =
2209 cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2211 for (
unsigned i = 0; i != NumMaskElts; ++i)
2213 NewMask[i] = Mask[i];
2225 unsigned NumElts = cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2249 Constant *C0 =
nullptr, *C1 =
nullptr;
2250 bool ConstantsAreOp1;
2253 ConstantsAreOp1 =
false;
2258 ConstantsAreOp1 =
true;
2265 bool DropNSW =
false;
2266 if (ConstantsAreOp1 && Opc0 != Opc1) {
2270 if (Opc0 == Instruction::Shl || Opc1 == Instruction::Shl)
2273 assert(isa<Constant>(AltB0.Op1) &&
"Expecting constant with alt binop");
2274 Opc0 = AltB0.Opcode;
2275 C0 = cast<Constant>(AltB0.Op1);
2277 assert(isa<Constant>(AltB1.Op1) &&
"Expecting constant with alt binop");
2278 Opc1 = AltB1.Opcode;
2279 C1 = cast<Constant>(AltB1.Op1);
2283 if (Opc0 != Opc1 || !C0 || !C1)
2296 bool MightCreatePoisonOrUB =
2299 if (MightCreatePoisonOrUB)
2322 if (MightCreatePoisonOrUB && !ConstantsAreOp1)
2343 if (
auto *NewI = dyn_cast<Instruction>(NewBO)) {
2344 NewI->copyIRFlags(B0);
2345 NewI->andIRFlags(B1);
2347 NewI->setHasNoSignedWrap(
false);
2349 NewI->dropPoisonGeneratingFlags();
2368 Type *SrcType =
X->getType();
2370 cast<FixedVectorType>(SrcType)->getNumElements() !=
2371 cast<FixedVectorType>(DestType)->getNumElements() ||
2376 "Expected a shuffle that decreases length");
2383 for (
unsigned i = 0, e = Mask.size(); i != e; ++i) {
2386 uint64_t LSBIndex = IsBigEndian ? (i + 1) * TruncRatio - 1 : i * TruncRatio;
2387 assert(LSBIndex <= INT32_MAX &&
"Overflowed 32-bits");
2388 if (Mask[i] != (
int)LSBIndex)
2414 unsigned NarrowNumElts =
2415 cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2418 cast<FixedVectorType>(NarrowCond->
getType())->getNumElements() !=
2420 !cast<ShuffleVectorInst>(
Cond)->isIdentityWithPadding())
2433 auto *S0 = dyn_cast<Instruction>(Shuf.
getOperand(0));
2438 bool IsFNeg = S0->getOpcode() == Instruction::FNeg;
2448 Intrinsic::fabs, Shuf.
getType());
2455 auto *S1 = dyn_cast<Instruction>(Shuf.
getOperand(1));
2458 S0->getOpcode() != S1->getOpcode() ||
2459 (!S0->hasOneUse() && !S1->hasOneUse()))
2466 NewF = UnaryOperator::CreateFNeg(NewShuf);
2469 Intrinsic::fabs, Shuf.
getType());
2481 auto *Cast0 = dyn_cast<CastInst>(Shuf.
getOperand(0));
2482 auto *Cast1 = dyn_cast<CastInst>(Shuf.
getOperand(1));
2483 if (!Cast0 || !Cast1 || Cast0->getOpcode() != Cast1->getOpcode() ||
2484 Cast0->getSrcTy() != Cast1->getSrcTy())
2490 switch (CastOpcode) {
2491 case Instruction::FPToSI:
2492 case Instruction::FPToUI:
2493 case Instruction::SIToFP:
2494 case Instruction::UIToFP:
2502 VectorType *CastSrcTy = cast<VectorType>(Cast0->getSrcTy());
2505 if (ShufTy->getElementCount().getKnownMinValue() >
2506 ShufOpTy->getElementCount().getKnownMinValue())
2510 assert(isa<FixedVectorType>(CastSrcTy) && isa<FixedVectorType>(ShufOpTy) &&
2511 "Expected fixed vector operands for casts and binary shuffle");
2512 if (CastSrcTy->getPrimitiveSizeInBits() > ShufOpTy->getPrimitiveSizeInBits())
2516 if (!Cast0->hasOneUse() && !Cast1->hasOneUse())
2520 Value *
X = Cast0->getOperand(0);
2521 Value *
Y = Cast1->getOperand(0);
2536 X->getType()->getPrimitiveSizeInBits() ==
2562 unsigned NumElts = cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2564 assert(NumElts < Mask.size() &&
2565 "Identity with extract must have less elements than its inputs");
2567 for (
unsigned i = 0; i != NumElts; ++i) {
2569 int MaskElt = Mask[i];
2570 NewMask[i] = ExtractMaskElt ==
PoisonMaskElem ? ExtractMaskElt : MaskElt;
2583 int NumElts = Mask.size();
2584 int InpNumElts = cast<FixedVectorType>(V0->
getType())->getNumElements();
2609 if (NumElts != InpNumElts)
2613 auto isShufflingScalarIntoOp1 = [&](
Value *&Scalar,
ConstantInt *&IndexC) {
2621 int NewInsIndex = -1;
2622 for (
int i = 0; i != NumElts; ++i) {
2628 if (Mask[i] == NumElts + i)
2632 if (NewInsIndex != -1 || Mask[i] != IndexC->getSExtValue())
2639 assert(NewInsIndex != -1 &&
"Did not fold shuffle with unused operand?");
2651 if (isShufflingScalarIntoOp1(Scalar, IndexC))
2659 if (isShufflingScalarIntoOp1(Scalar, IndexC))
2669 auto *Shuffle0 = dyn_cast<ShuffleVectorInst>(Shuf.
getOperand(0));
2670 auto *Shuffle1 = dyn_cast<ShuffleVectorInst>(Shuf.
getOperand(1));
2671 if (!Shuffle0 || !Shuffle0->isIdentityWithPadding() ||
2672 !Shuffle1 || !Shuffle1->isIdentityWithPadding())
2680 Value *
X = Shuffle0->getOperand(0);
2681 Value *
Y = Shuffle1->getOperand(0);
2682 if (
X->getType() !=
Y->getType() ||
2685 cast<FixedVectorType>(Shuffle0->getType())->getNumElements()) ||
2686 !
isPowerOf2_32(cast<FixedVectorType>(
X->getType())->getNumElements()) ||
2691 "Unexpected operand for identity shuffle");
2697 int NarrowElts = cast<FixedVectorType>(
X->getType())->getNumElements();
2698 int WideElts = cast<FixedVectorType>(Shuffle0->getType())->getNumElements();
2699 assert(WideElts > NarrowElts &&
"Unexpected types for identity with padding");
2703 for (
int i = 0, e = Mask.size(); i != e; ++i) {
2709 if (Mask[i] < WideElts) {
2710 if (Shuffle0->getMaskValue(Mask[i]) == -1)
2713 if (Shuffle1->getMaskValue(Mask[i] - WideElts) == -1)
2720 if (Mask[i] < WideElts) {
2721 assert(Mask[i] < NarrowElts &&
"Unexpected shuffle mask");
2722 NewMask[i] = Mask[i];
2724 assert(Mask[i] < (WideElts + NarrowElts) &&
"Unexpected shuffle mask");
2725 NewMask[i] = Mask[i] - (WideElts - NarrowElts);
2747 if (
X->getType() !=
Y->getType())
2750 auto *BinOp = cast<BinaryOperator>(Op0);
2755 if (
auto NewBOI = dyn_cast<Instruction>(NewBO))
2756 NewBOI->copyIRFlags(BinOp);
2775 unsigned VWidth = cast<FixedVectorType>(SVI.
getType())->getNumElements();
2776 unsigned LHSWidth = cast<FixedVectorType>(
LHS->
getType())->getNumElements();
2787 X->getType()->isVectorTy() &&
X->getType() ==
Y->getType() &&
2788 X->getType()->getScalarSizeInBits() ==
2805 X->getType()->isVectorTy() && VWidth == LHSWidth) {
2807 auto *XType = cast<FixedVectorType>(
X->getType());
2808 unsigned XNumElts = XType->getNumElements();
2810 if (XNumElts >= VWidth) {
2811 assert(XNumElts % VWidth == 0 &&
"Unexpected vector bitcast");
2814 assert(VWidth % XNumElts == 0 &&
"Unexpected vector bitcast");
2818 if (!ScaledMask.
empty()) {
2822 ScaledMask, XType, ShufQuery))
2830 "Shuffle with 2 undef ops not simplified?");
2858 APInt UndefElts(VWidth, 0);
2910 bool MadeChange =
false;
2913 unsigned MaskElems = Mask.size();
2914 auto *SrcTy = cast<FixedVectorType>(V->getType());
2915 unsigned VecBitWidth = SrcTy->getPrimitiveSizeInBits().getFixedValue();
2917 assert(SrcElemBitWidth &&
"vector elements must have a bitwidth");
2918 unsigned SrcNumElems = SrcTy->getNumElements();
2923 if (!BC->use_empty())
2927 unsigned BegIdx = Mask.front();
2928 Type *TgtTy = BC->getDestTy();
2930 if (!TgtElemBitWidth)
2932 unsigned TgtNumElems = VecBitWidth / TgtElemBitWidth;
2933 bool VecBitWidthsEqual = VecBitWidth == TgtNumElems * TgtElemBitWidth;
2934 bool BegIsAligned = 0 == ((SrcElemBitWidth * BegIdx) % TgtElemBitWidth);
2935 if (!VecBitWidthsEqual)
2940 if (!BegIsAligned) {
2944 for (
unsigned I = 0,
E = MaskElems,
Idx = BegIdx;
I !=
E; ++
Idx, ++
I)
2945 ShuffleMask[
I] =
Idx;
2950 unsigned SrcElemsPerTgtElem = TgtElemBitWidth / SrcElemBitWidth;
2951 assert(SrcElemsPerTgtElem);
2952 BegIdx /= SrcElemsPerTgtElem;
2953 bool BCAlreadyExists = NewBCs.
contains(CastSrcTy);
2958 if (!BCAlreadyExists)
2959 NewBCs[CastSrcTy] = NewBC;
3016 LHSShuffle =
nullptr;
3019 RHSShuffle =
nullptr;
3020 if (!LHSShuffle && !RHSShuffle)
3021 return MadeChange ? &SVI :
nullptr;
3023 Value* LHSOp0 =
nullptr;
3024 Value* LHSOp1 =
nullptr;
3025 Value* RHSOp0 =
nullptr;
3026 unsigned LHSOp0Width = 0;
3027 unsigned RHSOp0Width = 0;
3031 LHSOp0Width = cast<FixedVectorType>(LHSOp0->
getType())->getNumElements();
3035 RHSOp0Width = cast<FixedVectorType>(RHSOp0->
getType())->getNumElements();
3046 else if (LHSOp0Width == LHSWidth) {
3051 if (RHSShuffle && RHSOp0Width == LHSWidth) {
3055 if (LHSOp0 == RHSOp0) {
3060 if (newLHS ==
LHS && newRHS ==
RHS)
3061 return MadeChange ? &SVI :
nullptr;
3067 if (RHSShuffle && newRHS !=
RHS)
3070 unsigned newLHSWidth = (newLHS !=
LHS) ? LHSOp0Width : LHSWidth;
3076 for (
unsigned i = 0; i < VWidth; ++i) {
3081 }
else if (Mask[i] < (
int)LHSWidth) {
3086 if (newLHS !=
LHS) {
3087 eltMask = LHSMask[Mask[i]];
3090 if (eltMask >= (
int)LHSOp0Width && isa<PoisonValue>(LHSOp1))
3103 else if (newRHS !=
RHS) {
3104 eltMask = RHSMask[Mask[i]-LHSWidth];
3107 if (eltMask >= (
int)RHSOp0Width) {
3109 "should have been check above");
3113 eltMask = Mask[i]-LHSWidth;
3121 if (eltMask >= 0 && newRHS !=
nullptr && newLHS != newRHS)
3122 eltMask += newLHSWidth;
3127 if (SplatElt >= 0 && SplatElt != eltMask)
3137 if (
isSplat || newMask == LHSMask || newMask == RHSMask || newMask == Mask) {
3143 return MadeChange ? &SVI :
nullptr;
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file provides internal interfaces used to implement the InstCombine.
static Instruction * foldConstantInsEltIntoShuffle(InsertElementInst &InsElt)
insertelt (shufflevector X, CVec, Mask|insertelt X, C1, CIndex1), C, CIndex --> shufflevector X,...
static Value * evaluateInDifferentElementOrder(Value *V, ArrayRef< int > Mask, IRBuilderBase &Builder)
static bool collectSingleShuffleElements(Value *V, Value *LHS, Value *RHS, SmallVectorImpl< int > &Mask)
If V is a shuffle of values that ONLY returns elements from either LHS or RHS, return the shuffle mas...
static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl< int > &Mask, Value *PermittedRHS, InstCombinerImpl &IC, bool &Rerun)
static APInt findDemandedEltsByAllUsers(Value *V)
Find union of elements of V demanded by all its users.
static Instruction * foldTruncInsEltPair(InsertElementInst &InsElt, bool IsBigEndian, InstCombiner::BuilderTy &Builder)
If we are inserting 2 halves of a value into adjacent elements of a vector, try to convert to a singl...
static Instruction * foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf)
static Instruction * foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf)
static Instruction * foldIdentityExtractShuffle(ShuffleVectorInst &Shuf)
Try to fold an extract subvector operation.
static Instruction * foldInsEltIntoSplat(InsertElementInst &InsElt)
Try to fold an insert element into an existing splat shuffle by changing the shuffle's mask to includ...
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Instruction * foldShuffleWithInsert(ShuffleVectorInst &Shuf, InstCombinerImpl &IC)
Try to replace a shuffle with an insertelement or try to replace a shuffle operand with the operand o...
static Instruction * canonicalizeInsertSplat(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
If we have an insert of a scalar to a non-zero element of an undefined vector and then shuffle that v...
static Instruction * foldTruncShuffle(ShuffleVectorInst &Shuf, bool IsBigEndian)
Convert a narrowing shuffle of a bitcasted vector into a vector truncate.
static bool replaceExtractElements(InsertElementInst *InsElt, ExtractElementInst *ExtElt, InstCombinerImpl &IC)
If we have insertion into a vector that is wider than the vector that we are extracting from,...
static bool cheapToScalarize(Value *V, Value *EI)
Return true if the value is cheaper to scalarize than it is to leave as a vector operation.
static Value * buildNew(Instruction *I, ArrayRef< Value * > NewOps, IRBuilderBase &Builder)
Rebuild a new instruction just like 'I' but with the new operands given.
static bool canEvaluateShuffled(Value *V, ArrayRef< int > Mask, unsigned Depth=5)
Return true if we can evaluate the specified expression tree if the vector elements were shuffled in ...
static Instruction * foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf)
A select shuffle of a select shuffle with a shared operand can be reduced to a single select shuffle.
static Instruction * hoistInsEltConst(InsertElementInst &InsElt2, InstCombiner::BuilderTy &Builder)
If we have an insertelement instruction feeding into another insertelement and the 2nd is inserting a...
static APInt findDemandedEltsBySingleUser(Value *V, Instruction *UserInstr)
Find elements of V demanded by UserInstr.
static Instruction * foldShuffleOfUnaryOps(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
Canonicalize FP negate/abs after shuffle.
static Instruction * foldCastShuffle(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
Canonicalize casts after shuffle.
static Instruction * narrowInsElt(InsertElementInst &InsElt, InstCombiner::BuilderTy &Builder)
If both the base vector and the inserted element are extended from the same type, do the insert eleme...
static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf)
static Instruction * foldInsSequenceIntoSplat(InsertElementInst &InsElt)
Turn a chain of inserts that splats a value into an insert + shuffle: insertelt(insertelt(insertelt(i...
static Instruction * foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt)
Try to fold an extract+insert element into an existing identity shuffle by changing the shuffle's mas...
static ConstantInt * getPreferredVectorIndex(ConstantInt *IndexC)
Given a constant index for a extractelement or insertelement instruction, return it with the canonica...
static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI, ArrayRef< int > Mask)
static BinopElts getAlternateBinop(BinaryOperator *BO, const DataLayout &DL)
Binops may be transformed into binops with different opcodes and operands.
This file provides the interface for the instcombine pass implementation.
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the SmallBitVector class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
If both arms of a vector select are concatenated vectors, split the select, and concatenate the resul...
static const uint32_t IV[8]
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name=Twine(), Instruction *InsertBefore=nullptr)
Construct a binary instruction, given the opcode and the two operands.
BinaryOps getOpcode() const
static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Value *CopyO, const Twine &Name="", Instruction *InsertBefore=nullptr)
This class represents a no-op cast from one type to another.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static CmpInst * Create(OtherOps Op, Predicate predicate, Value *S1, Value *S2, const Twine &Name="", Instruction *InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate and the two operands.
static ConstantAggregateZero * get(Type *Ty)
static Constant * getShuffleVector(Constant *V1, Constant *V2, ArrayRef< int > Mask, Type *OnlyIfReducedTy=nullptr)
static Constant * getShl(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)
Return the identity constant for a binary opcode.
This is the shared class of boolean and integer constants.
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
void setIsInBounds(bool b=true)
Set or clear the inbounds flag on this GEP instruction.
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Common base class shared among various IRBuilders.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
static InsertElementInst * Create(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
This instruction inserts a struct field of array element value into an aggregate value.
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &UndefElts, unsigned Depth=0, bool AllowMultipleUsers=false) override
The specified value produces a vector with any number of elements.
Instruction * foldSelectShuffle(ShuffleVectorInst &Shuf)
Try to fold shuffles that are the equivalent of a vector select.
Instruction * visitInsertValueInst(InsertValueInst &IV)
Try to find redundant insertvalue instructions, like the following ones: %0 = insertvalue { i8,...
Instruction * visitInsertElementInst(InsertElementInst &IE)
Instruction * visitExtractElementInst(ExtractElementInst &EI)
Instruction * simplifyBinOpSplats(ShuffleVectorInst &SVI)
Instruction * foldAggregateConstructionIntoAggregateReuse(InsertValueInst &OrigIVI)
Look for chain of insertvalue's that fully define an aggregate, and trace back the values inserted,...
Instruction * visitShuffleVectorInst(ShuffleVectorInst &SVI)
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
void addToWorklist(Instruction *I)
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
static Constant * getSafeVectorConstantForBinop(BinaryOperator::BinaryOps Opcode, Constant *In, bool IsRHSConstant)
Some binary operators require special handling to avoid poison and undefined behavior.
void addValue(Value *V)
Add value to the worklist if it is an instruction.
bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
void andIRFlags(const Value *V)
Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction.
void setFastMathFlags(FastMathFlags FMF)
Convenience function for setting multiple fast-math flags on this instruction, which must be an opera...
const BasicBlock * getParent() const
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
bool isExact() const LLVM_READONLY
Determine whether the exact flag is set.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
unsigned getBitWidth() const
Get the number of bits in this IntegerType.
A wrapper class for inspecting calls to intrinsic functions.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
This instruction constructs a fixed permutation of two input vectors.
bool changesLength() const
Return true if this shuffle returns a vector with a different number of elements than its source vect...
static bool isSelectMask(ArrayRef< int > Mask)
Return true if this shuffle mask chooses elements from its source vectors without lane crossings.
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
static bool isIdentityMask(ArrayRef< int > Mask)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
VectorType * getType() const
Overload to return most specific vector type.
bool increasesLength() const
Return true if this shuffle returns a vector with a greater number of elements than its source vector...
bool isIdentityWithExtract() const
Return true if this shuffle extracts the first N elements of exactly one source vector.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
bool isSelect() const
Return true if this shuffle chooses elements from its source vectors without lane crossings and all o...
static void commuteShuffleMask(MutableArrayRef< int > Mask, unsigned InVecNumElts)
Change values in a shuffle permute mask assuming the two vector operands of length InVecNumElts have ...
void commute()
Swap the operands and adjust the mask to preserve the semantics of the instruction.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool all() const
Returns true if all bits are set.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
unsigned getStructNumElements() const
uint64_t getArrayNumElements() const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeID getTypeID() const
Return the type id for the type.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
static UnaryOperator * CreateFNegFMF(Value *Op, Instruction *FMFSource, const Twine &Name="", Instruction *InsertBefore=nullptr)
UnaryOps getOpcode() const
static UnaryOperator * CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO, const Twine &Name="", Instruction *InsertBefore=nullptr)
'undef' values are things that do not have specified contents.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const Value * DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) const
Translate PHI node to its predecessor from the given basic block.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Type * getElementType() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
class_match< PoisonValue > m_Poison()
Match an arbitrary poison constant.
specific_intval< false > m_SpecificInt(APInt V)
Match a specific integer value or vector with all elements equal to the value.
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
CastClass_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
CastClass_match< OpTy, Instruction::SExt > m_SExt(const OpTy &Op)
Matches SExt.
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
CastClass_match< OpTy, Instruction::ZExt > m_ZExt(const OpTy &Op)
Matches ZExt.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
OneUse_match< T > m_OneUse(const T &SubPattern)
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
CastClass_match< OpTy, Instruction::Trunc > m_Trunc(const OpTy &Op)
Matches Trunc.
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
CastClass_match< OpTy, Instruction::FPExt > m_FPExt(const OpTy &Op)
class_match< UnaryOperator > m_UnOp()
Match an arbitrary unary operation and ignore it.
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
This is an optimization pass for GlobalISel generic memory operations.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Value * simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef< int > Mask, Type *RetTy, const SimplifyQuery &Q)
Given operands for a ShuffleVectorInst, fold the result or return null.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Value * simplifyInsertValueInst(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const SimplifyQuery &Q)
Given operands for an InsertValueInst, fold the result or return null.
constexpr int PoisonMaskElem
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
Value * simplifyInsertElementInst(Value *Vec, Value *Elt, Value *Idx, const SimplifyQuery &Q)
Given operands for an InsertElement, fold the result or return null.
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool pred_empty(const BasicBlock *BB)
bool MaskedValueIsZero(const Value *V, const APInt &Mask, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if 'V & Mask' is known to be zero.
Value * simplifyExtractElementInst(Value *Vec, Value *Idx, const SimplifyQuery &Q)
Given operands for an ExtractElementInst, fold the result or return null.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
These are the ingredients in an alternate form binary operator as described below.
BinopElts(BinaryOperator::BinaryOps Opc=(BinaryOperator::BinaryOps) 0, Value *V0=nullptr, Value *V1=nullptr)
BinaryOperator::BinaryOps Opcode
SimplifyQuery getWithInstruction(Instruction *I) const