44#define DEBUG_TYPE "instcombine"
47using namespace PatternMatch;
50 "Number of aggregate reconstructions turned into reuse of the "
51 "original aggregate");
62 if (
auto *
C = dyn_cast<Constant>(V))
63 return CEI ||
C->getSplatValue();
65 if (CEI &&
match(V, m_Intrinsic<Intrinsic::experimental_stepvector>())) {
66 ElementCount EC = cast<VectorType>(V->getType())->getElementCount();
108 for (
auto *U : PN->
users()) {
114 }
else if (!PHIUser) {
115 PHIUser = cast<Instruction>(U);
128 !(isa<BinaryOperator>(PHIUser)) ||
142 if (PHIInVal == PHIUser) {
147 unsigned opId = (B0->
getOperand(0) == PN) ? 1 : 0;
160 Instruction *pos = dyn_cast<Instruction>(PHIInVal);
162 if (pos && !isa<PHINode>(pos)) {
174 for (
auto *E : Extracts) {
191 cast<VectorType>(
Ext.getVectorOperandType())->getElementCount();
198 if (
X->getType()->isIntegerTy()) {
199 assert(isa<FixedVectorType>(
Ext.getVectorOperand()->getType()) &&
200 "Expected fixed vector type for bitcast from scalar integer");
207 unsigned ShiftAmountC = ExtIndexC * DestWidth;
209 (isDesirableIntType(
X->getType()->getPrimitiveSizeInBits()) &&
210 Ext.getVectorOperand()->hasOneUse())) {
222 if (!
X->getType()->isVectorTy())
228 auto *SrcTy = cast<VectorType>(
X->getType());
230 if (NumSrcElts == NumElts)
235 "Src and Dst must be the same sort of vector type");
251 unsigned NarrowingRatio =
254 if (ExtIndexC / NarrowingRatio != InsIndexC) {
260 if (
X->hasOneUse() &&
Ext.getVectorOperand()->hasOneUse()) {
278 unsigned Chunk = ExtIndexC % NarrowingRatio;
280 Chunk = NarrowingRatio - 1 - Chunk;
285 bool NeedSrcBitcast = SrcTy->getScalarType()->isFloatingPointTy();
287 if (NeedSrcBitcast && NeedDestBitcast)
290 unsigned SrcWidth = SrcTy->getScalarSizeInBits();
291 unsigned ShAmt = Chunk * DestWidth;
296 if (!
X->hasOneUse() || !
Ext.getVectorOperand()->hasOneUse())
297 if (NeedSrcBitcast || NeedDestBitcast)
300 if (NeedSrcBitcast) {
307 if (!
Ext.getVectorOperand()->hasOneUse())
312 if (NeedDestBitcast) {
324 unsigned VWidth = cast<FixedVectorType>(V->getType())->getNumElements();
330 case Instruction::ExtractElement: {
334 if (EEIIndexC && EEIIndexC->
getValue().
ult(VWidth)) {
339 case Instruction::ShuffleVector: {
341 unsigned MaskNumElts =
342 cast<FixedVectorType>(UserInstr->
getType())->getNumElements();
344 UsedElts =
APInt(VWidth, 0);
345 for (
unsigned i = 0; i < MaskNumElts; i++) {
347 if (MaskVal == -1u || MaskVal >= 2 * VWidth)
349 if (Shuffle->
getOperand(0) == V && (MaskVal < VWidth))
352 ((MaskVal >= VWidth) && (MaskVal < 2 * VWidth)))
353 UsedElts.
setBit(MaskVal - VWidth);
368 unsigned VWidth = cast<FixedVectorType>(V->getType())->getNumElements();
370 APInt UnionUsedElts(VWidth, 0);
371 for (
const Use &U : V->uses()) {
372 if (
Instruction *
I = dyn_cast<Instruction>(U.getUser())) {
383 return UnionUsedElts;
414 if (SI->getCondition()->getType()->isIntegerTy() &&
421 auto *IndexC = dyn_cast<ConstantInt>(
Index);
428 unsigned NumElts = EC.getKnownMinValue();
434 if (IID == Intrinsic::experimental_stepvector &&
435 IndexC->getValue().ult(NumElts)) {
441 if (IndexC->getValue().getActiveBits() <=
BitWidth)
442 Idx = ConstantInt::get(Ty, IndexC->getValue().zextOrTrunc(
BitWidth));
451 if (!EC.isScalable() && IndexC->getValue().uge(NumElts))
459 if (
auto *Phi = dyn_cast<PHINode>(SrcVec))
460 if (
Instruction *ScalarPHI = scalarizePHI(EI, Phi))
490 CmpInst *SrcCmpInst = cast<CmpInst>(SrcVec);
495 if (
auto *
I = dyn_cast<Instruction>(SrcVec)) {
496 if (
auto *IE = dyn_cast<InsertElementInst>(
I)) {
500 if (isa<Constant>(IE->getOperand(2)) && IndexC)
502 }
else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(
I)) {
503 auto *VecType = cast<VectorType>(
GEP->getType());
505 uint64_t IdxVal = IndexC ? IndexC->getZExtValue() : 0;
506 if (IndexC && IdxVal < EC.getKnownMinValue() &&
GEP->hasOneUse()) {
517 return isa<VectorType>(V->getType());
519 if (VectorOps == 1) {
520 Value *NewPtr =
GEP->getPointerOperand();
521 if (isa<VectorType>(NewPtr->
getType()))
525 for (
unsigned I = 1;
I !=
GEP->getNumOperands(); ++
I) {
527 if (isa<VectorType>(
Op->getType()))
534 GEP->getSourceElementType(), NewPtr, NewOps);
539 }
else if (
auto *SVI = dyn_cast<ShuffleVectorInst>(
I)) {
543 if (isa<FixedVectorType>(SVI->getType()) && isa<ConstantInt>(
Index)) {
545 SVI->getMaskValue(cast<ConstantInt>(
Index)->getZExtValue());
547 unsigned LHSWidth = cast<FixedVectorType>(SVI->getOperand(0)->getType())
552 if (SrcIdx < (
int)LHSWidth)
553 Src = SVI->getOperand(0);
556 Src = SVI->getOperand(1);
560 Src, ConstantInt::get(Int64Ty, SrcIdx,
false));
562 }
else if (
auto *CI = dyn_cast<CastInst>(
I)) {
566 if (CI->hasOneUse() && (CI->getOpcode() != Instruction::BitCast)) {
578 unsigned NumElts = EC.getKnownMinValue();
582 if (!EC.isScalable() && NumElts != 1) {
586 APInt PoisonElts(NumElts, 0);
587 APInt DemandedElts(NumElts, 0);
588 DemandedElts.
setBit(IndexC->getZExtValue());
597 APInt PoisonElts(NumElts, 0);
599 SrcVec, DemandedElts, PoisonElts, 0 ,
619 "Invalid CollectSingleShuffleElements");
620 unsigned NumElts = cast<FixedVectorType>(V->getType())->getNumElements();
623 Mask.assign(NumElts, -1);
628 for (
unsigned i = 0; i != NumElts; ++i)
634 for (
unsigned i = 0; i != NumElts; ++i)
635 Mask.push_back(i + NumElts);
641 Value *VecOp = IEI->getOperand(0);
642 Value *ScalarOp = IEI->getOperand(1);
643 Value *IdxOp = IEI->getOperand(2);
645 if (!isa<ConstantInt>(IdxOp))
647 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
649 if (isa<PoisonValue>(ScalarOp)) {
654 Mask[InsertedIdx] = -1;
659 unsigned ExtractedIdx =
660 cast<ConstantInt>(EI->
getOperand(1))->getZExtValue();
661 unsigned NumLHSElts =
662 cast<FixedVectorType>(
LHS->
getType())->getNumElements();
671 Mask[InsertedIdx % NumElts] = ExtractedIdx;
674 Mask[InsertedIdx % NumElts] = ExtractedIdx + NumLHSElts;
692 auto *InsVecType = cast<FixedVectorType>(InsElt->
getType());
694 unsigned NumInsElts = InsVecType->getNumElements();
695 unsigned NumExtElts = ExtVecType->getNumElements();
698 if (InsVecType->getElementType() != ExtVecType->getElementType() ||
699 NumExtElts >= NumInsElts)
707 for (
unsigned i = 0; i < NumExtElts; ++i)
709 for (
unsigned i = NumExtElts; i < NumInsElts; ++i)
713 auto *ExtVecOpInst = dyn_cast<Instruction>(ExtVecOp);
714 BasicBlock *InsertionBlock = (ExtVecOpInst && !isa<PHINode>(ExtVecOpInst))
727 if (InsertionBlock != InsElt->
getParent())
744 if (ExtVecOpInst && !isa<PHINode>(ExtVecOpInst))
745 WideVec->insertAfter(ExtVecOpInst);
779 assert(V->getType()->isVectorTy() &&
"Invalid shuffle!");
780 unsigned NumElts = cast<FixedVectorType>(V->getType())->getNumElements();
783 Mask.assign(NumElts, -1);
784 return std::make_pair(
788 if (isa<ConstantAggregateZero>(V)) {
789 Mask.assign(NumElts, 0);
790 return std::make_pair(V,
nullptr);
795 Value *VecOp = IEI->getOperand(0);
796 Value *ScalarOp = IEI->getOperand(1);
797 Value *IdxOp = IEI->getOperand(2);
800 if (isa<ConstantInt>(EI->
getOperand(1)) && isa<ConstantInt>(IdxOp)) {
801 unsigned ExtractedIdx =
802 cast<ConstantInt>(EI->
getOperand(1))->getZExtValue();
803 unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
807 if (EI->
getOperand(0) == PermittedRHS || PermittedRHS ==
nullptr) {
810 assert(LR.second ==
nullptr || LR.second ==
RHS);
820 for (
unsigned i = 0; i < NumElts; ++i)
822 return std::make_pair(V,
nullptr);
825 unsigned NumLHSElts =
826 cast<FixedVectorType>(
RHS->
getType())->getNumElements();
827 Mask[InsertedIdx % NumElts] = NumLHSElts + ExtractedIdx;
828 return std::make_pair(LR.first,
RHS);
831 if (VecOp == PermittedRHS) {
834 unsigned NumLHSElts =
837 for (
unsigned i = 0; i != NumElts; ++i)
838 Mask.push_back(i == InsertedIdx ? ExtractedIdx : NumLHSElts + i);
839 return std::make_pair(EI->
getOperand(0), PermittedRHS);
847 return std::make_pair(EI->
getOperand(0), PermittedRHS);
853 for (
unsigned i = 0; i != NumElts; ++i)
855 return std::make_pair(V,
nullptr);
881 assert(NumAggElts > 0 &&
"Aggregate should have elements.");
885 static constexpr auto NotFound = std::nullopt;
886 static constexpr auto FoundMismatch =
nullptr;
893 auto KnowAllElts = [&AggElts]() {
901 static const int DepthLimit = 2 * NumAggElts;
906 Depth < DepthLimit && CurrIVI && !KnowAllElts();
907 CurrIVI = dyn_cast<InsertValueInst>(CurrIVI->getAggregateOperand()),
909 auto *InsertedValue =
910 dyn_cast<Instruction>(CurrIVI->getInsertedValueOperand());
917 if (Indices.
size() != 1)
923 std::optional<Instruction *> &Elt = AggElts[Indices.
front()];
924 Elt = Elt.value_or(InsertedValue);
937 enum class AggregateDescription {
953 auto Describe = [](std::optional<Value *> SourceAggregate) {
954 if (SourceAggregate == NotFound)
955 return AggregateDescription::NotFound;
956 if (*SourceAggregate == FoundMismatch)
957 return AggregateDescription::FoundMismatch;
958 return AggregateDescription::Found;
966 auto FindSourceAggregate =
967 [&](
Instruction *Elt,
unsigned EltIdx, std::optional<BasicBlock *> UseBB,
968 std::optional<BasicBlock *> PredBB) -> std::optional<Value *> {
975 auto *EVI = dyn_cast_or_null<ExtractValueInst>(Elt);
979 Value *SourceAggregate = EVI->getAggregateOperand();
982 if (SourceAggregate->
getType() != AggTy)
983 return FoundMismatch;
985 if (EVI->getNumIndices() != 1 || EltIdx != EVI->getIndices().front())
986 return FoundMismatch;
988 return SourceAggregate;
994 auto FindCommonSourceAggregate =
995 [&](std::optional<BasicBlock *> UseBB,
996 std::optional<BasicBlock *> PredBB) -> std::optional<Value *> {
997 std::optional<Value *> SourceAggregate;
1000 assert(Describe(SourceAggregate) != AggregateDescription::FoundMismatch &&
1001 "We don't store nullptr in SourceAggregate!");
1002 assert((Describe(SourceAggregate) == AggregateDescription::Found) ==
1004 "SourceAggregate should be valid after the first element,");
1009 std::optional<Value *> SourceAggregateForElement =
1010 FindSourceAggregate(*
I.value(),
I.index(), UseBB, PredBB);
1017 if (Describe(SourceAggregateForElement) != AggregateDescription::Found)
1018 return SourceAggregateForElement;
1022 switch (Describe(SourceAggregate)) {
1023 case AggregateDescription::NotFound:
1025 SourceAggregate = SourceAggregateForElement;
1027 case AggregateDescription::Found:
1030 if (*SourceAggregateForElement != *SourceAggregate)
1031 return FoundMismatch;
1033 case AggregateDescription::FoundMismatch:
1038 assert(Describe(SourceAggregate) == AggregateDescription::Found &&
1039 "Must be a valid Value");
1040 return *SourceAggregate;
1043 std::optional<Value *> SourceAggregate;
1046 SourceAggregate = FindCommonSourceAggregate(std::nullopt,
1048 if (Describe(SourceAggregate) != AggregateDescription::NotFound) {
1049 if (Describe(SourceAggregate) == AggregateDescription::FoundMismatch)
1051 ++NumAggregateReconstructionsSimplified;
1064 for (
const std::optional<Instruction *> &
I : AggElts) {
1088 static const int PredCountLimit = 64;
1095 if (Preds.
size() >= PredCountLimit)
1105 std::pair<
decltype(SourceAggregates)::iterator,
bool>
IV =
1106 SourceAggregates.
insert({Pred,
nullptr});
1114 SourceAggregate = FindCommonSourceAggregate(UseBB, Pred);
1115 if (Describe(SourceAggregate) != AggregateDescription::Found)
1117 IV.first->second = *SourceAggregate;
1126 Builder.SetInsertPoint(UseBB, UseBB->getFirstNonPHIIt());
1128 Builder.CreatePHI(AggTy, Preds.size(), OrigIVI.getName() +
".merged");
1130 PHI->addIncoming(SourceAggregates[Pred], Pred);
1132 ++NumAggregateReconstructionsSimplified;
1133 return replaceInstUsesWith(OrigIVI,
PHI);
1145 I.getAggregateOperand(),
I.getInsertedValueOperand(),
I.getIndices(),
1149 bool IsRedundant =
false;
1158 while (V->hasOneUse() &&
Depth < 10) {
1159 User *U = V->user_back();
1160 auto UserInsInst = dyn_cast<InsertValueInst>(U);
1161 if (!UserInsInst || U->getOperand(0) != V)
1163 if (UserInsInst->getIndices() == FirstIndices) {
1191 if (MaskSize != VecSize)
1196 for (
int i = 0; i != MaskSize; ++i) {
1198 if (Elt != -1 && Elt != i && Elt != i + VecSize)
1217 if (isa<ScalableVectorType>(VecTy))
1219 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements();
1223 if (NumElements == 1)
1238 auto *NextIE = dyn_cast<InsertElementInst>(CurrIE->
getOperand(0));
1242 if (CurrIE != &InsElt &&
1243 (!CurrIE->
hasOneUse() && (NextIE !=
nullptr || !
Idx->isZero())))
1246 ElementPresent[
Idx->getZExtValue()] =
true;
1252 if (FirstIE == &InsElt)
1260 if (!ElementPresent.
all())
1266 Constant *Zero = ConstantInt::get(Int64Ty, 0);
1267 if (!cast<ConstantInt>(FirstIE->
getOperand(2))->isZero())
1273 for (
unsigned i = 0; i != NumElements; ++i)
1274 if (!ElementPresent[i])
1284 auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0));
1285 if (!Shuf || !Shuf->isZeroEltSplat())
1290 if (isa<ScalableVectorType>(Shuf->getType()))
1300 Value *Op0 = Shuf->getOperand(0);
1308 unsigned NumMaskElts =
1309 cast<FixedVectorType>(Shuf->getType())->getNumElements();
1311 for (
unsigned i = 0; i != NumMaskElts; ++i)
1312 NewMask[i] = i == IdxC ? 0 : Shuf->getMaskValue(i);
1321 auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0));
1323 !(Shuf->isIdentityWithExtract() || Shuf->isIdentityWithPadding()))
1328 if (isa<ScalableVectorType>(Shuf->getType()))
1339 Value *
X = Shuf->getOperand(0);
1347 unsigned NumMaskElts =
1348 cast<FixedVectorType>(Shuf->getType())->getNumElements();
1351 for (
unsigned i = 0; i != NumMaskElts; ++i) {
1354 NewMask[i] = OldMask[i];
1355 }
else if (OldMask[i] == (
int)IdxC) {
1361 "Unexpected shuffle mask element for identity shuffle");
1380 auto *InsElt1 = dyn_cast<InsertElementInst>(InsElt2.
getOperand(0));
1381 if (!InsElt1 || !InsElt1->hasOneUse())
1388 match(InsElt1->getOperand(1),
m_Value(
Y)) && !isa<Constant>(
Y) &&
1402 auto *Inst = dyn_cast<Instruction>(InsElt.
getOperand(0));
1405 if (!Inst || !Inst->hasOneUse())
1407 if (
auto *Shuf = dyn_cast<ShuffleVectorInst>(InsElt.
getOperand(0))) {
1410 Constant *ShufConstVec, *InsEltScalar;
1434 unsigned NumElts = Mask.size();
1437 for (
unsigned I = 0;
I != NumElts; ++
I) {
1438 if (
I == InsEltIndex) {
1439 NewShufElts[
I] = InsEltScalar;
1440 NewMaskElts[
I] = InsEltIndex + NumElts;
1444 NewMaskElts[
I] = Mask[
I];
1448 if (!NewShufElts[
I])
1456 }
else if (
auto *IEI = dyn_cast<InsertElementInst>(Inst)) {
1461 if (isa<ScalableVectorType>(InsElt.
getType()))
1464 cast<FixedVectorType>(InsElt.
getType())->getNumElements();
1475 auto ValI = std::begin(Val);
1482 Mask[
I] = NumElts +
I;
1487 for (
unsigned I = 0;
I < NumElts; ++
I) {
1519 CastOpcode = Instruction::FPExt;
1521 CastOpcode = Instruction::SExt;
1523 CastOpcode = Instruction::ZExt;
1528 if (
X->getType()->getScalarType() !=
Y->getType())
1555 auto *VTy = dyn_cast<FixedVectorType>(InsElt.
getType());
1556 Value *Scalar0, *BaseVec;
1558 if (!VTy || (VTy->getNumElements() & 1) ||
1567 if (Index0 + 1 != Index1 || Index0 & 1)
1584 Type *SrcTy =
X->getType();
1586 unsigned VecEltWidth = VTy->getScalarSizeInBits();
1587 if (ScalarWidth != VecEltWidth * 2 || ShAmt != VecEltWidth)
1596 uint64_t NewIndex = IsBigEndian ? Index1 / 2 : Index0 / 2;
1602 Value *VecOp = IE.getOperand(0);
1603 Value *ScalarOp = IE.getOperand(1);
1604 Value *IdxOp = IE.getOperand(2);
1611 if (
auto *IndexC = dyn_cast<ConstantInt>(IdxOp)) {
1615 Value *BaseVec, *OtherScalar;
1620 !isa<Constant>(OtherScalar) && OtherIndexVal > IndexC->getZExtValue()) {
1652 cast<VectorType>(VecSrc->
getType())->getElementType() ==
1664 uint64_t InsertedIdx, ExtractedIdx;
1666 if (isa<FixedVectorType>(IE.getType()) &&
1670 isa<FixedVectorType>(ExtVecOp->
getType()) &&
1672 cast<FixedVectorType>(ExtVecOp->
getType())->getNumElements()) {
1688 if (!Insert.hasOneUse())
1690 auto *InsertUser = dyn_cast<InsertElementInst>(Insert.user_back());
1697 if (isShuffleRootCandidate(IE)) {
1708 if (LR.first != &IE && LR.second != &IE) {
1710 if (LR.second ==
nullptr)
1718 if (
auto VecTy = dyn_cast<FixedVectorType>(VecOp->
getType())) {
1719 unsigned VWidth = VecTy->getNumElements();
1720 APInt PoisonElts(VWidth, 0);
1743 return IdentityShuf;
1757 unsigned Depth = 5) {
1759 if (isa<Constant>(V))
1764 if (!
I)
return false;
1767 if (!
I->hasOneUse())
1770 if (
Depth == 0)
return false;
1772 switch (
I->getOpcode()) {
1773 case Instruction::UDiv:
1774 case Instruction::SDiv:
1775 case Instruction::URem:
1776 case Instruction::SRem:
1783 case Instruction::Add:
1784 case Instruction::FAdd:
1785 case Instruction::Sub:
1786 case Instruction::FSub:
1787 case Instruction::Mul:
1788 case Instruction::FMul:
1789 case Instruction::FDiv:
1790 case Instruction::FRem:
1791 case Instruction::Shl:
1792 case Instruction::LShr:
1793 case Instruction::AShr:
1794 case Instruction::And:
1795 case Instruction::Or:
1796 case Instruction::Xor:
1797 case Instruction::ICmp:
1798 case Instruction::FCmp:
1799 case Instruction::Trunc:
1800 case Instruction::ZExt:
1801 case Instruction::SExt:
1802 case Instruction::FPToUI:
1803 case Instruction::FPToSI:
1804 case Instruction::UIToFP:
1805 case Instruction::SIToFP:
1806 case Instruction::FPTrunc:
1807 case Instruction::FPExt:
1808 case Instruction::GetElementPtr: {
1811 Type *ITy =
I->getType();
1813 Mask.size() > cast<FixedVectorType>(ITy)->getNumElements())
1815 for (
Value *Operand :
I->operands()) {
1821 case Instruction::InsertElement: {
1822 ConstantInt *CI = dyn_cast<ConstantInt>(
I->getOperand(2));
1823 if (!CI)
return false;
1828 bool SeenOnce =
false;
1829 for (
int I : Mask) {
1830 if (
I == ElementNumber) {
1847 switch (
I->getOpcode()) {
1848 case Instruction::Add:
1849 case Instruction::FAdd:
1850 case Instruction::Sub:
1851 case Instruction::FSub:
1852 case Instruction::Mul:
1853 case Instruction::FMul:
1854 case Instruction::UDiv:
1855 case Instruction::SDiv:
1856 case Instruction::FDiv:
1857 case Instruction::URem:
1858 case Instruction::SRem:
1859 case Instruction::FRem:
1860 case Instruction::Shl:
1861 case Instruction::LShr:
1862 case Instruction::AShr:
1863 case Instruction::And:
1864 case Instruction::Or:
1865 case Instruction::Xor: {
1867 assert(NewOps.
size() == 2 &&
"binary operator with #ops != 2");
1869 NewOps[0], NewOps[1]);
1870 if (
auto *NewI = dyn_cast<Instruction>(New)) {
1871 if (isa<OverflowingBinaryOperator>(BO)) {
1875 if (isa<PossiblyExactOperator>(BO)) {
1876 NewI->setIsExact(BO->
isExact());
1878 if (isa<FPMathOperator>(BO))
1879 NewI->copyFastMathFlags(
I);
1883 case Instruction::ICmp:
1884 assert(NewOps.
size() == 2 &&
"icmp with #ops != 2");
1885 return Builder.
CreateICmp(cast<ICmpInst>(
I)->getPredicate(), NewOps[0],
1887 case Instruction::FCmp:
1888 assert(NewOps.
size() == 2 &&
"fcmp with #ops != 2");
1889 return Builder.
CreateFCmp(cast<FCmpInst>(
I)->getPredicate(), NewOps[0],
1891 case Instruction::Trunc:
1892 case Instruction::ZExt:
1893 case Instruction::SExt:
1894 case Instruction::FPToUI:
1895 case Instruction::FPToSI:
1896 case Instruction::UIToFP:
1897 case Instruction::SIToFP:
1898 case Instruction::FPTrunc:
1899 case Instruction::FPExt: {
1903 I->getType()->getScalarType(),
1904 cast<VectorType>(NewOps[0]->getType())->getElementCount());
1905 assert(NewOps.
size() == 1 &&
"cast with #ops != 1");
1909 case Instruction::GetElementPtr: {
1912 return Builder.
CreateGEP(cast<GEPOperator>(
I)->getSourceElementType(),
1914 cast<GEPOperator>(
I)->isInBounds());
1924 assert(V->getType()->isVectorTy() &&
"can't reorder non-vector elements");
1925 Type *EltTy = V->getType()->getScalarType();
1927 if (isa<PoisonValue>(V))
1933 if (isa<ConstantAggregateZero>(V))
1936 if (
Constant *
C = dyn_cast<Constant>(V))
1941 switch (
I->getOpcode()) {
1942 case Instruction::Add:
1943 case Instruction::FAdd:
1944 case Instruction::Sub:
1945 case Instruction::FSub:
1946 case Instruction::Mul:
1947 case Instruction::FMul:
1948 case Instruction::UDiv:
1949 case Instruction::SDiv:
1950 case Instruction::FDiv:
1951 case Instruction::URem:
1952 case Instruction::SRem:
1953 case Instruction::FRem:
1954 case Instruction::Shl:
1955 case Instruction::LShr:
1956 case Instruction::AShr:
1957 case Instruction::And:
1958 case Instruction::Or:
1959 case Instruction::Xor:
1960 case Instruction::ICmp:
1961 case Instruction::FCmp:
1962 case Instruction::Trunc:
1963 case Instruction::ZExt:
1964 case Instruction::SExt:
1965 case Instruction::FPToUI:
1966 case Instruction::FPToSI:
1967 case Instruction::UIToFP:
1968 case Instruction::SIToFP:
1969 case Instruction::FPTrunc:
1970 case Instruction::FPExt:
1971 case Instruction::Select:
1972 case Instruction::GetElementPtr: {
1976 cast<FixedVectorType>(
I->getType())->getNumElements());
1977 for (
int i = 0, e =
I->getNumOperands(); i != e; ++i) {
1982 if (
I->getOperand(i)->getType()->isVectorTy())
1985 V =
I->getOperand(i);
1987 NeedsRebuild |= (V !=
I->getOperand(i));
1993 case Instruction::InsertElement: {
1994 int Element = cast<ConstantInt>(
I->getOperand(2))->getLimitedValue();
2001 for (
int e = Mask.size();
Index != e; ++
Index) {
2002 if (Mask[
Index] == Element) {
2032 unsigned MaskElems = Mask.size();
2033 unsigned BegIdx = Mask.front();
2034 unsigned EndIdx = Mask.back();
2035 if (BegIdx > EndIdx || EndIdx >= LHSElems || EndIdx - BegIdx != MaskElems - 1)
2037 for (
unsigned I = 0;
I != MaskElems; ++
I)
2038 if (
static_cast<unsigned>(Mask[
I]) != BegIdx +
I)
2051 Opcode(Opc), Op0(V0), Op1(V1) {}
2052 operator bool()
const {
return Opcode != 0; }
2063 case Instruction::Shl: {
2068 return {Instruction::Mul, BO0, ShlOne};
2072 case Instruction::Or: {
2076 return {Instruction::Add, BO0, BO1};
2079 case Instruction::Sub:
2094 assert(Shuf.
isSelect() &&
"Must have select-equivalent shuffle");
2099 unsigned NumElts = Mask.size();
2102 auto *ShufOp = dyn_cast<ShuffleVectorInst>(Op0);
2103 if (ShufOp && ShufOp->isSelect() &&
2104 (ShufOp->getOperand(0) == Op1 || ShufOp->getOperand(1) == Op1)) {
2109 ShufOp = dyn_cast<ShuffleVectorInst>(Op1);
2110 if (!ShufOp || !ShufOp->isSelect() ||
2111 (ShufOp->getOperand(0) != Op0 && ShufOp->getOperand(1) != Op0))
2114 Value *
X = ShufOp->getOperand(0), *
Y = ShufOp->getOperand(1);
2116 ShufOp->getShuffleMask(Mask1);
2117 assert(Mask1.
size() == NumElts &&
"Vector size changed with select shuffle");
2130 for (
unsigned i = 0; i != NumElts; ++i)
2131 NewMask[i] = Mask[i] < (
signed)NumElts ? Mask[i] : Mask1[i];
2136 "Unexpected shuffle mask");
2142 assert(Shuf.
isSelect() &&
"Must have select-equivalent shuffle");
2159 auto *BO = cast<BinaryOperator>(Op0IsBinop ? Op0 : Op1);
2165 Value *
X = Op0IsBinop ? Op1 : Op0;
2186 bool MightCreatePoisonOrUB =
2189 if (MightCreatePoisonOrUB)
2230 unsigned NumMaskElts =
2231 cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2233 for (
unsigned i = 0; i != NumMaskElts; ++i)
2235 NewMask[i] = Mask[i];
2247 unsigned NumElts = cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2272 Constant *C0 =
nullptr, *C1 =
nullptr;
2273 bool ConstantsAreOp1;
2276 ConstantsAreOp1 =
false;
2281 ConstantsAreOp1 =
true;
2288 bool DropNSW =
false;
2289 if (ConstantsAreOp1 && Opc0 != Opc1) {
2293 if (Opc0 == Instruction::Shl || Opc1 == Instruction::Shl)
2296 assert(isa<Constant>(AltB0.Op1) &&
"Expecting constant with alt binop");
2297 Opc0 = AltB0.Opcode;
2298 C0 = cast<Constant>(AltB0.Op1);
2300 assert(isa<Constant>(AltB1.Op1) &&
"Expecting constant with alt binop");
2301 Opc1 = AltB1.Opcode;
2302 C1 = cast<Constant>(AltB1.Op1);
2306 if (Opc0 != Opc1 || !C0 || !C1)
2319 bool MightCreatePoisonOrUB =
2322 if (MightCreatePoisonOrUB)
2345 if (MightCreatePoisonOrUB && !ConstantsAreOp1)
2366 if (
auto *NewI = dyn_cast<Instruction>(NewBO)) {
2367 NewI->copyIRFlags(B0);
2368 NewI->andIRFlags(B1);
2370 NewI->setHasNoSignedWrap(
false);
2372 NewI->dropPoisonGeneratingFlags();
2391 Type *SrcType =
X->getType();
2393 cast<FixedVectorType>(SrcType)->getNumElements() !=
2394 cast<FixedVectorType>(DestType)->getNumElements() ||
2399 "Expected a shuffle that decreases length");
2406 for (
unsigned i = 0, e = Mask.size(); i != e; ++i) {
2409 uint64_t LSBIndex = IsBigEndian ? (i + 1) * TruncRatio - 1 : i * TruncRatio;
2410 assert(LSBIndex <= INT32_MAX &&
"Overflowed 32-bits");
2411 if (Mask[i] != (
int)LSBIndex)
2437 unsigned NarrowNumElts =
2438 cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2441 cast<FixedVectorType>(NarrowCond->
getType())->getNumElements() !=
2443 !cast<ShuffleVectorInst>(
Cond)->isIdentityWithPadding())
2456 auto *S0 = dyn_cast<Instruction>(Shuf.
getOperand(0));
2461 bool IsFNeg = S0->getOpcode() == Instruction::FNeg;
2471 Intrinsic::fabs, Shuf.
getType());
2481 S0->getOpcode() !=
S1->getOpcode() ||
2482 (!S0->hasOneUse() && !
S1->hasOneUse()))
2489 NewF = UnaryOperator::CreateFNeg(NewShuf);
2492 Intrinsic::fabs, Shuf.
getType());
2504 auto *Cast0 = dyn_cast<CastInst>(Shuf.
getOperand(0));
2505 auto *Cast1 = dyn_cast<CastInst>(Shuf.
getOperand(1));
2506 if (!Cast0 || !Cast1 || Cast0->getOpcode() != Cast1->getOpcode() ||
2507 Cast0->getSrcTy() != Cast1->getSrcTy())
2513 switch (CastOpcode) {
2514 case Instruction::FPToSI:
2515 case Instruction::FPToUI:
2516 case Instruction::SIToFP:
2517 case Instruction::UIToFP:
2525 VectorType *CastSrcTy = cast<VectorType>(Cast0->getSrcTy());
2528 if (ShufTy->getElementCount().getKnownMinValue() >
2529 ShufOpTy->getElementCount().getKnownMinValue())
2533 assert(isa<FixedVectorType>(CastSrcTy) && isa<FixedVectorType>(ShufOpTy) &&
2534 "Expected fixed vector operands for casts and binary shuffle");
2535 if (CastSrcTy->getPrimitiveSizeInBits() > ShufOpTy->getPrimitiveSizeInBits())
2539 if (!Cast0->hasOneUse() && !Cast1->hasOneUse())
2543 Value *
X = Cast0->getOperand(0);
2544 Value *
Y = Cast1->getOperand(0);
2559 X->getType()->getPrimitiveSizeInBits() ==
2585 unsigned NumElts = cast<FixedVectorType>(Shuf.
getType())->getNumElements();
2587 assert(NumElts < Mask.size() &&
2588 "Identity with extract must have less elements than its inputs");
2590 for (
unsigned i = 0; i != NumElts; ++i) {
2592 int MaskElt = Mask[i];
2593 NewMask[i] = ExtractMaskElt ==
PoisonMaskElem ? ExtractMaskElt : MaskElt;
2606 int NumElts = Mask.size();
2607 int InpNumElts = cast<FixedVectorType>(V0->
getType())->getNumElements();
2632 if (NumElts != InpNumElts)
2636 auto isShufflingScalarIntoOp1 = [&](
Value *&Scalar,
ConstantInt *&IndexC) {
2644 int NewInsIndex = -1;
2645 for (
int i = 0; i != NumElts; ++i) {
2651 if (Mask[i] == NumElts + i)
2655 if (NewInsIndex != -1 || Mask[i] != IndexC->getSExtValue())
2662 assert(NewInsIndex != -1 &&
"Did not fold shuffle with unused operand?");
2665 IndexC = ConstantInt::get(IndexC->getIntegerType(), NewInsIndex);
2674 if (isShufflingScalarIntoOp1(Scalar, IndexC))
2682 if (isShufflingScalarIntoOp1(Scalar, IndexC))
2692 auto *Shuffle0 = dyn_cast<ShuffleVectorInst>(Shuf.
getOperand(0));
2693 auto *Shuffle1 = dyn_cast<ShuffleVectorInst>(Shuf.
getOperand(1));
2694 if (!Shuffle0 || !Shuffle0->isIdentityWithPadding() ||
2695 !Shuffle1 || !Shuffle1->isIdentityWithPadding())
2703 Value *
X = Shuffle0->getOperand(0);
2704 Value *
Y = Shuffle1->getOperand(0);
2705 if (
X->getType() !=
Y->getType() ||
2708 cast<FixedVectorType>(Shuffle0->getType())->getNumElements()) ||
2709 !
isPowerOf2_32(cast<FixedVectorType>(
X->getType())->getNumElements()) ||
2714 "Unexpected operand for identity shuffle");
2720 int NarrowElts = cast<FixedVectorType>(
X->getType())->getNumElements();
2721 int WideElts = cast<FixedVectorType>(Shuffle0->getType())->getNumElements();
2722 assert(WideElts > NarrowElts &&
"Unexpected types for identity with padding");
2726 for (
int i = 0, e = Mask.size(); i != e; ++i) {
2732 if (Mask[i] < WideElts) {
2733 if (Shuffle0->getMaskValue(Mask[i]) == -1)
2736 if (Shuffle1->getMaskValue(Mask[i] - WideElts) == -1)
2743 if (Mask[i] < WideElts) {
2744 assert(Mask[i] < NarrowElts &&
"Unexpected shuffle mask");
2745 NewMask[i] = Mask[i];
2747 assert(Mask[i] < (WideElts + NarrowElts) &&
"Unexpected shuffle mask");
2748 NewMask[i] = Mask[i] - (WideElts - NarrowElts);
2770 if (
X->getType() !=
Y->getType())
2773 auto *BinOp = cast<BinaryOperator>(Op0);
2778 if (
auto NewBOI = dyn_cast<Instruction>(NewBO))
2779 NewBOI->copyIRFlags(BinOp);
2803 unsigned VWidth = cast<FixedVectorType>(SVI.
getType())->getNumElements();
2804 unsigned LHSWidth = cast<FixedVectorType>(
LHS->
getType())->getNumElements();
2815 X->getType()->isVectorTy() &&
X->getType() ==
Y->getType() &&
2816 X->getType()->getScalarSizeInBits() ==
2833 X->getType()->isVectorTy() && VWidth == LHSWidth) {
2835 auto *XType = cast<FixedVectorType>(
X->getType());
2836 unsigned XNumElts = XType->getNumElements();
2838 if (XNumElts >= VWidth) {
2839 assert(XNumElts % VWidth == 0 &&
"Unexpected vector bitcast");
2842 assert(VWidth % XNumElts == 0 &&
"Unexpected vector bitcast");
2846 if (!ScaledMask.
empty()) {
2850 ScaledMask, XType, ShufQuery))
2858 "Shuffle with 2 undef ops not simplified?");
2886 APInt PoisonElts(VWidth, 0);
2938 bool MadeChange =
false;
2941 unsigned MaskElems = Mask.size();
2942 auto *SrcTy = cast<FixedVectorType>(V->getType());
2943 unsigned VecBitWidth = SrcTy->getPrimitiveSizeInBits().getFixedValue();
2945 assert(SrcElemBitWidth &&
"vector elements must have a bitwidth");
2946 unsigned SrcNumElems = SrcTy->getNumElements();
2951 if (!BC->use_empty())
2955 unsigned BegIdx = Mask.front();
2956 Type *TgtTy = BC->getDestTy();
2958 if (!TgtElemBitWidth)
2960 unsigned TgtNumElems = VecBitWidth / TgtElemBitWidth;
2961 bool VecBitWidthsEqual = VecBitWidth == TgtNumElems * TgtElemBitWidth;
2962 bool BegIsAligned = 0 == ((SrcElemBitWidth * BegIdx) % TgtElemBitWidth);
2963 if (!VecBitWidthsEqual)
2968 if (!BegIsAligned) {
2972 for (
unsigned I = 0, E = MaskElems,
Idx = BegIdx;
I != E; ++
Idx, ++
I)
2973 ShuffleMask[
I] =
Idx;
2978 unsigned SrcElemsPerTgtElem = TgtElemBitWidth / SrcElemBitWidth;
2979 assert(SrcElemsPerTgtElem);
2980 BegIdx /= SrcElemsPerTgtElem;
2981 bool BCAlreadyExists = NewBCs.
contains(CastSrcTy);
2986 if (!BCAlreadyExists)
2987 NewBCs[CastSrcTy] = NewBC;
3045 LHSShuffle =
nullptr;
3048 RHSShuffle =
nullptr;
3049 if (!LHSShuffle && !RHSShuffle)
3050 return MadeChange ? &SVI :
nullptr;
3052 Value* LHSOp0 =
nullptr;
3053 Value* LHSOp1 =
nullptr;
3054 Value* RHSOp0 =
nullptr;
3055 unsigned LHSOp0Width = 0;
3056 unsigned RHSOp0Width = 0;
3060 LHSOp0Width = cast<FixedVectorType>(LHSOp0->
getType())->getNumElements();
3064 RHSOp0Width = cast<FixedVectorType>(RHSOp0->
getType())->getNumElements();
3075 else if (LHSOp0Width == LHSWidth) {
3080 if (RHSShuffle && RHSOp0Width == LHSWidth) {
3084 if (LHSOp0 == RHSOp0) {
3089 if (newLHS ==
LHS && newRHS ==
RHS)
3090 return MadeChange ? &SVI :
nullptr;
3096 if (RHSShuffle && newRHS !=
RHS)
3099 unsigned newLHSWidth = (newLHS !=
LHS) ? LHSOp0Width : LHSWidth;
3105 for (
unsigned i = 0; i < VWidth; ++i) {
3110 }
else if (Mask[i] < (
int)LHSWidth) {
3115 if (newLHS !=
LHS) {
3116 eltMask = LHSMask[Mask[i]];
3119 if (eltMask >= (
int)LHSOp0Width && isa<PoisonValue>(LHSOp1))
3132 else if (newRHS !=
RHS) {
3133 eltMask = RHSMask[Mask[i]-LHSWidth];
3136 if (eltMask >= (
int)RHSOp0Width) {
3138 "should have been check above");
3142 eltMask = Mask[i]-LHSWidth;
3150 if (eltMask >= 0 && newRHS !=
nullptr && newLHS != newRHS)
3151 eltMask += newLHSWidth;
3156 if (SplatElt >= 0 && SplatElt != eltMask)
3166 if (
isSplat || newMask == LHSMask || newMask == RHSMask || newMask == Mask) {
3172 return MadeChange ? &SVI :
nullptr;
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file provides internal interfaces used to implement the InstCombine.
static Instruction * foldConstantInsEltIntoShuffle(InsertElementInst &InsElt)
insertelt (shufflevector X, CVec, Mask|insertelt X, C1, CIndex1), C, CIndex --> shufflevector X,...
static Value * evaluateInDifferentElementOrder(Value *V, ArrayRef< int > Mask, IRBuilderBase &Builder)
static bool collectSingleShuffleElements(Value *V, Value *LHS, Value *RHS, SmallVectorImpl< int > &Mask)
If V is a shuffle of values that ONLY returns elements from either LHS or RHS, return the shuffle mas...
static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl< int > &Mask, Value *PermittedRHS, InstCombinerImpl &IC, bool &Rerun)
static APInt findDemandedEltsByAllUsers(Value *V)
Find union of elements of V demanded by all its users.
static Instruction * foldTruncInsEltPair(InsertElementInst &InsElt, bool IsBigEndian, InstCombiner::BuilderTy &Builder)
If we are inserting 2 halves of a value into adjacent elements of a vector, try to convert to a singl...
static Instruction * foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf, const SimplifyQuery &SQ)
static Instruction * foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf)
static Instruction * foldIdentityExtractShuffle(ShuffleVectorInst &Shuf)
Try to fold an extract subvector operation.
static Instruction * foldInsEltIntoSplat(InsertElementInst &InsElt)
Try to fold an insert element into an existing splat shuffle by changing the shuffle's mask to includ...
std::pair< Value *, Value * > ShuffleOps
We are building a shuffle to create V, which is a sequence of insertelement, extractelement pairs.
static Instruction * foldShuffleWithInsert(ShuffleVectorInst &Shuf, InstCombinerImpl &IC)
Try to replace a shuffle with an insertelement or try to replace a shuffle operand with the operand o...
static Instruction * canonicalizeInsertSplat(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
If we have an insert of a scalar to a non-zero element of an undefined vector and then shuffle that v...
static Instruction * foldTruncShuffle(ShuffleVectorInst &Shuf, bool IsBigEndian)
Convert a narrowing shuffle of a bitcasted vector into a vector truncate.
static bool replaceExtractElements(InsertElementInst *InsElt, ExtractElementInst *ExtElt, InstCombinerImpl &IC)
If we have insertion into a vector that is wider than the vector that we are extracting from,...
static bool cheapToScalarize(Value *V, Value *EI)
Return true if the value is cheaper to scalarize than it is to leave as a vector operation.
static Value * buildNew(Instruction *I, ArrayRef< Value * > NewOps, IRBuilderBase &Builder)
Rebuild a new instruction just like 'I' but with the new operands given.
static bool canEvaluateShuffled(Value *V, ArrayRef< int > Mask, unsigned Depth=5)
Return true if we can evaluate the specified expression tree if the vector elements were shuffled in ...
static Instruction * foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf)
A select shuffle of a select shuffle with a shared operand can be reduced to a single select shuffle.
static Instruction * hoistInsEltConst(InsertElementInst &InsElt2, InstCombiner::BuilderTy &Builder)
If we have an insertelement instruction feeding into another insertelement and the 2nd is inserting a...
static APInt findDemandedEltsBySingleUser(Value *V, Instruction *UserInstr)
Find elements of V demanded by UserInstr.
static Instruction * foldShuffleOfUnaryOps(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
Canonicalize FP negate/abs after shuffle.
static Instruction * foldCastShuffle(ShuffleVectorInst &Shuf, InstCombiner::BuilderTy &Builder)
Canonicalize casts after shuffle.
static Instruction * narrowInsElt(InsertElementInst &InsElt, InstCombiner::BuilderTy &Builder)
If both the base vector and the inserted element are extended from the same type, do the insert eleme...
static bool isShuffleEquivalentToSelect(ShuffleVectorInst &Shuf)
static Instruction * foldInsSequenceIntoSplat(InsertElementInst &InsElt)
Turn a chain of inserts that splats a value into an insert + shuffle: insertelt(insertelt(insertelt(i...
static Instruction * foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt)
Try to fold an extract+insert element into an existing identity shuffle by changing the shuffle's mas...
static ConstantInt * getPreferredVectorIndex(ConstantInt *IndexC)
Given a constant index for a extractelement or insertelement instruction, return it with the canonica...
static bool isShuffleExtractingFromLHS(ShuffleVectorInst &SVI, ArrayRef< int > Mask)
static BinopElts getAlternateBinop(BinaryOperator *BO, const DataLayout &DL)
Binops may be transformed into binops with different opcodes and operands.
This file provides the interface for the instcombine pass implementation.
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the SmallBitVector class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
If both arms of a vector select are concatenated vectors, split the select, and concatenate the resul...
static const uint32_t IV[8]
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const Function * getParent() const
Return the enclosing method, or null if none.
InstListType::iterator iterator
Instruction iterators...
static BinaryOperator * Create(BinaryOps Op, Value *S1, Value *S2, const Twine &Name, BasicBlock::iterator InsertBefore)
Construct a binary instruction, given the opcode and the two operands.
BinaryOps getOpcode() const
static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Value *CopyO, const Twine &Name, BasicBlock::iterator InsertBefore)
This class represents a no-op cast from one type to another.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr, BasicBlock::iterator InsertBefore)
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name, BasicBlock::iterator InsertBefore)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
This class is the base class for the comparison instructions.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
static CmpInst * CreateWithCopiedFlags(OtherOps Op, Predicate Pred, Value *S1, Value *S2, const Instruction *FlagsSource, const Twine &Name="", Instruction *InsertBefore=nullptr)
Construct a compare instruction, given the opcode, the predicate, the two operands and the instructio...
OtherOps getOpcode() const
Get the opcode casted to the right type.
static ConstantAggregateZero * get(Type *Ty)
static Constant * getShuffleVector(Constant *V1, Constant *V2, ArrayRef< int > Mask, Type *OnlyIfReducedTy=nullptr)
static Constant * getShl(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
static Constant * getBinOpIdentity(unsigned Opcode, Type *Ty, bool AllowRHSConstant=false, bool NSZ=false)
Return the identity constant for a binary opcode.
This is the shared class of boolean and integer constants.
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
const APInt & getValue() const
Return the constant as an APInt value reference.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr, BasicBlock::iterator InsertBefore)
void setIsInBounds(bool b=true)
Set or clear the inbounds flag on this GEP instruction.
Common base class shared among various IRBuilders.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
static InsertElementInst * Create(Value *Vec, Value *NewElt, Value *Idx, const Twine &NameStr, BasicBlock::iterator InsertBefore)
This instruction inserts a struct field of array element value into an aggregate value.
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI, bool FoldWithMultiUse=false)
Given an instruction with a select as one operand and a constant as the other operand,...
Value * SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &PoisonElts, unsigned Depth=0, bool AllowMultipleUsers=false) override
The specified value produces a vector with any number of elements.
Instruction * foldSelectShuffle(ShuffleVectorInst &Shuf)
Try to fold shuffles that are the equivalent of a vector select.
Instruction * visitInsertValueInst(InsertValueInst &IV)
Try to find redundant insertvalue instructions, like the following ones: %0 = insertvalue { i8,...
Instruction * visitInsertElementInst(InsertElementInst &IE)
Instruction * visitExtractElementInst(ExtractElementInst &EI)
Instruction * simplifyBinOpSplats(ShuffleVectorInst &SVI)
Instruction * foldAggregateConstructionIntoAggregateReuse(InsertValueInst &OrigIVI)
Look for chain of insertvalue's that fully define an aggregate, and trace back the values inserted,...
Instruction * visitShuffleVectorInst(ShuffleVectorInst &SVI)
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
InstructionWorklist & Worklist
A worklist of the instructions that need to be simplified.
Instruction * InsertNewInstWith(Instruction *New, BasicBlock::iterator Old)
Same as InsertNewInstBefore, but also sets the debug loc.
void addToWorklist(Instruction *I)
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
static Constant * getSafeVectorConstantForBinop(BinaryOperator::BinaryOps Opcode, Constant *In, bool IsRHSConstant)
Some binary operators require special handling to avoid poison and undefined behavior.
const SimplifyQuery & getSimplifyQuery() const
void addValue(Value *V)
Add value to the worklist if it is an instruction.
bool hasNoUnsignedWrap() const LLVM_READONLY
Determine whether the no unsigned wrap flag is set.
bool hasNoSignedWrap() const LLVM_READONLY
Determine whether the no signed wrap flag is set.
void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
void andIRFlags(const Value *V)
Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction.
void setFastMathFlags(FastMathFlags FMF)
Convenience function for setting multiple fast-math flags on this instruction, which must be an opera...
const BasicBlock * getParent() const
Instruction * user_back()
Specialize the methods defined in Value, as we know that an instruction can only be used by other ins...
bool isExact() const LLVM_READONLY
Determine whether the exact flag is set.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void dropPoisonGeneratingFlags()
Drops flags that may cause this instruction to evaluate to poison despite having non-poison inputs.
A wrapper class for inspecting calls to intrinsic functions.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
In order to facilitate speculative execution, many instructions do not invoke immediate undefined beh...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
This class represents the LLVM 'select' instruction.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr, BasicBlock::iterator InsertBefore, Instruction *MDFrom=nullptr)
This instruction constructs a fixed permutation of two input vectors.
bool changesLength() const
Return true if this shuffle returns a vector with a different number of elements than its source vect...
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
static bool isSelectMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from its source vectors without lane crossings.
VectorType * getType() const
Overload to return most specific vector type.
bool increasesLength() const
Return true if this shuffle returns a vector with a greater number of elements than its source vector...
bool isIdentityWithExtract() const
Return true if this shuffle extracts the first N elements of exactly one source vector.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
bool isSelect() const
Return true if this shuffle chooses elements from its source vectors without lane crossings and all o...
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
static void commuteShuffleMask(MutableArrayRef< int > Mask, unsigned InVecNumElts)
Change values in a shuffle permute mask assuming the two vector operands of length InVecNumElts have ...
void commute()
Swap the operands and adjust the mask to preserve the semantics of the instruction.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool all() const
Returns true if all bits are set.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class represents a truncation of integer types.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
bool isVectorTy() const
True if this is an instance of VectorType.
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
unsigned getStructNumElements() const
uint64_t getArrayNumElements() const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeID getTypeID() const
Return the type id for the type.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
static UnaryOperator * CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO, const Twine &Name, BasicBlock::iterator InsertBefore)
static UnaryOperator * CreateFNegFMF(Value *Op, Instruction *FMFSource, const Twine &Name, BasicBlock::iterator InsertBefore)
UnaryOps getOpcode() const
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
const Value * DoPHITranslation(const BasicBlock *CurBB, const BasicBlock *PredBB) const
Translate PHI node to its predecessor from the given basic block.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
LLVMContext & getContext() const
All values hold a context through their type.
StringRef getName() const
Return a constant reference to the value's name.
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Type * getElementType() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
class_match< PoisonValue > m_Poison()
Match an arbitrary poison constant.
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
bool match(Val *V, const Pattern &P)
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
TwoOps_match< Val_t, Idx_t, Instruction::ExtractElement > m_ExtractElt(const Val_t &Val, const Idx_t &Idx)
Matches ExtractElementInst.
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
CastOperator_match< OpTy, Instruction::Trunc > m_Trunc(const OpTy &Op)
Matches Trunc.
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
OneUse_match< T > m_OneUse(const T &SubPattern)
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
CastInst_match< OpTy, FPExtInst > m_FPExt(const OpTy &Op)
OneOps_match< OpTy, Instruction::Load > m_Load(const OpTy &Op)
Matches LoadInst.
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
class_match< UnaryOperator > m_UnOp()
Match an arbitrary unary operation and ignore it.
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::LShr > m_LShr(const LHS &L, const RHS &R)
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
auto m_Undef()
Match an arbitrary undef constant.
CastInst_match< OpTy, SExtInst > m_SExt(const OpTy &Op)
Matches SExt.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)
Combine two pattern matchers matching L || R.
This is an optimization pass for GlobalISel generic memory operations.
bool MaskedValueIsZero(const Value *V, const APInt &Mask, const SimplifyQuery &DL, unsigned Depth=0)
Return true if 'V & Mask' is known to be zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Value * simplifyShuffleVectorInst(Value *Op0, Value *Op1, ArrayRef< int > Mask, Type *RetTy, const SimplifyQuery &Q)
Given operands for a ShuffleVectorInst, fold the result or return null.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Value * simplifyInsertValueInst(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const SimplifyQuery &Q)
Given operands for an InsertValueInst, fold the result or return null.
constexpr int PoisonMaskElem
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
Value * simplifyInsertElementInst(Value *Vec, Value *Elt, Value *Idx, const SimplifyQuery &Q)
Given operands for an InsertElement, fold the result or return null.
bool isSafeToSpeculativelyExecute(const Instruction *I, const Instruction *CtxI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool pred_empty(const BasicBlock *BB)
bool isKnownNeverNaN(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
Value * simplifyExtractElementInst(Value *Vec, Value *Idx, const SimplifyQuery &Q)
Given operands for an ExtractElementInst, fold the result or return null.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
These are the ingredients in an alternate form binary operator as described below.
BinopElts(BinaryOperator::BinaryOps Opc=(BinaryOperator::BinaryOps) 0, Value *V0=nullptr, Value *V1=nullptr)
BinaryOperator::BinaryOps Opcode
SimplifyQuery getWithInstruction(const Instruction *I) const