46#define DEBUG_TYPE "instcombine"
50using namespace PatternMatch;
67 IsEq = Pred == ICmpInst::ICMP_EQ;
68 else if (Pred == FCmpInst::FCMP_OEQ)
70 else if (Pred == FCmpInst::FCMP_UNE)
100 if (isa<FPMathOperator>(BO))
124 const APInt *SelTC, *SelFC;
131 if (SelType->
isVectorTy() != Cmp->getType()->isVectorTy())
136 bool CreateAnd =
false;
142 V = Cmp->getOperand(0);
158 if (Pred == ICmpInst::ICMP_NE)
165 const APInt &TC = *SelTC;
166 const APInt &FC = *SelFC;
167 if (!TC.
isZero() && !FC.isZero()) {
172 if (CreateAnd && !Cmp->hasOneUse())
179 Constant *TCC = ConstantInt::get(SelType, TC);
180 Constant *FCC = ConstantInt::get(SelType, FC);
181 Constant *MaskC = ConstantInt::get(SelType, AndMask);
182 for (
auto Opc : {Instruction::Or, Instruction::Xor, Instruction::Add,
202 unsigned ValZeros = ValC.
logBase2();
203 unsigned AndZeros = AndMask.
logBase2();
204 bool ShouldNotVal = !TC.
isZero();
209 if (CreateAnd && ShouldNotVal && ValZeros != AndZeros)
214 V = Builder.
CreateAnd(V, ConstantInt::get(V->getType(), AndMask));
218 if (ValZeros > AndZeros) {
220 V = Builder.
CreateShl(V, ValZeros - AndZeros);
221 }
else if (ValZeros < AndZeros) {
222 V = Builder.
CreateLShr(V, AndZeros - ValZeros);
247 switch (
I->getOpcode()) {
248 case Instruction::Add:
249 case Instruction::FAdd:
250 case Instruction::Mul:
251 case Instruction::FMul:
252 case Instruction::And:
253 case Instruction::Or:
254 case Instruction::Xor:
256 case Instruction::Sub:
257 case Instruction::FSub:
258 case Instruction::FDiv:
259 case Instruction::Shl:
260 case Instruction::LShr:
261 case Instruction::AShr:
291 if (
auto *CondVTy = dyn_cast<VectorType>(CondTy)) {
293 CondVTy->getElementCount() !=
294 cast<VectorType>(FIOpndTy)->getElementCount())
305 if (TI->
getOpcode() != Instruction::BitCast &&
318 SI.getName() +
".v", &SI);
323 Value *OtherOpT, *OtherOpF;
326 bool Swapped =
false) ->
Value * {
327 assert(!(Commute && Swapped) &&
328 "Commute and Swapped can't set at the same time");
333 MatchIsOpZero =
true;
338 MatchIsOpZero =
false;
343 if (!Commute && !Swapped)
352 MatchIsOpZero =
true;
357 MatchIsOpZero =
false;
371 FMF |= SI.getFastMathFlags();
374 if (
auto *NewSelI = dyn_cast<Instruction>(NewSel))
375 NewSelI->setFastMathFlags(FMF);
376 Instruction *NewFNeg = UnaryOperator::CreateFNeg(NewSel);
385 auto *
TII = dyn_cast<IntrinsicInst>(TI);
386 auto *FII = dyn_cast<IntrinsicInst>(FI);
387 if (
TII && FII &&
TII->getIntrinsicID() == FII->getIntrinsicID()) {
389 if (
Value *MatchOp = getCommonOp(TI, FI,
true)) {
401 if (
TII->getIntrinsicID() == Intrinsic::ldexp) {
402 Value *LdexpVal0 =
TII->getArgOperand(0);
403 Value *LdexpExp0 =
TII->getArgOperand(1);
404 Value *LdexpVal1 = FII->getArgOperand(0);
405 Value *LdexpExp1 = FII->getArgOperand(1);
409 FMF &= cast<FPMathOperator>(FII)->getFastMathFlags();
416 TII->
getType(), Intrinsic::ldexp, {SelectVal, SelectExp});
429 bool Swapped = TPred != FPred;
433 SI.getName() +
".v", &SI);
448 (!isa<BinaryOperator>(TI) && !isa<GetElementPtrInst>(TI)) ||
470 auto *BO = dyn_cast<BinaryOperator>(TI);
474 if (BO->getOpcode() == Instruction::SDiv ||
475 BO->getOpcode() == Instruction::SRem || MatchIsOpZero)
481 SI.getName() +
".v", &SI);
482 Value *Op0 = MatchIsOpZero ? MatchOp : NewSI;
483 Value *Op1 = MatchIsOpZero ? NewSI : MatchOp;
484 if (
auto *BO = dyn_cast<BinaryOperator>(TI)) {
490 if (
auto *TGEP = dyn_cast<GetElementPtrInst>(TI)) {
491 auto *FGEP = cast<GetElementPtrInst>(FI);
492 Type *ElementType = TGEP->getSourceElementType();
494 ElementType, Op0, Op1, TGEP->getNoWrapFlags() & FGEP->getNoWrapFlags());
515 auto *TVI = dyn_cast<BinaryOperator>(TrueVal);
516 if (!TVI || !TVI->hasOneUse() || isa<Constant>(FalseVal))
520 unsigned OpToFold = 0;
521 if ((SFO & 1) && FalseVal == TVI->getOperand(0))
523 else if ((SFO & 2) && FalseVal == TVI->getOperand(1))
533 if (isa<FPMathOperator>(&SI))
534 FMF = SI.getFastMathFlags();
536 TVI->getOpcode(), TVI->getType(),
true, FMF.
noSignedZeros());
537 Value *OOp = TVI->getOperand(2 - OpToFold);
542 if (isa<Constant>(OOp) &&
543 (!OOpIsAPInt || !
isSelect01(
C->getUniqueInteger(), *OOpC)))
552 if (isa<FPMathOperator>(&SI) &&
557 Swapped ? OOp :
C,
"", &SI);
558 if (isa<FPMathOperator>(&SI))
559 cast<Instruction>(NewSel)->setFastMathFlags(FMF);
567 if (
Instruction *R = TryFoldSelectIntoOp(SI, TrueVal, FalseVal,
false))
570 if (
Instruction *R = TryFoldSelectIntoOp(SI, FalseVal, TrueVal,
true))
587 if (!(Cmp->hasOneUse() && Cmp->getOperand(0)->hasOneUse() &&
619 Constant *One = ConstantInt::get(SelType, 1);
624 return new ZExtInst(ICmpNeZero, SelType);
646 const APInt *C2, *C1;
656 auto *FI = dyn_cast<Instruction>(FVal);
660 FI->setHasNoSignedWrap(
false);
661 FI->setHasNoUnsignedWrap(
false);
696 const auto *Ashr = cast<Instruction>(FalseVal);
698 bool IsExact = Ashr->isExact() && cast<Instruction>(TrueVal)->isExact();
730 if (!TrueVal->getType()->isIntOrIntVectorTy() ||
765 BinOp = cast<BinaryOperator>(FalseVal);
769 BinOp = cast<BinaryOperator>(TrueVal);
779 if (IdentityC ==
nullptr || !IdentityC->isNullValue())
784 bool NeedShift = C1Log != C2Log;
785 bool NeedZExtTrunc =
Y->getType()->getScalarSizeInBits() !=
786 V->getType()->getScalarSizeInBits();
789 if ((NeedShift + NeedXor + NeedZExtTrunc +
NeedAnd) >
796 V = Builder.
CreateAnd(V, ConstantInt::get(V->getType(), C1));
802 }
else if (C1Log > C2Log) {
829 Constant *OrC = ConstantInt::get(Ty, *
C);
831 return BinaryOperator::CreateOr(
T, NewSel);
838 Constant *OrC = ConstantInt::get(Ty, *
C);
840 return BinaryOperator::CreateOr(
F, NewSel);
857 auto *CondVal = SI.getCondition();
858 auto *TrueVal = SI.getTrueValue();
859 auto *FalseVal = SI.getFalseValue();
877 auto *TrueValC = dyn_cast<Constant>(TrueVal);
878 if (TrueValC ==
nullptr ||
880 !isa<Instruction>(FalseVal))
883 auto *ZeroC = cast<Constant>(cast<Instruction>(CondVal)->getOperand(1));
891 auto *FalseValI = cast<Instruction>(FalseVal);
894 IC.
replaceOperand(*FalseValI, FalseValI->getOperand(0) ==
Y ? 0 : 1, FrY);
901 const Value *TrueVal,
902 const Value *FalseVal,
923 ConstantInt::get(
A->getType(), 1));
937 "Unexpected isUnsigned predicate!");
943 bool IsNegative =
false;
956 if (IsNegative && !TrueVal->hasOneUse() && !ICI->
hasOneUse())
969 if (!Cmp->hasOneUse())
973 Value *Cmp0 = Cmp->getOperand(0);
974 Value *Cmp1 = Cmp->getOperand(1);
995 Intrinsic::uadd_sat, Cmp0, ConstantInt::get(Cmp0->
getType(), 1));
1006 ConstantInt::get(Cmp0->
getType(), *
C));
1016 ConstantInt::get(Cmp0->
getType(), *
C));
1026 ConstantInt::get(Cmp0->
getType(), *
C));
1073 auto *TI = dyn_cast<Instruction>(TVal);
1074 auto *FI = dyn_cast<Instruction>(FVal);
1080 Value *
A = Cmp->getOperand(0);
1081 Value *
B = Cmp->getOperand(1);
1094 (TI->hasNoSignedWrap() || TI->hasNoUnsignedWrap()) &&
1095 (FI->hasNoSignedWrap() || FI->hasNoUnsignedWrap())) {
1102 TI->setHasNoUnsignedWrap(
false);
1103 if (!TI->hasNoSignedWrap())
1104 TI->setHasNoSignedWrap(TI->hasOneUse());
1132 if (!
match(FalseVal,
1136 if (!
match(Ctlz, m_Intrinsic<Intrinsic::ctlz>()))
1143 auto *
II = cast<IntrinsicInst>(Ctlz);
1180 Value *Count =
nullptr;
1188 if (!
match(Count, m_Intrinsic<Intrinsic::cttz>(
m_Value(
X))) &&
1208 II->dropPoisonGeneratingAnnotations();
1227 if (!
TrueVal->getType()->isIntOrIntVectorTy())
1254 IntrinsicID = Intrinsic::umin;
1257 IntrinsicID = Intrinsic::umax;
1260 IntrinsicID = Intrinsic::smin;
1263 IntrinsicID = Intrinsic::smax;
1280 assert(!isa<Constant>(Old) &&
"Only replace non-constant values");
1282 auto *
I = dyn_cast<Instruction>(V);
1283 if (!
I || !
I->hasOneUse() ||
1287 bool Changed =
false;
1288 for (
Use &U :
I->operands()) {
1318 if (!
Cmp.isEquality())
1323 bool Swapped =
false;
1329 Value *CmpLHS =
Cmp.getOperand(0), *CmpRHS =
Cmp.getOperand(1);
1330 auto ReplaceOldOpWithNewOp = [&](
Value *OldOp,
1338 if (TrueVal == OldOp)
1374 if (
Instruction *R = ReplaceOldOpWithNewOp(CmpLHS, CmpRHS))
1376 if (
Instruction *R = ReplaceOldOpWithNewOp(CmpRHS, CmpLHS))
1379 auto *FalseInst = dyn_cast<Instruction>(FalseVal);
1394 &DropFlags) == TrueVal ||
1397 &DropFlags) == TrueVal) {
1399 I->dropPoisonGeneratingAnnotations();
1440 if (!isa<SelectInst>(Sel1)) {
1481 if (Cmp00->
getType() !=
X->getType() &&
X->hasOneUse())
1489 else if (!
match(Cmp00,
1497 Value *ReplacementLow, *ReplacementHigh;
1534 std::swap(ReplacementLow, ReplacementHigh);
1540 "Unexpected predicate type.");
1548 "Unexpected predicate type.");
1550 std::swap(ThresholdLowIncl, ThresholdHighExcl);
1566 if (
X->getType() != Sel0.
getType()) {
1576 assert(ReplacementLow && ReplacementHigh &&
1577 "Constant folding of ImmConstant cannot fail");
1583 Value *MaybeReplacedLow =
1589 ShouldReplaceHigh, ReplacementHigh, MaybeReplacedLow);
1633 Value *SelVal0, *SelVal1;
1642 auto MatchesSelectValue = [SelVal0, SelVal1](
Constant *
C) {
1643 return C->isElementWiseEqual(SelVal0) ||
C->isElementWiseEqual(SelVal1);
1647 if (MatchesSelectValue(C0))
1651 auto FlippedStrictness =
1653 if (!FlippedStrictness)
1657 if (!MatchesSelectValue(FlippedStrictness->second))
1666 Cmp.getName() +
".inv");
1677 if (!
Cmp->hasOneUse())
1707 Value *TVal =
SI.getTrueValue();
1708 Value *FVal =
SI.getFalseValue();
1734 const APInt *BinOpC;
1771 const unsigned AndOps = Instruction::And, OrOps = Instruction::Or,
1772 XorOps = Instruction::Xor, NoOps = 0;
1773 enum NotMask {
None = 0, NotInner, NotRHS };
1775 auto matchFalseVal = [&](
unsigned OuterOpc,
unsigned InnerOpc,
1778 if (OuterOpc == NoOps)
1781 if (NotMask == NotInner) {
1784 }
else if (NotMask == NotRHS) {
1788 return match(FalseVal,
1799 if (matchFalseVal(OrOps, XorOps,
None) ||
1800 matchFalseVal(XorOps, XorOps,
None))
1805 if (matchFalseVal(XorOps, OrOps,
None) ||
1806 matchFalseVal(AndOps, OrOps, NotRHS))
1817 if (matchFalseVal(XorOps, XorOps,
None) ||
1818 matchFalseVal(AndOps, XorOps, NotInner))
1823 if (matchFalseVal(XorOps, AndOps,
None) ||
1824 matchFalseVal(AndOps, AndOps, NotInner))
1835 if (matchFalseVal(XorOps, OrOps,
None) ||
1836 matchFalseVal(AndOps, OrOps, NotRHS))
1841 if (matchFalseVal(OrOps, AndOps,
None) ||
1842 matchFalseVal(XorOps, AndOps,
None))
1857 canonicalizeSPF(*ICI,
SI.getTrueValue(),
SI.getFalseValue(), *
this))
1860 if (
Value *V = foldSelectInstWithICmpConst(SI, ICI,
Builder))
1863 if (
Value *V = canonicalizeClampLike(SI, *ICI,
Builder, *
this))
1867 tryToReuseConstantFromSelectInComparison(SI, *ICI, *
this))
1874 bool Changed =
false;
1880 if (CmpRHS != CmpLHS && isa<Constant>(CmpRHS) && !isa<Constant>(CmpLHS)) {
1892 if (
Instruction *NewSel = foldSelectICmpEq(SI, ICI, *
this))
1907 SI.swapProfMetadata();
1913 if (
TrueVal->getType()->isIntOrIntVectorTy()) {
1920 bool IsBitTest =
false;
1928 Y = &MinSignedValue;
1930 TrueWhenUnset =
false;
1933 Y = &MinSignedValue;
1935 TrueWhenUnset =
true;
1940 if (TrueWhenUnset && TrueVal ==
X &&
1944 else if (!TrueWhenUnset && FalseVal ==
X &&
1948 else if (TrueWhenUnset && FalseVal ==
X &&
1952 else if (!TrueWhenUnset && TrueVal ==
X &&
1980 if (
Value *V = foldSelectCttzCtlz(ICI, TrueVal, FalseVal, *
this))
1992 return Changed ? &
SI :
nullptr;
2004static bool canSelectOperandBeMappingIntoPredBlock(
const Value *V,
2009 if (!
I)
return true;
2013 const PHINode *CondPHI = cast<PHINode>(
SI.getCondition());
2015 if (
const PHINode *VP = dyn_cast<PHINode>(
I))
2016 if (VP->getParent() == CondPHI->
getParent())
2040 if (
C ==
A ||
C ==
B) {
2055 Value *CondVal =
SI.getCondition();
2058 auto *TI = dyn_cast<Instruction>(TrueVal);
2059 auto *FI = dyn_cast<Instruction>(FalseVal);
2060 if (!TI || !FI || !TI->hasOneUse() || !FI->hasOneUse())
2064 if ((TI->getOpcode() == Instruction::Sub &&
2065 FI->getOpcode() == Instruction::Add) ||
2066 (TI->getOpcode() == Instruction::FSub &&
2067 FI->getOpcode() == Instruction::FAdd)) {
2070 }
else if ((FI->getOpcode() == Instruction::Sub &&
2071 TI->getOpcode() == Instruction::Add) ||
2072 (FI->getOpcode() == Instruction::FSub &&
2073 TI->getOpcode() == Instruction::FAdd)) {
2079 Value *OtherAddOp =
nullptr;
2080 if (SubOp->getOperand(0) == AddOp->
getOperand(0)) {
2082 }
else if (SubOp->getOperand(0) == AddOp->
getOperand(1)) {
2090 if (
SI.getType()->isFPOrFPVectorTy()) {
2091 NegVal = Builder.
CreateFNeg(SubOp->getOperand(1));
2092 if (
Instruction *NegInst = dyn_cast<Instruction>(NegVal)) {
2094 Flags &= SubOp->getFastMathFlags();
2095 NegInst->setFastMathFlags(Flags);
2098 NegVal = Builder.
CreateNeg(SubOp->getOperand(1));
2101 Value *NewTrueOp = OtherAddOp;
2102 Value *NewFalseOp = NegVal;
2106 SI.getName() +
".p", &SI);
2108 if (
SI.getType()->isFPOrFPVectorTy()) {
2110 BinaryOperator::CreateFAdd(SubOp->getOperand(0), NewSel);
2113 Flags &= SubOp->getFastMathFlags();
2117 return BinaryOperator::CreateAdd(SubOp->getOperand(0), NewSel);
2130 Value *CondVal =
SI.getCondition();
2142 auto IsSignedSaturateLimit = [&](
Value *Limit,
bool IsAdd) {
2152 auto IsZeroOrOne = [](
const APInt &
C) {
return C.isZero() ||
C.isOne(); };
2169 IsMinMax(TrueVal, FalseVal))
2176 IsMinMax(FalseVal, TrueVal))
2182 IsMinMax(TrueVal, FalseVal))
2187 IsMinMax(FalseVal, TrueVal))
2192 IsMinMax(FalseVal, TrueVal))
2197 IsMinMax(TrueVal, FalseVal))
2205 if (
II->getIntrinsicID() == Intrinsic::uadd_with_overflow &&
2208 NewIntrinsicID = Intrinsic::uadd_sat;
2209 else if (
II->getIntrinsicID() == Intrinsic::usub_with_overflow &&
2212 NewIntrinsicID = Intrinsic::usub_sat;
2213 else if (
II->getIntrinsicID() == Intrinsic::sadd_with_overflow &&
2214 IsSignedSaturateLimit(TrueVal,
true))
2223 NewIntrinsicID = Intrinsic::sadd_sat;
2224 else if (
II->getIntrinsicID() == Intrinsic::ssub_with_overflow &&
2225 IsSignedSaturateLimit(TrueVal,
false))
2234 NewIntrinsicID = Intrinsic::ssub_sat;
2255 if (ExtOpcode != Instruction::ZExt && ExtOpcode != Instruction::SExt)
2261 Type *SmallType =
X->getType();
2263 auto *
Cmp = dyn_cast<CmpInst>(
Cond);
2265 (!Cmp ||
Cmp->getOperand(0)->getType() != SmallType))
2273 Value *TruncCVal = cast<Value>(TruncC);
2289 Value *CondVal =
SI.getCondition();
2291 auto *CondValTy = dyn_cast<FixedVectorType>(CondVal->
getType());
2295 unsigned NumElts = CondValTy->getNumElements();
2297 Mask.reserve(NumElts);
2298 for (
unsigned i = 0; i != NumElts; ++i) {
2308 Mask.push_back(i + NumElts);
2309 }
else if (isa<UndefValue>(Elt)) {
2329 auto *Ty = dyn_cast<VectorType>(Sel.
getType());
2361 if (TVal ==
A || TVal ==
B || FVal ==
A || FVal ==
B)
2378 if (TSrc ==
C && FSrc ==
D) {
2382 }
else if (TSrc ==
D && FSrc ==
C) {
2421 auto *Extract = dyn_cast<ExtractValueInst>(V);
2424 if (Extract->getIndices()[0] !=
I)
2426 return dyn_cast<AtomicCmpXchgInst>(Extract->getAggregateOperand());
2432 if (
auto *
Select = dyn_cast<SelectInst>(
SI.user_back()))
2433 if (
Select->getCondition() ==
SI.getCondition())
2434 if (
Select->getFalseValue() ==
SI.getTrueValue() ||
2435 Select->getTrueValue() ==
SI.getFalseValue())
2439 auto *CmpXchg = isExtractFromCmpXchg(
SI.getCondition(), 1);
2446 if (
auto *
X = isExtractFromCmpXchg(
SI.getTrueValue(), 0))
2447 if (
X == CmpXchg &&
X->getCompareOperand() ==
SI.getFalseValue())
2448 return SI.getFalseValue();
2453 if (
auto *
X = isExtractFromCmpXchg(
SI.getFalseValue(), 0))
2454 if (
X == CmpXchg &&
X->getCompareOperand() ==
SI.getTrueValue())
2455 return SI.getFalseValue();
2479 Value *SV0, *SV1, *SA0, *SA1;
2488 if (Or0->
getOpcode() == BinaryOperator::LShr) {
2494 Or1->
getOpcode() == BinaryOperator::LShr &&
2495 "Illegal or(shift,shift) pair");
2510 bool IsFshl = (ShAmt == SA0);
2512 if ((IsFshl && TVal != SV0) || (!IsFshl && TVal != SV1))
2532 Intrinsic::ID IID = IsFshl ? Intrinsic::fshl : Intrinsic::fshr;
2553 assert(TC != FC &&
"Expected equal select arms to simplify");
2557 bool IsTrueIfSignSet;
2575 Value *MagArg = ConstantFP::get(SelType,
abs(*TC));
2582 if (!isa<VectorType>(Sel.
getType()))
2593 if (
auto *
I = dyn_cast<Instruction>(V))
2594 I->copyIRFlags(&Sel);
2605 return createSelReverse(
C,
X,
Y);
2609 return createSelReverse(
C,
X, FVal);
2614 return createSelReverse(
C, TVal,
Y);
2617 auto *VecTy = dyn_cast<FixedVectorType>(Sel.
getType());
2621 unsigned NumElts = VecTy->getNumElements();
2622 APInt PoisonElts(NumElts, 0);
2636 cast<ShuffleVectorInst>(TVal)->isSelect()) {
2650 cast<ShuffleVectorInst>(FVal)->isSelect()) {
2671 auto *IDomNode = DT[BB]->getIDom();
2677 Value *IfTrue, *IfFalse;
2693 if (TrueSucc == FalseSucc)
2714 if (
auto *
Insn = dyn_cast<Instruction>(Inputs[Pred]))
2733 if (
auto *
I = dyn_cast<Instruction>(V))
2734 CandidateBlocks.
insert(
I->getParent());
2737 if (
auto *PN = foldSelectToPhiImpl(Sel, BB, DT, Builder))
2750 Value *CondVal =
SI.getCondition();
2755 Value *
Op, *RemRes, *Remainder;
2757 bool TrueIfSigned =
false;
2771 return BinaryOperator::CreateAnd(
Op,
Add);
2783 return FoldToBitwiseAnd(Remainder);
2792 return FoldToBitwiseAnd(ConstantInt::get(RemRes->
getType(), 2));
2828 Value *InnerCondVal =
SI.getCondition();
2829 Value *InnerTrueVal =
SI.getTrueValue();
2830 Value *InnerFalseVal =
SI.getFalseValue();
2832 "The type of inner condition must match with the outer.");
2834 return *Implied ? InnerTrueVal : InnerFalseVal;
2841 assert(
Op->getType()->isIntOrIntVectorTy(1) &&
2842 "Op must be either i1 or vector of i1.");
2843 if (
SI.getCondition()->getType() !=
Op->getType())
2845 if (
Value *V = simplifyNestedSelectsUsingImpliedCond(SI,
Op, IsAnd,
DL))
2856 Value *CondVal =
SI.getCondition();
2858 bool ChangedFMF =
false;
2859 for (
bool Swap : {
false,
true}) {
2889 FastMathFlags FMF = cast<FPMathOperator>(TrueVal)->getFastMathFlags();
2890 if (FMF.
noNaNs() && !
SI.hasNoNaNs()) {
2891 SI.setHasNoNaNs(
true);
2894 if (FMF.
noInfs() && !
SI.hasNoInfs()) {
2895 SI.setHasNoInfs(
true);
2909 if (!
SI.hasNoSignedZeros() || !
SI.hasNoNaNs())
2926 Instruction *NewFNeg = UnaryOperator::CreateFNeg(Fabs);
2935 for (
bool Swap : {
false,
true}) {
2951 if (Swap == TrueIfSigned && !CondVal->
hasOneUse() && !
TrueVal->hasOneUse())
2957 if (Swap != TrueIfSigned)
2962 return ChangedFMF ? &
SI :
nullptr;
2980foldRoundUpIntegerWithPow2Alignment(
SelectInst &SI,
2984 Value *XBiasedHighBits =
SI.getFalseValue();
2997 const APInt *LowBitMaskCst;
3002 const APInt *BiasCst, *HighBitMaskCst;
3003 if (!
match(XBiasedHighBits,
3006 !
match(XBiasedHighBits,
3011 if (!LowBitMaskCst->
isMask())
3014 APInt InvertedLowBitMaskCst = ~*LowBitMaskCst;
3015 if (InvertedLowBitMaskCst != *HighBitMaskCst)
3018 APInt AlignmentCst = *LowBitMaskCst + 1;
3020 if (*BiasCst != AlignmentCst && *BiasCst != *LowBitMaskCst)
3025 if (*BiasCst == *LowBitMaskCst &&
impliesPoison(XBiasedHighBits,
X))
3026 return XBiasedHighBits;
3031 Type *Ty =
X->getType();
3032 Value *XOffset = Builder.
CreateAdd(
X, ConstantInt::get(Ty, *LowBitMaskCst),
3033 X->getName() +
".biased");
3034 Value *
R = Builder.
CreateAnd(XOffset, ConstantInt::get(Ty, *HighBitMaskCst));
3040struct DecomposedSelect {
3052foldSelectOfSymmetricSelect(
SelectInst &OuterSelVal,
3055 Value *OuterCond, *InnerCond, *InnerTrueVal, *InnerFalseVal;
3083 DecomposedSelect OuterSel;
3090 std::swap(OuterSel.TrueVal, OuterSel.FalseVal);
3098 Value *InnerSelVal = IsAndVariant ? OuterSel.FalseVal : OuterSel.TrueVal;
3102 [](
Value *V) {
return V->hasOneUse(); }))
3106 DecomposedSelect InnerSel;
3107 if (!
match(InnerSelVal,
3114 std::swap(InnerSel.TrueVal, InnerSel.FalseVal);
3116 Value *AltCond =
nullptr;
3117 auto matchOuterCond = [OuterSel, IsAndVariant, &AltCond](
auto m_InnerCond) {
3122 return IsAndVariant ?
match(OuterSel.Cond,
3132 if (matchOuterCond(
m_Specific(InnerSel.Cond))) {
3137 std::swap(InnerSel.TrueVal, InnerSel.FalseVal);
3138 InnerSel.Cond = NotInnerCond;
3143 AltCond, IsAndVariant ? OuterSel.TrueVal : InnerSel.FalseVal,
3144 IsAndVariant ? InnerSel.TrueVal : OuterSel.FalseVal);
3147 IsAndVariant ? SelInner : InnerSel.TrueVal,
3148 !IsAndVariant ? SelInner : InnerSel.FalseVal);
3152 Value *CondVal =
SI.getCondition();
3155 Type *SelType =
SI.getType();
3174 return BinaryOperator::CreateOr(CondVal, FalseVal);
3184 if (
auto *LHS = dyn_cast<FCmpInst>(CondVal))
3185 if (
auto *RHS = dyn_cast<FCmpInst>(FalseVal))
3186 if (
Value *V = foldLogicOfFCmps(LHS, RHS,
false,
3194 bool CondLogicAnd = isa<SelectInst>(CondVal);
3195 bool FalseLogicAnd = isa<SelectInst>(FalseVal);
3196 auto AndFactorization = [&](
Value *Common,
Value *InnerCond,
3202 if (FalseLogicAnd || (CondLogicAnd && Common ==
A))
3205 return BinaryOperator::CreateAnd(Common, InnerSel);
3209 return AndFactorization(
A,
B,
D);
3211 return AndFactorization(
A,
B,
C);
3213 return AndFactorization(
B,
A,
D);
3215 return AndFactorization(
B,
A,
C, CondLogicAnd && FalseLogicAnd);
3222 return BinaryOperator::CreateAnd(CondVal, TrueVal);
3232 if (
auto *LHS = dyn_cast<FCmpInst>(CondVal))
3233 if (
auto *RHS = dyn_cast<FCmpInst>(TrueVal))
3234 if (
Value *V = foldLogicOfFCmps(LHS, RHS,
true,
3242 bool CondLogicOr = isa<SelectInst>(CondVal);
3243 bool TrueLogicOr = isa<SelectInst>(TrueVal);
3244 auto OrFactorization = [&](
Value *Common,
Value *InnerCond,
3250 if (TrueLogicOr || (CondLogicOr && Common ==
A))
3253 return BinaryOperator::CreateOr(Common, InnerSel);
3257 return OrFactorization(
A,
B,
D);
3259 return OrFactorization(
A,
B,
C);
3261 return OrFactorization(
B,
A,
D);
3263 return OrFactorization(
B,
A,
C, CondLogicOr && TrueLogicOr);
3306 return BinaryOperator::CreateXor(
A,
B);
3340 auto *FI =
new FreezeInst(*
Y, (*Y)->getName() +
".fr");
3346 if (
auto *ICmp0 = dyn_cast<ICmpInst>(CondVal))
3347 if (
auto *ICmp1 = dyn_cast<ICmpInst>(Op1))
3348 if (
auto *V = foldAndOrOfICmps(ICmp0, ICmp1, SI, IsAnd,
3359 if (Res && *Res ==
false)
3365 if (Res && *Res ==
false)
3374 if (Res && *Res ==
true)
3380 if (Res && *Res ==
true)
3393 auto *SelCond = dyn_cast<SelectInst>(CondVal);
3394 auto *SelFVal = dyn_cast<SelectInst>(FalseVal);
3395 bool MayNeedFreeze = SelCond && SelFVal &&
3396 match(SelFVal->getTrueValue(),
3409 auto *SelCond = dyn_cast<SelectInst>(CondVal);
3410 auto *SelFVal = dyn_cast<SelectInst>(FalseVal);
3411 bool MayNeedFreeze = SelCond && SelFVal &&
3412 match(SelCond->getTrueValue(),
3428 bool &ShouldDropNUW) {
3451 ShouldDropNUW =
false;
3457 auto MatchForward = [&](
Value *CommonAncestor) {
3458 const APInt *
C =
nullptr;
3459 if (CtlzOp == CommonAncestor)
3466 ShouldDropNUW =
true;
3477 const APInt *
C =
nullptr;
3478 Value *CommonAncestor;
3479 if (MatchForward(Cond0)) {
3483 if (!MatchForward(CommonAncestor))
3520 Type *SelType =
SI.getType();
3527 Value *Cond0, *Ctlz, *CtlzOp;
3543 !isSafeToRemoveBitCeilSelect(Pred, Cond0, Cond1, CtlzOp,
BitWidth,
3548 cast<Instruction>(CtlzOp)->setHasNoUnsignedWrap(
false);
3567 Value *TV =
SI.getTrueValue();
3568 Value *FV =
SI.getFalseValue();
3591 bool Replace =
false;
3625 FastMathFlags FMF = cast<FPMathOperator>(TrueVal)->getFastMathFlags();
3647 if (
auto *
I = dyn_cast<Instruction>(V)) {
3648 if (isa<PHINode>(
I)) {
3654 return Op->getType()->isIntOrIntVectorTy() &&
3655 hasAffectedValue(Op, Affected, Depth + 1);
3663 Value *CondVal =
SI.getCondition();
3666 Type *SelType =
SI.getType();
3675 if (
Instruction *
I = canonicalizeScalarSelectOfVecs(SI, *
this))
3711 return new ZExtInst(CondVal, SelType);
3715 return new SExtInst(CondVal, SelType);
3720 return new ZExtInst(NotCond, SelType);
3726 return new SExtInst(NotCond, SelType);
3730 auto *SIFPOp = dyn_cast<FPMathOperator>(&SI);
3732 if (
auto *FCmp = dyn_cast<FCmpInst>(CondVal)) {
3734 Value *Cmp0 = FCmp->getOperand(0), *Cmp1 = FCmp->getOperand(1);
3736 if ((Cmp0 == TrueVal && Cmp1 == FalseVal) ||
3737 (Cmp0 == FalseVal && Cmp1 == TrueVal)) {
3749 FCmp->getName() +
".inv");
3769 Value *MatchCmp0 =
nullptr;
3770 Value *MatchCmp1 =
nullptr;
3782 if (Cmp0 == MatchCmp0 &&
3783 matchFMulByZeroIfResultEqZero(*
this, Cmp0, Cmp1, MatchCmp1, MatchCmp0,
3784 SI, SIFPOp->hasNoSignedZeros()))
3794 if (SIFPOp->hasNoNaNs() && SIFPOp->hasNoSignedZeros()) {
3807 if (
Instruction *Fabs = foldSelectWithFCmpToFabs(SI, *
this))
3811 if (
ICmpInst *ICI = dyn_cast<ICmpInst>(CondVal))
3825 auto *TI = dyn_cast<Instruction>(TrueVal);
3826 auto *FI = dyn_cast<Instruction>(FalseVal);
3827 if (TI && FI && TI->getOpcode() == FI->
getOpcode())
3846 if (isa<VectorType>(CondVal->
getType()) && !isa<VectorType>(
Idx->getType()))
3858 if (
auto *TrueGep = dyn_cast<GetElementPtrInst>(TrueVal))
3859 if (
auto *NewGep = SelectGepWithBase(TrueGep, FalseVal,
false))
3861 if (
auto *FalseGep = dyn_cast<GetElementPtrInst>(FalseVal))
3862 if (
auto *NewGep = SelectGepWithBase(FalseGep, TrueVal,
true))
3878 RHS2, SI, SPF, RHS))
3882 RHS2, SI, SPF, LHS))
3891 bool IsCastNeeded =
LHS->
getType() != SelType;
3892 Value *CmpLHS = cast<CmpInst>(CondVal)->getOperand(0);
3893 Value *CmpRHS = cast<CmpInst>(CondVal)->getOperand(1);
3896 ((CmpLHS != LHS && CmpLHS != RHS) ||
3897 (CmpRHS != LHS && CmpRHS != RHS)))) {
3906 cast<FPMathOperator>(
SI.getCondition())->getFastMathFlags();
3922 if (
auto *PN = dyn_cast<PHINode>(
SI.getCondition()))
3924 if (canSelectOperandBeMappingIntoPredBlock(TrueVal, SI) &&
3925 canSelectOperandBeMappingIntoPredBlock(FalseVal, SI))
3929 if (
SelectInst *TrueSI = dyn_cast<SelectInst>(TrueVal)) {
3930 if (TrueSI->getCondition()->getType() == CondVal->
getType()) {
3933 if (
Value *V = simplifyNestedSelectsUsingImpliedCond(
3934 *TrueSI, CondVal,
true,
DL))
3941 if (TrueSI->getFalseValue() == FalseVal && TrueSI->hasOneUse()) {
3949 if (
SelectInst *FalseSI = dyn_cast<SelectInst>(FalseVal)) {
3950 if (FalseSI->getCondition()->getType() == CondVal->
getType()) {
3953 if (
Value *V = simplifyNestedSelectsUsingImpliedCond(
3954 *FalseSI, CondVal,
false,
DL))
3958 if (FalseSI->getTrueValue() == TrueVal && FalseSI->hasOneUse()) {
3975 if (
auto *TrueBOSI = dyn_cast<SelectInst>(TrueBO->
getOperand(0))) {
3976 if (TrueBOSI->getCondition() == CondVal) {
3982 if (
auto *TrueBOSI = dyn_cast<SelectInst>(TrueBO->
getOperand(1))) {
3983 if (TrueBOSI->getCondition() == CondVal) {
3994 if (
auto *FalseBOSI = dyn_cast<SelectInst>(FalseBO->
getOperand(0))) {
3995 if (FalseBOSI->getCondition() == CondVal) {
4001 if (
auto *FalseBOSI = dyn_cast<SelectInst>(FalseBO->
getOperand(1))) {
4002 if (FalseBOSI->getCondition() == CondVal) {
4015 SI.swapProfMetadata();
4030 if (Known.One.isOne())
4032 if (Known.Zero.isOne())
4040 if (
Value *V = foldSelectCmpXchg(SI))
4058 if (
Value *V = foldRoundUpIntegerWithPow2Alignment(SI,
Builder))
4068 auto *MaskedInst = cast<IntrinsicInst>(TrueVal);
4069 if (isa<UndefValue>(MaskedInst->getArgOperand(3)))
4070 MaskedInst->setArgOperand(3, FalseVal );
4085 bool CanMergeSelectIntoLoad =
false;
4089 if (CanMergeSelectIntoLoad) {
4090 auto *MaskedInst = cast<IntrinsicInst>(FalseVal);
4091 if (isa<UndefValue>(MaskedInst->getArgOperand(3)))
4092 MaskedInst->setArgOperand(3, TrueVal );
4121 auto FoldSelectWithAndOrCond = [&](
bool IsAnd,
Value *
A,
4129 if (
ICmpInst *Cmp = dyn_cast<ICmpInst>(
B))
4130 if (
Value *V = canonicalizeSPF(*Cmp, TrueVal, FalseVal, *
this))
4132 IsAnd ? FalseVal : V);
4140 if (
Instruction *
I = FoldSelectWithAndOrCond(
true, LHS, RHS))
4142 if (
Instruction *
I = FoldSelectWithAndOrCond(
true, RHS, LHS))
4145 if (
Instruction *
I = FoldSelectWithAndOrCond(
false, LHS, RHS))
4147 if (
Instruction *
I = FoldSelectWithAndOrCond(
false, RHS, LHS))
4153 if (
Instruction *
I = FoldSelectWithAndOrCond(
true, LHS, RHS))
4156 if (
Instruction *
I = FoldSelectWithAndOrCond(
false, LHS, RHS))
4163 return BinaryOperator::CreateXor(CondVal, FalseVal);
4168 (!isa<Constant>(TrueVal) || !isa<Constant>(FalseVal))) {
4172 CC.AffectedValues.insert(V);
4175 if (!
CC.AffectedValues.empty()) {
4176 if (!isa<Constant>(TrueVal) &&
4177 hasAffectedValue(TrueVal,
CC.AffectedValues, 0)) {
4185 if (!isa<Constant>(FalseVal) &&
4186 hasAffectedValue(FalseVal,
CC.AffectedValues, 0)) {
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
amdgpu AMDGPU Register Bank Select
This file implements a class to represent arbitrary precision integral constant values and operations...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file provides internal interfaces used to implement the InstCombine.
static Value * canonicalizeSaturatedAdd(ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder)
static Instruction * foldSetClearBits(SelectInst &Sel, InstCombiner::BuilderTy &Builder)
Canonicalize a set or clear of a masked set of constant bits to select-of-constants form.
static Instruction * foldSelectICmpAndAnd(Type *SelType, const ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder)
We want to turn: (select (icmp eq (and X, Y), 0), (and (lshr X, Z), 1), 1) into: zext (icmp ne i32 (a...
static unsigned getSelectFoldableOperands(BinaryOperator *I)
We want to turn code that looks like this: C = or A, B D = select cond, C, A into: C = select cond,...
static Instruction * foldSelectZeroOrMul(SelectInst &SI, InstCombinerImpl &IC)
static Value * canonicalizeSaturatedSubtract(const ICmpInst *ICI, const Value *TrueVal, const Value *FalseVal, InstCombiner::BuilderTy &Builder)
Transform patterns such as (a > b) ? a - b : 0 into usub.sat(a, b).
static Value * foldAbsDiff(ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder)
Try to match patterns with select and subtract as absolute difference.
static Instruction * foldSelectBinOpIdentity(SelectInst &Sel, const TargetLibraryInfo &TLI, InstCombinerImpl &IC)
Replace a select operand based on an equality comparison with the identity constant of a binop.
static Value * foldSelectICmpAndZeroShl(const ICmpInst *Cmp, Value *TVal, Value *FVal, InstCombiner::BuilderTy &Builder)
We want to turn: (select (icmp eq (and X, C1), 0), 0, (shl [nsw/nuw] X, C2)); iff C1 is a mask and th...
static Value * foldSelectICmpAndBinOp(const ICmpInst *IC, Value *TrueVal, Value *FalseVal, InstCombiner::BuilderTy &Builder)
We want to turn: (select (icmp eq (and X, C1