57 if (
F.getFnAttribute(
"disable-tail-calls").getValueAsBool())
63 AttrBuilder CallerAttrs(
F.getContext(),
F.getAttributes().getRetAttrs());
64 for (
const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
65 Attribute::DereferenceableOrNull, Attribute::NoAlias,
66 Attribute::NonNull, Attribute::NoUndef})
73 if (CallerAttrs.
contains(Attribute::ZExt) ||
74 CallerAttrs.
contains(Attribute::SExt))
85 for (
unsigned I = 0,
E = ArgLocs.
size();
I !=
E; ++
I) {
101 Register ArgReg = cast<RegisterSDNode>(
Value->getOperand(1))->getReg();
102 if (
MRI.getLiveInPhysReg(ArgReg) != Reg)
112 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
113 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
114 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
115 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
116 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
117 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
118 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
119 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
120 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
121 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
122 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
123 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
124 Alignment = Call->getParamStackAlign(ArgIdx);
127 "multiple ABI attributes?");
143std::pair<SDValue, SDValue>
153 Args.reserve(Ops.
size());
156 for (
unsigned i = 0; i < Ops.
size(); ++i) {
159 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.
getContext());
162 Entry.IsZExt = !Entry.IsSExt;
166 Entry.IsSExt = Entry.IsZExt =
false;
168 Args.push_back(Entry);
171 if (LC == RTLIB::UNKNOWN_LIBCALL)
179 bool zeroExtend = !signExtend;
183 signExtend = zeroExtend =
false;
194 return LowerCallTo(CLI);
198 std::vector<EVT> &MemOps,
unsigned Limit,
const MemOp &Op,
unsigned DstAS,
200 if (Limit != ~
unsigned(0) && Op.isMemcpyWithFixedDstAlign() &&
201 Op.getSrcAlign() < Op.getDstAlign())
211 if (Op.isFixedDstAlign())
229 unsigned NumMemOps = 0;
233 while (VTSize >
Size) {
265 if (NumMemOps && Op.allowOverlap() && NewVTSize <
Size &&
267 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() :
Align(1),
277 if (++NumMemOps > Limit)
280 MemOps.push_back(VT);
295 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
305 bool IsSignaling)
const {
311 &&
"Unsupported setcc type!");
314 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
315 bool ShouldInvertCC =
false;
319 LC1 = (VT ==
MVT::f32) ? RTLIB::OEQ_F32 :
321 (VT ==
MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
325 LC1 = (VT ==
MVT::f32) ? RTLIB::UNE_F32 :
327 (VT ==
MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
331 LC1 = (VT ==
MVT::f32) ? RTLIB::OGE_F32 :
333 (VT ==
MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
337 LC1 = (VT ==
MVT::f32) ? RTLIB::OLT_F32 :
339 (VT ==
MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
343 LC1 = (VT ==
MVT::f32) ? RTLIB::OLE_F32 :
345 (VT ==
MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
349 LC1 = (VT ==
MVT::f32) ? RTLIB::OGT_F32 :
351 (VT ==
MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
354 ShouldInvertCC =
true;
357 LC1 = (VT ==
MVT::f32) ? RTLIB::UO_F32 :
359 (VT ==
MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
363 ShouldInvertCC =
true;
366 LC1 = (VT ==
MVT::f32) ? RTLIB::UO_F32 :
368 (VT ==
MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
369 LC2 = (VT ==
MVT::f32) ? RTLIB::OEQ_F32 :
371 (VT ==
MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
375 ShouldInvertCC =
true;
378 LC1 = (VT ==
MVT::f32) ? RTLIB::OGE_F32 :
380 (VT ==
MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
383 LC1 = (VT ==
MVT::f32) ? RTLIB::OGT_F32 :
385 (VT ==
MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
388 LC1 = (VT ==
MVT::f32) ? RTLIB::OLE_F32 :
390 (VT ==
MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
393 LC1 = (VT ==
MVT::f32) ? RTLIB::OLT_F32 :
395 (VT ==
MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
403 SDValue Ops[2] = {NewLHS, NewRHS};
408 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
413 if (ShouldInvertCC) {
415 CCCode = getSetCCInverse(CCCode, RetVT);
418 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
425 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
428 CCCode = getSetCCInverse(CCCode, RetVT);
429 NewLHS = DAG.
getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
443 if (!isPositionIndependent())
457 unsigned JTEncoding = getJumpTableEncoding();
482 if (!
TM.shouldAssumeDSOLocal(*GV->
getParent(), GV))
486 if (isPositionIndependent())
502 const APInt &DemandedElts,
505 unsigned Opcode = Op.getOpcode();
508 if (targetShrinkDemandedConstant(Op,
DemandedBits, DemandedElts, TLO))
518 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
519 if (!Op1C || Op1C->isOpaque())
523 const APInt &
C = Op1C->getAPIntValue();
528 EVT VT = Op.getValueType();
544 EVT VT = Op.getValueType();
555 const APInt &Demanded,
557 assert(Op.getNumOperands() == 2 &&
558 "ShrinkDemandedOp only supports binary operators!");
559 assert(Op.getNode()->getNumValues() == 1 &&
560 "ShrinkDemandedOp only supports nodes with one result!");
566 if (Op.getValueType().isVector())
571 if (!Op.getNode()->hasOneUse())
578 unsigned SmallVTBits = DemandedSize;
586 Op.getOpcode(), dl, SmallVT,
589 assert(DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?");
604 bool Simplified = SimplifyDemandedBits(Op,
DemandedBits, Known, TLO);
613 const APInt &DemandedElts,
621 SimplifyDemandedBits(Op,
DemandedBits, DemandedElts, Known, TLO);
633 bool AssumeSingleUse)
const {
634 EVT VT = Op.getValueType();
650 EVT VT = Op.getValueType();
668 switch (Op.getOpcode()) {
674 EVT SrcVT = Src.getValueType();
675 EVT DstVT = Op.getValueType();
681 if (NumSrcEltBits == NumDstEltBits)
682 if (
SDValue V = SimplifyMultipleUseDemandedBits(
686 if (SrcVT.
isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
687 unsigned Scale = NumDstEltBits / NumSrcEltBits;
691 for (
unsigned i = 0; i != Scale; ++i) {
692 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
693 unsigned BitOffset = EltOffset * NumSrcEltBits;
696 DemandedSrcBits |= Sub;
697 for (
unsigned j = 0; j != NumElts; ++j)
699 DemandedSrcElts.
setBit((j * Scale) + i);
703 if (
SDValue V = SimplifyMultipleUseDemandedBits(
704 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
709 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
710 unsigned Scale = NumSrcEltBits / NumDstEltBits;
714 for (
unsigned i = 0; i != NumElts; ++i)
715 if (DemandedElts[i]) {
716 unsigned Offset = (i % Scale) * NumDstEltBits;
718 DemandedSrcElts.
setBit(i / Scale);
721 if (
SDValue V = SimplifyMultipleUseDemandedBits(
722 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
736 return Op.getOperand(0);
738 return Op.getOperand(1);
749 return Op.getOperand(0);
751 return Op.getOperand(1);
761 return Op.getOperand(0);
763 return Op.getOperand(1);
769 if (
const APInt *MaxSA =
771 SDValue Op0 = Op.getOperand(0);
772 unsigned ShAmt = MaxSA->getZExtValue();
773 unsigned NumSignBits =
776 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
782 SDValue Op0 = Op.getOperand(0);
783 SDValue Op1 = Op.getOperand(1);
804 SDValue Op0 = Op.getOperand(0);
805 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
811 if (NumSignBits >= (
BitWidth - ExBits + 1))
823 SDValue Src = Op.getOperand(0);
824 EVT SrcVT = Src.getValueType();
825 EVT DstVT = Op.getValueType();
826 if (IsLE && DemandedElts == 1 &&
838 SDValue Vec = Op.getOperand(0);
839 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
842 !DemandedElts[CIdx->getZExtValue()])
850 SDValue Vec = Op.getOperand(0);
851 SDValue Sub = Op.getOperand(1);
856 if (DemandedSubElts == 0)
862 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
866 bool AllUndef =
true, IdentityLHS =
true, IdentityRHS =
true;
867 for (
unsigned i = 0; i != NumElts; ++i) {
868 int M = ShuffleMask[i];
869 if (M < 0 || !DemandedElts[i])
872 IdentityLHS &= (M == (int)i);
873 IdentityRHS &= ((M - NumElts) == i);
877 return DAG.
getUNDEF(Op.getValueType());
879 return Op.getOperand(0);
881 return Op.getOperand(1);
891 if (
SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
901 unsigned Depth)
const {
902 EVT VT = Op.getValueType();
909 return SimplifyMultipleUseDemandedBits(Op,
DemandedBits, DemandedElts, DAG,
915 unsigned Depth)
const {
917 return SimplifyMultipleUseDemandedBits(Op,
DemandedBits, DemandedElts, DAG,
926 const APInt &DemandedElts,
929 "SRL or SRA node is required here!");
932 if (!N1C || !N1C->
isOne())
978 unsigned ShiftOpc = Op.getOpcode();
979 bool IsSigned =
false;
983 unsigned NumSigned = std::min(NumSignedA, NumSignedB) - 1;
988 unsigned NumZero = std::min(NumZeroA, NumZeroB);
994 if (NumZero >= 2 && NumSigned < NumZero) {
999 if (NumSigned >= 1) {
1007 if (NumZero >= 1 && NumSigned < NumZero) {
1027 EVT VT = Op.getValueType();
1036 if (((!IsSigned &&
Add->getFlags().hasNoUnsignedWrap() &&
1038 (IsSigned &&
Add->getFlags().hasNoSignedWrap() &&
1064 unsigned Depth,
bool AssumeSingleUse)
const {
1067 "Mask size mismatches value type size!");
1072 EVT VT = Op.getValueType();
1074 unsigned NumElts = OriginalDemandedElts.
getBitWidth();
1076 "Unexpected vector size");
1079 APInt DemandedElts = OriginalDemandedElts;
1100 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
1105 bool HasMultiUse =
false;
1106 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1115 }
else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1124 switch (Op.getOpcode()) {
1128 if (!DemandedElts[0])
1132 SDValue Src = Op.getOperand(0);
1133 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1135 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO,
Depth + 1))
1140 if (DemandedElts == 1)
1150 auto *LD = cast<LoadSDNode>(Op);
1151 if (getTargetConstantFromLoad(LD)) {
1157 EVT MemVT = LD->getMemoryVT();
1167 SDValue Vec = Op.getOperand(0);
1168 SDValue Scl = Op.getOperand(1);
1169 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1174 APInt DemandedVecElts(DemandedElts);
1176 unsigned Idx = CIdx->getZExtValue();
1180 if (!DemandedElts[
Idx])
1187 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO,
Depth + 1))
1193 if (SimplifyDemandedBits(Vec,
DemandedBits, DemandedVecElts, KnownVec, TLO,
1197 if (!!DemandedVecElts)
1207 SDValue Src = Op.getOperand(0);
1208 SDValue Sub = Op.getOperand(1);
1212 APInt DemandedSrcElts = DemandedElts;
1216 if (SimplifyDemandedBits(Sub,
DemandedBits, DemandedSubElts, KnownSub, TLO,
1219 if (SimplifyDemandedBits(Src,
DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1225 if (!!DemandedSubElts)
1227 if (!!DemandedSrcElts)
1233 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1235 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1237 if (NewSub || NewSrc) {
1238 NewSub = NewSub ? NewSub : Sub;
1239 NewSrc = NewSrc ? NewSrc : Src;
1251 SDValue Src = Op.getOperand(0);
1252 if (Src.getValueType().isScalableVector())
1255 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1258 if (SimplifyDemandedBits(Src,
DemandedBits, DemandedSrcElts, Known, TLO,
1264 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1279 EVT SubVT = Op.getOperand(0).getValueType();
1280 unsigned NumSubVecs = Op.getNumOperands();
1282 for (
unsigned i = 0; i != NumSubVecs; ++i) {
1283 APInt DemandedSubElts =
1284 DemandedElts.
extractBits(NumSubElts, i * NumSubElts);
1285 if (SimplifyDemandedBits(Op.getOperand(i),
DemandedBits, DemandedSubElts,
1286 Known2, TLO,
Depth + 1))
1289 if (!!DemandedSubElts)
1296 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1299 APInt DemandedLHS, DemandedRHS;
1304 if (!!DemandedLHS || !!DemandedRHS) {
1305 SDValue Op0 = Op.getOperand(0);
1306 SDValue Op1 = Op.getOperand(1);
1310 if (!!DemandedLHS) {
1311 if (SimplifyDemandedBits(Op0,
DemandedBits, DemandedLHS, Known2, TLO,
1316 if (!!DemandedRHS) {
1317 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedRHS, Known2, TLO,
1324 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1326 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1328 if (DemandedOp0 || DemandedOp1) {
1329 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1330 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1338 SDValue Op0 = Op.getOperand(0);
1339 SDValue Op1 = Op.getOperand(1);
1365 LHSKnown.
One == ~RHSC->getAPIntValue()) {
1377 unsigned NumSubElts =
1394 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1399 Known2, TLO,
Depth + 1))
1422 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1424 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1426 if (DemandedOp0 || DemandedOp1) {
1427 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1428 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1438 SDValue Op0 = Op.getOperand(0);
1439 SDValue Op1 = Op.getOperand(1);
1441 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1445 if (SimplifyDemandedBits(Op0, ~Known.
One &
DemandedBits, DemandedElts,
1446 Known2, TLO,
Depth + 1))
1465 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1467 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1469 if (DemandedOp0 || DemandedOp1) {
1470 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1471 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1482 for (
int I = 0;
I != 2; ++
I) {
1483 SDValue X = Op.getOperand(
I).getOperand(0);
1484 SDValue C1 = Op.getOperand(
I).getOperand(1);
1485 SDValue Alt = Op.getOperand(1 -
I).getOperand(0);
1486 SDValue C2 = Op.getOperand(1 -
I).getOperand(1);
1488 for (
int J = 0; J != 2; ++J) {
1508 SDValue Op0 = Op.getOperand(0);
1509 SDValue Op1 = Op.getOperand(1);
1511 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1515 if (SimplifyDemandedBits(Op0,
DemandedBits, DemandedElts, Known2, TLO,
1543 if (
C->getAPIntValue() == Known2.
One) {
1552 if (!
C->isAllOnes() &&
DemandedBits.isSubsetOf(
C->getAPIntValue())) {
1564 if (ShiftC->getAPIntValue().ult(
BitWidth)) {
1565 uint64_t ShiftAmt = ShiftC->getZExtValue();
1568 : Ones.
lshr(ShiftAmt);
1586 if (!
C || !
C->isAllOnes())
1592 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1594 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1596 if (DemandedOp0 || DemandedOp1) {
1597 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1598 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1608 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, Known, TLO,
1611 if (SimplifyDemandedBits(Op.getOperand(1),
DemandedBits, Known2, TLO,
1625 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, DemandedElts,
1626 Known, TLO,
Depth + 1))
1628 if (SimplifyDemandedBits(Op.getOperand(1),
DemandedBits, DemandedElts,
1629 Known2, TLO,
Depth + 1))
1638 if (SimplifyDemandedBits(Op.getOperand(3),
DemandedBits, Known, TLO,
1641 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, Known2, TLO,
1655 SDValue Op0 = Op.getOperand(0);
1656 SDValue Op1 = Op.getOperand(1);
1683 SDValue Op0 = Op.getOperand(0);
1684 SDValue Op1 = Op.getOperand(1);
1687 if (
const APInt *SA =
1689 unsigned ShAmt = SA->getZExtValue();
1699 if (
const APInt *SA2 =
1701 unsigned C1 = SA2->getZExtValue();
1703 int Diff = ShAmt - C1;
1722 if (ShAmt < InnerBits &&
DemandedBits.getActiveBits() <= InnerBits &&
1723 isTypeDesirableForOp(
ISD::SHL, InnerVT)) {
1742 if (
const APInt *SA2 =
1744 unsigned InnerShAmt = SA2->getZExtValue();
1745 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1747 (InnerBits - InnerShAmt + ShAmt) &&
1761 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1765 Known.
Zero <<= ShAmt;
1766 Known.
One <<= ShAmt;
1772 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1773 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1789 if (
unsigned CTLZ =
DemandedBits.countLeadingZeros()) {
1791 if (SimplifyDemandedBits(Op0, DemandedFromOp, DemandedElts, Known, TLO,
1794 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
1797 Flags.setNoSignedWrap(
false);
1798 Flags.setNoUnsignedWrap(
false);
1799 Op->setFlags(Flags);
1809 if (
const APInt *MaxSA =
1811 unsigned ShAmt = MaxSA->getZExtValue();
1812 unsigned NumSignBits =
1815 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1821 SDValue Op0 = Op.getOperand(0);
1822 SDValue Op1 = Op.getOperand(1);
1827 DemandedElts,
Depth + 1))
1830 if (
const APInt *SA =
1832 unsigned ShAmt = SA->getZExtValue();
1842 if (
const APInt *SA2 =
1844 unsigned C1 = SA2->getZExtValue();
1846 int Diff = ShAmt - C1;
1862 if (Op->getFlags().hasExact())
1866 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1877 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1878 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1888 SDValue Op0 = Op.getOperand(0);
1889 SDValue Op1 = Op.getOperand(1);
1908 DemandedElts,
Depth + 1))
1911 if (
const APInt *SA =
1913 unsigned ShAmt = SA->getZExtValue();
1921 if (Op->getFlags().hasExact())
1929 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1941 Flags.setExact(Op->getFlags().hasExact());
1959 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1960 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1971 SDValue Op0 = Op.getOperand(0);
1972 SDValue Op1 = Op.getOperand(1);
1973 SDValue Op2 = Op.getOperand(2);
1974 bool IsFSHL = (Op.getOpcode() ==
ISD::FSHL);
1977 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
1982 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1,
DemandedBits, DemandedElts,
1983 Known, TLO,
Depth + 1))
1992 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1995 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
2009 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2010 Op0, Demanded0, DemandedElts, TLO.
DAG,
Depth + 1);
2011 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2012 Op1, Demanded1, DemandedElts, TLO.
DAG,
Depth + 1);
2013 if (DemandedOp0 || DemandedOp1) {
2014 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2015 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2026 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
2027 Known2, TLO,
Depth + 1))
2034 SDValue Op0 = Op.getOperand(0);
2035 SDValue Op1 = Op.getOperand(1);
2036 bool IsROTL = (Op.getOpcode() ==
ISD::ROTL);
2043 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
2049 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2059 DemandedBits.countTrailingZeros() >= (IsROTL ? Amt : RevAmt)) {
2064 DemandedBits.countLeadingZeros() >= (IsROTL ? RevAmt : Amt)) {
2073 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
2081 SDValue Op0 = Op.getOperand(0);
2082 SDValue Op1 = Op.getOperand(1);
2087 return TLO.
CombineTo(Op, *IsULE ? Op0 : Op1);
2089 return TLO.
CombineTo(Op, *IsULT ? Op0 : Op1);
2094 SDValue Op0 = Op.getOperand(0);
2095 SDValue Op1 = Op.getOperand(1);
2100 return TLO.
CombineTo(Op, *IsUGE ? Op0 : Op1);
2102 return TLO.
CombineTo(Op, *IsUGT ? Op0 : Op1);
2106 SDValue Src = Op.getOperand(0);
2108 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2116 SDValue Src = Op.getOperand(0);
2135 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2143 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2162 SDValue Op0 = Op.getOperand(0);
2163 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2168 unsigned MinSignedBits =
2170 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2173 if (!AlreadySignExtended) {
2191 InputDemandedBits.
setBit(ExVTBits - 1);
2193 if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO,
2202 if (Known.
Zero[ExVTBits - 1])
2206 if (Known.
One[ExVTBits - 1]) {
2216 EVT HalfVT = Op.getOperand(0).getValueType();
2224 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO,
Depth + 1))
2227 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO,
Depth + 1))
2230 Known = KnownHi.
concat(KnownLo);
2238 SDValue Src = Op.getOperand(0);
2239 EVT SrcVT = Src.getValueType();
2248 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2259 APInt InDemandedElts = DemandedElts.
zext(InElts);
2260 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2268 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2269 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2278 SDValue Src = Op.getOperand(0);
2279 EVT SrcVT = Src.getValueType();
2288 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2299 APInt InDemandedElts = DemandedElts.
zext(InElts);
2303 InDemandedBits.
setBit(InBits - 1);
2305 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2323 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2324 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2333 SDValue Src = Op.getOperand(0);
2334 EVT SrcVT = Src.getValueType();
2341 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2346 APInt InDemandedElts = DemandedElts.
zext(InElts);
2347 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2355 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2356 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2361 SDValue Src = Op.getOperand(0);
2365 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2367 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2373 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2374 Src, TruncMask, DemandedElts, TLO.
DAG,
Depth + 1))
2379 switch (Src.getOpcode()) {
2390 if (Src.getNode()->hasOneUse()) {
2391 const APInt *ShAmtC =
2422 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2424 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask |
DemandedBits, Known,
2429 Known.
Zero |= ~InMask;
2430 Known.
One &= (~Known.Zero);
2434 SDValue Src = Op.getOperand(0);
2436 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2437 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2445 if (
auto *CIdx = dyn_cast<ConstantSDNode>(
Idx))
2446 if (CIdx->getAPIntValue().ult(NumSrcElts))
2453 DemandedSrcBits = DemandedSrcBits.
trunc(EltBitWidth);
2455 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2461 if (
SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2462 Src, DemandedSrcBits, DemandedSrcElts, TLO.
DAG,
Depth + 1)) {
2477 SDValue Src = Op.getOperand(0);
2478 EVT SrcVT = Src.getValueType();
2495 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2496 if (!OpVTLegal && OpVTSizeInBits > 32)
2498 unsigned ShVal = Op.getValueSizeInBits() - 1;
2508 unsigned Scale =
BitWidth / NumSrcEltBits;
2512 for (
unsigned i = 0; i != Scale; ++i) {
2513 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2514 unsigned BitOffset = EltOffset * NumSrcEltBits;
2517 DemandedSrcBits |= Sub;
2518 for (
unsigned j = 0; j != NumElts; ++j)
2519 if (DemandedElts[j])
2520 DemandedSrcElts.
setBit((j * Scale) + i);
2524 APInt KnownSrcUndef, KnownSrcZero;
2525 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2526 KnownSrcZero, TLO,
Depth + 1))
2530 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2531 KnownSrcBits, TLO,
Depth + 1))
2533 }
else if (IsLE && (NumSrcEltBits %
BitWidth) == 0) {
2535 unsigned Scale = NumSrcEltBits /
BitWidth;
2539 for (
unsigned i = 0; i != NumElts; ++i)
2540 if (DemandedElts[i]) {
2543 DemandedSrcElts.
setBit(i / Scale);
2547 APInt KnownSrcUndef, KnownSrcZero;
2548 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2549 KnownSrcZero, TLO,
Depth + 1))
2554 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2555 KnownSrcBits, TLO,
Depth + 1))
2574 if (
C &&
C->getAPIntValue().countTrailingZeros() == CTZ) {
2584 if (Op.getOperand(0) == Op.getOperand(1) &&
DemandedBits.ult(4)) {
2594 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2596 unsigned DemandedBitsLZ =
DemandedBits.countLeadingZeros();
2598 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2600 SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2604 if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2607 Flags.setNoSignedWrap(
false);
2608 Flags.setNoUnsignedWrap(
false);
2609 Op->setFlags(Flags);
2616 isa<ConstantSDNode>(Op0) && cast<ConstantSDNode>(Op0)->isZero())
2621 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2622 Op0, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
2623 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2624 Op1, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
2625 if (DemandedOp0 || DemandedOp1) {
2626 Flags.setNoSignedWrap(
false);
2627 Flags.setNoUnsignedWrap(
false);
2628 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2629 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2631 TLO.
DAG.
getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2643 if (
C && !
C->isAllOnes() && !
C->isOne() &&
2644 (
C->getAPIntValue() | HighMask).isAllOnes()) {
2648 Flags.setNoSignedWrap(
false);
2649 Flags.setNoUnsignedWrap(
false);
2657 auto getShiftLeftAmt = [&HighMask](
SDValue Mul) ->
unsigned {
2684 if (
unsigned ShAmt = getShiftLeftAmt(Op0))
2687 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
2688 return foldMul(
ISD::SUB, Op1.getOperand(0), Op0, ShAmt);
2692 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
2693 return foldMul(
ISD::ADD, Op1.getOperand(0), Op0, ShAmt);
2705 if (Op.getValueType().isScalableVector())
2707 if (SimplifyDemandedBitsForTargetNode(Op,
DemandedBits, DemandedElts,
2720 if (!isTargetCanonicalConstantNode(Op) &&
2723 const SDNode *
N = Op.getNode();
2749 const APInt &DemandedElts,
2755 APInt KnownUndef, KnownZero;
2757 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2769 const APInt &UndefOp0,
2770 const APInt &UndefOp1) {
2773 "Vector binop only");
2778 UndefOp1.
getBitWidth() == NumElts &&
"Bad type for undef analysis");
2780 auto getUndefOrConstantElt = [&](
SDValue V,
unsigned Index,
2781 const APInt &UndefVals) {
2782 if (UndefVals[
Index])
2785 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2789 auto *
C = dyn_cast<ConstantSDNode>(Elt);
2790 if (isa<ConstantFPSDNode>(Elt) || Elt.
isUndef() || (
C && !
C->isOpaque()))
2798 for (
unsigned i = 0; i != NumElts; ++i) {
2817 bool AssumeSingleUse)
const {
2818 EVT VT = Op.getValueType();
2819 unsigned Opcode = Op.getOpcode();
2820 APInt DemandedElts = OriginalDemandedElts;
2835 "Mask size mismatches value type element count!");
2844 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
2848 if (DemandedElts == 0) {
2863 auto SimplifyDemandedVectorEltsBinOp = [&](
SDValue Op0,
SDValue Op1) {
2864 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2866 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2868 if (NewOp0 || NewOp1) {
2870 Opcode,
SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2878 if (!DemandedElts[0]) {
2882 SDValue ScalarSrc = Op.getOperand(0);
2886 EVT SrcVT = Src.getValueType();
2898 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2907 SDValue Src = Op.getOperand(0);
2908 EVT SrcVT = Src.getValueType();
2917 if (NumSrcElts == NumElts)
2918 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2919 KnownZero, TLO,
Depth + 1);
2921 APInt SrcDemandedElts, SrcZero, SrcUndef;
2925 if ((NumElts % NumSrcElts) == 0) {
2926 unsigned Scale = NumElts / NumSrcElts;
2928 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2938 for (
unsigned i = 0; i != NumElts; ++i)
2939 if (DemandedElts[i]) {
2940 unsigned Ofs = (i % Scale) * EltSizeInBits;
2941 SrcDemandedBits.
setBits(Ofs, Ofs + EltSizeInBits);
2945 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2953 for (
unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
2957 for (
unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
2958 unsigned Elt = Scale * SrcElt + SubElt;
2959 if (DemandedElts[Elt])
2967 for (
unsigned i = 0; i != NumSrcElts; ++i) {
2968 if (SrcDemandedElts[i]) {
2970 KnownZero.
setBits(i * Scale, (i + 1) * Scale);
2972 KnownUndef.
setBits(i * Scale, (i + 1) * Scale);
2980 if ((NumSrcElts % NumElts) == 0) {
2981 unsigned Scale = NumSrcElts / NumElts;
2983 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2989 for (
unsigned i = 0; i != NumElts; ++i) {
2990 if (DemandedElts[i]) {
3005 [&](
SDValue Elt) { return Op.getOperand(0) != Elt; })) {
3007 bool Updated =
false;
3008 for (
unsigned i = 0; i != NumElts; ++i) {
3009 if (!DemandedElts[i] && !Ops[i].
isUndef()) {
3019 for (
unsigned i = 0; i != NumElts; ++i) {
3021 if (
SrcOp.isUndef()) {
3023 }
else if (EltSizeInBits ==
SrcOp.getScalarValueSizeInBits() &&
3031 EVT SubVT = Op.getOperand(0).getValueType();
3032 unsigned NumSubVecs = Op.getNumOperands();
3034 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3035 SDValue SubOp = Op.getOperand(i);
3037 APInt SubUndef, SubZero;
3038 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
3041 KnownUndef.
insertBits(SubUndef, i * NumSubElts);
3042 KnownZero.
insertBits(SubZero, i * NumSubElts);
3047 bool FoundNewSub =
false;
3049 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3050 SDValue SubOp = Op.getOperand(i);
3052 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3053 SubOp, SubElts, TLO.
DAG,
Depth + 1);
3054 DemandedSubOps.
push_back(NewSubOp ? NewSubOp : SubOp);
3055 FoundNewSub = NewSubOp ?
true : FoundNewSub;
3068 SDValue Src = Op.getOperand(0);
3069 SDValue Sub = Op.getOperand(1);
3073 APInt DemandedSrcElts = DemandedElts;
3076 APInt SubUndef, SubZero;
3077 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
3082 if (!DemandedSrcElts && !Src.isUndef())
3087 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
3095 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3096 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3097 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3098 Sub, DemandedSubElts, TLO.
DAG,
Depth + 1);
3099 if (NewSrc || NewSub) {
3100 NewSrc = NewSrc ? NewSrc : Src;
3101 NewSub = NewSub ? NewSub : Sub;
3103 NewSub, Op.getOperand(2));
3111 SDValue Src = Op.getOperand(0);
3112 if (Src.getValueType().isScalableVector())
3115 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3118 APInt SrcUndef, SrcZero;
3119 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3127 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3128 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3138 SDValue Vec = Op.getOperand(0);
3139 SDValue Scl = Op.getOperand(1);
3140 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3144 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
3145 unsigned Idx = CIdx->getZExtValue();
3146 if (!DemandedElts[
Idx])
3149 APInt DemandedVecElts(DemandedElts);
3151 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
3152 KnownZero, TLO,
Depth + 1))
3161 APInt VecUndef, VecZero;
3162 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
3169 SDValue Sel = Op.getOperand(0);
3175 APInt UndefSel, UndefZero;
3176 if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, UndefZero, TLO,
3181 APInt DemandedLHS(DemandedElts);
3182 APInt DemandedRHS(DemandedElts);
3183 APInt UndefLHS, ZeroLHS;
3184 APInt UndefRHS, ZeroRHS;
3185 if (SimplifyDemandedVectorElts(
LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3188 if (SimplifyDemandedVectorElts(
RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3192 KnownUndef = UndefLHS & UndefRHS;
3193 KnownZero = ZeroLHS & ZeroRHS;
3197 APInt DemandedSel = DemandedElts & ~KnownZero;
3198 if (DemandedSel != DemandedElts)
3199 if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, UndefZero, TLO,
3208 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
3211 APInt DemandedLHS(NumElts, 0);
3212 APInt DemandedRHS(NumElts, 0);
3213 for (
unsigned i = 0; i != NumElts; ++i) {
3214 int M = ShuffleMask[i];
3215 if (M < 0 || !DemandedElts[i])
3217 assert(0 <= M && M < (
int)(2 * NumElts) &&
"Shuffle index out of range");
3218 if (M < (
int)NumElts)
3221 DemandedRHS.
setBit(M - NumElts);
3225 APInt UndefLHS, ZeroLHS;
3226 APInt UndefRHS, ZeroRHS;
3227 if (SimplifyDemandedVectorElts(
LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3230 if (SimplifyDemandedVectorElts(
RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3235 bool Updated =
false;
3236 bool IdentityLHS =
true, IdentityRHS =
true;
3238 for (
unsigned i = 0; i != NumElts; ++i) {
3239 int &M = NewMask[i];
3242 if (!DemandedElts[i] || (M < (
int)NumElts && UndefLHS[M]) ||
3243 (M >= (
int)NumElts && UndefRHS[M - NumElts])) {
3247 IdentityLHS &= (M < 0) || (M == (
int)i);
3248 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3253 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.
LegalOps) {
3255 buildLegalVectorShuffle(VT,
DL,
LHS,
RHS, NewMask, TLO.
DAG);
3261 for (
unsigned i = 0; i != NumElts; ++i) {
3262 int M = ShuffleMask[i];
3265 }
else if (M < (
int)NumElts) {
3271 if (UndefRHS[M - NumElts])
3273 if (ZeroRHS[M - NumElts])
3282 APInt SrcUndef, SrcZero;
3283 SDValue Src = Op.getOperand(0);
3284 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3285 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
3286 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3293 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3294 DemandedSrcElts == 1) {
3307 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() ==
ISD::AND &&
3308 Op->isOnlyUserOf(Src.getNode()) &&
3309 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3311 EVT SrcVT = Src.getValueType();
3318 ISD::AND,
DL, SrcVT, {Src.getOperand(1), Mask})) {
3330 SDValue Op0 = Op.getOperand(0);
3331 SDValue Op1 = Op.getOperand(1);
3332 if (Op0 == Op1 && Op->isOnlyUserOf(Op0.
getNode())) {
3333 APInt UndefLHS, ZeroLHS;
3334 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3348 SDValue Op0 = Op.getOperand(0);
3349 SDValue Op1 = Op.getOperand(1);
3351 APInt UndefRHS, ZeroRHS;
3352 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3355 APInt UndefLHS, ZeroLHS;
3356 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3360 KnownZero = ZeroLHS & ZeroRHS;
3366 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3375 SDValue Op0 = Op.getOperand(0);
3376 SDValue Op1 = Op.getOperand(1);
3378 APInt UndefRHS, ZeroRHS;
3379 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3382 APInt UndefLHS, ZeroLHS;
3383 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3387 KnownZero = ZeroLHS;
3388 KnownUndef = UndefLHS & UndefRHS;
3393 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3401 SDValue Op0 = Op.getOperand(0);
3402 SDValue Op1 = Op.getOperand(1);
3404 APInt SrcUndef, SrcZero;
3405 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
3410 APInt DemandedElts0 = DemandedElts & ~SrcZero;
3411 if (SimplifyDemandedVectorElts(Op0, DemandedElts0, KnownUndef, KnownZero,
3415 KnownUndef &= DemandedElts0;
3416 KnownZero &= DemandedElts0;
3421 if (DemandedElts.
isSubsetOf(SrcZero | KnownZero | SrcUndef | KnownUndef))
3428 KnownZero |= SrcZero;
3429 KnownUndef &= SrcUndef;
3430 KnownUndef &= ~KnownZero;
3434 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3441 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
3442 KnownZero, TLO,
Depth + 1))
3454 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
3455 KnownZero, TLO,
Depth))
3460 if (SimplifyDemandedBits(Op,
DemandedBits, OriginalDemandedElts, Known,
3461 TLO,
Depth, AssumeSingleUse))
3467 assert((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero");
3481 const APInt &DemandedElts,
3483 unsigned Depth)
const {
3488 "Should use MaskedValueIsZero if you don't know whether Op"
3489 " is a target node!");
3496 unsigned Depth)
const {
3508 unsigned Depth)
const {
3517 unsigned Depth)
const {
3522 "Should use ComputeNumSignBits if you don't know whether Op"
3523 " is a target node!");
3540 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3541 " is a target node!");
3552 "Should use SimplifyDemandedBits if you don't know whether Op"
3553 " is a target node!");
3554 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.
DAG,
Depth);
3566 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3567 " is a target node!");
3594 bool PoisonOnly,
unsigned Depth)
const {
3600 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3601 " is a target node!");
3607 bool PoisonOnly,
bool ConsiderFlags,
unsigned Depth)
const {
3612 "Should use canCreateUndefOrPoison if you don't know whether Op"
3613 " is a target node!");
3621 unsigned Depth)
const {
3626 "Should use isKnownNeverNaN if you don't know whether Op"
3627 " is a target node!");
3632 const APInt &DemandedElts,
3635 unsigned Depth)
const {
3640 "Should use isSplatValue if you don't know whether Op"
3641 " is a target node!");
3656 CVal = CN->getAPIntValue();
3657 EltWidth =
N.getValueType().getScalarSizeInBits();
3664 CVal = CVal.
trunc(EltWidth);
3670 return CVal.
isOne();
3712 return (
N->isOne() && !SExt) || (SExt && (
N->getValueType(0) !=
MVT::i1));
3715 return N->isAllOnes() && SExt;
3724 DAGCombinerInfo &DCI)
const {
3752 auto *AndC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
3753 if (AndC &&
isNullConstant(N1) && AndC->getAPIntValue().isPowerOf2() &&
3756 AndC->getAPIntValue().getActiveBits());
3787 if (DCI.isBeforeLegalizeOps() ||
3799 auto *YConst = dyn_cast<ConstantSDNode>(
Y);
3800 if (YConst && YConst->isZero())
3822SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3827 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3836 if (!(C01 = dyn_cast<ConstantSDNode>(N0->
getOperand(1))))
3840 EVT XVT =
X.getValueType();
3864 auto checkConstants = [&
I1, &I01]() ->
bool {
3869 if (checkConstants()) {
3877 if (!checkConstants())
3883 const unsigned KeptBits =
I1.logBase2();
3884 const unsigned KeptBitsMinusOne = I01.
logBase2();
3887 if (KeptBits != (KeptBitsMinusOne + 1))
3911SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3913 DAGCombinerInfo &DCI,
const SDLoc &
DL)
const {
3916 "Should be a comparison with 0.");
3918 "Valid only for [in]equality comparisons.");
3920 unsigned NewShiftOpcode;
3931 unsigned OldShiftOpcode = V.getOpcode();
3932 switch (OldShiftOpcode) {
3944 C = V.getOperand(0);
3949 Y = V.getOperand(1);
3953 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3954 X, XC,
CC,
Y, OldShiftOpcode, NewShiftOpcode, DAG);
3971 EVT VT =
X.getValueType();
3986 DAGCombinerInfo &DCI)
const {
3989 "Unexpected binop");
4016 !DCI.isBeforeLegalize());
4019 if (!DCI.isCalledByLegalizer())
4020 DCI.AddToWorklist(YShl1.
getNode());
4035 if (CTPOP.getOpcode() !=
ISD::CTPOP || !CTPOP.hasOneUse())
4038 EVT CTVT = CTPOP.getValueType();
4039 SDValue CTOp = CTPOP.getOperand(0);
4059 for (
unsigned i = 0; i <
Passes; i++) {
4100 auto getRotateSource = [](
SDValue X) {
4102 return X.getOperand(0);
4109 if (
SDValue R = getRotateSource(N0))
4142 if (!C1 || !C1->
isZero())
4151 if (!ShAmtC || ShAmtC->getAPIntValue().uge(
BitWidth))
4155 unsigned ShAmt = ShAmtC->getZExtValue();
4164 if (
Or.getOperand(0) ==
Other) {
4165 X =
Or.getOperand(0);
4166 Y =
Or.getOperand(1);
4169 if (
Or.getOperand(1) ==
Other) {
4170 X =
Or.getOperand(1);
4171 Y =
Or.getOperand(0);
4181 if (matchOr(F0, F1)) {
4188 if (matchOr(F1, F0)) {
4204 const SDLoc &dl)
const {
4214 bool N0ConstOrSplat =
4216 bool N1ConstOrSplat =
4227 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
4233 if (!N0ConstOrSplat && !N1ConstOrSplat &&
4238 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
4247 const APInt &C1 = N1C->getAPIntValue();
4267 return DAG.
getNode(LogicOp, dl, VT, IsXZero, IsYZero);
4297 if (
auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode())) {
4298 const APInt &C1 = N1C->getAPIntValue();
4313 if (
auto *
C = dyn_cast<ConstantSDNode>(N0->
getOperand(1)))
4314 if ((
C->getAPIntValue()+1).isPowerOf2()) {
4315 MinBits =
C->getAPIntValue().countTrailingOnes();
4323 }
else if (
auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
4326 MinBits = LN0->getMemoryVT().getSizeInBits();
4330 MinBits = LN0->getMemoryVT().getSizeInBits();
4341 MinBits >= ReqdBits) {
4343 if (isTypeDesirableForOp(
ISD::SETCC, MinVT)) {
4346 if (MinBits == 1 && C1 == 1)
4378 cast<CondCodeSDNode>(TopSetCC.
getOperand(2))->get(),
4399 unsigned bestWidth = 0, bestOffset = 0;
4402 unsigned maskWidth = origWidth;
4408 for (
unsigned width = origWidth / 2; width>=8; width /= 2) {
4410 for (
unsigned offset=0; offset<origWidth/width; offset++) {
4411 if (Mask.isSubsetOf(newMask)) {
4413 bestOffset = (
uint64_t)offset * (width/8);
4415 bestOffset = (origWidth/width - offset - 1) * (width/8);
4416 bestMask = Mask.lshr(offset * (width/8) * 8);
4429 if (bestOffset != 0)