57 if (
F.getFnAttribute(
"disable-tail-calls").getValueAsBool())
63 AttrBuilder CallerAttrs(
F.getContext(),
F.getAttributes().getRetAttrs());
64 for (
const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
65 Attribute::DereferenceableOrNull, Attribute::NoAlias,
66 Attribute::NonNull, Attribute::NoUndef})
73 if (CallerAttrs.
contains(Attribute::ZExt) ||
74 CallerAttrs.
contains(Attribute::SExt))
85 for (
unsigned I = 0,
E = ArgLocs.
size();
I !=
E; ++
I) {
101 Register ArgReg = cast<RegisterSDNode>(
Value->getOperand(1))->getReg();
102 if (
MRI.getLiveInPhysReg(ArgReg) != Reg)
112 IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
113 IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
114 IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
115 IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
116 IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
117 IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
118 IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
119 IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
120 IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
121 IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
122 IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
123 IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
124 Alignment = Call->getParamStackAlign(ArgIdx);
127 "multiple ABI attributes?");
143std::pair<SDValue, SDValue>
153 Args.reserve(Ops.
size());
156 for (
unsigned i = 0; i < Ops.
size(); ++i) {
159 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.
getContext());
162 Entry.IsZExt = !Entry.IsSExt;
166 Entry.IsSExt = Entry.IsZExt =
false;
168 Args.push_back(Entry);
171 if (LC == RTLIB::UNKNOWN_LIBCALL)
179 bool zeroExtend = !signExtend;
183 signExtend = zeroExtend =
false;
194 return LowerCallTo(CLI);
198 std::vector<EVT> &MemOps,
unsigned Limit,
const MemOp &Op,
unsigned DstAS,
200 if (Limit != ~
unsigned(0) && Op.isMemcpyWithFixedDstAlign() &&
201 Op.getSrcAlign() < Op.getDstAlign())
206 if (VT == MVT::Other) {
211 if (Op.isFixedDstAlign())
229 unsigned NumMemOps = 0;
233 while (VTSize >
Size) {
244 else if (NewVT == MVT::i64 &&
256 if (NewVT == MVT::i8)
265 if (NumMemOps && Op.allowOverlap() && NewVTSize <
Size &&
267 VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() :
Align(1),
277 if (++NumMemOps > Limit)
280 MemOps.push_back(VT);
295 return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
305 bool IsSignaling)
const {
310 assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
311 &&
"Unsupported setcc type!");
314 RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
315 bool ShouldInvertCC =
false;
319 LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
320 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
321 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
325 LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
326 (VT == MVT::f64) ? RTLIB::UNE_F64 :
327 (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
331 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
332 (VT == MVT::f64) ? RTLIB::OGE_F64 :
333 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
337 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
338 (VT == MVT::f64) ? RTLIB::OLT_F64 :
339 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
343 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
344 (VT == MVT::f64) ? RTLIB::OLE_F64 :
345 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
349 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
350 (VT == MVT::f64) ? RTLIB::OGT_F64 :
351 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
354 ShouldInvertCC =
true;
357 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
358 (VT == MVT::f64) ? RTLIB::UO_F64 :
359 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
363 ShouldInvertCC =
true;
366 LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
367 (VT == MVT::f64) ? RTLIB::UO_F64 :
368 (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
369 LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
370 (VT == MVT::f64) ? RTLIB::OEQ_F64 :
371 (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
375 ShouldInvertCC =
true;
378 LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
379 (VT == MVT::f64) ? RTLIB::OGE_F64 :
380 (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
383 LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
384 (VT == MVT::f64) ? RTLIB::OGT_F64 :
385 (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
388 LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
389 (VT == MVT::f64) ? RTLIB::OLE_F64 :
390 (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
393 LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
394 (VT == MVT::f64) ? RTLIB::OLT_F64 :
395 (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
403 SDValue Ops[2] = {NewLHS, NewRHS};
408 auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
413 if (ShouldInvertCC) {
415 CCCode = getSetCCInverse(CCCode, RetVT);
418 if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
425 auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
428 CCCode = getSetCCInverse(CCCode, RetVT);
429 NewLHS = DAG.
getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
443 if (!isPositionIndependent())
457 unsigned JTEncoding = getJumpTableEncoding();
482 if (!
TM.shouldAssumeDSOLocal(*GV->
getParent(), GV))
486 if (isPositionIndependent())
502 const APInt &DemandedElts,
505 unsigned Opcode = Op.getOpcode();
508 if (targetShrinkDemandedConstant(Op,
DemandedBits, DemandedElts, TLO))
518 auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
519 if (!Op1C || Op1C->isOpaque())
523 const APInt &
C = Op1C->getAPIntValue();
528 EVT VT = Op.getValueType();
544 EVT VT = Op.getValueType();
557 assert(Op.getNumOperands() == 2 &&
558 "ShrinkDemandedOp only supports binary operators!");
559 assert(Op.getNode()->getNumValues() == 1 &&
560 "ShrinkDemandedOp only supports nodes with one result!");
562 EVT VT = Op.getValueType();
572 if (!Op.getNode()->hasOneUse())
585 Op.getOpcode(), dl, SmallVT,
588 assert(DemandedSize <= SmallVTBits &&
"Narrowed below demanded bits?");
603 bool Simplified = SimplifyDemandedBits(Op,
DemandedBits, Known, TLO);
612 const APInt &DemandedElts,
620 SimplifyDemandedBits(Op,
DemandedBits, DemandedElts, Known, TLO);
632 bool AssumeSingleUse)
const {
633 EVT VT = Op.getValueType();
649 EVT VT = Op.getValueType();
667 switch (Op.getOpcode()) {
673 EVT SrcVT = Src.getValueType();
674 EVT DstVT = Op.getValueType();
680 if (NumSrcEltBits == NumDstEltBits)
681 if (
SDValue V = SimplifyMultipleUseDemandedBits(
685 if (SrcVT.
isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
686 unsigned Scale = NumDstEltBits / NumSrcEltBits;
690 for (
unsigned i = 0; i != Scale; ++i) {
691 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
692 unsigned BitOffset = EltOffset * NumSrcEltBits;
695 DemandedSrcBits |= Sub;
696 for (
unsigned j = 0; j != NumElts; ++j)
698 DemandedSrcElts.
setBit((j * Scale) + i);
702 if (
SDValue V = SimplifyMultipleUseDemandedBits(
703 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
708 if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
709 unsigned Scale = NumSrcEltBits / NumDstEltBits;
713 for (
unsigned i = 0; i != NumElts; ++i)
714 if (DemandedElts[i]) {
715 unsigned Offset = (i % Scale) * NumDstEltBits;
717 DemandedSrcElts.
setBit(i / Scale);
720 if (
SDValue V = SimplifyMultipleUseDemandedBits(
721 Src, DemandedSrcBits, DemandedSrcElts, DAG,
Depth + 1))
735 return Op.getOperand(0);
737 return Op.getOperand(1);
748 return Op.getOperand(0);
750 return Op.getOperand(1);
760 return Op.getOperand(0);
762 return Op.getOperand(1);
768 if (
const APInt *MaxSA =
770 SDValue Op0 = Op.getOperand(0);
771 unsigned ShAmt = MaxSA->getZExtValue();
772 unsigned NumSignBits =
775 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
781 SDValue Op0 = Op.getOperand(0);
782 SDValue Op1 = Op.getOperand(1);
803 SDValue Op0 = Op.getOperand(0);
804 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
811 if (NumSignBits >= (
BitWidth - ExBits + 1))
823 SDValue Src = Op.getOperand(0);
824 EVT SrcVT = Src.getValueType();
825 EVT DstVT = Op.getValueType();
826 if (IsLE && DemandedElts == 1 &&
838 SDValue Vec = Op.getOperand(0);
839 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
842 !DemandedElts[CIdx->getZExtValue()])
850 SDValue Vec = Op.getOperand(0);
851 SDValue Sub = Op.getOperand(1);
856 if (DemandedSubElts == 0)
862 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
866 bool AllUndef =
true, IdentityLHS =
true, IdentityRHS =
true;
867 for (
unsigned i = 0; i != NumElts; ++i) {
868 int M = ShuffleMask[i];
869 if (M < 0 || !DemandedElts[i])
872 IdentityLHS &= (M == (int)i);
873 IdentityRHS &= ((M - NumElts) == i);
877 return DAG.
getUNDEF(Op.getValueType());
879 return Op.getOperand(0);
881 return Op.getOperand(1);
891 if (
SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
901 unsigned Depth)
const {
902 EVT VT = Op.getValueType();
909 return SimplifyMultipleUseDemandedBits(Op,
DemandedBits, DemandedElts, DAG,
915 unsigned Depth)
const {
917 return SimplifyMultipleUseDemandedBits(Op,
DemandedBits, DemandedElts, DAG,
926 const APInt &DemandedElts,
929 "SRL or SRA node is required here!");
932 if (!N1C || !N1C->
isOne())
978 unsigned ShiftOpc = Op.getOpcode();
979 bool IsSigned =
false;
983 unsigned NumSigned = std::min(NumSignedA, NumSignedB) - 1;
988 unsigned NumZero = std::min(NumZeroA, NumZeroB);
994 if (NumZero >= 2 && NumSigned < NumZero) {
999 if (NumSigned >= 1) {
1007 if (NumZero >= 1 && NumSigned < NumZero) {
1027 EVT VT = Op.getValueType();
1040 Add.getOperand(1)) ==
1068 unsigned Depth,
bool AssumeSingleUse)
const {
1071 "Mask size mismatches value type size!");
1076 EVT VT = Op.getValueType();
1078 unsigned NumElts = OriginalDemandedElts.
getBitWidth();
1080 "Unexpected vector size");
1083 APInt DemandedElts = OriginalDemandedElts;
1104 cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
1109 bool HasMultiUse =
false;
1110 if (!AssumeSingleUse && !Op.getNode()->hasOneUse()) {
1119 }
else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
1128 switch (Op.getOpcode()) {
1132 if (!DemandedElts[0])
1136 SDValue Src = Op.getOperand(0);
1137 unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
1139 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO,
Depth + 1))
1144 if (DemandedElts == 1)
1154 auto *LD = cast<LoadSDNode>(Op);
1155 if (getTargetConstantFromLoad(LD)) {
1161 EVT MemVT = LD->getMemoryVT();
1171 SDValue Vec = Op.getOperand(0);
1172 SDValue Scl = Op.getOperand(1);
1173 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1178 APInt DemandedVecElts(DemandedElts);
1180 unsigned Idx = CIdx->getZExtValue();
1184 if (!DemandedElts[
Idx])
1191 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO,
Depth + 1))
1197 if (SimplifyDemandedBits(Vec,
DemandedBits, DemandedVecElts, KnownVec, TLO,
1201 if (!!DemandedVecElts)
1211 SDValue Src = Op.getOperand(0);
1212 SDValue Sub = Op.getOperand(1);
1216 APInt DemandedSrcElts = DemandedElts;
1220 if (SimplifyDemandedBits(Sub,
DemandedBits, DemandedSubElts, KnownSub, TLO,
1223 if (SimplifyDemandedBits(Src,
DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1229 if (!!DemandedSubElts)
1231 if (!!DemandedSrcElts)
1237 SDValue NewSub = SimplifyMultipleUseDemandedBits(
1239 SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1241 if (NewSub || NewSrc) {
1242 NewSub = NewSub ? NewSub : Sub;
1243 NewSrc = NewSrc ? NewSrc : Src;
1255 SDValue Src = Op.getOperand(0);
1256 if (Src.getValueType().isScalableVector())
1259 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1262 if (SimplifyDemandedBits(Src,
DemandedBits, DemandedSrcElts, Known, TLO,
1268 SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1283 EVT SubVT = Op.getOperand(0).getValueType();
1284 unsigned NumSubVecs = Op.getNumOperands();
1286 for (
unsigned i = 0; i != NumSubVecs; ++i) {
1287 APInt DemandedSubElts =
1288 DemandedElts.
extractBits(NumSubElts, i * NumSubElts);
1289 if (SimplifyDemandedBits(Op.getOperand(i),
DemandedBits, DemandedSubElts,
1290 Known2, TLO,
Depth + 1))
1293 if (!!DemandedSubElts)
1300 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1303 APInt DemandedLHS, DemandedRHS;
1308 if (!!DemandedLHS || !!DemandedRHS) {
1309 SDValue Op0 = Op.getOperand(0);
1310 SDValue Op1 = Op.getOperand(1);
1314 if (!!DemandedLHS) {
1315 if (SimplifyDemandedBits(Op0,
DemandedBits, DemandedLHS, Known2, TLO,
1320 if (!!DemandedRHS) {
1321 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedRHS, Known2, TLO,
1328 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1330 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1332 if (DemandedOp0 || DemandedOp1) {
1333 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1334 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1342 SDValue Op0 = Op.getOperand(0);
1343 SDValue Op1 = Op.getOperand(1);
1369 LHSKnown.
One == ~RHSC->getAPIntValue()) {
1381 unsigned NumSubElts =
1398 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1403 Known2, TLO,
Depth + 1))
1426 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1428 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1430 if (DemandedOp0 || DemandedOp1) {
1431 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1432 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1442 SDValue Op0 = Op.getOperand(0);
1443 SDValue Op1 = Op.getOperand(1);
1445 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1449 if (SimplifyDemandedBits(Op0, ~Known.
One &
DemandedBits, DemandedElts,
1450 Known2, TLO,
Depth + 1))
1469 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1471 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1473 if (DemandedOp0 || DemandedOp1) {
1474 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1475 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1486 for (
int I = 0;
I != 2; ++
I) {
1487 SDValue X = Op.getOperand(
I).getOperand(0);
1488 SDValue C1 = Op.getOperand(
I).getOperand(1);
1489 SDValue Alt = Op.getOperand(1 -
I).getOperand(0);
1490 SDValue C2 = Op.getOperand(1 -
I).getOperand(1);
1492 for (
int J = 0; J != 2; ++J) {
1512 SDValue Op0 = Op.getOperand(0);
1513 SDValue Op1 = Op.getOperand(1);
1515 if (SimplifyDemandedBits(Op1,
DemandedBits, DemandedElts, Known, TLO,
1519 if (SimplifyDemandedBits(Op0,
DemandedBits, DemandedElts, Known2, TLO,
1547 if (
C->getAPIntValue() == Known2.
One) {
1556 if (!
C->isAllOnes() &&
DemandedBits.isSubsetOf(
C->getAPIntValue())) {
1568 if (ShiftC->getAPIntValue().ult(
BitWidth)) {
1569 uint64_t ShiftAmt = ShiftC->getZExtValue();
1572 : Ones.
lshr(ShiftAmt);
1590 if (!
C || !
C->isAllOnes())
1596 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1598 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1600 if (DemandedOp0 || DemandedOp1) {
1601 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1602 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1612 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, Known, TLO,
1615 if (SimplifyDemandedBits(Op.getOperand(1),
DemandedBits, Known2, TLO,
1629 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, DemandedElts,
1630 Known, TLO,
Depth + 1))
1632 if (SimplifyDemandedBits(Op.getOperand(1),
DemandedBits, DemandedElts,
1633 Known2, TLO,
Depth + 1))
1642 if (SimplifyDemandedBits(Op.getOperand(3),
DemandedBits, Known, TLO,
1645 if (SimplifyDemandedBits(Op.getOperand(2),
DemandedBits, Known2, TLO,
1659 SDValue Op0 = Op.getOperand(0);
1660 SDValue Op1 = Op.getOperand(1);
1687 SDValue Op0 = Op.getOperand(0);
1688 SDValue Op1 = Op.getOperand(1);
1691 if (
const APInt *SA =
1693 unsigned ShAmt = SA->getZExtValue();
1703 if (
const APInt *SA2 =
1705 unsigned C1 = SA2->getZExtValue();
1707 int Diff = ShAmt - C1;
1726 if (ShAmt < InnerBits &&
DemandedBits.getActiveBits() <= InnerBits &&
1727 isTypeDesirableForOp(
ISD::SHL, InnerVT)) {
1743 if (
const APInt *SA2 =
1745 unsigned InnerShAmt = SA2->getZExtValue();
1746 if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1748 (InnerBits - InnerShAmt + ShAmt) &&
1762 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1766 Known.
Zero <<= ShAmt;
1767 Known.
One <<= ShAmt;
1773 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1774 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1792 if (SimplifyDemandedBits(Op0, DemandedFromOp, DemandedElts, Known, TLO,
1795 if (
Flags.hasNoSignedWrap() ||
Flags.hasNoUnsignedWrap()) {
1798 Flags.setNoSignedWrap(
false);
1799 Flags.setNoUnsignedWrap(
false);
1800 Op->setFlags(
Flags);
1810 if (
const APInt *MaxSA =
1812 unsigned ShAmt = MaxSA->getZExtValue();
1813 unsigned NumSignBits =
1816 if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1822 SDValue Op0 = Op.getOperand(0);
1823 SDValue Op1 = Op.getOperand(1);
1828 DemandedElts,
Depth + 1))
1831 if (
const APInt *SA =
1833 unsigned ShAmt = SA->getZExtValue();
1843 if (
const APInt *SA2 =
1845 unsigned C1 = SA2->getZExtValue();
1847 int Diff = ShAmt - C1;
1863 if (Op->getFlags().hasExact())
1867 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1878 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1879 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1889 SDValue Op0 = Op.getOperand(0);
1890 SDValue Op1 = Op.getOperand(1);
1909 DemandedElts,
Depth + 1))
1912 if (
const APInt *SA =
1914 unsigned ShAmt = SA->getZExtValue();
1922 if (Op->getFlags().hasExact())
1930 if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1942 Flags.setExact(Op->getFlags().hasExact());
1960 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1961 Op0, InDemandedMask, DemandedElts, TLO.
DAG,
Depth + 1);
1972 SDValue Op0 = Op.getOperand(0);
1973 SDValue Op1 = Op.getOperand(1);
1974 SDValue Op2 = Op.getOperand(2);
1975 bool IsFSHL = (Op.getOpcode() ==
ISD::FSHL);
1978 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
1983 if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1,
DemandedBits, DemandedElts,
1984 Known, TLO,
Depth + 1))
1993 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1996 if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
2009 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2010 Op0, Demanded0, DemandedElts, TLO.
DAG,
Depth + 1);
2011 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2012 Op1, Demanded1, DemandedElts, TLO.
DAG,
Depth + 1);
2013 if (DemandedOp0 || DemandedOp1) {
2014 DemandedOp0 = DemandedOp0 ? DemandedOp0 : Op0;
2015 DemandedOp1 = DemandedOp1 ? DemandedOp1 : Op1;
2026 if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
2027 Known2, TLO,
Depth + 1))
2034 SDValue Op0 = Op.getOperand(0);
2035 SDValue Op1 = Op.getOperand(1);
2036 bool IsROTL = (Op.getOpcode() ==
ISD::ROTL);
2043 unsigned Amt = SA->getAPIntValue().urem(
BitWidth);
2049 if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
2059 DemandedBits.countr_zero() >= (IsROTL ? Amt : RevAmt)) {
2064 DemandedBits.countl_zero() >= (IsROTL ? RevAmt : Amt)) {
2073 if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
2081 SDValue Op0 = Op.getOperand(0);
2082 SDValue Op1 = Op.getOperand(1);
2087 return TLO.
CombineTo(Op, *IsULE ? Op0 : Op1);
2089 return TLO.
CombineTo(Op, *IsULT ? Op0 : Op1);
2094 SDValue Op0 = Op.getOperand(0);
2095 SDValue Op1 = Op.getOperand(1);
2100 return TLO.
CombineTo(Op, *IsUGE ? Op0 : Op1);
2102 return TLO.
CombineTo(Op, *IsUGT ? Op0 : Op1);
2106 SDValue Src = Op.getOperand(0);
2108 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2116 SDValue Src = Op.getOperand(0);
2135 unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
2143 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
2162 SDValue Op0 = Op.getOperand(0);
2163 EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2168 unsigned MinSignedBits =
2170 bool AlreadySignExtended = ExVTBits >= MinSignedBits;
2173 if (!AlreadySignExtended) {
2191 InputDemandedBits.
setBit(ExVTBits - 1);
2193 if (SimplifyDemandedBits(Op0, InputDemandedBits, DemandedElts, Known, TLO,
2202 if (Known.
Zero[ExVTBits - 1])
2206 if (Known.
One[ExVTBits - 1]) {
2216 EVT HalfVT = Op.getOperand(0).getValueType();
2224 if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO,
Depth + 1))
2227 if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO,
Depth + 1))
2230 Known = KnownHi.
concat(KnownLo);
2238 SDValue Src = Op.getOperand(0);
2239 EVT SrcVT = Src.getValueType();
2248 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2259 APInt InDemandedElts = DemandedElts.
zext(InElts);
2260 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2268 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2269 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2278 SDValue Src = Op.getOperand(0);
2279 EVT SrcVT = Src.getValueType();
2288 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2299 APInt InDemandedElts = DemandedElts.
zext(InElts);
2303 InDemandedBits.
setBit(InBits - 1);
2305 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2323 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2324 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2333 SDValue Src = Op.getOperand(0);
2334 EVT SrcVT = Src.getValueType();
2341 if (IsLE && IsVecInReg && DemandedElts == 1 &&
2346 APInt InDemandedElts = DemandedElts.
zext(InElts);
2347 if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2355 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2356 Src, InDemandedBits, InDemandedElts, TLO.
DAG,
Depth + 1))
2361 SDValue Src = Op.getOperand(0);
2365 unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2367 if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2373 if (
SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2374 Src, TruncMask, DemandedElts, TLO.
DAG,
Depth + 1))
2379 switch (Src.getOpcode()) {
2390 if (Src.getNode()->hasOneUse()) {
2391 const APInt *ShAmtC =
2422 EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2424 if (SimplifyDemandedBits(Op.getOperand(0), ~InMask |
DemandedBits, Known,
2429 Known.
Zero |= ~InMask;
2430 Known.
One &= (~Known.Zero);
2434 SDValue Src = Op.getOperand(0);
2436 ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2437 unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2445 if (
auto *CIdx = dyn_cast<ConstantSDNode>(
Idx))
2446 if (CIdx->getAPIntValue().ult(NumSrcElts))
2453 DemandedSrcBits = DemandedSrcBits.
trunc(EltBitWidth);
2455 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2461 if (
SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2462 Src, DemandedSrcBits, DemandedSrcElts, TLO.
DAG,
Depth + 1)) {
2477 SDValue Src = Op.getOperand(0);
2478 EVT SrcVT = Src.getValueType();
2488 if ((OpVTLegal || i32Legal) && VT.
isSimple() && SrcVT != MVT::f16 &&
2489 SrcVT != MVT::f128) {
2491 EVT Ty = OpVTLegal ? VT : MVT::i32;
2495 unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2496 if (!OpVTLegal && OpVTSizeInBits > 32)
2498 unsigned ShVal = Op.getValueSizeInBits() - 1;
2508 unsigned Scale =
BitWidth / NumSrcEltBits;
2512 for (
unsigned i = 0; i != Scale; ++i) {
2513 unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2514 unsigned BitOffset = EltOffset * NumSrcEltBits;
2517 DemandedSrcBits |= Sub;
2518 for (
unsigned j = 0; j != NumElts; ++j)
2519 if (DemandedElts[j])
2520 DemandedSrcElts.
setBit((j * Scale) + i);
2524 APInt KnownSrcUndef, KnownSrcZero;
2525 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2526 KnownSrcZero, TLO,
Depth + 1))
2530 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2531 KnownSrcBits, TLO,
Depth + 1))
2533 }
else if (IsLE && (NumSrcEltBits %
BitWidth) == 0) {
2535 unsigned Scale = NumSrcEltBits /
BitWidth;
2539 for (
unsigned i = 0; i != NumElts; ++i)
2540 if (DemandedElts[i]) {
2543 DemandedSrcElts.
setBit(i / Scale);
2547 APInt KnownSrcUndef, KnownSrcZero;
2548 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2549 KnownSrcZero, TLO,
Depth + 1))
2554 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2555 KnownSrcBits, TLO,
Depth + 1))
2574 if (
C &&
C->getAPIntValue().countr_zero() == CTZ) {
2584 if (Op.getOperand(0) == Op.getOperand(1) &&
DemandedBits.ult(4)) {
2594 SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2599 if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, KnownOp0, TLO,
2601 SimplifyDemandedBits(Op1, LoMask, DemandedElts, KnownOp1, TLO,
2605 if (
Flags.hasNoSignedWrap() ||
Flags.hasNoUnsignedWrap()) {
2608 Flags.setNoSignedWrap(
false);
2609 Flags.setNoUnsignedWrap(
false);
2610 Op->setFlags(
Flags);
2617 isa<ConstantSDNode>(Op0) && cast<ConstantSDNode>(Op0)->isZero())
2622 SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2623 Op0, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
2624 SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2625 Op1, LoMask, DemandedElts, TLO.
DAG,
Depth + 1);
2626 if (DemandedOp0 || DemandedOp1) {
2627 Flags.setNoSignedWrap(
false);
2628 Flags.setNoUnsignedWrap(
false);
2629 Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2630 Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2644 if (
C && !
C->isAllOnes() && !
C->isOne() &&
2645 (
C->getAPIntValue() | HighMask).isAllOnes()) {
2649 Flags.setNoSignedWrap(
false);
2650 Flags.setNoUnsignedWrap(
false);
2658 auto getShiftLeftAmt = [&HighMask](
SDValue Mul) ->
unsigned {
2685 if (
unsigned ShAmt = getShiftLeftAmt(Op0))
2688 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
2689 return foldMul(
ISD::SUB, Op1.getOperand(0), Op0, ShAmt);
2693 if (
unsigned ShAmt = getShiftLeftAmt(Op1))
2694 return foldMul(
ISD::ADD, Op1.getOperand(0), Op0, ShAmt);
2702 Flags.hasNoSignedWrap(), KnownOp0,
2713 if (Op.getValueType().isScalableVector())
2715 if (SimplifyDemandedBitsForTargetNode(Op,
DemandedBits, DemandedElts,
2728 if (!isTargetCanonicalConstantNode(Op) &&
2731 const SDNode *
N = Op.getNode();
2757 const APInt &DemandedElts,
2763 APInt KnownUndef, KnownZero;
2765 SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2777 const APInt &UndefOp0,
2778 const APInt &UndefOp1) {
2781 "Vector binop only");
2786 UndefOp1.
getBitWidth() == NumElts &&
"Bad type for undef analysis");
2788 auto getUndefOrConstantElt = [&](
SDValue V,
unsigned Index,
2789 const APInt &UndefVals) {
2790 if (UndefVals[
Index])
2793 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2797 auto *
C = dyn_cast<ConstantSDNode>(Elt);
2798 if (isa<ConstantFPSDNode>(Elt) || Elt.
isUndef() || (
C && !
C->isOpaque()))
2806 for (
unsigned i = 0; i != NumElts; ++i) {
2825 bool AssumeSingleUse)
const {
2826 EVT VT = Op.getValueType();
2827 unsigned Opcode = Op.getOpcode();
2828 APInt DemandedElts = OriginalDemandedElts;
2843 "Mask size mismatches value type element count!");
2852 if (!AssumeSingleUse && !Op.getNode()->hasOneUse())
2856 if (DemandedElts == 0) {
2871 auto SimplifyDemandedVectorEltsBinOp = [&](
SDValue Op0,
SDValue Op1) {
2872 SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2874 SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2876 if (NewOp0 || NewOp1) {
2878 Opcode,
SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2886 if (!DemandedElts[0]) {
2890 SDValue ScalarSrc = Op.getOperand(0);
2894 EVT SrcVT = Src.getValueType();
2906 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2915 SDValue Src = Op.getOperand(0);
2916 EVT SrcVT = Src.getValueType();
2925 if (NumSrcElts == NumElts)
2926 return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2927 KnownZero, TLO,
Depth + 1);
2929 APInt SrcDemandedElts, SrcZero, SrcUndef;
2933 if ((NumElts % NumSrcElts) == 0) {
2934 unsigned Scale = NumElts / NumSrcElts;
2936 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2946 for (
unsigned i = 0; i != NumElts; ++i)
2947 if (DemandedElts[i]) {
2948 unsigned Ofs = (i % Scale) * EltSizeInBits;
2949 SrcDemandedBits.
setBits(Ofs, Ofs + EltSizeInBits);
2953 if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2961 for (
unsigned SubElt = 0; SubElt != Scale; ++SubElt) {
2965 for (
unsigned SrcElt = 0; SrcElt != NumSrcElts; ++SrcElt) {
2966 unsigned Elt = Scale * SrcElt + SubElt;
2967 if (DemandedElts[Elt])
2975 for (
unsigned i = 0; i != NumSrcElts; ++i) {
2976 if (SrcDemandedElts[i]) {
2978 KnownZero.
setBits(i * Scale, (i + 1) * Scale);
2980 KnownUndef.
setBits(i * Scale, (i + 1) * Scale);
2988 if ((NumSrcElts % NumElts) == 0) {
2989 unsigned Scale = NumSrcElts / NumElts;
2991 if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2997 for (
unsigned i = 0; i != NumElts; ++i) {
2998 if (DemandedElts[i]) {
3013 [&](
SDValue Elt) { return Op.getOperand(0) != Elt; })) {
3015 bool Updated =
false;
3016 for (
unsigned i = 0; i != NumElts; ++i) {
3017 if (!DemandedElts[i] && !Ops[i].
isUndef()) {
3027 for (
unsigned i = 0; i != NumElts; ++i) {
3029 if (
SrcOp.isUndef()) {
3031 }
else if (EltSizeInBits ==
SrcOp.getScalarValueSizeInBits() &&
3039 EVT SubVT = Op.getOperand(0).getValueType();
3040 unsigned NumSubVecs = Op.getNumOperands();
3042 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3043 SDValue SubOp = Op.getOperand(i);
3045 APInt SubUndef, SubZero;
3046 if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
3049 KnownUndef.
insertBits(SubUndef, i * NumSubElts);
3050 KnownZero.
insertBits(SubZero, i * NumSubElts);
3055 bool FoundNewSub =
false;
3057 for (
unsigned i = 0; i != NumSubVecs; ++i) {
3058 SDValue SubOp = Op.getOperand(i);
3060 SDValue NewSubOp = SimplifyMultipleUseDemandedVectorElts(
3061 SubOp, SubElts, TLO.
DAG,
Depth + 1);
3062 DemandedSubOps.
push_back(NewSubOp ? NewSubOp : SubOp);
3063 FoundNewSub = NewSubOp ?
true : FoundNewSub;
3076 SDValue Src = Op.getOperand(0);
3077 SDValue Sub = Op.getOperand(1);
3081 APInt DemandedSrcElts = DemandedElts;
3084 APInt SubUndef, SubZero;
3085 if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
3090 if (!DemandedSrcElts && !Src.isUndef())
3095 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
3103 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3104 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3105 SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
3106 Sub, DemandedSubElts, TLO.
DAG,
Depth + 1);
3107 if (NewSrc || NewSub) {
3108 NewSrc = NewSrc ? NewSrc : Src;
3109 NewSub = NewSub ? NewSub : Sub;
3111 NewSub, Op.getOperand(2));
3119 SDValue Src = Op.getOperand(0);
3120 if (Src.getValueType().isScalableVector())
3123 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3126 APInt SrcUndef, SrcZero;
3127 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3135 SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
3136 Src, DemandedSrcElts, TLO.
DAG,
Depth + 1);
3146 SDValue Vec = Op.getOperand(0);
3147 SDValue Scl = Op.getOperand(1);
3148 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
3152 if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
3153 unsigned Idx = CIdx->getZExtValue();
3154 if (!DemandedElts[
Idx])
3157 APInt DemandedVecElts(DemandedElts);
3159 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
3160 KnownZero, TLO,
Depth + 1))
3169 APInt VecUndef, VecZero;
3170 if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
3177 SDValue Sel = Op.getOperand(0);
3183 APInt UndefSel, UndefZero;
3184 if (SimplifyDemandedVectorElts(Sel, DemandedElts, UndefSel, UndefZero, TLO,
3189 APInt DemandedLHS(DemandedElts);
3190 APInt DemandedRHS(DemandedElts);
3191 APInt UndefLHS, ZeroLHS;
3192 APInt UndefRHS, ZeroRHS;
3193 if (SimplifyDemandedVectorElts(
LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3196 if (SimplifyDemandedVectorElts(
RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3200 KnownUndef = UndefLHS & UndefRHS;
3201 KnownZero = ZeroLHS & ZeroRHS;
3205 APInt DemandedSel = DemandedElts & ~KnownZero;
3206 if (DemandedSel != DemandedElts)
3207 if (SimplifyDemandedVectorElts(Sel, DemandedSel, UndefSel, UndefZero, TLO,
3216 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
3219 APInt DemandedLHS(NumElts, 0);
3220 APInt DemandedRHS(NumElts, 0);
3221 for (
unsigned i = 0; i != NumElts; ++i) {
3222 int M = ShuffleMask[i];
3223 if (M < 0 || !DemandedElts[i])
3225 assert(0 <= M && M < (
int)(2 * NumElts) &&
"Shuffle index out of range");
3226 if (M < (
int)NumElts)
3229 DemandedRHS.
setBit(M - NumElts);
3233 APInt UndefLHS, ZeroLHS;
3234 APInt UndefRHS, ZeroRHS;
3235 if (SimplifyDemandedVectorElts(
LHS, DemandedLHS, UndefLHS, ZeroLHS, TLO,
3238 if (SimplifyDemandedVectorElts(
RHS, DemandedRHS, UndefRHS, ZeroRHS, TLO,
3243 bool Updated =
false;
3244 bool IdentityLHS =
true, IdentityRHS =
true;
3246 for (
unsigned i = 0; i != NumElts; ++i) {
3247 int &M = NewMask[i];
3250 if (!DemandedElts[i] || (M < (
int)NumElts && UndefLHS[M]) ||
3251 (M >= (
int)NumElts && UndefRHS[M - NumElts])) {
3255 IdentityLHS &= (M < 0) || (M == (
int)i);
3256 IdentityRHS &= (M < 0) || ((M - NumElts) == i);
3261 if (Updated && !IdentityLHS && !IdentityRHS && !TLO.
LegalOps) {
3263 buildLegalVectorShuffle(VT,
DL,
LHS,
RHS, NewMask, TLO.
DAG);
3269 for (
unsigned i = 0; i != NumElts; ++i) {
3270 int M = ShuffleMask[i];
3273 }
else if (M < (
int)NumElts) {
3279 if (UndefRHS[M - NumElts])
3281 if (ZeroRHS[M - NumElts])
3290 APInt SrcUndef, SrcZero;
3291 SDValue Src = Op.getOperand(0);
3292 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3293 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
3294 if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
3301 Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
3302 DemandedSrcElts == 1) {
3315 if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() ==
ISD::AND &&
3316 Op->isOnlyUserOf(Src.getNode()) &&
3317 Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
3319 EVT SrcVT = Src.getValueType();
3326 ISD::AND,
DL, SrcVT, {Src.getOperand(1), Mask})) {
3338 SDValue Op0 = Op.getOperand(0);
3339 SDValue Op1 = Op.getOperand(1);
3340 if (Op0 == Op1 && Op->isOnlyUserOf(Op0.
getNode())) {
3341 APInt UndefLHS, ZeroLHS;
3342 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3356 SDValue Op0 = Op.getOperand(0);
3357 SDValue Op1 = Op.getOperand(1);
3359 APInt UndefRHS, ZeroRHS;
3360 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3363 APInt UndefLHS, ZeroLHS;
3364 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3368 KnownZero = ZeroLHS & ZeroRHS;
3374 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3383 SDValue Op0 = Op.getOperand(0);
3384 SDValue Op1 = Op.getOperand(1);
3386 APInt UndefRHS, ZeroRHS;
3387 if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
3390 APInt UndefLHS, ZeroLHS;
3391 if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
3395 KnownZero = ZeroLHS;
3396 KnownUndef = UndefLHS & UndefRHS;
3401 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3409 SDValue Op0 = Op.getOperand(0);
3410 SDValue Op1 = Op.getOperand(1);
3412 APInt SrcUndef, SrcZero;
3413 if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
3418 APInt DemandedElts0 = DemandedElts & ~SrcZero;
3419 if (SimplifyDemandedVectorElts(Op0, DemandedElts0, KnownUndef, KnownZero,
3423 KnownUndef &= DemandedElts0;
3424 KnownZero &= DemandedElts0;
3429 if (DemandedElts.
isSubsetOf(SrcZero | KnownZero | SrcUndef | KnownUndef))
3436 KnownZero |= SrcZero;
3437 KnownUndef &= SrcUndef;
3438 KnownUndef &= ~KnownZero;
3442 if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
3449 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
3450 KnownZero, TLO,
Depth + 1))
3462 if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
3463 KnownZero, TLO,
Depth))
3468 if (SimplifyDemandedBits(Op,
DemandedBits, OriginalDemandedElts, Known,
3469 TLO,
Depth, AssumeSingleUse))
3475 assert((KnownUndef & KnownZero) == 0 &&
"Elements flagged as undef AND zero");
3489 const APInt &DemandedElts,
3491 unsigned Depth)
const {
3496 "Should use MaskedValueIsZero if you don't know whether Op"
3497 " is a target node!");
3504 unsigned Depth)
const {
3516 unsigned Depth)
const {
3525 unsigned Depth)
const {
3530 "Should use ComputeNumSignBits if you don't know whether Op"
3531 " is a target node!");
3548 "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3549 " is a target node!");
3560 "Should use SimplifyDemandedBits if you don't know whether Op"
3561 " is a target node!");
3562 computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.
DAG,
Depth);
3574 "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3575 " is a target node!");
3602 bool PoisonOnly,
unsigned Depth)
const {
3608 "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3609 " is a target node!");
3615 bool PoisonOnly,
bool ConsiderFlags,
unsigned Depth)
const {
3620 "Should use canCreateUndefOrPoison if you don't know whether Op"
3621 " is a target node!");
3629 unsigned Depth)
const {
3634 "Should use isKnownNeverNaN if you don't know whether Op"
3635 " is a target node!");
3640 const APInt &DemandedElts,
3643 unsigned Depth)
const {
3648 "Should use isSplatValue if you don't know whether Op"
3649 " is a target node!");
3664 CVal = CN->getAPIntValue();
3665 EltWidth =
N.getValueType().getScalarSizeInBits();
3672 CVal = CVal.
trunc(EltWidth);
3678 return CVal.
isOne();
3720 return (
N->isOne() && !SExt) || (SExt && (
N->getValueType(0) != MVT::i1));
3723 return N->isAllOnes() && SExt;
3732 DAGCombinerInfo &DCI)
const {
3760 auto *AndC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
3761 if (AndC &&
isNullConstant(N1) && AndC->getAPIntValue().isPowerOf2() &&
3764 AndC->getAPIntValue().getActiveBits());
3795 if (DCI.isBeforeLegalizeOps() ||
3807 auto *YConst = dyn_cast<ConstantSDNode>(
Y);
3808 if (YConst && YConst->isZero())
3830SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3835 if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3844 if (!(C01 = dyn_cast<ConstantSDNode>(N0->
getOperand(1))))
3848 EVT XVT =
X.getValueType();
3872 auto checkConstants = [&
I1, &I01]() ->
bool {
3877 if (checkConstants()) {
3885 if (!checkConstants())
3891 const unsigned KeptBits =
I1.logBase2();
3892 const unsigned KeptBitsMinusOne = I01.
logBase2();
3895 if (KeptBits != (KeptBitsMinusOne + 1))
3919SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3921 DAGCombinerInfo &DCI,
const SDLoc &
DL)
const {
3923 "Should be a comparison with 0.");
3925 "Valid only for [in]equality comparisons.");
3927 unsigned NewShiftOpcode;
3938 unsigned OldShiftOpcode =
V.getOpcode();
3939 switch (OldShiftOpcode) {
3951 C =
V.getOperand(0);
3956 Y =
V.getOperand(1);
3960 return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3961 X, XC,
CC,
Y, OldShiftOpcode, NewShiftOpcode, DAG);
3978 EVT VT =
X.getValueType();
3993 DAGCombinerInfo &DCI)
const {
3996 "Unexpected binop");
4023 !DCI.isBeforeLegalize());
4026 if (!DCI.isCalledByLegalizer())
4027 DCI.AddToWorklist(YShl1.
getNode());
4042 if (CTPOP.getOpcode() !=
ISD::CTPOP || !CTPOP.hasOneUse())
4045 EVT CTVT = CTPOP.getValueType();
4046 SDValue CTOp = CTPOP.getOperand(0);
4066 for (
unsigned i = 0; i <
Passes; i++) {
4107 auto getRotateSource = [](
SDValue X) {
4109 return X.getOperand(0);
4116 if (
SDValue R = getRotateSource(N0))
4149 if (!C1 || !C1->
isZero())
4158 if (!ShAmtC || ShAmtC->getAPIntValue().uge(
BitWidth))
4162 unsigned ShAmt = ShAmtC->getZExtValue();
4171 if (
Or.getOperand(0) ==
Other) {
4172 X =
Or.getOperand(0);
4173 Y =
Or.getOperand(1);
4176 if (
Or.getOperand(1) ==
Other) {
4177 X =
Or.getOperand(1);
4178 Y =
Or.getOperand(0);
4188 if (matchOr(F0, F1)) {
4195 if (matchOr(F1, F0)) {
4211 const SDLoc &dl)
const {
4221 bool N0ConstOrSplat =
4223 bool N1ConstOrSplat =
4234 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
4240 if (!N0ConstOrSplat && !N1ConstOrSplat &&
4245 return DAG.
getSetCC(dl, VT, N1, N0, SwappedCC);
4254 const APInt &C1 = N1C->getAPIntValue();
4274 return DAG.
getNode(LogicOp, dl, VT, IsXZero, IsYZero);
4304 if (
auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode())) {
4305 const APInt &C1 = N1C->getAPIntValue();
4320 if (
auto *
C = dyn_cast<ConstantSDNode>(N0->
getOperand(1)))
4321 if ((
C->getAPIntValue()+1).isPowerOf2()) {
4322 MinBits =
C->getAPIntValue().countr_one();
4330 }
else if (
auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
4333 MinBits = LN0->getMemoryVT().getSizeInBits();
4337 MinBits = LN0->getMemoryVT().getSizeInBits();
4348 MinBits >= ReqdBits) {
4350 if (isTypeDesirableForOp(
ISD::SETCC, MinVT)) {
4353 if (MinBits == 1 && C1 == 1)
4372 if (TopSetCC.
getValueType() == MVT::i1 && VT == MVT::i1 &&
4385 cast<CondCodeSDNode>(TopSetCC.
getOperand(2))->get(),
4406 unsigned bestWidth = 0, bestOffset = 0;
4409 unsigned maskWidth = origWidth;
4415 for (
unsigned width = origWidth / 2; width>=8; width /= 2) {
4417 for (
unsigned offset=0; offset<origWidth/width; offset++) {
4418 if (Mask.isSubsetOf(newMask)) {
4420 bestOffset = (
uint64_t)offset * (width/8);
4422 bestOffset = (origWidth/width - offset - 1) * (width/8);
4423 bestMask = Mask.lshr(offset * (width/8) * 8);
4436 if (bestOffset != 0)