79#define DEBUG_TYPE "dagcombine"
81STATISTIC(NodesCombined ,
"Number of dag nodes combined");
82STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
83STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
84STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
85STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
87STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
91 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
95 cl::desc(
"Enable DAG combiner's use of TBAA"));
100 cl::desc(
"Only use DAG-combiner alias analysis in this"
108 cl::desc(
"Bypass the profitability model of load slicing"),
113 cl::desc(
"DAG combiner may split indexing from loads"));
117 cl::desc(
"DAG combiner enable merging multiple stores "
118 "into a wider store"));
122 cl::desc(
"Limit the number of operands to inline for Token Factors"));
126 cl::desc(
"Limit the number of times for the same StoreNode and RootNode "
127 "to bail out in store merging dependence check"));
131 cl::desc(
"DAG combiner enable reducing the width of load/op/store "
136 cl::desc(
"DAG combiner enable load/<replace bytes>/store with "
137 "a narrower store"));
142 "Enable merging extends and rounds into FCOPYSIGN on vector types"));
152 bool LegalDAG =
false;
153 bool LegalOperations =
false;
154 bool LegalTypes =
false;
156 bool DisableGenericCombines;
197 void AddUsersToWorklist(
SDNode *
N) {
203 void AddToWorklistWithUsers(
SDNode *
N) {
204 AddUsersToWorklist(
N);
211 void clearAddedDanglingWorklistEntries() {
213 while (!PruningList.
empty()) {
216 recursivelyDeleteUnusedNodes(
N);
220 SDNode *getNextWorklistEntry() {
222 clearAddedDanglingWorklistEntries();
226 while (!
N && !Worklist.
empty()) {
231 bool GoodWorklistEntry = WorklistMap.
erase(
N);
232 (void)GoodWorklistEntry;
233 assert(GoodWorklistEntry &&
234 "Found a worklist entry without a corresponding map entry!");
244 : DAG(
D), TLI(
D.getTargetLoweringInfo()),
245 STI(
D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) {
249 MaximumLegalStoreInBits = 0;
255 VT.getSizeInBits().getKnownMinValue() >= MaximumLegalStoreInBits)
256 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinValue();
259 void ConsiderForPruning(
SDNode *
N) {
266 void AddToWorklist(
SDNode *
N) {
268 "Deleted Node added to Worklist");
275 ConsiderForPruning(
N);
277 if (WorklistMap.
insert(std::make_pair(
N, Worklist.
size())).second)
282 void removeFromWorklist(
SDNode *
N) {
285 StoreRootCountMap.
erase(
N);
287 auto It = WorklistMap.
find(
N);
288 if (It == WorklistMap.
end())
292 Worklist[It->second] =
nullptr;
293 WorklistMap.
erase(It);
296 void deleteAndRecombine(
SDNode *
N);
297 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
305 return CombineTo(
N, &Res, 1, AddTo);
312 return CombineTo(
N, To, 2, AddTo);
318 unsigned MaximumLegalStoreInBits;
323 bool SimplifyDemandedBits(
SDValue Op) {
324 unsigned BitWidth =
Op.getScalarValueSizeInBits();
336 AddToWorklist(
Op.getNode());
338 CommitTargetLoweringOpt(TLO);
345 bool SimplifyDemandedVectorElts(
SDValue Op) {
347 if (
Op.getValueType().isScalableVector())
350 unsigned NumElts =
Op.getValueType().getVectorNumElements();
352 return SimplifyDemandedVectorElts(Op, DemandedElts);
356 const APInt &DemandedElts,
357 bool AssumeSingleUse =
false);
358 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
359 bool AssumeSingleUse =
false);
361 bool CombineToPreIndexedLoadStore(
SDNode *
N);
362 bool CombineToPostIndexedLoadStore(
SDNode *
N);
387 void ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad);
515 bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
SDNode *
N);
539 template <
class MatchContextClass>
545 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
554 SDValue reassociateReduction(
unsigned ResOpc,
unsigned Opc,
const SDLoc &
DL,
568 bool NotExtCompare =
false);
569 SDValue convertSelectOfFPConstantsToLoadOffset(
584 const SDLoc &
DL,
bool foldBooleans);
588 SDValue &
CC,
bool MatchStrict =
false)
const;
589 bool isOneUseSetCC(
SDValue N)
const;
618 bool DemandHighBits =
true);
622 unsigned PosOpcode,
unsigned NegOpcode,
626 unsigned PosOpcode,
unsigned NegOpcode,
642 SDValue VecIn2,
unsigned LeftIdx,
677 int64_t OffsetFromBase;
680 : MemNode(
N), OffsetFromBase(
Offset) {}
685 StoreSource getStoreSource(
SDValue StoreVal) {
689 return StoreSource::Constant;
692 return StoreSource::Extract;
694 return StoreSource::Load;
696 return StoreSource::Unknown;
704 bool isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
711 EVT LoadResultTy,
EVT &ExtVT);
716 EVT &MemVT,
unsigned ShAmt = 0);
724 bool BackwardsPropagateMask(
SDNode *
N);
741 EVT MemVT,
unsigned NumStores,
742 bool IsConstantSrc,
bool UseVector,
756 bool checkMergeStoreCandidatesForDependencies(
764 int64_t ElementSizeBytes)
const;
769 unsigned NumConsecutiveStores,
770 EVT MemVT,
SDNode *Root,
bool AllowVectors);
777 unsigned NumConsecutiveStores,
EVT MemVT,
783 unsigned NumConsecutiveStores,
EVT MemVT,
784 SDNode *Root,
bool AllowVectors,
785 bool IsNonTemporalStore,
bool IsNonTemporalLoad);
804 bool hasOperation(
unsigned Opcode,
EVT VT) {
816 EVT getShiftAmountTy(
EVT LHSTy) {
823 bool isTypeLegal(
const EVT &VT) {
824 if (!LegalTypes)
return true;
829 EVT getSetCCResultType(
EVT VT)
const {
844 explicit WorklistRemover(DAGCombiner &dc)
845 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
848 DC.removeFromWorklist(
N);
856 explicit WorklistInserter(DAGCombiner &dc)
857 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
864class EmptyMatchContext {
870 : DAG(DAG), TLI(TLI) {}
877 template <
typename... ArgT>
SDValue getNode(ArgT &&...Args) {
878 return DAG.
getNode(std::forward<ArgT>(Args)...);
881 bool isOperationLegalOrCustom(
unsigned Op,
EVT VT,
882 bool LegalOnly =
false)
const {
887class VPMatchContext {
895 : DAG(DAG), TLI(TLI), RootMaskOp(), RootVectorLenOp() {
900 if (
auto RootVLenPos =
902 RootVectorLenOp = Root->
getOperand(*RootVLenPos);
920 if (RootMaskOp != MaskOp &&
927 if (RootVectorLenOp != OpVal.
getOperand(*VLenPos))
941 {Operand, RootMaskOp, RootVectorLenOp});
950 {N1, N2, RootMaskOp, RootVectorLenOp});
959 {N1, N2, N3, RootMaskOp, RootVectorLenOp});
967 return DAG.
getNode(VPOpcode,
DL, VT, {Operand, RootMaskOp, RootVectorLenOp},
976 return DAG.
getNode(VPOpcode,
DL, VT, {N1, N2, RootMaskOp, RootVectorLenOp},
986 {N1, N2, N3, RootMaskOp, RootVectorLenOp},
Flags);
989 bool isOperationLegalOrCustom(
unsigned Op,
EVT VT,
990 bool LegalOnly =
false)
const {
1003 ((DAGCombiner*)
DC)->AddToWorklist(
N);
1008 return ((DAGCombiner*)DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
1013 return ((DAGCombiner*)DC)->CombineTo(
N, Res, AddTo);
1018 return ((DAGCombiner*)DC)->CombineTo(
N, Res0, Res1, AddTo);
1023 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(
N);
1028 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
1035void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
1036 removeFromWorklist(
N);
1044 if (Op->hasOneUse() || Op->getNumValues() > 1)
1045 AddToWorklist(Op.getNode());
1054 unsigned Bits =
Offset + std::max(
LHS.getBitWidth(),
RHS.getBitWidth());
1067 LHS =
N.getOperand(0);
1068 RHS =
N.getOperand(1);
1069 CC =
N.getOperand(2);
1076 LHS =
N.getOperand(1);
1077 RHS =
N.getOperand(2);
1078 CC =
N.getOperand(3);
1090 LHS =
N.getOperand(0);
1091 RHS =
N.getOperand(1);
1092 CC =
N.getOperand(4);
1099bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
1101 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N->hasOneUse())
1113 MaskForTy = 0xFFULL;
1116 MaskForTy = 0xFFFFULL;
1119 MaskForTy = 0xFFFFFFFFULL;
1138 return !(Const->isOpaque() && NoOpaques);
1141 unsigned BitWidth =
N.getScalarValueSizeInBits();
1142 for (
const SDValue &Op :
N->op_values()) {
1146 if (!Const || Const->getAPIntValue().getBitWidth() !=
BitWidth ||
1147 (Const->isOpaque() && NoOpaques))
1166 !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
1169bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1186 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1190 const APInt &C2APIntVal = C2->getAPIntValue();
1194 if (
auto *C1 = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
1198 const APInt &C1APIntVal = C1->getAPIntValue();
1199 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1202 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1205 if (
auto *LoadStore = dyn_cast<MemSDNode>(
Node)) {
1211 AM.
BaseOffs = C2APIntVal.getSExtValue();
1213 unsigned AS =
LoadStore->getAddressSpace();
1225 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(N0.
getOperand(1)))
1238 AM.
BaseOffs = C2APIntVal.getSExtValue();
1240 unsigned AS =
LoadStore->getAddressSpace();
1253SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &
DL,
1267 return DAG.
getNode(Opc,
DL, VT, N00, OpNode);
1274 return DAG.
getNode(Opc,
DL, VT, OpNode, N01);
1284 if (N1 == N00 || N1 == N01)
1329 if (!
Flags.hasAllowReassociation() || !
Flags.hasNoSignedZeros())
1332 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N0, N1))
1334 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N1, N0))
1342SDValue DAGCombiner::reassociateReduction(
unsigned RedOpc,
unsigned Opc,
1360 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1364 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1365 for (
unsigned i = 0, e = NumTo; i !=
e; ++i)
1366 assert((!To[i].getNode() ||
1367 N->getValueType(i) == To[i].getValueType()) &&
1368 "Cannot combine value to value of different type!");
1370 WorklistRemover DeadNodes(*
this);
1374 for (
unsigned i = 0, e = NumTo; i !=
e; ++i) {
1375 if (To[i].getNode())
1376 AddToWorklistWithUsers(To[i].getNode());
1384 deleteAndRecombine(
N);
1402 recursivelyDeleteUnusedNodes(TLO.
Old.
getNode());
1408 const APInt &DemandedElts,
1409 bool AssumeSingleUse) {
1417 AddToWorklist(
Op.getNode());
1419 CommitTargetLoweringOpt(TLO);
1426bool DAGCombiner::SimplifyDemandedVectorElts(
SDValue Op,
1427 const APInt &DemandedElts,
1428 bool AssumeSingleUse) {
1430 APInt KnownUndef, KnownZero;
1432 TLO, 0, AssumeSingleUse))
1436 AddToWorklist(
Op.getNode());
1438 CommitTargetLoweringOpt(TLO);
1442void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad) {
1444 EVT VT =
Load->getValueType(0);
1453 AddToWorklist(Trunc.
getNode());
1454 recursivelyDeleteUnusedNodes(Load);
1462 EVT MemVT =
LD->getMemoryVT();
1464 :
LD->getExtensionType();
1467 LD->getChain(),
LD->getBasePtr(),
1468 MemVT,
LD->getMemOperand());
1471 unsigned Opc =
Op.getOpcode();
1475 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1479 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1485 return DAG.
getNode(ExtOpc,
DL, PVT, Op);
1497 EVT OldVT =
Op.getValueType();
1499 bool Replace =
false;
1500 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1503 AddToWorklist(NewOp.
getNode());
1506 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1512 EVT OldVT =
Op.getValueType();
1514 bool Replace =
false;
1515 SDValue NewOp = PromoteOperand(Op, PVT, Replace);
1518 AddToWorklist(NewOp.
getNode());
1521 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1529 if (!LegalOperations)
1532 EVT VT =
Op.getValueType();
1538 unsigned Opc =
Op.getOpcode();
1546 assert(PVT != VT &&
"Don't know what type to promote to!");
1550 bool Replace0 =
false;
1552 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1554 bool Replace1 =
false;
1556 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1568 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1571 CombineTo(
Op.getNode(), RV);
1597 if (!LegalOperations)
1600 EVT VT =
Op.getValueType();
1606 unsigned Opc =
Op.getOpcode();
1614 assert(PVT != VT &&
"Don't know what type to promote to!");
1618 bool Replace =
false;
1621 N0 = SExtPromoteOperand(N0, PVT);
1623 N0 = ZExtPromoteOperand(N0, PVT);
1625 N0 = PromoteOperand(N0, PVT, Replace);
1636 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1646 if (!LegalOperations)
1649 EVT VT =
Op.getValueType();
1655 unsigned Opc =
Op.getOpcode();
1663 assert(PVT != VT &&
"Don't know what type to promote to!");
1673bool DAGCombiner::PromoteLoad(
SDValue Op) {
1674 if (!LegalOperations)
1680 EVT VT =
Op.getValueType();
1686 unsigned Opc =
Op.getOpcode();
1694 assert(PVT != VT &&
"Don't know what type to promote to!");
1699 EVT MemVT =
LD->getMemoryVT();
1701 :
LD->getExtensionType();
1703 LD->getChain(),
LD->getBasePtr(),
1704 MemVT,
LD->getMemOperand());
1713 AddToWorklist(
Result.getNode());
1714 recursivelyDeleteUnusedNodes(
N);
1727bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1728 if (!
N->use_empty())
1738 if (
N->use_empty()) {
1739 for (
const SDValue &ChildN :
N->op_values())
1740 Nodes.
insert(ChildN.getNode());
1742 removeFromWorklist(
N);
1747 }
while (!Nodes.
empty());
1762 WorklistInserter AddNodes(*
this);
1766 AddToWorklist(&
Node);
1774 while (
SDNode *
N = getNextWorklistEntry()) {
1778 if (recursivelyDeleteUnusedNodes(
N))
1781 WorklistRemover DeadNodes(*
this);
1789 for (
SDNode *LN : UpdatedNodes)
1790 AddToWorklistWithUsers(LN);
1802 for (
const SDValue &ChildN :
N->op_values())
1803 if (!CombinedNodes.
count(ChildN.getNode()))
1804 AddToWorklist(ChildN.getNode());
1822 "Node was deleted but visit returned new node!");
1830 N->getNumValues() == 1 &&
"Type mismatch");
1841 AddUsersToWorklist(RV.
getNode());
1848 recursivelyDeleteUnusedNodes(
N);
1857 switch (
N->getOpcode()) {
2001#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
2002#include "llvm/IR/VPIntrinsics.def"
2003 return visitVPOp(
N);
2010 if (!DisableGenericCombines)
2016 "Node was deleted but visit returned NULL!");
2023 DagCombineInfo(DAG, Level,
false,
this);
2031 switch (
N->getOpcode()) {
2039 RV = PromoteIntBinOp(
SDValue(
N, 0));
2044 RV = PromoteIntShiftOp(
SDValue(
N, 0));
2065 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
2080 if (
unsigned NumOps =
N->getNumOperands()) {
2081 if (
N->getOperand(0).getValueType() ==
MVT::Other)
2082 return N->getOperand(0);
2083 if (
N->getOperand(NumOps-1).getValueType() ==
MVT::Other)
2084 return N->getOperand(NumOps-1);
2085 for (
unsigned i = 1; i < NumOps-1; ++i)
2086 if (
N->getOperand(i).getValueType() ==
MVT::Other)
2087 return N->getOperand(i);
2095 if (
N->getNumOperands() == 2) {
2097 return N->getOperand(0);
2099 return N->getOperand(1);
2114 AddToWorklist(*(
N->use_begin()));
2119 bool Changed =
false;
2126 for (
unsigned i = 0; i < TFs.
size(); ++i) {
2131 for (
unsigned j = i;
j < TFs.
size();
j++)
2142 switch (
Op.getOpcode()) {
2160 if (SeenOps.
insert(
Op.getNode()).second)
2171 for (
unsigned i = 1, e = TFs.
size(); i < e; i++)
2172 AddToWorklist(TFs[i]);
2184 bool DidPruneOps =
false;
2186 unsigned NumLeftToConsider = 0;
2187 for (
const SDValue &Op : Ops) {
2188 Worklist.
push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
2192 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
2198 unsigned OrigOpNumber = 0;
2199 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() !=
Op)
2201 assert((OrigOpNumber != Ops.size()) &&
2202 "expected to find TokenFactor Operand");
2204 for (
unsigned i = CurIdx + 1; i < Worklist.
size(); ++i) {
2205 if (Worklist[i].second == OrigOpNumber) {
2206 Worklist[i].second = OpNumber;
2209 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2210 OpWorkCount[OrigOpNumber] = 0;
2211 NumLeftToConsider--;
2214 if (SeenChains.
insert(Op).second) {
2215 OpWorkCount[OpNumber]++;
2216 Worklist.
push_back(std::make_pair(Op, OpNumber));
2220 for (
unsigned i = 0; i < Worklist.
size() && i < 1024; ++i) {
2222 if (NumLeftToConsider <= 1)
2224 auto CurNode = Worklist[i].first;
2225 auto CurOpNumber = Worklist[i].second;
2226 assert((OpWorkCount[CurOpNumber] > 0) &&
2227 "Node should not appear in worklist");
2228 switch (CurNode->getOpcode()) {
2234 NumLeftToConsider++;
2238 AddToWorklist(i,
Op.getNode(), CurOpNumber);
2244 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
2247 if (
auto *MemNode = dyn_cast<MemSDNode>(CurNode))
2248 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2251 OpWorkCount[CurOpNumber]--;
2252 if (OpWorkCount[CurOpNumber] == 0)
2253 NumLeftToConsider--;
2266 for (
const SDValue &Op : Ops) {
2267 if (SeenChains.
count(
Op.getNode()) == 0)
2282 WorklistRemover DeadNodes(*
this);
2288 AddUsersToWorklist(
N);
2292 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i)
2295 }
while (!
N->use_empty());
2296 deleteAndRecombine(
N);
2304 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2315 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2317 VT = LD->getMemoryVT();
2318 AS = LD->getAddressSpace();
2320 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2322 VT = ST->getMemoryVT();
2323 AS = ST->getAddressSpace();
2325 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2327 VT = LD->getMemoryVT();
2328 AS = LD->getAddressSpace();
2330 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2332 VT = ST->getMemoryVT();
2333 AS = ST->getAddressSpace();
2348 }
else if (
N->getOpcode() ==
ISD::SUB) {
2370 bool ShouldCommuteOperands) {
2375 if (ShouldCommuteOperands)
2383 unsigned Opcode =
N->getOpcode();
2387 EVT VT =
N->getValueType(0);
2394 unsigned OpNo = ShouldCommuteOperands ? 0 : 1;
2412 "Unexpected binary operator");
2429 unsigned SelOpNo = 0;
2454 bool CanFoldNonConst =
2460 if (!CanFoldNonConst &&
2468 if (CanFoldNonConst) {
2486 NewCT = SelOpNo ? DAG.
getNode(BinOpcode,
DL, VT, CBO, CT)
2487 : DAG.
getNode(BinOpcode,
DL, VT, CT, CBO);
2488 if (!CanFoldNonConst && !NewCT.
isUndef() &&
2493 NewCF = SelOpNo ? DAG.
getNode(BinOpcode,
DL, VT, CBO, CF)
2494 : DAG.
getNode(BinOpcode,
DL, VT, CF, CBO);
2495 if (!CanFoldNonConst && !NewCF.
isUndef() &&
2508 "Expecting add or sub");
2513 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2514 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2515 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2516 auto *CN = dyn_cast<ConstantSDNode>(
C);
2521 if (Z.getOperand(0).getOpcode() !=
ISD::SETCC ||
2522 Z.getOperand(0).getValueType() !=
MVT::i1)
2526 SDValue SetCC = Z.getOperand(0);
2537 EVT VT =
C.getValueType();
2549 "Expecting add or sub");
2553 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2554 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2555 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2578 {ConstantOp, DAG.getConstant(1, DL, VT)})) {
2588 unsigned Opcode = V.getOpcode();
2632 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2664 if ((!LegalOperations ||
2667 X.getScalarValueSizeInBits() == 1) {
2683 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2687 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD,
DL,
N, N0, N1)) {
2788 return (!Max && !Op) ||
2789 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2841 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2844 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2856 if (
SDValue Combined = visitADDLike(
N))
2892 APInt NewStep = C0 + C1;
2902 APInt NewStep = SV0 + SV1;
2911 unsigned Opcode =
N->getOpcode();
2928 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
2932 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2953 bool Masked =
false;
2958 V = V.getOperand(0);
2964 V = V.getOperand(0);
2972 if (V.getResNo() != 1)
2979 EVT VT = V->getValueType(0);
3095 DAG.
getVTList(VT, Carry.getValueType()), N0,
3108 if (!
N->hasAnyUseOfValue(1))
3141 if (Force && isa<ConstantSDNode>(V))
3151 EVT VT = V.getValueType();
3153 bool IsFlip =
false;
3156 IsFlip = Const->isOne();
3159 IsFlip = Const->isAllOnes();
3162 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
3167 return V.getOperand(0);
3179 EVT CarryVT =
N->getValueType(1);
3183 if (!
N->hasAnyUseOfValue(1))
3190 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
3210 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
3213 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
3247 SDValue CarryIn =
N->getOperand(2);
3266 SDValue CarryIn =
N->getOperand(2);
3277 if (!LegalOperations ||
3287 AddToWorklist(CarryExt.
getNode());
3293 if (
SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn,
N))
3296 if (
SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn,
N))
3304 SDValue Ops[] = {N1, N0, CarryIn};
3316 SDValue CarryIn =
N->getOperand(2);
3327 if (!LegalOperations ||
3474 unsigned CarryInOperandNum =
3476 if (Opcode ==
ISD::USUBO && CarryInOperandNum != 1)
3561 "Illegal truncation");
3587 !(!LegalOperations || hasOperation(
ISD::USUBSAT, DstVT)))
3590 EVT SubVT =
N->getValueType(0);
3648 auto PeekThroughFreeze = [](
SDValue N) {
3650 return N->getOperand(0);
3656 if (PeekThroughFreeze(N0) == PeekThroughFreeze(N1))
3665 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3673 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3700 if (
N->getFlags().hasNoUnsignedWrap())
3706 if (
N->getFlags().hasNoSignedWrap())
3849 if (
SDValue V = foldSubToUSubSat(VT,
N))
3911 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3928 if (GA->getGlobal() == GB->getGlobal())
3990 DAG.
getVTList(VT, Carry.getValueType()), NegX, Zero,
3998 if (!C0->isOpaque()) {
3999 const APInt &C0Val = C0->getAPIntValue();
4000 const APInt &MaybeOnes = ~DAG.computeKnownBits(N1).Zero;
4001 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))
4007 auto MatchSubMaxMin = [&](
unsigned Max,
unsigned Min,
unsigned Abd) {
4015 if (!hasOperation(Abd, VT))
4047 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4069 if (!
N->hasAnyUseOfValue(1))
4096 EVT CarryVT =
N->getValueType(1);
4100 if (!
N->hasAnyUseOfValue(1))
4132 SDValue CarryIn =
N->getOperand(2);