78 #define DEBUG_TYPE "dagcombine"
80 STATISTIC(NodesCombined ,
"Number of dag nodes combined");
81 STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
82 STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
83 STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
84 STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
85 STATISTIC(SlicedLoads,
"Number of load sliced");
86 STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
90 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
94 cl::desc(
"Enable DAG combiner's use of TBAA"));
99 cl::desc(
"Only use DAG-combiner alias analysis in this"
107 cl::desc(
"Bypass the profitability model of load slicing"),
112 cl::desc(
"DAG combiner may split indexing from loads"));
116 cl::desc(
"DAG combiner enable merging multiple stores "
117 "into a wider store"));
121 cl::desc(
"Limit the number of operands to inline for Token Factors"));
125 cl::desc(
"Limit the number of times for the same StoreNode and RootNode "
126 "to bail out in store merging dependence check"));
130 cl::desc(
"DAG combiner enable reducing the width of load/op/store "
135 cl::desc(
"DAG combiner enable load/<replace bytes>/store with "
136 "a narrower store"));
146 bool LegalDAG =
false;
147 bool LegalOperations =
false;
148 bool LegalTypes =
false;
150 bool DisableGenericCombines;
191 void AddUsersToWorklist(
SDNode *
N) {
197 void AddToWorklistWithUsers(
SDNode *
N) {
198 AddUsersToWorklist(
N);
205 void clearAddedDanglingWorklistEntries() {
207 while (!PruningList.
empty()) {
210 recursivelyDeleteUnusedNodes(
N);
214 SDNode *getNextWorklistEntry() {
216 clearAddedDanglingWorklistEntries();
220 while (!
N && !Worklist.empty()) {
225 bool GoodWorklistEntry = WorklistMap.
erase(
N);
226 (void)GoodWorklistEntry;
227 assert(GoodWorklistEntry &&
228 "Found a worklist entry without a corresponding map entry!");
238 : DAG(
D), TLI(
D.getTargetLoweringInfo()),
239 STI(
D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL),
AA(
AA) {
243 MaximumLegalStoreInBits = 0;
249 VT.getSizeInBits().getKnownMinSize() >= MaximumLegalStoreInBits)
250 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinSize();
253 void ConsiderForPruning(
SDNode *
N) {
260 void AddToWorklist(
SDNode *
N) {
262 "Deleted Node added to Worklist");
269 ConsiderForPruning(
N);
271 if (WorklistMap.
insert(std::make_pair(
N, Worklist.size())).second)
272 Worklist.push_back(
N);
276 void removeFromWorklist(
SDNode *
N) {
279 StoreRootCountMap.
erase(
N);
281 auto It = WorklistMap.
find(
N);
282 if (It == WorklistMap.
end())
286 Worklist[It->second] =
nullptr;
287 WorklistMap.
erase(It);
290 void deleteAndRecombine(
SDNode *
N);
291 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
299 return CombineTo(
N, &Res, 1, AddTo);
306 return CombineTo(
N, To, 2, AddTo);
312 unsigned MaximumLegalStoreInBits;
318 unsigned BitWidth =
Op.getScalarValueSizeInBits();
330 AddToWorklist(
Op.getNode());
332 CommitTargetLoweringOpt(TLO);
339 bool SimplifyDemandedVectorElts(
SDValue Op) {
341 if (
Op.getValueType().isScalableVector())
344 unsigned NumElts =
Op.getValueType().getVectorNumElements();
346 return SimplifyDemandedVectorElts(
Op, DemandedElts);
350 const APInt &DemandedElts,
351 bool AssumeSingleUse =
false);
352 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
353 bool AssumeSingleUse =
false);
355 bool CombineToPreIndexedLoadStore(
SDNode *
N);
356 bool CombineToPostIndexedLoadStore(
SDNode *
N);
522 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
542 bool NotExtCompare =
false);
543 SDValue convertSelectOfFPConstantsToLoadOffset(
557 const SDLoc &
DL,
bool foldBooleans);
561 SDValue &CC,
bool MatchStrict =
false)
const;
562 bool isOneUseSetCC(
SDValue N)
const;
587 bool DemandHighBits =
true);
591 unsigned PosOpcode,
unsigned NegOpcode,
595 unsigned PosOpcode,
unsigned NegOpcode,
610 SDValue VecIn2,
unsigned LeftIdx,
645 int64_t OffsetFromBase;
648 : MemNode(
N), OffsetFromBase(
Offset) {}
653 StoreSource getStoreSource(
SDValue StoreVal) {
660 return StoreSource::Extract;
672 bool isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
679 EVT LoadResultTy,
EVT &ExtVT);
684 EVT &MemVT,
unsigned ShAmt = 0);
692 bool BackwardsPropagateMask(
SDNode *
N);
704 EVT MemVT,
unsigned NumStores,
705 bool IsConstantSrc,
bool UseVector,
719 bool checkMergeStoreCandidatesForDependencies(
727 int64_t ElementSizeBytes)
const;
732 unsigned NumConsecutiveStores,
733 EVT MemVT,
SDNode *Root,
bool AllowVectors);
740 unsigned NumConsecutiveStores,
EVT MemVT,
746 unsigned NumConsecutiveStores,
EVT MemVT,
747 SDNode *Root,
bool AllowVectors,
748 bool IsNonTemporalStore,
bool IsNonTemporalLoad);
767 bool hasOperation(
unsigned Opcode,
EVT VT) {
779 EVT getShiftAmountTy(
EVT LHSTy) {
786 bool isTypeLegal(
const EVT &VT) {
787 if (!LegalTypes)
return true;
792 EVT getSetCCResultType(
EVT VT)
const {
807 explicit WorklistRemover(DAGCombiner &dc)
811 DC.removeFromWorklist(
N);
819 explicit WorklistInserter(DAGCombiner &dc)
824 void NodeInserted(
SDNode *
N)
override {
DC.ConsiderForPruning(
N); }
834 ((DAGCombiner*)
DC)->AddToWorklist(
N);
839 return ((DAGCombiner*)
DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
844 return ((DAGCombiner*)
DC)->CombineTo(
N, Res, AddTo);
849 return ((DAGCombiner*)
DC)->CombineTo(
N, Res0, Res1, AddTo);
854 return ((DAGCombiner*)
DC)->recursivelyDeleteUnusedNodes(
N);
859 return ((DAGCombiner*)
DC)->CommitTargetLoweringOpt(TLO);
866 void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
867 removeFromWorklist(
N);
875 if (
Op->hasOneUse() ||
Op->getNumValues() > 1)
876 AddToWorklist(
Op.getNode());
896 SDValue &CC,
bool MatchStrict)
const {
898 LHS =
N.getOperand(0);
899 RHS =
N.getOperand(1);
900 CC =
N.getOperand(2);
907 LHS =
N.getOperand(1);
908 RHS =
N.getOperand(2);
909 CC =
N.getOperand(3);
921 LHS =
N.getOperand(0);
922 RHS =
N.getOperand(1);
923 CC =
N.getOperand(4);
930 bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
932 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N->hasOneUse())
947 MaskForTy = 0xFFFFULL;
950 MaskForTy = 0xFFFFFFFFULL;
969 return !(Const->isOpaque() && NoOpaques);
972 unsigned BitWidth =
N.getScalarValueSizeInBits();
977 if (!Const || Const->getAPIntValue().getBitWidth() !=
BitWidth ||
978 (Const->isOpaque() && NoOpaques))
997 !cast<ConstantSDNode>(
LD->getOperand(2))->isOpaque());
1000 bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1017 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1021 const APInt &C2APIntVal = C2->getAPIntValue();
1025 if (
auto *
C1 = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
1029 const APInt &C1APIntVal =
C1->getAPIntValue();
1030 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1033 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1035 for (
SDNode *Node :
N->uses()) {
1036 if (
auto *
LoadStore = dyn_cast<MemSDNode>(Node)) {
1042 AM.
BaseOffs = C2APIntVal.getSExtValue();
1044 unsigned AS =
LoadStore->getAddressSpace();
1056 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(N0.
getOperand(1)))
1060 for (
SDNode *Node :
N->uses()) {
1061 auto *
LoadStore = dyn_cast<MemSDNode>(Node);
1069 AM.
BaseOffs = C2APIntVal.getSExtValue();
1071 unsigned AS =
LoadStore->getAddressSpace();
1084 SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &
DL,
1098 return DAG.
getNode(Opc,
DL, VT, N00, OpNode);
1105 return DAG.
getNode(Opc,
DL, VT, OpNode, N01);
1115 if (N1 == N00 || N1 == N01)
1163 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N0, N1))
1165 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N1, N0))
1172 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1176 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1177 for (
unsigned i = 0,
e = NumTo;
i !=
e; ++
i)
1179 N->getValueType(
i) == To[
i].getValueType()) &&
1180 "Cannot combine value to value of different type!");
1182 WorklistRemover DeadNodes(*
this);
1186 for (
unsigned i = 0,
e = NumTo;
i !=
e; ++
i) {
1187 if (To[
i].getNode())
1188 AddToWorklistWithUsers(To[
i].getNode());
1196 deleteAndRecombine(
N);
1209 WorklistRemover DeadNodes(*
this);
1225 const APInt &DemandedElts,
1226 bool AssumeSingleUse) {
1234 AddToWorklist(
Op.getNode());
1236 CommitTargetLoweringOpt(TLO);
1243 bool DAGCombiner::SimplifyDemandedVectorElts(
SDValue Op,
1244 const APInt &DemandedElts,
1245 bool AssumeSingleUse) {
1247 APInt KnownUndef, KnownZero;
1249 TLO, 0, AssumeSingleUse))
1253 AddToWorklist(
Op.getNode());
1255 CommitTargetLoweringOpt(TLO);
1259 void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *
Load,
SDNode *ExtLoad) {
1261 EVT VT =
Load->getValueType(0);
1266 WorklistRemover DeadNodes(*
this);
1269 deleteAndRecombine(
Load);
1270 AddToWorklist(Trunc.
getNode());
1278 EVT MemVT =
LD->getMemoryVT();
1280 :
LD->getExtensionType();
1283 LD->getChain(),
LD->getBasePtr(),
1284 MemVT,
LD->getMemOperand());
1287 unsigned Opc =
Op.getOpcode();
1291 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1295 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1313 EVT OldVT =
Op.getValueType();
1315 bool Replace =
false;
1316 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1319 AddToWorklist(NewOp.
getNode());
1322 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1328 EVT OldVT =
Op.getValueType();
1330 bool Replace =
false;
1331 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1334 AddToWorklist(NewOp.
getNode());
1337 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1345 if (!LegalOperations)
1348 EVT VT =
Op.getValueType();
1354 unsigned Opc =
Op.getOpcode();
1362 assert(PVT != VT &&
"Don't know what type to promote to!");
1366 bool Replace0 =
false;
1368 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1370 bool Replace1 =
false;
1372 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1384 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1387 CombineTo(
Op.getNode(), RV);
1413 if (!LegalOperations)
1416 EVT VT =
Op.getValueType();
1422 unsigned Opc =
Op.getOpcode();
1430 assert(PVT != VT &&
"Don't know what type to promote to!");
1434 bool Replace =
false;
1437 N0 = SExtPromoteOperand(N0, PVT);
1439 N0 = ZExtPromoteOperand(N0, PVT);
1441 N0 = PromoteOperand(N0, PVT, Replace);
1452 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1462 if (!LegalOperations)
1465 EVT VT =
Op.getValueType();
1471 unsigned Opc =
Op.getOpcode();
1479 assert(PVT != VT &&
"Don't know what type to promote to!");
1489 bool DAGCombiner::PromoteLoad(
SDValue Op) {
1490 if (!LegalOperations)
1496 EVT VT =
Op.getValueType();
1502 unsigned Opc =
Op.getOpcode();
1510 assert(PVT != VT &&
"Don't know what type to promote to!");
1515 EVT MemVT =
LD->getMemoryVT();
1517 :
LD->getExtensionType();
1519 LD->getChain(),
LD->getBasePtr(),
1520 MemVT,
LD->getMemOperand());
1525 WorklistRemover DeadNodes(*
this);
1528 deleteAndRecombine(
N);
1529 AddToWorklist(
Result.getNode());
1541 bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1542 if (!
N->use_empty())
1552 if (
N->use_empty()) {
1553 for (
const SDValue &ChildN :
N->op_values())
1554 Nodes.
insert(ChildN.getNode());
1556 removeFromWorklist(
N);
1561 }
while (!Nodes.
empty());
1576 WorklistInserter AddNodes(*
this);
1580 AddToWorklist(&Node);
1588 while (
SDNode *
N = getNextWorklistEntry()) {
1592 if (recursivelyDeleteUnusedNodes(
N))
1595 WorklistRemover DeadNodes(*
this);
1603 for (
SDNode *LN : UpdatedNodes)
1604 AddToWorklistWithUsers(LN);
1616 for (
const SDValue &ChildN :
N->op_values())
1617 if (!CombinedNodes.
count(ChildN.getNode()))
1618 AddToWorklist(ChildN.getNode());
1636 "Node was deleted but visit returned new node!");
1644 N->getNumValues() == 1 &&
"Type mismatch");
1655 AddUsersToWorklist(RV.
getNode());
1662 recursivelyDeleteUnusedNodes(
N);
1671 switch (
N->getOpcode()) {
1812 #define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
1813 #include "llvm/IR/VPIntrinsics.def"
1814 return visitVPOp(
N);
1821 if (!DisableGenericCombines)
1827 "Node was deleted but visit returned NULL!");
1834 DagCombineInfo(DAG,
Level,
false,
this);
1842 switch (
N->getOpcode()) {
1850 RV = PromoteIntBinOp(
SDValue(
N, 0));
1855 RV = PromoteIntShiftOp(
SDValue(
N, 0));
1872 N->getNumValues() == 1) {
1877 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
1892 if (
unsigned NumOps =
N->getNumOperands()) {
1893 if (
N->getOperand(0).getValueType() ==
MVT::Other)
1894 return N->getOperand(0);
1895 if (
N->getOperand(NumOps-1).getValueType() ==
MVT::Other)
1896 return N->getOperand(NumOps-1);
1897 for (
unsigned i = 1;
i < NumOps-1; ++
i)
1899 return N->getOperand(
i);
1907 if (
N->getNumOperands() == 2) {
1909 return N->getOperand(0);
1911 return N->getOperand(1);
1926 AddToWorklist(*(
N->use_begin()));
1931 bool Changed =
false;
1938 for (
unsigned i = 0;
i < TFs.size(); ++
i) {
1943 for (
unsigned j =
i;
j < TFs.size();
j++)
1954 switch (
Op.getOpcode()) {
1964 TFs.push_back(
Op.getNode());
1972 if (SeenOps.
insert(
Op.getNode()).second)
1983 for (
unsigned i = 1,
e = TFs.size();
i <
e;
i++)
1984 AddToWorklist(TFs[
i]);
1996 bool DidPruneOps =
false;
1998 unsigned NumLeftToConsider = 0;
2000 Worklist.push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
2001 OpWorkCount.push_back(1);
2004 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
2010 unsigned OrigOpNumber = 0;
2011 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() !=
Op)
2013 assert((OrigOpNumber != Ops.size()) &&
2014 "expected to find TokenFactor Operand");
2016 for (
unsigned i = CurIdx + 1;
i < Worklist.size(); ++
i) {
2017 if (Worklist[
i].second == OrigOpNumber) {
2018 Worklist[
i].second = OpNumber;
2021 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2022 OpWorkCount[OrigOpNumber] = 0;
2023 NumLeftToConsider--;
2026 if (SeenChains.
insert(
Op).second) {
2027 OpWorkCount[OpNumber]++;
2028 Worklist.push_back(std::make_pair(
Op, OpNumber));
2032 for (
unsigned i = 0;
i < Worklist.size() &&
i < 1024; ++
i) {
2034 if (NumLeftToConsider <= 1)
2036 auto CurNode = Worklist[
i].first;
2037 auto CurOpNumber = Worklist[
i].second;
2038 assert((OpWorkCount[CurOpNumber] > 0) &&
2039 "Node should not appear in worklist");
2040 switch (CurNode->getOpcode()) {
2046 NumLeftToConsider++;
2049 for (
const SDValue &
Op : CurNode->op_values())
2050 AddToWorklist(
i,
Op.getNode(), CurOpNumber);
2056 AddToWorklist(
i, CurNode->getOperand(0).getNode(), CurOpNumber);
2059 if (
auto *MemNode = dyn_cast<MemSDNode>(CurNode))
2060 AddToWorklist(
i, MemNode->getChain().getNode(), CurOpNumber);
2063 OpWorkCount[CurOpNumber]--;
2064 if (OpWorkCount[CurOpNumber] == 0)
2065 NumLeftToConsider--;
2079 if (SeenChains.
count(
Op.getNode()) == 0)
2080 PrunedOps.push_back(
Op);
2094 WorklistRemover DeadNodes(*
this);
2100 AddUsersToWorklist(
N);
2104 for (
unsigned i = 0,
e =
N->getNumOperands();
i !=
e; ++
i)
2105 Ops.push_back(
N->getOperand(
i));
2107 }
while (!
N->use_empty());
2108 deleteAndRecombine(
N);
2116 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2127 if (
LD->isIndexed() ||
LD->getBasePtr().getNode() !=
N)
2129 VT =
LD->getMemoryVT();
2130 AS =
LD->getAddressSpace();
2132 if (
ST->isIndexed() ||
ST->getBasePtr().getNode() !=
N)
2134 VT =
ST->getMemoryVT();
2135 AS =
ST->getAddressSpace();
2137 if (
LD->isIndexed() ||
LD->getBasePtr().getNode() !=
N)
2139 VT =
LD->getMemoryVT();
2140 AS =
LD->getAddressSpace();
2142 if (
ST->isIndexed() ||
ST->getBasePtr().getNode() !=
N)
2144 VT =
ST->getMemoryVT();
2145 AS =
ST->getAddressSpace();
2156 AM.BaseOffs = Offset->getSExtValue();
2160 }
else if (
N->getOpcode() ==
ISD::SUB) {
2161 AM.HasBaseReg =
true;
2165 AM.BaseOffs = -Offset->getSExtValue();
2182 bool ShouldCommuteOperands) {
2187 if (ShouldCommuteOperands)
2194 unsigned Opcode =
N->getOpcode();
2195 EVT VT =
N->getValueType(0);
2203 auto isIdentityConstantForOpcode = [](
unsigned Opcode,
SDValue V) {
2207 return C->isZero() &&
C->isNegative();
2209 return C->isZero() && !
C->isNegative();
2212 return C->isExactlyValue(1.0);
2232 if (isIdentityConstantForOpcode(Opcode, TVal)) {
2238 if (isIdentityConstantForOpcode(Opcode, FVal)) {
2249 "Unexpected binary operator");
2266 unsigned SelOpNo = 0;
2291 bool CanFoldNonConst =
2297 if (!CanFoldNonConst &&
2307 : DAG.
getNode(BinOpcode,
DL, VT, CT, CBO);
2308 if (!CanFoldNonConst && !NewCT.
isUndef() &&
2314 : DAG.
getNode(BinOpcode,
DL, VT, CF, CBO);
2315 if (!CanFoldNonConst && !NewCF.
isUndef() &&
2327 "Expecting add or sub");
2332 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2333 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2334 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2335 auto *CN = dyn_cast<ConstantSDNode>(
C);
2340 if (Z.getOperand(0).getOpcode() !=
ISD::SETCC ||
2341 Z.getOperand(0).getValueType() !=
MVT::i1)
2345 SDValue SetCC = Z.getOperand(0);
2356 EVT VT =
C.getValueType();
2368 "Expecting add or sub");
2372 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2373 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2374 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2399 {ConstantOp, DAG.getConstant(1, DL, VT)}))
2439 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2471 if ((!LegalOperations ||
2474 X.getScalarValueSizeInBits() == 1) {
2490 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2494 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD,
DL,
N, N0, N1)) {
2511 if (
SDValue Add = ReassociateAddOr(N0, N1))
2513 if (
SDValue Add = ReassociateAddOr(N1, N0))
2580 return (!Max && !
Op) ||
2581 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2633 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2636 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2648 if (
SDValue Combined = visitADDLike(
N))
2694 APInt NewStep = SV0 + SV1;
2703 unsigned Opcode =
N->getOpcode();
2720 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
2724 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2745 bool Masked =
false;
2877 DAG.
getVTList(VT, Carry.getValueType()), N0,
2890 if (!
N->hasAnyUseOfValue(1))
2923 if (Force && isa<ConstantSDNode>(V))
2935 bool IsFlip =
false;
2938 IsFlip = Const->isOne();
2941 IsFlip = Const->isAllOnes();
2944 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
2961 EVT CarryVT =
N->getValueType(1);
2965 if (!
N->hasAnyUseOfValue(1))
2972 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
2992 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
2995 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
3029 SDValue CarryIn =
N->getOperand(2);
3048 SDValue CarryIn =
N->getOperand(2);
3059 if (!LegalOperations ||
3069 AddToWorklist(CarryExt.
getNode());
3075 if (
SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn,
N))
3078 if (
SDValue Combined = visitADDCARRYLike(N1, N0, CarryIn,
N))
3087 SDValue CarryIn =
N->getOperand(2);
3098 if (!LegalOperations ||
3245 unsigned CarryInOperandNum =
3247 if (Opcode ==
ISD::USUBO && CarryInOperandNum != 1)
3332 "Illegal truncation");
3358 !(!LegalOperations || hasOperation(
ISD::USUBSAT, DstVT)))
3361 EVT SubVT =
N->getValueType(0);
3419 auto PeekThroughFreeze = [](
SDValue N) {
3421 return N->getOperand(0);
3427 if (PeekThroughFreeze(N0) == PeekThroughFreeze(N1))
3436 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3444 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3471 if (
N->getFlags().hasNoUnsignedWrap())
3477 if (
N->getFlags().hasNoSignedWrap())
3623 if (
SDValue V = foldSubToUSubSat(VT,
N))
3685 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3702 if (GA->getGlobal() == GB->getGlobal())
3758 DAG.
getVTList(VT, Carry.getValueType()), NegX, Zero,
3786 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3808 if (!
N->hasAnyUseOfValue(1))
3835 EVT CarryVT =
N->getValueType(1);
3839 if (!
N->hasAnyUseOfValue(1))
3871 SDValue CarryIn =
N->getOperand(2);
3883 SDValue CarryIn =
N->getOperand(2);
3887 if (!LegalOperations ||
3898 SDValue CarryIn =
N->getOperand(2);
3902 if (!LegalOperations ||
3953 bool N1IsConst =
false;
3954 bool N1IsOpaqueConst =
false;
3959 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3965 "Splat APInt should be element width");
3967 N1IsConst = isa<ConstantSDNode>(N1);
3969 ConstValue1 = cast<ConstantSDNode>(N1)->getAPIntValue();
3970 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
3975 if (N1IsConst && ConstValue1.
isZero())
3979 if (N1IsConst && ConstValue1.
isOne())
3982 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3986 if (N1IsConst && ConstValue1.
isAllOnes())
3994 SDValue LogBase2 = BuildLogBase2(N1,
DL);
4002 unsigned Log2Val = (-ConstValue1).logBase2();
4036 if ((MulC - 1).isPowerOf2())
4038 else if ((MulC + 1).isPowerOf2())
4043 MathOp ==
ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
4046 "multiply-by-constant generated out of bounds shift");
4050 TZeros ? DAG.
getNode(MathOp,
DL, VT, Shl,
4102 const APInt &
C1 = NC1->getAPIntValue();
4111 APInt NewStep = C0 * MulVal;
4124 if (!V || V->isZero()) {
4138 for (
unsigned I = 0;
I != NumElts; ++
I)
4159 switch (
NodeType.getSimpleVT().SimpleTy) {
4160 default:
return false;
4161 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
4162 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
4163 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
4164 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
4165 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
4173 if (Node->use_empty())
4176 unsigned Opcode = Node->getOpcode();
4181 EVT VT = Node->getValueType(0);
4195 unsigned OtherOpcode = 0;
4206 SDValue Op0 = Node->getOperand(0);
4207 SDValue Op1 = Node->getOperand(1);
4216 unsigned UserOpc =
User->getOpcode();
4217 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
4221 if (UserOpc == OtherOpcode) {
4223 combined = DAG.
getNode(DivRemOpc,
SDLoc(Node), VTs, Op0, Op1);
4224 }
else if (UserOpc == DivRemOpc) {
4227 assert(UserOpc == Opcode);
4232 CombineTo(
User, combined);
4243 EVT VT =
N->getValueType(0);
4246 unsigned Opc =
N->getOpcode();
4255 if (DAG.
isUndef(Opc, {N0, N1}))
4266 if (N0C && N0C->
isZero())
4289 EVT VT =
N->getValueType(0);
4299 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4316 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4324 if (
SDValue V = visitSDIVLike(N0, N1,
N)) {
4331 AddToWorklist(
Mul.getNode());
4333 CombineTo(RemNode, Sub);
4353 if (
C->isZero() ||
C->isOpaque())
4355 if (
C->getAPIntValue().isPowerOf2())
4357 if (
C->getAPIntValue().isNegatedPowerOf2())
4367 EVT VT =
N->getValueType(0);
4392 AddToWorklist(Sign.
getNode());
4398 AddToWorklist(
Add.getNode());