85#define DEBUG_TYPE "dagcombine"
87STATISTIC(NodesCombined ,
"Number of dag nodes combined");
88STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
89STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
90STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
91STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
93STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
96 "Controls whether a DAG combine is performed for a node");
100 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
104 cl::desc(
"Enable DAG combiner's use of TBAA"));
109 cl::desc(
"Only use DAG-combiner alias analysis in this"
117 cl::desc(
"Bypass the profitability model of load slicing"),
122 cl::desc(
"DAG combiner may split indexing from loads"));
126 cl::desc(
"DAG combiner enable merging multiple stores "
127 "into a wider store"));
131 cl::desc(
"Limit the number of operands to inline for Token Factors"));
135 cl::desc(
"Limit the number of times for the same StoreNode and RootNode "
136 "to bail out in store merging dependence check"));
140 cl::desc(
"DAG combiner enable reducing the width of load/op/store "
145 cl::desc(
"DAG combiner enable load/<replace bytes>/store with "
146 "a narrower store"));
151 "Enable merging extends and rounds into FCOPYSIGN on vector types"));
161 bool LegalDAG =
false;
162 bool LegalOperations =
false;
163 bool LegalTypes =
false;
165 bool DisableGenericCombines;
207 void AddUsersToWorklist(
SDNode *
N) {
213 void AddToWorklistWithUsers(
SDNode *
N) {
214 AddUsersToWorklist(
N);
221 void clearAddedDanglingWorklistEntries() {
223 while (!PruningList.
empty()) {
226 recursivelyDeleteUnusedNodes(
N);
230 SDNode *getNextWorklistEntry() {
232 clearAddedDanglingWorklistEntries();
236 while (!
N && !Worklist.
empty()) {
241 bool GoodWorklistEntry = WorklistMap.
erase(
N);
242 (void)GoodWorklistEntry;
243 assert(GoodWorklistEntry &&
244 "Found a worklist entry without a corresponding map entry!");
254 : DAG(
D), TLI(
D.getTargetLoweringInfo()),
255 STI(
D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) {
259 MaximumLegalStoreInBits = 0;
265 VT.getSizeInBits().getKnownMinValue() >= MaximumLegalStoreInBits)
266 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinValue();
269 void ConsiderForPruning(
SDNode *
N) {
276 void AddToWorklist(
SDNode *
N,
bool IsCandidateForPruning =
true) {
278 "Deleted Node added to Worklist");
285 if (IsCandidateForPruning)
286 ConsiderForPruning(
N);
288 if (WorklistMap.
insert(std::make_pair(
N, Worklist.
size())).second)
293 void removeFromWorklist(
SDNode *
N) {
296 StoreRootCountMap.
erase(
N);
298 auto It = WorklistMap.
find(
N);
299 if (It == WorklistMap.
end())
303 Worklist[It->second] =
nullptr;
304 WorklistMap.
erase(It);
307 void deleteAndRecombine(
SDNode *
N);
308 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
316 return CombineTo(
N, &Res, 1, AddTo);
323 return CombineTo(
N, To, 2, AddTo);
329 unsigned MaximumLegalStoreInBits;
335 unsigned BitWidth =
Op.getScalarValueSizeInBits();
341 EVT VT =
Op.getValueType();
345 return SimplifyDemandedBits(
Op,
DemandedBits, DemandedElts,
false);
351 bool SimplifyDemandedVectorElts(
SDValue Op) {
353 if (
Op.getValueType().isScalableVector())
356 unsigned NumElts =
Op.getValueType().getVectorNumElements();
358 return SimplifyDemandedVectorElts(
Op, DemandedElts);
362 const APInt &DemandedElts,
363 bool AssumeSingleUse =
false);
364 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
365 bool AssumeSingleUse =
false);
367 bool CombineToPreIndexedLoadStore(
SDNode *
N);
368 bool CombineToPostIndexedLoadStore(
SDNode *
N);
393 void ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad);
530 bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
SDNode *
N);
560 template <
class MatchContextClass>
562 template <
class MatchContextClass>
567 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
576 SDValue reassociateReduction(
unsigned RedOpc,
unsigned Opc,
const SDLoc &
DL,
590 bool NotExtCompare =
false);
591 SDValue convertSelectOfFPConstantsToLoadOffset(
606 const SDLoc &
DL,
bool foldBooleans);
610 SDValue &
CC,
bool MatchStrict =
false)
const;
611 bool isOneUseSetCC(
SDValue N)
const;
636 bool KnownNeverZero =
false,
637 bool InexpensiveOnly =
false,
638 std::optional<EVT> OutVT = std::nullopt);
648 bool DemandHighBits =
true);
652 unsigned PosOpcode,
unsigned NegOpcode,
656 unsigned PosOpcode,
unsigned NegOpcode,
672 SDValue VecIn2,
unsigned LeftIdx,
707 int64_t OffsetFromBase;
710 : MemNode(
N), OffsetFromBase(
Offset) {}
715 StoreSource getStoreSource(
SDValue StoreVal) {
719 return StoreSource::Constant;
723 return StoreSource::Constant;
724 return StoreSource::Unknown;
727 return StoreSource::Extract;
729 return StoreSource::Load;
731 return StoreSource::Unknown;
739 bool isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
746 EVT LoadResultTy,
EVT &ExtVT);
751 EVT &MemVT,
unsigned ShAmt = 0);
759 bool BackwardsPropagateMask(
SDNode *
N);
776 EVT MemVT,
unsigned NumStores,
777 bool IsConstantSrc,
bool UseVector,
791 bool checkMergeStoreCandidatesForDependencies(
799 int64_t ElementSizeBytes)
const;
804 unsigned NumConsecutiveStores,
805 EVT MemVT,
SDNode *Root,
bool AllowVectors);
812 unsigned NumConsecutiveStores,
EVT MemVT,
818 unsigned NumConsecutiveStores,
EVT MemVT,
819 SDNode *Root,
bool AllowVectors,
820 bool IsNonTemporalStore,
bool IsNonTemporalLoad);
839 bool hasOperation(
unsigned Opcode,
EVT VT) {
851 EVT getShiftAmountTy(
EVT LHSTy) {
858 bool isTypeLegal(
const EVT &VT) {
859 if (!LegalTypes)
return true;
864 EVT getSetCCResultType(
EVT VT)
const {
879 explicit WorklistRemover(DAGCombiner &dc)
880 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
883 DC.removeFromWorklist(
N);
891 explicit WorklistInserter(DAGCombiner &dc)
892 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
906 ((DAGCombiner*)
DC)->AddToWorklist(
N);
911 return ((DAGCombiner*)DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
916 return ((DAGCombiner*)DC)->CombineTo(
N, Res, AddTo);
921 return ((DAGCombiner*)DC)->CombineTo(
N, Res0, Res1, AddTo);
926 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(
N);
931 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
938void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
939 removeFromWorklist(
N);
947 if (
Op->hasOneUse() ||
Op->getNumValues() > 1)
948 AddToWorklist(
Op.getNode());
957 unsigned Bits =
Offset + std::max(
LHS.getBitWidth(),
RHS.getBitWidth());
970 LHS =
N.getOperand(0);
971 RHS =
N.getOperand(1);
972 CC =
N.getOperand(2);
979 LHS =
N.getOperand(1);
980 RHS =
N.getOperand(2);
981 CC =
N.getOperand(3);
993 LHS =
N.getOperand(0);
994 RHS =
N.getOperand(1);
995 CC =
N.getOperand(4);
1002bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
1004 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N->hasOneUse())
1016 MaskForTy = 0xFFULL;
1019 MaskForTy = 0xFFFFULL;
1022 MaskForTy = 0xFFFFFFFFULL;
1041 return !(Const->isOpaque() && NoOpaques);
1044 unsigned BitWidth =
N.getScalarValueSizeInBits();
1049 if (!Const || Const->getAPIntValue().getBitWidth() !=
BitWidth ||
1050 (Const->isOpaque() && NoOpaques))
1069 !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
1072bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1089 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1093 const APInt &C2APIntVal = C2->getAPIntValue();
1097 if (
auto *C1 = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
1101 const APInt &C1APIntVal = C1->getAPIntValue();
1102 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1105 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1108 if (
auto *LoadStore = dyn_cast<MemSDNode>(
Node)) {
1114 AM.
BaseOffs = C2APIntVal.getSExtValue();
1116 unsigned AS =
LoadStore->getAddressSpace();
1128 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(N0.
getOperand(1)))
1141 AM.
BaseOffs = C2APIntVal.getSExtValue();
1143 unsigned AS =
LoadStore->getAddressSpace();
1156SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &
DL,
1170 Flags.hasNoUnsignedWrap())
1176 return DAG.
getNode(Opc,
DL, VT, N00, OpNode, NewFlags);
1183 return DAG.
getNode(Opc,
DL, VT, OpNode, N01, NewFlags);
1193 if (N1 == N00 || N1 == N01)
1239 if (CC1 == CC00 && CC1 != CC01) {
1241 return DAG.
getNode(Opc,
DL, VT, OpNode, N01, Flags);
1243 if (CC1 == CC01 && CC1 != CC00) {
1245 return DAG.
getNode(Opc,
DL, VT, OpNode, N00, Flags);
1263 if (!
Flags.hasAllowReassociation() || !
Flags.hasNoSignedZeros())
1266 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N0, N1, Flags))
1268 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N1, N0, Flags))
1276SDValue DAGCombiner::reassociateReduction(
unsigned RedOpc,
unsigned Opc,
1294 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1298 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1299 for (
unsigned i = 0, e = NumTo; i !=
e; ++i)
1300 assert((!To[i].getNode() ||
1301 N->getValueType(i) == To[i].getValueType()) &&
1302 "Cannot combine value to value of different type!");
1304 WorklistRemover DeadNodes(*
this);
1308 for (
unsigned i = 0, e = NumTo; i !=
e; ++i) {
1309 if (To[i].getNode())
1310 AddToWorklistWithUsers(To[i].getNode());
1318 deleteAndRecombine(
N);
1336 recursivelyDeleteUnusedNodes(TLO.
Old.
getNode());
1342 const APInt &DemandedElts,
1343 bool AssumeSingleUse) {
1351 AddToWorklist(
Op.getNode());
1353 CommitTargetLoweringOpt(TLO);
1360bool DAGCombiner::SimplifyDemandedVectorElts(
SDValue Op,
1361 const APInt &DemandedElts,
1362 bool AssumeSingleUse) {
1364 APInt KnownUndef, KnownZero;
1366 TLO, 0, AssumeSingleUse))
1370 AddToWorklist(
Op.getNode());
1372 CommitTargetLoweringOpt(TLO);
1376void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad) {
1378 EVT VT =
Load->getValueType(0);
1387 AddToWorklist(Trunc.
getNode());
1388 recursivelyDeleteUnusedNodes(Load);
1396 EVT MemVT =
LD->getMemoryVT();
1398 :
LD->getExtensionType();
1401 LD->getChain(),
LD->getBasePtr(),
1402 MemVT,
LD->getMemOperand());
1405 unsigned Opc =
Op.getOpcode();
1409 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1413 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1431 EVT OldVT =
Op.getValueType();
1433 bool Replace =
false;
1434 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1437 AddToWorklist(NewOp.
getNode());
1440 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1446 EVT OldVT =
Op.getValueType();
1448 bool Replace =
false;
1449 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1452 AddToWorklist(NewOp.
getNode());
1455 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1463 if (!LegalOperations)
1466 EVT VT =
Op.getValueType();
1472 unsigned Opc =
Op.getOpcode();
1480 assert(PVT != VT &&
"Don't know what type to promote to!");
1484 bool Replace0 =
false;
1486 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1488 bool Replace1 =
false;
1490 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1502 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1505 CombineTo(
Op.getNode(), RV);
1531 if (!LegalOperations)
1534 EVT VT =
Op.getValueType();
1540 unsigned Opc =
Op.getOpcode();
1548 assert(PVT != VT &&
"Don't know what type to promote to!");
1552 bool Replace =
false;
1555 N0 = SExtPromoteOperand(N0, PVT);
1557 N0 = ZExtPromoteOperand(N0, PVT);
1559 N0 = PromoteOperand(N0, PVT, Replace);
1570 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1580 if (!LegalOperations)
1583 EVT VT =
Op.getValueType();
1589 unsigned Opc =
Op.getOpcode();
1597 assert(PVT != VT &&
"Don't know what type to promote to!");
1607bool DAGCombiner::PromoteLoad(
SDValue Op) {
1608 if (!LegalOperations)
1614 EVT VT =
Op.getValueType();
1620 unsigned Opc =
Op.getOpcode();
1628 assert(PVT != VT &&
"Don't know what type to promote to!");
1633 EVT MemVT =
LD->getMemoryVT();
1635 :
LD->getExtensionType();
1637 LD->getChain(),
LD->getBasePtr(),
1638 MemVT,
LD->getMemOperand());
1647 AddToWorklist(
Result.getNode());
1648 recursivelyDeleteUnusedNodes(
N);
1661bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1662 if (!
N->use_empty())
1672 if (
N->use_empty()) {
1673 for (
const SDValue &ChildN :
N->op_values())
1674 Nodes.
insert(ChildN.getNode());
1676 removeFromWorklist(
N);
1681 }
while (!Nodes.
empty());
1696 WorklistInserter AddNodes(*
this);
1705 AddToWorklist(&
Node,
Node.use_empty());
1713 while (
SDNode *
N = getNextWorklistEntry()) {
1717 if (recursivelyDeleteUnusedNodes(
N))
1720 WorklistRemover DeadNodes(*
this);
1728 for (
SDNode *LN : UpdatedNodes)
1729 AddToWorklistWithUsers(LN);
1740 for (
const SDValue &ChildN :
N->op_values())
1741 if (!CombinedNodes.
count(ChildN.getNode()))
1742 AddToWorklist(ChildN.getNode());
1761 "Node was deleted but visit returned new node!");
1769 N->getNumValues() == 1 &&
"Type mismatch");
1779 AddToWorklistWithUsers(RV.
getNode());
1785 recursivelyDeleteUnusedNodes(
N);
1795 switch (
N->getOpcode()) {
1884 case ISD::FMA:
return visitFMA<EmptyMatchContext>(
N);
1949#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
1950#include "llvm/IR/VPIntrinsics.def"
1951 return visitVPOp(
N);
1962 if (!DisableGenericCombines)
1968 "Node was deleted but visit returned NULL!");
1975 DagCombineInfo(DAG, Level,
false,
this);
1983 switch (
N->getOpcode()) {
1991 RV = PromoteIntBinOp(
SDValue(
N, 0));
1996 RV = PromoteIntShiftOp(
SDValue(
N, 0));
2017 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
2032 if (
unsigned NumOps =
N->getNumOperands()) {
2033 if (
N->getOperand(0).getValueType() == MVT::Other)
2034 return N->getOperand(0);
2035 if (
N->getOperand(NumOps-1).getValueType() == MVT::Other)
2036 return N->getOperand(NumOps-1);
2037 for (
unsigned i = 1; i < NumOps-1; ++i)
2038 if (
N->getOperand(i).getValueType() == MVT::Other)
2039 return N->getOperand(i);
2047 if (
N->getNumOperands() == 2) {
2049 return N->getOperand(0);
2051 return N->getOperand(1);
2066 AddToWorklist(*(
N->use_begin()));
2071 bool Changed =
false;
2078 for (
unsigned i = 0; i < TFs.
size(); ++i) {
2083 for (
unsigned j = i;
j < TFs.
size();
j++)
2094 switch (
Op.getOpcode()) {
2112 if (SeenOps.
insert(
Op.getNode()).second)
2123 for (
unsigned i = 1, e = TFs.
size(); i < e; i++)
2124 AddToWorklist(TFs[i]);
2136 bool DidPruneOps =
false;
2138 unsigned NumLeftToConsider = 0;
2140 Worklist.
push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
2144 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
2150 unsigned OrigOpNumber = 0;
2151 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() !=
Op)
2153 assert((OrigOpNumber != Ops.size()) &&
2154 "expected to find TokenFactor Operand");
2156 for (
unsigned i = CurIdx + 1; i < Worklist.
size(); ++i) {
2157 if (Worklist[i].second == OrigOpNumber) {
2158 Worklist[i].second = OpNumber;
2161 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2162 OpWorkCount[OrigOpNumber] = 0;
2163 NumLeftToConsider--;
2166 if (SeenChains.
insert(
Op).second) {
2167 OpWorkCount[OpNumber]++;
2172 for (
unsigned i = 0; i < Worklist.
size() && i < 1024; ++i) {
2174 if (NumLeftToConsider <= 1)
2176 auto CurNode = Worklist[i].first;
2177 auto CurOpNumber = Worklist[i].second;
2178 assert((OpWorkCount[CurOpNumber] > 0) &&
2179 "Node should not appear in worklist");
2180 switch (CurNode->getOpcode()) {
2186 NumLeftToConsider++;
2189 for (
const SDValue &
Op : CurNode->op_values())
2190 AddToWorklist(i,
Op.getNode(), CurOpNumber);
2196 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
2199 if (
auto *MemNode = dyn_cast<MemSDNode>(CurNode))
2200 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2203 OpWorkCount[CurOpNumber]--;
2204 if (OpWorkCount[CurOpNumber] == 0)
2205 NumLeftToConsider--;
2219 if (SeenChains.
count(
Op.getNode()) == 0)
2234 WorklistRemover DeadNodes(*
this);
2240 AddUsersToWorklist(
N);
2244 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i)
2247 }
while (!
N->use_empty());
2248 deleteAndRecombine(
N);
2256 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2266 Op =
N->getOperand(0);
2272 N.getValueType().getScalarType() != MVT::i1 ||
2273 cast<CondCodeSDNode>(
N.getOperand(2))->get() !=
ISD::SETNE)
2300 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2302 VT = LD->getMemoryVT();
2303 AS = LD->getAddressSpace();
2305 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2307 VT = ST->getMemoryVT();
2308 AS = ST->getAddressSpace();
2310 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2312 VT = LD->getMemoryVT();
2313 AS = LD->getAddressSpace();
2315 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2317 VT = ST->getMemoryVT();
2318 AS = ST->getAddressSpace();
2333 }
else if (
N->getOpcode() ==
ISD::SUB) {
2355 bool ShouldCommuteOperands) {
2360 if (ShouldCommuteOperands)
2372 unsigned Opcode =
N->getOpcode();
2373 EVT VT =
N->getValueType(0);
2380 unsigned OpNo = ShouldCommuteOperands ? 0 : 1;
2398 "Unexpected binary operator");
2415 unsigned SelOpNo = 0;
2451 bool CanFoldNonConst =
2457 if (!CanFoldNonConst &&
2465 if (CanFoldNonConst) {
2502 "Expecting add or sub");
2507 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2508 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2509 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2510 auto *CN = dyn_cast<ConstantSDNode>(
C);
2515 if (Z.getOperand(0).getOpcode() !=
ISD::SETCC ||
2516 Z.getOperand(0).getValueType() != MVT::i1)
2520 SDValue SetCC = Z.getOperand(0);
2531 EVT VT =
C.getValueType();
2564 "Expecting add or sub");
2568 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2569 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2570 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2592 {ConstantOp, DAG.getConstant(1, DL, VT)})) {
2594 Not.getOperand(0), ShAmt);
2637 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2669 if ((!LegalOperations ||
2672 X.getScalarValueSizeInBits() == 1) {
2688 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2692 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD,
DL,
N, N0, N1)) {
2786 return (!Max && !
Op) ||
2787 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2828 !
N->getFlags().hasNoSignedWrap()))) {
2841 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2844 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2878 if (
SDValue Combined = visitADDLike(
N))
2895 Flags.setDisjoint(
true);
2921 APInt NewStep = C0 + C1;
2931 APInt NewStep = SV0 + SV1;
2940 unsigned Opcode =
N->getOpcode();
2958 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
2962 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2982 bool ForceCarryReconstruction =
false) {
2983 bool Masked =
false;
2988 V = V.getOperand(0);
2993 if (ForceCarryReconstruction)
2997 V = V.getOperand(0);
3001 if (ForceCarryReconstruction && V.getValueType() == MVT::i1)
3008 if (V.getResNo() != 1)
3015 EVT VT = V->getValueType(0);
3123 if (TN->
getVT() == MVT::i1) {
3140 DAG.
getVTList(VT, Carry.getValueType()), N0,
3153 if (!
N->hasAnyUseOfValue(1))
3186 if (Force && isa<ConstantSDNode>(V))
3196 EVT VT = V.getValueType();
3198 bool IsFlip =
false;
3201 IsFlip = Const->isOne();
3204 IsFlip = Const->isAllOnes();
3207 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
3212 return V.getOperand(0);
3224 EVT CarryVT =
N->getValueType(1);
3228 if (!
N->hasAnyUseOfValue(1))
3235 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
3260 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
3263 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
3297 SDValue CarryIn =
N->getOperand(2);
3316 SDValue CarryIn =
N->getOperand(2);
3327 if (!LegalOperations ||
3337 AddToWorklist(CarryExt.
getNode());
3343 if (
SDValue Combined = visitUADDO_CARRYLike(N0, N1, CarryIn,
N))
3346 if (
SDValue Combined = visitUADDO_CARRYLike(N1, N0, CarryIn,
N))
3354 SDValue Ops[] = {N1, N0, CarryIn};
3493 EVT CarryOutType =
N->getValueType(0);
3509 unsigned CarryInOperandNum =
3511 if (Opcode ==
ISD::USUBO && CarryInOperandNum != 1)
3605 SDValue CarryIn =
N->getOperand(2);
3616 if (!LegalOperations ||
3621 if (
SDValue Combined = visitSADDO_CARRYLike(N0, N1, CarryIn,
N))
3624 if (
SDValue Combined = visitSADDO_CARRYLike(N1, N0, CarryIn,
N))
3636 "Illegal truncation");
3662 !(!LegalOperations || hasOperation(
ISD::USUBSAT, DstVT)))
3665 EVT SubVT =
N->getValueType(0);
3724 auto PeekThroughFreeze = [](
SDValue N) {
3726 return N->getOperand(0);
3732 if (PeekThroughFreeze(N0) == PeekThroughFreeze(N1))
3741 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3749 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3772 if (
N->getFlags().hasNoUnsignedWrap())
3778 if (
N->getFlags().hasNoSignedWrap())
3955 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3957 if (
C->getAPIntValue() == (
BitWidth - 1))
3967 if (GA->getGlobal() == GB->getGlobal())
3975 if (TN->
getVT() == MVT::i1) {
4028 DAG.
getVTList(VT, Carry.getValueType()), NegX, Zero,
4036 if (!C0->isOpaque()) {
4037 const APInt &C0Val = C0->getAPIntValue();
4038 const APInt &MaybeOnes = ~DAG.computeKnownBits(N1).Zero;
4039 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))
4060 unsigned Opcode =
N->getOpcode();
4081 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4107 if (!
N->hasAnyUseOfValue(1))
4134 EVT CarryVT =
N->getValueType(1);
4138 if (!
N->hasAnyUseOfValue(1))
4173 SDValue CarryIn =
N->getOperand(2);
4185 SDValue CarryIn =
N->getOperand(2);
4189 if (!LegalOperations ||
4200 SDValue CarryIn =
N->getOperand(2);
4204 if (!LegalOperations ||
4255 bool N1IsConst =
false;
4256 bool N1IsOpaqueConst =
false;
4261 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4267 "Splat APInt should be element width");
4269 N1IsConst = isa<ConstantSDNode>(N1);
4272 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
4277 if (N1IsConst && ConstValue1.
isZero())
4281 if (N1IsConst && ConstValue1.
isOne())
4284 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4288 if (N1IsConst && ConstValue1.
isAllOnes())
4294 if (
SDValue LogBase2 = BuildLogBase2(N1,
DL)) {
4303 unsigned Log2Val = (-ConstValue1).logBase2();
4321 if (LoHi->hasAnyUseOfValue(1))
4324 if (LoHi->hasAnyUseOfValue(1))
4351 unsigned TZeros = MulC == 2 ? 0 : MulC.
countr_zero();
4353 if ((MulC - 1).isPowerOf2())
4355 else if ((MulC + 1).isPowerOf2())
4360 MathOp ==
ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
4363 "multiply-by-constant generated out of bounds shift");
4367 TZeros ? DAG.
getNode(MathOp,
DL, VT, Shl,
4428 APInt NewStep = C0 * MulVal;
4441 if (!V ||
V->isZero()) {
4455 for (
unsigned I = 0;
I != NumElts; ++
I)
4482 EVT NodeType =
Node->getValueType(0);
4483 if (!NodeType.isSimple())
4485 switch (NodeType.getSimpleVT().SimpleTy) {
4486 default:
return false;
4487 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
4488 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
4489 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
4490 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
4491 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
4499 if (
Node->use_empty())
4502 unsigned Opcode =
Node->getOpcode();
4507 EVT VT =
Node->getValueType(0);
4521 unsigned OtherOpcode = 0;
4542 unsigned UserOpc =
User->getOpcode();
4543 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
4547 if (UserOpc == OtherOpcode) {
4550 }
else if (UserOpc == DivRemOpc) {
4553 assert(UserOpc == Opcode);
4558 CombineTo(
User, combined);
4569 EVT VT =
N->getValueType(0);
4572 unsigned Opc =
N->getOpcode();
4581 if (DAG.
isUndef(Opc, {N0, N1}))
4592 if (N0C && N0C->
isZero())
4615 EVT VT =
N->getValueType(0);
4625 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4642 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4650 if (
SDValue V = visitSDIVLike(N0, N1,
N)) {
4657 AddToWorklist(
Mul.getNode());
4659 CombineTo(RemNode, Sub);
4679 if (
C->isZero() ||
C->isOpaque())
4681 if (
C->getAPIntValue().isPowerOf2())
4683 if (
C->getAPIntValue().isNegatedPowerOf2())
4693 EVT VT =
N->getValueType(0);
4718 AddToWorklist(Sign.
getNode());
4724 AddToWorklist(
Add.getNode());
4735 Sra = DAG.
getSelect(
DL, VT, IsOneOrAllOnes, N0, Sra);
4763 EVT VT =
N->getValueType(0);
4773 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4787 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4790 if (
SDValue V = visitUDIVLike(N0, N1,
N)) {
4797 AddToWorklist(
Mul.getNode());
4799 CombineTo(RemNode, Sub);
4817 EVT VT =
N->getValueType(0);
4821 if (
SDValue LogBase2 = BuildLogBase2(N1,
DL)) {
4822 AddToWorklist(LogBase2.getNode());
4826 AddToWorklist(Trunc.
getNode());
4835 if (
SDValue LogBase2 = BuildLogBase2(N10,
DL)) {
4836 AddToWorklist(LogBase2.getNode());
4840 AddToWorklist(Trunc.
getNode());
4842 AddToWorklist(
Add.getNode());
4870 unsigned Opcode =
N->getOpcode();
4873 EVT VT =
N->getValueType(0);
4895 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4908 AddToWorklist(
Add.getNode());
4919 AddToWorklist(
Add.getNode());
4936 if (
SDValue OptimizedRem = buildOptimizedSREM(N0, N1,
N))
4937 return OptimizedRem;
4941 isSigned ? visitSDIVLike(N0, N1,
N) : visitUDIVLike(N0, N1,
N);
4947 CombineTo(DivNode, OptimizedDiv);
4950 AddToWorklist(OptimizedDiv.
getNode());
4951 AddToWorklist(
Mul.getNode());
4958 return DivRem.getValue(1);
4966 EVT VT =
N->getValueType(0);
4979 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5007 unsigned SimpleSize =
Simple.getSizeInBits();
5026 EVT VT =
N->getValueType(0);
5039 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5063 if (
SDValue LogBase2 = BuildLogBase2(N1,
DL)) {
5078 unsigned SimpleSize =
Simple.getSizeInBits();
5101 unsigned Opcode =
N->getOpcode();
5104 EVT VT =
N->getValueType(0);
5114 return DAG.
getNode(Opcode,
DL,
N->getVTList(), N1, N0);
5117 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5145 unsigned Opcode =
N->getOpcode();
5148 EVT VT =
N->getValueType(0);
5158 return DAG.
getNode(Opcode,
DL,
N->getVTList(), N1, N0);
5161 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5189SDValue DAGCombiner::SimplifyNodeWithTwoResults(
SDNode *
N,
unsigned LoOp,
5192 bool HiExists =
N->hasAnyUseOfValue(1);
5193 if (!HiExists && (!LegalOperations ||
5196 return CombineTo(
N, Res, Res);
5200 bool LoExists =
N->hasAnyUseOfValue(0);
5201 if (!LoExists && (!LegalOperations ||
5204 return CombineTo(
N, Res, Res);
5208 if (LoExists && HiExists)
5214 AddToWorklist(
Lo.getNode());
5217 (!LegalOperations ||
5219 return CombineTo(
N, LoOpt, LoOpt);
5224 AddToWorklist(
Hi.getNode());
5227 (!LegalOperations ||
5229 return CombineTo(
N, HiOpt, HiOpt);
5241 EVT VT =
N->getValueType(0);
5245 if (isa<ConstantSDNode>(N0) && isa<ConstantSDNode>(N1))
5257 unsigned SimpleSize =
Simple.getSizeInBits();
5270 return CombineTo(
N,
Lo,
Hi);
5283 EVT VT =
N->getValueType(0);
5287 if (isa<ConstantSDNode>(N0) && isa<ConstantSDNode>(N1))
5298 return CombineTo(
N, Zero, Zero);
5304 return CombineTo(
N, N0, Zero);
5311 unsigned SimpleSize =
Simple.getSizeInBits();
5324 return CombineTo(
N,
Lo,
Hi);
5337 EVT CarryVT =
N->getValueType(1);
5358 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
5370 N->getVTList(), N0, N0);
5377 return CombineTo(
N,
And, Cmp);
5415 unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3,
CC);
5453 N0CC = cast<CondCodeSDNode>(N0.
getOperand(4))->get();
5469 unsigned Opcode1 = isSignedMinMax(N00, N01, N02, N03, N0CC);
5470 if (!Opcode1 || Opcode0 == Opcode1)
5480 APInt MinCPlus1 = MinC + 1;
5481 if (-MaxC == MinCPlus1 && MinCPlus1.
isPowerOf2()) {
5538 unsigned BW = (C1 + 1).exactLogBase2();
5558 unsigned Opcode =
N->getOpcode();
5572 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
5576 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5580 if (
SDValue RMINMAX = reassociateOps(Opcode,
DL, N0, N1,
N->getFlags()))
5600 return DAG.
getNode(AltOpcode,
DL, VT, N0, N1);
5612 auto ReductionOpcode = [](
unsigned Opcode) {
5626 if (
SDValue SD = reassociateReduction(ReductionOpcode(Opcode), Opcode,
5640 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
5642 unsigned LogicOpcode =
N->getOpcode();
5657 EVT XVT =
X.getValueType();
5667 if (XVT !=
Y.getValueType())
5671 if ((VT.
isVector() || LegalOperations) &&
5684 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5694 if (XVT !=
Y.getValueType())
5706 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5727 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5742 return DAG.
getNode(HandOpcode,
DL, VT, Logic0, Logic1, S);
5755 if (XVT.
isInteger() && XVT ==
Y.getValueType() &&
5759 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5776 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
5777 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
5778 assert(
X.getValueType() ==
Y.getValueType() &&
5779 "Inputs to shuffles are not the same type");
5785 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
5786 !SVN0->getMask().equals(SVN1->getMask()))
5822 SDValue LL, LR, RL, RR, N0CC, N1CC;
5823 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
5824 !isSetCCEquivalent(N1, RL, RR, N1CC))
5828 "Unexpected operand types for bitwise logic op");
5831 "Unexpected operand types for setcc");
5847 if (LR == RR && CC0 == CC1 && IsInteger) {
5852 bool AndEqZero = IsAnd && CC1 ==
ISD::SETEQ && IsZero;
5854 bool AndGtNeg1 = IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
5856 bool OrNeZero = !IsAnd && CC1 ==
ISD::SETNE && IsZero;
5858 bool OrLtZero = !IsAnd && CC1 ==
ISD::SETLT && IsZero;
5864 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
5866 AddToWorklist(
Or.getNode());
5871 bool AndEqNeg1 = IsAnd && CC1 ==
ISD::SETEQ && IsNeg1;
5873 bool AndLtZero = IsAnd && CC1 ==
ISD::SETLT && IsZero;
5875 bool OrNeNeg1 = !IsAnd && CC1 ==
ISD::SETNE && IsNeg1;
5877 bool OrGtNeg1 = !IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
5883 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
5885 AddToWorklist(
And.getNode());
5899 AddToWorklist(
Add.getNode());
5926 return !C0->
isOpaque() && !C1->isOpaque() && (CMax - CMin).isPowerOf2();
5944 if (LL == RR && LR == RL) {
5951 if (LL == RL && LR == RR) {
5955 (!LegalOperations ||
5977 bool isFMAXNUMFMINNUM_IEEE,
5978 bool isFMAXNUMFMINNUM) {
5989 isFMAXNUMFMINNUM_IEEE
5997 isFMAXNUMFMINNUM_IEEE
6015 isFMAXNUMFMINNUM_IEEE
6024 isFMAXNUMFMINNUM_IEEE
6034 "Invalid Op to combine SETCC with");
6045 LogicOp,
LHS.getNode(),
RHS.getNode());
6080 (isFMAXNUMFMINNUM_IEEE || isFMAXNUMFMINNUM))) &&
6086 SDValue CommonValue, Operand1, Operand2;
6094 }
else if (LHS1 == RHS1) {
6107 }
else if (RHS0 == LHS1) {
6124 bool IsSigned = isSignedIntSetCC(
CC);
6136 DAG, isFMAXNUMFMINNUM_IEEE, isFMAXNUMFMINNUM);
6140 DAG.
getNode(NewOpcode,
DL, OpVT, Operand1, Operand2);
6141 return DAG.
getSetCC(
DL, VT, MinMaxValue, CommonValue,
CC);
6151 LHS0 == RHS0 && LHS1C && RHS1C && OpVT.
isInteger()) {
6152 const APInt &APLhs = LHS1C->getAPIntValue();
6153 const APInt &APRhs = RHS1C->getAPIntValue();
6157 if (APLhs == (-APRhs) &&
6168 }
else if (TargetPreference &
6189 APInt Dif = MaxC - MinC;
6223 EVT CondVT =
Cond.getValueType();
6234 EVT OpVT =
T.getValueType();
6253 if (
SDValue V = foldLogicOfSetCCs(
true, N0, N1,
DL))
6271 APInt SRLC = SRLI->getAPIntValue();
6283 CombineTo(N0.
getNode(), NewAdd);
6297 EVT LoadResultTy,
EVT &ExtVT) {
6306 if (ExtVT == LoadedVT &&
6307 (!LegalOperations ||
6323 if (LegalOperations &&
6359 if (LdStMemVT.
bitsLT(MemVT))
6364 assert(ShAmt % 8 == 0 &&
"ShAmt is byte offset");
6365 const unsigned ByteShAmt = ShAmt / 8;
6376 if (PtrType == MVT::Untyped || PtrType.
isExtended())
6379 if (isa<LoadSDNode>(LDST)) {
6383 if (!
SDValue(Load, 0).hasOneUse())
6386 if (LegalOperations &&
6395 if (
Load->getNumValues() > 2)
6408 assert(isa<StoreSDNode>(LDST) &&
"It is not a Load nor a Store SDNode");
6414 if (LegalOperations &&
6421bool DAGCombiner::SearchForAndLoads(
SDNode *
N,
6429 if (
Op.getValueType().isVector())
6433 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
6435 (
Mask->getAPIntValue() &
C->getAPIntValue()) !=
C->getAPIntValue())
6440 if (!
Op.hasOneUse())
6443 switch(
Op.getOpcode()) {
6445 auto *
Load = cast<LoadSDNode>(
Op);
6447 if (isAndLoadExtLoad(Mask, Load,
Load->getValueType(0), ExtVT) &&
6465 unsigned ActiveBits =
Mask->getAPIntValue().countr_one();
6468 cast<VTSDNode>(
Op.getOperand(1))->getVT() :
6469 Op.getOperand(0).getValueType();
6480 if (!SearchForAndLoads(
Op.getNode(), Loads, NodesWithConsts, Mask,
6491 NodeToMask =
Op.getNode();
6494 for (
unsigned i = 0, e = NodeToMask->
getNumValues(); i < e; ++i) {
6496 if (VT != MVT::Glue && VT != MVT::Other) {
6498 NodeToMask =
nullptr;
6510bool DAGCombiner::BackwardsPropagateMask(
SDNode *
N) {
6511 auto *
Mask = dyn_cast<ConstantSDNode>(
N->getOperand(1));
6515 if (!
Mask->getAPIntValue().isMask())
6519 if (isa<LoadSDNode>(
N->getOperand(0)))
6524 SDNode *FixupNode =
nullptr;
6525 if (SearchForAndLoads(
N, Loads, NodesWithConsts, Mask, FixupNode)) {
6538 SDValue(FixupNode, 0), MaskOp);
6540 if (
And.getOpcode() == ISD ::AND)
6545 for (
auto *LogicN : NodesWithConsts) {
6549 if (isa<ConstantSDNode>(Op0))
6553 if (isa<ConstantSDNode>(Op1))
6557 if (isa<ConstantSDNode>(Op0) && !isa<ConstantSDNode>(Op1))
6564 for (
auto *Load : Loads) {
6569 if (
And.getOpcode() == ISD ::AND)
6572 SDValue NewLoad = reduceLoadWidth(
And.getNode());
6574 "Shouldn't be masking the load if it can't be narrowed");
6575 CombineTo(Load, NewLoad, NewLoad.
getValue(1));
6588SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(
SDNode *
N) {
6599 unsigned OuterShift;
6600 unsigned InnerShift;
6602 auto matchMask = [&OuterShift, &InnerShift, &
Y](
SDValue M) ->
bool {
6605 OuterShift =
M->getOpcode();
6614 Y =
M->getOperand(1);
6621 else if (matchMask(N0))
6627 EVT VT =
N->getValueType(0);
6644 SDValue And0 =
And->getOperand(0), And1 =
And->getOperand(1);
6654 bool FoundNot =
false;
6657 Src = Src.getOperand(0);
6663 Src = Src.getOperand(0);
6667 if (Src.getOpcode() !=
ISD::SRL || !Src.hasOneUse())
6671 EVT SrcVT = Src.getValueType();
6678 SDValue ShiftAmt = Src.getOperand(1);
6679 auto *ShiftAmtC = dyn_cast<ConstantSDNode>(ShiftAmt);
6680 if (!ShiftAmtC || !ShiftAmtC->getAPIntValue().ult(
BitWidth))
6684 Src = Src.getOperand(0);
6691 Src = Src.getOperand(0);
6715 EVT VT =
N->getValueType(0);
6741 unsigned LogicOpcode =
N->getOpcode();
6743 "Expected bitwise logic operation");
6749 unsigned ShiftOpcode = ShiftOp.
getOpcode();
6750 if (LogicOp.
getOpcode() != LogicOpcode ||
6774 EVT VT =
N->getValueType(0);
6778 return DAG.
getNode(LogicOpcode,
DL, VT, NewShift, Z);
6789 unsigned LogicOpcode =
N->getOpcode();
6791 "Expected bitwise logic operation");
6792 if (LeftHand.
getOpcode() != LogicOpcode ||
6813 EVT VT =
N->getValueType(0);
6815 return DAG.
getNode(LogicOpcode,
DL, VT, CombinedShifts, W);
6842 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
6856 auto *MLoad = dyn_cast<MaskedLoadSDNode>(N0);
6860 EVT LoadVT = MLoad->getMemoryVT();
6868 if (
Splat->getAPIntValue().isMask(ElementSize)) {
6870 ExtVT,
DL, MLoad->getChain(), MLoad->getBasePtr(),
6871 MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(),
6872 LoadVT, MLoad->getMemOperand(), MLoad->getAddressingMode(),
6874 bool LoadHasOtherUsers = !N0.
hasOneUse();
6875 CombineTo(
N, NewLoad);
6876 if (LoadHasOtherUsers)
6897 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
6911 return RHS->getAPIntValue().isSubsetOf(
LHS->getAPIntValue());
6978 unsigned EltBitWidth =
Vector->getValueType(0).getScalarSizeInBits();
6979 APInt SplatValue, SplatUndef;
6980 unsigned SplatBitSize;
6987 const bool IsBigEndian =
false;
6989 Vector->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
6990 HasAnyUndefs, EltBitWidth, IsBigEndian);
6994 if (IsSplat && (SplatBitSize % EltBitWidth) == 0) {
6997 SplatValue |= SplatUndef;
7004 for (
unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
7013 Load->getValueType(0),
7014 Load->getMemoryVT());
7022 switch (
Load->getExtensionType()) {
7023 default:
B =
false;
break;
7035 CombineTo(
N, (N0.
getNode() == Load) ? NewLoad : N0);
7040 Load->getChain(),
Load->getBasePtr(),
7041 Load->getOffset(),
Load->getMemoryVT(),
7042 Load->getMemOperand());
7044 if (
Load->getNumValues() == 3) {
7048 CombineTo(Load, To, 3,
true);
7050 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
7060 if (
SDValue Shuffle = XformToShuffleWithZero(
N))
7069 EVT ExtVT =
Ext->getValueType(0);
7086 if (
auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
7087 EVT MemVT = GN0->getMemoryVT();
7090 if (
SDValue(GN0, 0).hasOneUse() &&
7093 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
7094 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
7097 DAG.
getVTList(VT, MVT::Other), MemVT,
DL, Ops, GN0->getMemOperand(),
7100 CombineTo(
N, ZExtLoad);
7101 AddToWorklist(ZExtLoad.
getNode());
7110 if (
SDValue Res = reduceLoadWidth(
N))
7118 if (BackwardsPropagateMask(
N))
7122 if (
SDValue Combined = visitANDLike(N0, N1,
N))
7127 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
7164 auto *LN0 = cast<LoadSDNode>(N0);
7165 EVT MemVT = LN0->getMemoryVT();
7172 ((!LegalOperations && LN0->isSimple()) ||
7176 LN0->getBasePtr(), MemVT, LN0->getMemOperand());
7190 if (
SDValue Shifts = unfoldExtremeBitClearingToShifts(
N))
7206 auto *
C = dyn_cast<ConstantSDNode>(RHS);
7210 if (!
C->getAPIntValue().isMask(
7211 LHS.getOperand(0).getValueType().getFixedSizeInBits()))
7218 if (IsAndZeroExtMask(N0, N1))
7227 if (LegalOperations || VT.
isVector())
7236 bool DemandHighBits) {
7237 if (!LegalOperations)
7240 EVT VT =
N->getValueType(0);
7241 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
7247 bool LookPassAnd0 =
false;
7248 bool LookPassAnd1 =
false;
7263 LookPassAnd0 =
true;
7273 LookPassAnd1 =
true;
7299 LookPassAnd0 =
true;
7313 LookPassAnd1 =
true;
7322 if (OpSizeInBits > 16) {
7326 if (DemandHighBits && !LookPassAnd0)
7333 if (!LookPassAnd1) {
7334 unsigned HighBit = DemandHighBits ? OpSizeInBits : 24;
7342 if (OpSizeInBits > 16) {
7358 if (!
N->hasOneUse())
7361 unsigned Opc =
N.getOpcode();
7373 N1C = dyn_cast<ConstantSDNode>(
N.getOperand(1));
7375 N1C = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
7379 unsigned MaskByteOffset;
7383 case 0xFF: MaskByteOffset = 0;
break;
7384 case 0xFF00: MaskByteOffset = 1;
break;
7393 case 0xFF0000: MaskByteOffset = 2;
break;
7394 case 0xFF000000: MaskByteOffset = 3;
break;
7399 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
7405 if (!
C ||
C->getZExtValue() != 8)
7413 if (!
C ||
C->getZExtValue() != 8)
7419 if (MaskByteOffset != 0 && MaskByteOffset != 2)
7422 if (!
C ||
C->getZExtValue() != 8)
7427 if (MaskByteOffset != 1 && MaskByteOffset != 3)
7430 if (!
C ||
C->getZExtValue() != 8)
7434 if (Parts[MaskByteOffset])
7449 if (!
C ||
C->getAPIntValue() != 16)
7451 Parts[0] = Parts[1] =
N.getOperand(0).getOperand(0).getNode();
7466 "MatchBSwapHWordOrAndAnd: expecting i32");
7476 if (!Mask0 || !Mask1)
7487 if (!ShiftAmt0 || !ShiftAmt1)
7507 if (!LegalOperations)
7510 EVT VT =
N->getValueType(0);
7550 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
7578 if (
SDValue V = foldLogicOfSetCCs(
false, N0, N1,
DL))
7593 const APInt &LHSMask = N0O1C->getAPIntValue();
7594 const APInt &RHSMask = N1O1C->getAPIntValue();
7628 auto peekThroughResize = [](
SDValue V) {
7630 return V->getOperand(0);
7634 SDValue N0Resized = peekThroughResize(N0);
7636 SDValue N1Resized = peekThroughResize(N1);
7641 if (N00 == N1Resized || N01 == N1Resized)
7648 if (peekThroughResize(NotOperand) == N1Resized)
7656 if (peekThroughResize(NotOperand) == N1Resized)
7677 auto peekThroughZext = [](
SDValue V) {
7679 return V->getOperand(0);
7701 Lo.getScalarValueSizeInBits() == (BW / 2) &&
7702 Lo.getValueType() ==
Hi.getValueType()) {
7739 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
7753 auto *SV0 = dyn_cast<ShuffleVectorSDNode>(N0);
7754 auto *SV1 = dyn_cast<ShuffleVectorSDNode>(N1);
7761 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
7762 assert((!ZeroN00 || !ZeroN01) &&
"Both inputs zero!");
7763 assert((!ZeroN10 || !ZeroN11) &&
"Both inputs zero!");
7764 bool CanFold =
true;
7768 for (
int i = 0; i != NumElts; ++i) {
7769 int M0 = SV0->getMaskElt(i);
7770 int M1 = SV1->getMaskElt(i);
7773 bool M0Zero =
M0 < 0 || (ZeroN00 == (
M0 < NumElts));
7774 bool M1Zero =
M1 < 0 || (ZeroN10 == (
M1 < NumElts));
7778 if ((M0Zero &&
M1 < 0) || (M1Zero &&
M0 < 0))
7782 if (M0Zero == M1Zero) {
7787 assert((
M0 >= 0 ||
M1 >= 0) &&
"Undef index!");
7793 Mask[i] = M1Zero ?
M0 % NumElts : (
M1 % NumElts) + NumElts;
7802 return LegalShuffle;
7816 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
7827 if (
SDValue Combined = visitORLike(N0, N1,
DL))
7834 if (
SDValue BSwap = MatchBSwapHWord(
N, N0, N1))
7836 if (
SDValue BSwap = MatchBSwapHWordLow(
N, N0, N1))
7870 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
7874 if (
SDValue Rot = MatchRotate(N0, N1,
DL))
7877 if (
SDValue Load = MatchLoadCombine(
N))
7887 if (
SDValue Combined = visitADDLike(
N))
7892 if (LegalOperations || VT.
isVector())
7903 Mask =
Op.getOperand(1);
7904 return Op.getOperand(0);
7947 assert(OppShift && ExtractFrom &&
"Empty SDValue");
7975 bool IsMulOrDiv =
false;
7978 auto SelectOpcode = [&](
unsigned NeededShift,
unsigned MulOrDivVariant) {
7979 IsMulOrDiv = ExtractFrom.
getOpcode() == MulOrDivVariant;
7980 if (!IsMulOrDiv && ExtractFrom.
getOpcode() != NeededShift)
7982 Opcode = NeededShift;
8032 if (Rem != 0 || ResultAmt != OppLHSAmt)
8038 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.
zextOrTrunc(
8047 return DAG.
getNode(Opcode,
DL, ResVT, OppShiftLHS, NewShiftNode);
8101 unsigned MaskLoBits = 0;
8103 unsigned Bits =
Log2_64(EltSize);
8105 if (NegBits >= Bits) {
8128 if (PosBits >= MaskLoBits) {
8150 if ((Pos == NegOp1) ||
8174 return Width.
getLoBits(MaskLoBits) == 0;
8175 return Width == EltSize;
8185 SDValue InnerNeg,
bool HasPos,
8186 unsigned PosOpcode,
unsigned NegOpcode,
8198 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, Shifted,
8199 HasPos ? Pos : Neg);
8213 SDValue InnerNeg,
bool HasPos,
8214 unsigned PosOpcode,
unsigned NegOpcode,
8226 if (
matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, N0 == N1)) {
8227 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, N0, N1,
8228 HasPos ? Pos : Neg);
8235 auto IsBinOpImm = [](
SDValue Op,
unsigned BinOpc,
unsigned Imm) {
8236 if (
Op.getOpcode() != BinOpc)
8245 IsBinOpImm(InnerNeg,
ISD::XOR, EltBits - 1) &&
8254 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
8264 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
8279 EVT VT =
LHS.getValueType();
8284 bool HasROTL = hasOperation(
ISD::ROTL, VT);
8285 bool HasROTR = hasOperation(
ISD::ROTR, VT);
8286 bool HasFSHL = hasOperation(
ISD::FSHL, VT);
8287 bool HasFSHR = hasOperation(
ISD::FSHR, VT);
8298 if (LegalOperations && !HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
8303 LHS.getOperand(0).getValueType() ==
RHS.getOperand(0).getValueType()) {
8305 if (
SDValue Rot = MatchRotate(
LHS.getOperand(0),
RHS.getOperand(0),
DL)) {
8320 if (!LHSShift && !RHSShift)
8335 RHSShift = NewRHSShift;
8340 LHSShift = NewLHSShift;
8343 if (!RHSShift || !LHSShift)
8370 return (
LHS->getAPIntValue() +
RHS->getAPIntValue()) == EltSizeInBits;
8373 auto ApplyMasks = [&](
SDValue Res) {
8397 bool IsRotate = LHSShiftArg == RHSShiftArg;
8398 if (!IsRotate && !(HasFSHL || HasFSHR)) {
8407 if (CommonOp ==
Or.getOperand(0)) {
8409 Y =
Or.getOperand(1);
8412 if (CommonOp ==
Or.getOperand(1)) {
8414 Y =
Or.getOperand(0);
8421 if (matchOr(LHSShiftArg, RHSShiftArg)) {
8426 }
else if (matchOr(RHSShiftArg, LHSShiftArg)) {
8435 return ApplyMasks(Res);
8448 if (IsRotate && (HasROTL || HasROTR || !(HasFSHL || HasFSHR))) {
8449 bool UseROTL = !LegalOperations || HasROTL;
8451 UseROTL ? LHSShiftAmt : RHSShiftAmt);
8453 bool UseFSHL = !LegalOperations || HasFSHL;
8455 RHSShiftArg, UseFSHL ? LHSShiftAmt : RHSShiftAmt);
8458 return ApplyMasks(Res);
8463 if (!HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
8472 SDValue LExtOp0 = LHSShiftAmt;
8473 SDValue RExtOp0 = RHSShiftAmt;
8486 if (IsRotate && (HasROTL || HasROTR)) {
8488 MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt, LExtOp0,
8494 MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt, RExtOp0,
8501 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt, RHSShiftAmt,
8507 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
8558static std::optional<SDByteProvider>
8560 std::optional<uint64_t> VectorIndex,
8561 unsigned StartingIndex = 0) {
8565 return std::nullopt;
8569 if (
Depth && !
Op.hasOneUse() &&
8570 (
Op.getOpcode() !=
ISD::LOAD || !
Op.getValueType().isVector()))
8571 return std::nullopt;
8575 if (
Op.getOpcode() !=
ISD::LOAD && VectorIndex.has_value())
8576 return std::nullopt;
8580 return std::nullopt;
8582 assert(
Index < ByteWidth &&
"invalid index requested");
8585 switch (
Op.getOpcode()) {
8590 return std::nullopt;
8594 return std::nullopt;
8596 if (
LHS->isConstantZero())
8598 if (
RHS->isConstantZero())
8600 return std::nullopt;
8603 auto ShiftOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
8605 return std::nullopt;
8607 uint64_t BitShift = ShiftOp->getZExtValue();
8609 if (BitShift % 8 != 0)
8610 return std::nullopt;
8616 return Index < ByteShift
8626 if (NarrowBitWidth % 8 != 0)
8627 return std::nullopt;
8628 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
8630 if (
Index >= NarrowByteWidth)
8632 ? std::optional<SDByteProvider>(
8640 Depth + 1, VectorIndex, StartingIndex);
8642 auto OffsetOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
8644 return std::nullopt;
8646 VectorIndex = OffsetOp->getZExtValue();
8650 if (NarrowBitWidth % 8 != 0)
8651 return std::nullopt;
8652 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
8655 if (
Index >= NarrowByteWidth)
8656 return std::nullopt;
8664 if (*VectorIndex * NarrowByteWidth > StartingIndex)
8665 return std::nullopt;
8666 if ((*VectorIndex + 1) * NarrowByteWidth <= StartingIndex)
8667 return std::nullopt;
8670 VectorIndex, StartingIndex);
8673 auto L = cast<LoadSDNode>(
Op.getNode());
8674 if (!L->isSimple() || L->isIndexed())
8675 return std::nullopt;
8677 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
8678 if (NarrowBitWidth % 8 != 0)
8679 return std::nullopt;
8680 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
8685 if (
Index >= NarrowByteWidth)
8687 ? std::optional<SDByteProvider>(
8691 unsigned BPVectorIndex = VectorIndex.value_or(0U);
8696 return std::nullopt;
8711 int64_t FirstOffset) {
8713 unsigned Width = ByteOffsets.
size();
8715 return std::nullopt;
8717 bool BigEndian =
true, LittleEndian =
true;
8718 for (
unsigned i = 0; i < Width; i++) {
8719 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
8722 if (!BigEndian && !LittleEndian)
8723 return std::nullopt;
8726 assert((BigEndian != LittleEndian) &&
"It should be either big endian or"
8732 switch (
Value.getOpcode()) {
8775 EVT MemVT =
N->getMemoryVT();
8776 if (!(MemVT == MVT::i8 || MemVT == MVT::i16 || MemVT == MVT::i32) ||
8777 !
N->isSimple() ||
N->isIndexed())
8784 unsigned MaxWideNumBits = 64;
8785 unsigned MaxStores = MaxWideNumBits / NarrowNumBits;
8786 while (
auto *Store = dyn_cast<StoreSDNode>(Chain)) {
8794 if (
Store->getMemoryVT() != MemVT || !
Store->isSimple() ||
8798 Chain =
Store->getChain();
8799 if (MaxStores < Stores.
size())
8803 if (Stores.
size() < 2)
8808 unsigned NumStores = Stores.
size();
8809 unsigned WideNumBits = NumStores * NarrowNumBits;
8811 if (WideVT != MVT::i16 && WideVT != MVT::i32 && WideVT != MVT::i64)
8820 std::optional<BaseIndexOffset>
Base;
8821 for (
auto *Store : Stores) {
8832 isa<ConstantSDNode>(WideVal.
getOperand(1))) {
8840 if (ShiftAmtC % NarrowNumBits != 0)
8847 Offset = ShiftAmtC / NarrowNumBits;
8854 SourceValue = WideVal;
8861 SourceValue = WideVal;
8869 int64_t ByteOffsetFromBase = 0;
8872 else if (!
Base->equalBaseIndex(
Ptr, DAG, ByteOffsetFromBase))
8876 if (ByteOffsetFromBase < FirstOffset) {
8878 FirstOffset = ByteOffsetFromBase;
8882 if (Offset < 0 || Offset >= NumStores || OffsetMap[
Offset] !=
INT64_MAX)
8884 OffsetMap[
Offset] = ByteOffsetFromBase;
8888 assert(FirstStore &&
"First store must be set");
8895 if (!Allowed || !
Fast)
8900 auto checkOffsets = [&](
bool MatchLittleEndian) {
8901 if (MatchLittleEndian) {
8902 for (
unsigned i = 0; i != NumStores; ++i)
8903 if (OffsetMap[i] != i * (NarrowNumBits / 8) + FirstOffset)
8906 for (
unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --
j)
8907 if (OffsetMap[j] != i * (NarrowNumBits / 8) + FirstOffset)
8914 bool NeedBswap =
false;
8915 bool NeedRotate =
false;
8918 if (NarrowNumBits == 8 && checkOffsets(Layout.
isBigEndian()))
8920 else if (NumStores == 2 && checkOffsets(Layout.
isBigEndian()))
8929 "Unexpected store value to merge");
8938 }
else if (NeedRotate) {
8939 assert(WideNumBits % 2 == 0 &&
"Unexpected type for rotate");
8985 "Can only match load combining against OR nodes");
8988 EVT VT =
N->getValueType(0);
8989 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
8995 assert(
P.hasSrc() &&
"Must be a memory byte provider");
8996 auto *
Load = cast<LoadSDNode>(
P.Src.value());
8998 unsigned LoadBitWidth =
Load->getMemoryVT().getScalarSizeInBits();
9000 assert(LoadBitWidth % 8 == 0 &&
9001 "can only analyze providers for individual bytes not bit");
9002 unsigned LoadByteWidth = LoadBitWidth / 8;
9007 std::optional<BaseIndexOffset>
Base;
9011 std::optional<SDByteProvider> FirstByteProvider;
9017 unsigned ZeroExtendedBytes = 0;
9018 for (
int i = ByteWidth - 1; i >= 0; --i) {
9025 if (
P->isConstantZero()) {
9028 if (++ZeroExtendedBytes != (ByteWidth -
static_cast<unsigned>(i)))
9032 assert(
P->hasSrc() &&
"provenance should either be memory or zero");
9033 auto *
L = cast<LoadSDNode>(
P->Src.value());
9039 else if (Chain != LChain)
9044 int64_t ByteOffsetFromBase = 0;
9053 if (
L->getMemoryVT().isVector()) {
9054 unsigned LoadWidthInBit =
L->getMemoryVT().getScalarSizeInBits();
9055 if (LoadWidthInBit % 8 != 0)
9057 unsigned ByteOffsetFromVector =
P->SrcOffset * LoadWidthInBit / 8;
9058 Ptr.addToOffset(ByteOffsetFromVector);
9064 else if (!
Base->equalBaseIndex(
Ptr, DAG, ByteOffsetFromBase))
9068 ByteOffsetFromBase += MemoryByteOffset(*
P);
9069 ByteOffsets[i] = ByteOffsetFromBase;
9072 if (ByteOffsetFromBase < FirstOffset) {
9073 FirstByteProvider =
P;
9074 FirstOffset = ByteOffsetFromBase;
9080 assert(!Loads.
empty() &&
"All the bytes of the value must be loaded from "
9081 "memory, so there must be at least one load which produces the value");
9082 assert(
Base &&
"Base address of the accessed memory location must be set");
9085 bool NeedsZext = ZeroExtendedBytes > 0;
9096 if (LegalOperations &&
9104 ArrayRef(ByteOffsets).drop_back(ZeroExtendedBytes), FirstOffset);
9108 assert(FirstByteProvider &&
"must be set");
9112 if (MemoryByteOffset(*FirstByteProvider) != 0)
9114 auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value());
9121 bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
9128 if (NeedsBswap && (LegalOperations || NeedsZext) &&
9134 if (NeedsBswap && NeedsZext && LegalOperations &&
9142 *FirstLoad->getMemOperand(), &
Fast);
9143 if (!Allowed || !
Fast)
9148 Chain, FirstLoad->getBasePtr(),
9149 FirstLoad->getPointerInfo(), MemVT, FirstLoad->getAlign());
9162 SDLoc(
N), LegalOperations))
9190 EVT VT =
N->getValueType(0);
9212 M =
And.getOperand(XorIdx ? 0 : 1);
9218 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
9219 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
9225 if (isa<ConstantSDNode>(
M.getNode()))
9293 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
9305 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
9321 Flags.setDisjoint(
true);
9329 if (
SDValue Combined = visitADDLike(
N))
9336 isSetCCEquivalent(N0, LHS, RHS,
CC,
true)) {
9338 LHS.getValueType());
9339 if (!LegalOperations ||
9357 CombineTo(
N, SetCC);
9359 recursivelyDeleteUnusedNodes(N0.
getNode());
9375 AddToWorklist(
V.getNode());
9383 if (isOneUseSetCC(N01) || isOneUseSetCC(N00)) {
9388 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
9395 if (isa<ConstantSDNode>(N01) || isa<ConstantSDNode>(N00)) {
9400 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
9423 AddToWorklist(NotX.
getNode());
9432 SDValue A0 =
A.getOperand(0), A1 =
A.getOperand(1);
9434 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
9471 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
9482 if (
SDValue MM = unfoldMaskedMerge(
N))
9505 unsigned LogicOpcode = LogicOp.
getOpcode();
9511 unsigned ShiftOpcode = Shift->
getOpcode();
9514 assert(C1Node &&
"Expected a shift with constant operand");
9517 const APInt *&ShiftAmtVal) {
9518 if (V.getOpcode() != ShiftOpcode || !V.hasOneUse())
9526 ShiftOp = V.getOperand(0);
9531 if (ShiftAmtVal->getBitWidth() != C1Val.
getBitWidth())
9536 bool Overflow =
false;
9537 APInt NewShiftAmt = C1Val.
uadd_ov(*ShiftAmtVal, Overflow);
9542 if (NewShiftAmt.
uge(V.getScalarValueSizeInBits()))
9551 if (matchFirstShift(LogicOp.
getOperand(0),
X, C0Val))
9553 else if (matchFirstShift(LogicOp.
getOperand(1),
X, C0Val))
9565 return DAG.
getNode(LogicOpcode,
DL, VT, NewShift1, NewShift2,
9595 switch (
LHS.getOpcode()) {
9615 isa<ConstantSDNode>(BinOpLHSVal.
getOperand(1));
9619 if (!IsShiftByConstant && !IsCopyOrSelect)
9622 if (IsCopyOrSelect &&
N->hasOneUse())
9627 EVT VT =
N->getValueType(0);
9629 N->getOpcode(),
DL, VT, {LHS.getOperand(1), N->getOperand(1)})) {
9632 return DAG.
getNode(
LHS.getOpcode(),
DL, VT, NewShift, NewRHS);
9643 EVT TruncVT =
N->getValueType(0);
9644 if (
N->hasOneUse() &&
N->getOperand(0).hasOneUse() &&
9646 SDValue N01 =
N->getOperand(0).getOperand(1);
9649 SDValue N00 =
N->getOperand(0).getOperand(0);
9652 AddToWorklist(Trunc00.
getNode());
9653 AddToWorklist(Trunc01.
getNode());
9665 EVT VT =
N->getValueType(0);
9680 bool OutOfRange =
false;
9682 OutOfRange |=
C->getAPIntValue().uge(Bitsize);
9690 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, Amt);
9695 if (RotAmtC && RotAmtC->getAPIntValue() == 8 &&
9707 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, NewOp1);
9719 bool SameSide = (
N->getOpcode() == NextOp);
9728 CombineOp, dl, ShiftVT, {Norm1, Norm2})) {
9730 {CombinedShift, BitsizeC});
9732 ISD::UREM, dl, ShiftVT, {CombinedShift, BitsizeC});
9758 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
9781 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
9802 return (c1 + c2).uge(OpSizeInBits);
9812 return (c1 + c2).ult(OpSizeInBits);
9839 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
9840 (c1 + c2).uge(OpSizeInBits);
9852 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
9853 (c1 + c2).ult(OpSizeInBits);
9877 return c1.
ult(VT.getScalarSizeInBits()) && (c1 == c2);
9885 AddToWorklist(NewSHL.
getNode());
9893 const APInt &LHSC =
LHS->getAPIntValue();
9894 const APInt &RHSC =
RHS->getAPIntValue();
9895 return LHSC.
ult(OpSizeInBits) && RHSC.
ult(OpSizeInBits) &&
9967 AddToWorklist(Shl0.
getNode());
9971 Flags.setDisjoint(
true);
9987 {Add.getOperand(1)})) {
10007 if (
SDValue NewSHL = visitShiftByConstant(
N))
10026 APInt NewStep = C0 << ShlVal;
10041 "SRL or SRA node is required here!");
10050 SDValue ShiftOperand =
N->getOperand(0);
10061 if (!IsSignExt && !IsZeroExt)
10068 auto UserOfLowerBits = [NarrowVTSize](
SDNode *U) {
10073 if (!UShiftAmtSrc) {
10077 return UShiftAmt < NarrowVTSize;
10091 unsigned ActiveBits = IsSignExt
10092 ?
Constant->getAPIntValue().getSignificantBits()
10093 :
Constant->getAPIntValue().getActiveBits();
10094 if (ActiveBits > NarrowVTSize)
10111 "Cannot have a multiply node with two different operand types.");
10122 if (ShiftAmt != NarrowVTSize)
10144 bool IsSigned =
N->getOpcode() ==
ISD::SRA;
10151 unsigned Opcode =
N->getOpcode();
10156 EVT VT =
N->getValueType(0);
10206 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
10209 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
10225 APInt Sum = c1 + c2;
10226 unsigned ShiftSum =
10237 "Expected matchBinaryPredicate to return one element for "
10241 ShiftValue = ShiftValues[0];
10269 if ((ShiftAmt > 0) &&
10280 N->getValueType(0), Trunc);
10315 DAG.
getConstant(AddC->getAPIntValue().lshr(ShiftAmt).trunc(
10332 if (
SDValue NewOp1 = distributeTruncateThroughAnd(N1.
getNode()))
10349 if (LargeShift->getAPIntValue() == TruncBits) {
10370 if (
SDValue NewSRA = visitShiftByConstant(
N))
10379 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
10402 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
10405 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
10421 return (c1 + c2).uge(OpSizeInBits);
10431 return (c1 + c2).ult(OpSizeInBits);
10451 if (c1 + OpSizeInBits == InnerShiftSize) {
10452 if (c1 + c2 >= InnerShiftSize)
10462 c1 + c2 < InnerShiftSize) {
10467 OpSizeInBits - c2),
10482 const APInt &LHSC =
LHS->getAPIntValue();
10483 const APInt &RHSC =
RHS->getAPIntValue();
10484 return LHSC.
ult(OpSizeInBits) && RHSC.
ult(OpSizeInBits) &&
10526 AddToWorklist(SmallShift.
getNode());
10554 APInt UnknownBits = ~Known.Zero;
10571 AddToWorklist(
Op.getNode());
10580 if (
SDValue NewOp1 = distributeTruncateThroughAnd(N1.
getNode()))
10590 if (
SDValue NewSRL = visitShiftByConstant(
N))
10594 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
10621 if (
N->hasOneUse()) {
10630 AddToWorklist(
Use);
10642 EVT VT =
N->getValueType(0);
10654 return IsFSHL ? N0 : N1;
10656 auto IsUndefOrZero = [](
SDValue V) {
10673 return IsFSHL ? N0 : N1;
10679 if (IsUndefOrZero(N0))
10683 if (IsUndefOrZero(N1))
10695 auto *
LHS = dyn_cast<LoadSDNode>(N0);
10696 auto *
RHS = dyn_cast<LoadSDNode>(N1);
10697 if (LHS && RHS &&
LHS->isSimple() &&
RHS->isSimple() &&
10698 LHS->getAddressSpace() ==
RHS->getAddressSpace() &&
10708 RHS->getAddressSpace(), NewAlign,
10709 RHS->getMemOperand()->getFlags(), &
Fast) &&
10713 AddToWorklist(NewPtr.
getNode());
10715 VT,
DL,
RHS->getChain(), NewPtr,
10716 RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
10717 RHS->getMemOperand()->getFlags(),
RHS->getAAInfo());
10719 WorklistRemover DeadNodes(*
this);
10745 if (N0 == N1 && hasOperation(RotOpc, VT))
10791 EVT SrcVT =
N->getValueType(0);
10794 N =
N->getOperand(0).getNode();
10799 EVT VT =
N->getValueType(0);
10800 SDValue AbsOp1 =
N->getOperand(0);
10827 VT0 = cast<VTSDNode>(Op0.
getOperand(1))->getVT();
10828 VT1 = cast<VTSDNode>(Op1.
getOperand(1))->getVT();
10837 EVT MaxVT = VT0.
bitsGT(VT1) ? VT0 : VT1;
10838 if ((VT0 == MaxVT || Op0->
hasOneUse()) &&
10839 (VT1 == MaxVT || Op1->
hasOneUse()) && hasOperation(ABDOpcode, MaxVT)) {
10849 if (hasOperation(ABDOpcode, VT)) {
10859 EVT VT =
N->getValueType(0);
10894 EVT VT =
N->getValueType(0);
10917 auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
10919 if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
10920 ShAmt->getZExtValue() >= (BW / 2) &&
10921 (ShAmt->getZExtValue() % 16) == 0 && TLI.
isTypeLegal(HalfVT) &&
10923 (!LegalOperations || hasOperation(
ISD::BSWAP, HalfVT))) {
10925 if (
uint64_t NewShAmt = (ShAmt->getZExtValue() - (BW / 2)))
10940 auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
10941 if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
10942 ShAmt->getZExtValue() % 8 == 0) {
10957 EVT VT =
N->getValueType(0);
10971 EVT VT =
N->getValueType(0);
10988 EVT VT =
N->getValueType(0);
11000 EVT VT =
N->getValueType(0);
11017 EVT VT =
N->getValueType(0);
11029 EVT VT =
N->getValueType(0);
11041 const APInt &Amt = AmtC->getAPIntValue();
11042 if (Amt.
ult(NumBits)) {
11079 EVT VT =
LHS.getValueType();
11135 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
11141 True, DAG, LegalOperations, ForCodeSize);
11153 if (LHS == NegTrue) {
11157 RHS, DAG, LegalOperations, ForCodeSize);
11160 if (NegRHS == False) {
11162 False,
CC, TLI, DAG);
11182 EVT VT =
N->getValueType(0);
11184 VT !=
Cond.getOperand(0).getValueType())
11231 EVT VT =
N->getValueType(0);
11232 EVT CondVT =
Cond.getValueType();
11238 auto *C1 = dyn_cast<ConstantSDNode>(N1);
11239 auto *C2 = dyn_cast<ConstantSDNode>(N2);
11243 if (CondVT != MVT::i1 || LegalOperations) {
11258 C1->
isZero() && C2->isOne()) {
11273 assert(CondVT == MVT::i1 && !LegalOperations);
11276 if (C1->
isOne() && C2->isZero())
11284 if (C1->
isZero() && C2->isOne()) {
11291 if (C1->
isZero() && C2->isAllOnes()) {
11304 const APInt &C1Val = C1->getAPIntValue();
11305 const APInt &C2Val = C2->getAPIntValue();
11308 if (C1Val - 1 == C2Val) {
11314 if (C1Val + 1 == C2Val) {
11334 if (C2->isAllOnes()) {
11346template <
class MatchContextClass>
11349 N->getOpcode() == ISD::VP_SELECT) &&
11350 "Expected a (v)(vp.)select");
11352 SDValue T =
N->getOperand(1),
F =
N->getOperand(2);
11353 EVT VT =
N->getValueType(0);
11355 MatchContextClass matcher(DAG, TLI,
N);
11391 EVT VT =
N->getValueType(0);
11449 EVT VT =
N->getValueType(0);
11457 if (
SDValue V = foldBoolSelectToLogic<EmptyMatchContext>(
N, DAG))
11467 if (
SDValue V = foldSelectOfConstants(
N))
11471 if (SimplifySelectOps(
N, N1, N2))
11474 if (VT0 == MVT::i1) {
11483 bool normalizeToSequence =
11492 if (normalizeToSequence || !InnerSelect.
use_empty())
11494 InnerSelect, N2, Flags);
11497 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
11504 Cond1, N1, N2, Flags);
11505 if (normalizeToSequence || !InnerSelect.
use_empty())
11507 InnerSelect, Flags);
11510 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
11520 if (!normalizeToSequence) {
11526 if (
SDValue Combined = visitANDLike(N0, N1_0,
N)) {
11539 if (!normalizeToSequence) {
11545 if (
SDValue Combined = visitORLike(N0, N2_0,
DL))
11563 combineMinNumMaxNum(
DL, VT, Cond0, Cond1, N1, N2,
CC))
11574 auto *
C = dyn_cast<ConstantSDNode>(N2.
getOperand(1));
11575 auto *NotC = dyn_cast<ConstantSDNode>(Cond1);
11576 if (
C && NotC &&
C->getAPIntValue() == ~NotC->getAPIntValue()) {
11596 (!LegalOperations &&
11607 if (
SDValue NewSel = SimplifySelect(
DL, N0, N1, N2))
11612 if (
SDValue BinOp = foldSelectOfBinops(
N))
11628 EVT VT =
N->getValueType(0);
11636 if (
LHS->getNumOperands() != 2 ||
RHS->getNumOperands() != 2)
11645 for (
int i = 0; i < NumElems / 2; ++i) {
11646 if (
Cond->getOperand(i)->isUndef())
11649 if (BottomHalf ==
nullptr)
11650 BottomHalf = cast<ConstantSDNode>(
Cond.getOperand(i));
11651 else if (
Cond->getOperand(i).getNode() != BottomHalf)
11657 for (
int i = NumElems / 2; i < NumElems; ++i) {
11658 if (
Cond->getOperand(i)->isUndef())
11661 if (TopHalf ==
nullptr)
11662 TopHalf = cast<ConstantSDNode>(
Cond.getOperand(i));
11663 else if (
Cond->getOperand(i).getNode() != TopHalf)
11667 assert(TopHalf && BottomHalf &&
11668 "One half of the selector was all UNDEFs and the other was all the "
11669 "same value. This should have been addressed before this function.");
11672 BottomHalf->
isZero() ?
RHS->getOperand(0) :
LHS->getOperand(0),
11673 TopHalf->
isZero() ?
RHS->getOperand(1) :
LHS->getOperand(1));
11686 EVT VT = BasePtr.getValueType();
11690 SplatVal.getValueType() == VT) {
11821 MST1->isSimple() && MST1->getBasePtr() ==
Ptr &&
11824 MST1->getMemoryVT().getStoreSize()) ||
11828 CombineTo(MST1, MST1->getChain());
11845 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
11849 Value.getValueType().isInteger() &&
11850 (!isa<ConstantSDNode>(
Value) ||
11851 !cast<ConstantSDNode>(
Value)->isOpaque())) {
11852 APInt TruncDemandedBits =
11878 Value.getOperand(0).getValueType());
11889 auto *SST = cast<VPStridedStoreSDNode>(
N);
11892 if (
auto *CStride = dyn_cast<ConstantSDNode>(SST->getStride());
11893 CStride && CStride->getZExtValue() == EltVT.
getStoreSize()) {
11895 SST->getBasePtr(), SST->getOffset(), SST->getMask(),
11896 SST->getVectorLength(), SST->getMemoryVT(),
11897 SST->getMemOperand(), SST->getAddressingMode(),
11898 SST->isTruncatingStore(), SST->isCompressingStore());
11944 return CombineTo(
N, PassThru, MGT->
getChain());
11980 return CombineTo(
N, NewLd, NewLd.
getValue(1));
11984 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
11991 auto *SLD = cast<VPStridedLoadSDNode>(
N);
11994 if (
auto *CStride = dyn_cast<ConstantSDNode>(SLD->getStride());
11995 CStride && CStride->getZExtValue() == EltVT.
getStoreSize()) {
11997 SLD->getAddressingMode(), SLD->getExtensionType(), SLD->getValueType(0),
11998 SDLoc(
N), SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(),
11999 SLD->getMask(), SLD->getVectorLength(), SLD->getMemoryVT(),
12000 SLD->getMemOperand(), SLD->isExpandingLoad());
12001 return CombineTo(
N, NewLd, NewLd.
getValue(1));
12012 EVT VT =
N->getValueType(0);
12013 if (!
Cond.hasOneUse() ||
Cond.getScalarValueSizeInBits() != 1 ||
12022 bool AllAddOne =
true;
12023 bool AllSubOne =
true;
12025 for (
unsigned i = 0; i != Elts; ++i) {
12044 if (AllAddOne || AllSubOne) {
12079 if (
SDValue V = foldBoolSelectToLogic<VPMatchContext>(
N, DAG))
12089 EVT VT =
N->getValueType(0);
12095 if (
SDValue V = foldBoolSelectToLogic<EmptyMatchContext>(
N, DAG))
12121 bool isAbs =
false;
12140 AddToWorklist(Shift.
getNode());
12141 AddToWorklist(
Add.getNode());
12152 if (
SDValue FMinMax = combineMinNumMaxNum(
DL, VT, LHS, RHS, N1, N2,
CC))
12167 EVT NarrowVT =
LHS.getValueType();
12175 SetCCWidth != 1 && SetCCWidth < WideWidth &&
12201 if (hasOperation(ABDOpc, VT)) {
12208 return DAG.
getNode(ABDOpc,
DL, VT, LHS, RHS);
12215 return DAG.
getNode(ABDOpc,
DL, VT, LHS, RHS);
12250 (OpLHS == CondLHS || OpRHS == CondLHS))
12253 if (OpRHS.getOpcode() == CondRHS.getOpcode() &&
12256 CondLHS == OpLHS) {
12261 return Cond->getAPIntValue() ==
~Op->getAPIntValue();
12302 if (OpLHS == LHS) {
12318 return (!
Op && !
Cond) ||
12320 Cond->getAPIntValue() == (-
Op->getAPIntValue() - 1));
12351 if (SimplifySelectOps(
N, N1, N2))
12371 if (
SDValue V = foldVSelectOfConstants(
N))
12404 AddToWorklist(
SCC.getNode());
12408 if (
auto *SCCC = dyn_cast<ConstantSDNode>(
SCC.getNode()))
12409 return SCCC->isZero() ? N3 : N2;
12413 if (
SCC->isUndef())
12420 SCC.getOperand(1), N2, N3,
SCC.getOperand(2));
12427 if (SimplifySelectOps(
N, N2, N3))
12431 return SimplifySelectCC(
SDLoc(
N), N0, N1, N2, N3,
CC);
12439 N->hasOneUse() &&
N->use_begin()->getOpcode() ==
ISD::BRCOND;
12442 EVT VT =
N->getValueType(0);
12443 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
12451 SDValue NewSetCC = rebuildSetCC(Combined);
12479 A.getOperand(0) ==
B.getOperand(0);
12483 B.getOperand(0) ==
A;
12486 bool IsRotate =
false;
12489 if (IsAndWithShift(N0, N1)) {
12491 ShiftOrRotate = N1;
12492 }
else if (IsAndWithShift(N1, N0)) {
12494 ShiftOrRotate = N0;
12495 }
else if (IsRotateWithOp(N0, N1)) {
12498 ShiftOrRotate = N1;
12499 }
else if (IsRotateWithOp(N1, N0)) {
12502 ShiftOrRotate = N0;
12505 if (AndOrOp && ShiftOrRotate && ShiftOrRotate.hasOneUse() &&
12510 auto GetAPIntValue = [](
SDValue Op) -> std::optional<APInt> {
12513 if (CNode ==
nullptr)
12514 return std::nullopt;
12517 std::optional<APInt> AndCMask =
12518 IsRotate ? std::nullopt : GetAPIntValue(AndOrOp.
getOperand(1));
12519 std::optional<APInt> ShiftCAmt =
12520 GetAPIntValue(ShiftOrRotate.getOperand(1));
12524 if (ShiftCAmt && (IsRotate || AndCMask) && ShiftCAmt->ult(NumBits)) {
12525 unsigned ShiftOpc = ShiftOrRotate.getOpcode();
12527 bool CanTransform = IsRotate;
12528 if (!CanTransform) {
12530 CanTransform = *ShiftCAmt == (~*AndCMask).
popcount();
12532 CanTransform &= (*ShiftCAmt + AndCMask->popcount()) == NumBits;
12540 OpVT, ShiftOpc, ShiftCAmt->isPowerOf2(), *ShiftCAmt, AndCMask);
12542 if (CanTransform && NewShiftOpc != ShiftOpc) {
12545 DAG.
getNode(NewShiftOpc,
DL, OpVT, ShiftOrRotate.getOperand(0),
12546 ShiftOrRotate.getOperand(1));
12553 NumBits - ShiftCAmt->getZExtValue())
12554 :
APInt::getLowBitsSet(NumBits,
12555 NumBits - ShiftCAmt->getZExtValue());
12563 return DAG.
getSetCC(
DL, VT, NewAndOrOp, NewShiftOrRotate,
Cond);
12592 if (!
N.hasOneUse())
12595 if (!isa<LoadSDNode>(
N))
12621 unsigned Opcode =
N->getOpcode();
12623 EVT VT =
N->getValueType(0);
12628 "Expected EXTEND dag node in input!");
12669 unsigned Opcode =
N->getOpcode();
12671 EVT VT =
N->getValueType(0);
12674 "Expected EXTEND dag node in input!");
12679 if (isa<ConstantSDNode>(N0))
12680 return DAG.
getNode(Opcode,
DL, VT, N0);
12688 if (isa<ConstantSDNode>(Op1) && isa<ConstantSDNode>(Op2) &&
12699 unsigned FoldOpc = Opcode;
12722 for (
unsigned i = 0; i != NumElts; ++i) {
12724 if (
Op.isUndef()) {
12735 APInt C =
Op->getAsAPIntVal().zextOrTrunc(EVTBits);
12753 bool HasCopyToRegUses =
false;
12760 if (UI.getUse().getResNo() != N0.
getResNo())
12769 for (
unsigned i = 0; i != 2; ++i) {
12773 if (!isa<ConstantSDNode>(UseOp))
12787 HasCopyToRegUses =
true;
12790 if (HasCopyToRegUses) {
12791 bool BothLiveOut =
false;
12796 BothLiveOut =
true;
12803 return !ExtendNodes.
empty();
12813 for (
SDNode *SetCC : SetCCs) {
12816 for (
unsigned j = 0;
j != 2; ++
j) {
12818 if (SOp == OrigLoad)
12832 EVT DstVT =
N->getValueType(0);
12837 "Unexpected node type (not an extend)!");
12875 EVT SplitSrcVT = SrcVT;
12876 EVT SplitDstVT = DstVT;
12889 const unsigned NumSplits =
12896 for (
unsigned Idx = 0;
Idx < NumSplits;
Idx++) {
12915 AddToWorklist(NewChain.
getNode());
12917 CombineTo(
N, NewValue);
12923 ExtendSetCCUses(SetCCs, N0, NewValue, (
ISD::NodeType)
N->getOpcode());
12924 CombineTo(N0.
getNode(), Trunc, NewChain);
12932 EVT VT =
N->getValueType(0);
12933 EVT OrigVT =
N->getOperand(0).getValueType();
12955 EVT MemVT =
Load->getMemoryVT();
12976 Load->getChain(),
Load->getBasePtr(),
12977 Load->getMemoryVT(),
Load->getMemOperand());
12990 if (
SDValue(Load, 0).hasOneUse()) {
12994 Load->getValueType(0), ExtLoad);
12995 CombineTo(Load, Trunc, ExtLoad.
getValue(1));
12999 recursivelyDeleteUnusedNodes(N0.
getNode());
13008SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(
SDNode *Cast) {
13009 unsigned CastOpcode = Cast->
getOpcode();
13013 "Unexpected opcode for vector select narrowing/widening");
13053 bool LegalOperations,
SDNode *
N,
13064 if ((LegalOperations || !LN0->
isSimple() ||
13075 Combiner.recursivelyDeleteUnusedNodes(LN0);
13088 bool NonNegZExt =
false) {
13095 "Unexpected load type or opcode");
13112 !cast<LoadSDNode>(N0)->isSimple()) &&
13116 bool DoXform =
true;
13129 Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
13133 if (NoReplaceTrunc) {
13135 Combiner.recursivelyDeleteUnusedNodes(LN0);
13155 if ((LegalOperations || !cast<MaskedLoadSDNode>(N0)->
isSimple()) &&
13177 auto *ALoad = dyn_cast<AtomicSDNode>(N0);
13180 EVT MemoryVT = ALoad->getMemoryVT();
13189 EVT OrigVT = ALoad->getValueType(0);
13191 auto *NewALoad = cast<AtomicSDNode>(DAG.
getAtomic(
13193 ALoad->getBasePtr(), ALoad->getMemOperand()));
13194 NewALoad->setExtensionType(ExtLoadType);
13204 bool LegalOperations) {
13216 EVT VT =
N->getValueType(0);
13217 EVT XVT =
X.getValueType();
13233 return DAG.
getNode(ShiftOpcode,
DL, VT, NotX, ShiftAmount);
13247 EVT VT =
N->getValueType(0);
13257 if (VT.
isVector() && !LegalOperations &&
13276 if (SVT == MatchingVecType) {
13292 auto IsFreeToExtend = [&](
SDValue V) {
13300 cast<LoadSDNode>(V)->isSimple() &&
13310 if (UI.getUse().getResNo() != 0 ||
User == N0.
getNode())
13315 if (
User->getOpcode() != ExtOpcode ||
User->getValueType(0) != VT)
13321 if (IsFreeToExtend(N00) && IsFreeToExtend(N01)) {
13341 SDValue ExtTrueVal = (SetCCWidth == 1)
13345 if (
SDValue SCC = SimplifySelectCC(
DL, N00, N01, ExtTrueVal, Zero,
CC,
true))
13357 return DAG.
getSelect(
DL, VT, SetCC, ExtTrueVal, Zero);
13366 EVT VT =
N->getValueType(0);
13370 if (
SDValue FoldedVOp = SimplifyVCastOp(
N,
DL))
13408 if (NarrowLoad.getNode() != N0.
getNode()) {
13409 CombineTo(N0.
getNode(), NarrowLoad);
13411 AddToWorklist(oye);
13419 unsigned OpBits =
Op.getScalarValueSizeInBits();
13424 if (OpBits == DestBits) {
13427 if (NumSignBits > DestBits-MidBits)
13429 }
else if (OpBits < DestBits) {
13432 if (NumSignBits > OpBits-MidBits)
13437 if (NumSignBits > OpBits-MidBits)
13444 if (OpBits < DestBits)
13446 else if (OpBits > DestBits)
13466 if (
SDValue ExtLoad = CombineExtLoad(
N))
13501 bool NoReplaceTruncAnd = !N0.
hasOneUse();
13505 if (NoReplaceTruncAnd) {
13508 CombineTo(N0.
getNode(), TruncAnd);
13510 if (NoReplaceTrunc) {
13515 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
13533 Flags.setNonNeg(
true);
13537 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
13568 if (NewXor.getNode() == N0.
getNode()) {
13640 EVT VT =
N->getValueType(0);
13644 if (
SDValue FoldedVOp = SimplifyVCastOp(
N,
DL))
13676 APInt TruncatedBits =
13678 APInt(
Op.getScalarValueSizeInBits(), 0) :
13681 std::min(
Op.getScalarValueSizeInBits(),
13687 return ZExtOrTrunc;
13697 if (NarrowLoad.getNode() != N0.
getNode()) {
13698 CombineTo(N0.
getNode(), NarrowLoad);
13700 AddToWorklist(oye);
13708 if (
N->getFlags().hasNonNeg()) {
13715 if (OpBits == DestBits) {
13718 if (NumSignBits > DestBits - MidBits)
13720 }
else if (OpBits < DestBits) {
13724 if (NumSignBits > OpBits - MidBits)
13729 if (NumSignBits > OpBits - MidBits)
13741 AddToWorklist(
Op.getNode());
13745 return ZExtOrTrunc;
13751 AddToWorklist(
Op.getNode());
13787 if (
SDValue ExtLoad = CombineExtLoad(
N))
13807 bool DoXform =
true;
13811 auto *AndC = cast<ConstantSDNode>(N0.
getOperand(1));
13814 if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
13830 bool NoReplaceTruncAnd = !N0.
hasOneUse();
13834 if (NoReplaceTruncAnd) {
13837 CombineTo(N0.
getNode(), TruncAnd);
13839 if (NoReplaceTrunc) {
13844 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
13853 if (
SDValue ZExtLoad = CombineZExtLogicopShiftLoad(
N))
13869 if (!LegalOperations && VT.
isVector() &&
13901 if (
SDValue SCC = SimplifySelectCC(
13905 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
13914 if (
auto *ShAmtC = dyn_cast<ConstantSDNode>(ShAmt)) {
13921 if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) {
13925 APInt ShiftOutMask =
13927 ShAmtC->getAPIntValue().getZExtValue());
13945 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
13969 EVT VT =
N->getValueType(0);
14003 if (NarrowLoad.getNode() != N0.
getNode()) {
14004 CombineTo(N0.
getNode(), NarrowLoad);
14006 AddToWorklist(oye);
14024 assert(isa<ConstantSDNode>(
Y) &&
"Expected constant to be folded!");
14040 bool DoXform =
true;
14053 CombineTo(
N, ExtLoad);
14054 if (NoReplaceTrunc) {
14056 recursivelyDeleteUnusedNodes(LN0);
14060 CombineTo(LN0, Trunc, ExtLoad.
getValue(1));
14074 if (!LegalOperations || TLI.
isLoadExtLegal(ExtType, VT, MemVT)) {
14078 CombineTo(
N, ExtLoad);
14080 recursivelyDeleteUnusedNodes(LN0);
14094 if (VT.
isVector() && !LegalOperations) {
14106 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
14114 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
14119 if (
SDValue SCC = SimplifySelectCC(
14122 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
14136 unsigned Opcode =
N->getOpcode();
14139 EVT AssertVT = cast<VTSDNode>(N1)->getVT();
14143 AssertVT == cast<VTSDNode>(N0.
getOperand(1))->getVT())
14155 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
14156 EVT MinAssertVT = AssertVT.
bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
14170 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
14171 if (AssertVT.
bitsLT(BigA_AssertVT)) {
14185 Align AL = cast<AssertAlignSDNode>(
N)->getAlign();
14190 if (
auto *AAN = dyn_cast<AssertAlignSDNode>(N0))
14192 std::max(AL, AAN->getAlign()));
14202 unsigned AlignShift =
Log2(AL);
14207 if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
14208 if (LHSAlignShift < AlignShift)
14210 if (RHSAlignShift < AlignShift)
14225 unsigned Opc =
N->getOpcode();
14229 EVT VT =
N->getValueType(0);
14239 unsigned ShAmt = 0;
14244 unsigned ShiftedOffset = 0;
14249 ExtVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
14257 auto *LN = dyn_cast<LoadSDNode>(N0);
14258 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
14264 uint64_t MemoryWidth = LN->getMemoryVT().getScalarSizeInBits();
14265 if (MemoryWidth <= ShAmt)
14276 LN->getExtensionType() != ExtType)
14280 auto AndC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
14285 unsigned ActiveBits = 0;
14286 if (
Mask.isMask()) {
14287 ActiveBits =
Mask.countr_one();
14288 }
else if (
Mask.isShiftedMask(ShAmt, ActiveBits)) {
14289 ShiftedOffset = ShAmt;
14310 if (!
SRL.hasOneUse())
14315 auto *LN = dyn_cast<LoadSDNode>(
SRL.getOperand(0));
14316 auto *SRL1C = dyn_cast<ConstantSDNode>(
SRL.getOperand(1));
14323 ShAmt = SRL1C->getZExtValue();
14324 uint64_t MemoryWidth = LN->getMemoryVT().getSizeInBits();
14325 if (ShAmt >= MemoryWidth)
14352 isa<ConstantSDNode>(
Mask->getOperand(1))) {
14353 unsigned Offset, ActiveBits;
14354 const APInt& ShiftMask =
Mask->getConstantOperandAPInt(1);
14355 if (ShiftMask.
isMask()) {
14377 N0 =
SRL.getOperand(0);
14385 unsigned ShLeftAmt = 0;
14389 ShLeftAmt = N01->getZExtValue();
14395 if (!isa<LoadSDNode>(N0))
14402 !isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
14405 auto AdjustBigEndianShift = [&](
unsigned ShAmt) {
14406 unsigned LVTStoreBits =
14409 return LVTStoreBits - EVTStoreBits - ShAmt;
14414 unsigned PtrAdjustmentInBits =
14417 uint64_t PtrOff = PtrAdjustmentInBits / 8;
14421 Flags.setNoUnsignedWrap(
true);
14424 AddToWorklist(NewPtr.
getNode());
14439 WorklistRemover DeadNodes(*
this);
14444 if (ShLeftAmt != 0) {
14459 if (ShiftedOffset != 0) {
14476 EVT VT =
N->getValueType(0);
14477 EVT ExtVT = cast<VTSDNode>(N1)->getVT();
14506 if ((N00Bits <= ExtVTBits ||
14522 if ((N00Bits == ExtVTBits ||
14523 (!IsZext && (N00Bits < ExtVTBits ||
14525 (!LegalOperations ||
14550 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
14557 if (
auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1)))
14558 if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) {
14562 if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
14574 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
14575 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple() &&
14583 CombineTo(
N, ExtLoad);
14585 AddToWorklist(ExtLoad.
getNode());
14592 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
14593 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) &&
14600 CombineTo(
N, ExtLoad);
14608 if (ExtVT == Ld->getMemoryVT() && N0.
hasOneUse() &&
14612 VT,
SDLoc(
N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
14613 Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
14614 Ld->getAddressingMode(),
ISD::SEXTLOAD, Ld->isExpandingLoad());
14615 CombineTo(
N, ExtMaskedLoad);
14622 if (
auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
14623 if (
SDValue(GN0, 0).hasOneUse() &&
14624 ExtVT == GN0->getMemoryVT() &&
14626 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
14627 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
14633 CombineTo(
N, ExtLoad);
14635 AddToWorklist(ExtLoad.
getNode());
14658 (!LegalOperations ||
14672 bool LegalOperations) {
14673 unsigned InregOpcode =
N->getOpcode();
14677 EVT VT =
N->getValueType(0);
14679 Src.getValueType().getVectorElementType(),
14683 "Expected EXTEND_VECTOR_INREG dag node in input!");
14692 Src = Src.getOperand(0);
14693 if (Src.getValueType() != SrcVT)
14699 return DAG.
getNode(Opcode,
DL, VT, Src);
14704 EVT VT =
N->getValueType(0);
14730 EVT VT =
N->getValueType(0);
14768 EVT ExtVT = cast<VTSDNode>(ExtVal)->getVT();
14790 LegalTypes && !LegalOperations && N0->
hasOneUse() && VT != MVT::i1) {
14793 EVT TrTy =
N->getValueType(0);
14797 auto NewEltCnt = EltCnt * SizeRatio;
14803 if (isa<ConstantSDNode>(EltNo) &&
isTypeLegal(NVT)) {
14805 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
14836 AddToWorklist(Amt.
getNode());
14885 if (BuildVectEltTy == TruncVecEltTy) {
14889 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
14891 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
14892 "Invalid number of elements");
14895 for (
unsigned i = 0, e = BuildVecNumElts; i !=
e; i += TruncEltOffset)
14905 if (
SDValue Reduced = reduceLoadWidth(
N))
14911 auto *LN0 = cast<LoadSDNode>(N0);
14928 unsigned NumDefs = 0;
14932 if (!
X.isUndef()) {
14943 X.getValueType().getVectorElementCount()));
14949 if (NumDefs == 1) {
14950 assert(
V.getNode() &&
"The single defined operand is empty!");
14952 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
14958 AddToWorklist(
NV.getNode());
14973 (!LegalOperations ||
15001 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
15015 if (!LegalOperations && N0.
hasOneUse() &&
15048 if (!LegalOperations && N0.
hasOneUse() &&
15084 !LD1->hasOneUse() || !LD2->hasOneUse() ||
15085 LD1->getAddressSpace() != LD2->getAddressSpace())
15088 unsigned LD1Fast = 0;
15089 EVT LD1VT = LD1->getValueType(0);
15094 *LD1->getMemOperand(), &LD1Fast) && LD1Fast)
15095 return DAG.
getLoad(VT,
SDLoc(
N), LD1->getChain(), LD1->getBasePtr(),
15096 LD1->getPointerInfo(), LD1->getAlign());
15111 EVT VT =
N->getValueType(0);
15148 auto IsBitCastOrFree = [&TLI, FPOpcode](
SDValue Op,
EVT VT) {
15149 if (
Op.getOpcode() ==
ISD::BITCAST &&
Op.getOperand(0).getValueType() == VT)
15162 IsBitCastOrFree(LogicOp0, VT)) {
15165 NumFPLogicOpsConv++;
15176 EVT VT =
N->getValueType(0);
15192 cast<BuildVectorSDNode>(N0)->isConstant())
15193 return ConstantFoldBITCASTofBUILD_VECTOR(N0.
getNode(),
15201 if (!LegalOperations ||
15207 if (
C.getNode() !=
N)
15220 auto IsFreeBitcast = [VT](
SDValue V) {
15222 V.getOperand(0).getValueType() == VT) ||
15243 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) ||
15257 if (
SDValue V = foldBitcastedFPLogic(
N, DAG, TLI))
15277 AddToWorklist(NewConv.
getNode());
15280 if (N0.
getValueType() == MVT::ppcf128 && !LegalTypes) {
15287 AddToWorklist(FlipBit.
getNode());
15294 AddToWorklist(
Hi.getNode());
15296 AddToWorklist(FlipBit.
getNode());
15300 AddToWorklist(FlipBits.
getNode());
15330 AddToWorklist(
X.getNode());
15334 if (OrigXWidth < VTWidth) {
15336 AddToWorklist(
X.getNode());
15337 }
else if (OrigXWidth > VTWidth) {
15342 X.getValueType(),
X,
15344 X.getValueType()));
15345 AddToWorklist(
X.getNode());
15347 AddToWorklist(
X.getNode());
15350 if (N0.
getValueType() == MVT::ppcf128 && !LegalTypes) {
15353 AddToWorklist(Cst.
getNode());
15355 AddToWorklist(
X.getNode());
15357 AddToWorklist(XorResult.
getNode());
15361 SDLoc(XorResult)));
15362 AddToWorklist(XorResult64.
getNode());
15366 AddToWorklist(FlipBit.
getNode());
15369 AddToWorklist(FlipBits.
getNode());
15375 AddToWorklist(
X.getNode());
15380 AddToWorklist(Cst.
getNode());
15388 if (
SDValue CombineLD = CombineConsecutiveLoads(N0.
getNode(), VT))
15403 auto PeekThroughBitcast = [&](
SDValue Op) {
15405 Op.getOperand(0).getValueType() == VT)
15424 for (
int i = 0; i != MaskScale; ++i)
15425 NewMask.
push_back(M < 0 ? -1 : M * MaskScale + i);
15430 return LegalShuffle;
15437 EVT VT =
N->getValueType(0);
15438 return CombineConsecutiveLoads(
N, VT);
15465 bool AllowMultipleMaybePoisonOperands =
15496 bool HadMaybePoisonOperands = !MaybePoisonOperands.
empty();
15497 bool IsNewMaybePoisonOperand = MaybePoisonOperands.
insert(
Op);
15498 if (!HadMaybePoisonOperands)
15500 if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) {
15509 for (
SDValue MaybePoisonOperand : MaybePoisonOperands) {
15511 if (MaybePoisonOperand.getOpcode() ==
ISD::UNDEF)
15518 FrozenMaybePoisonOperand.
getOperand(0) == FrozenMaybePoisonOperand) {
15522 MaybePoisonOperand);
15532 N0 =
N->getOperand(0);
15544 if (
auto *SVN = dyn_cast<ShuffleVectorSDNode>(N0)) {
15553 "Can't create node that may be undef/poison!");
15560ConstantFoldBITCASTofBUILD_VECTOR(
SDNode *BV,
EVT DstEltVT) {
15564 if (SrcEltVT == DstEltVT)
return SDValue(BV, 0);
15571 if (SrcBitSize == DstBitSize) {
15576 if (
Op.getValueType() != SrcEltVT)
15579 AddToWorklist(Ops.
back().getNode());
15593 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
15601 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
15604 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
15612 auto *BVN = cast<BuildVectorSDNode>(BV);
15618 if (!BVN->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
15623 for (
unsigned I = 0, E = RawBits.
size();
I != E; ++
I) {
15624 if (UndefElements[
I])
15640 N->getFlags().hasAllowContract();
15645 return Options.NoInfsFPMath ||
N->getFlags().hasNoInfs();
15649template <
class MatchContextClass>
15653 EVT VT =
N->getValueType(0);
15655 MatchContextClass matcher(DAG, TLI,
N);
15658 bool UseVP = std::is_same_v<MatchContextClass, VPMatchContext>;
15663 bool HasFMAD = !UseVP && (LegalOperations && TLI.
isFMADLegal(DAG,
N));
15668 (!LegalOperations || matcher.isOperationLegalOrCustom(
ISD::FMA, VT));
15671 if (!HasFMAD && !HasFMA)
15674 bool CanReassociate =
15675 Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
15677 Options.UnsafeFPMath || HasFMAD);
15679 if (!AllowFusionGlobally && !
N->getFlags().hasAllowContract())
15705 return AllowFusionGlobally ||
N->getFlags().hasAllowContract();
15716 return matcher.getNode(PreferredFusedOpcode, SL, VT, N0.
getOperand(0),
15723 return matcher.getNode(PreferredFusedOpcode, SL, VT, N1.
getOperand(0),
15735 if (CanReassociate) {
15740 }
else if (isFusedOp(N1) && N1.
hasOneUse()) {
15746 while (E && isFusedOp(TmpFMA) && TmpFMA.
hasOneUse()) {
15771 PreferredFusedOpcode, SL, VT,
15785 PreferredFusedOpcode, SL, VT,
15797 return matcher.getNode(
15798 PreferredFusedOpcode, SL, VT,
X,
Y,
15799 matcher.getNode(PreferredFusedOpcode, SL, VT,
15803 if (isFusedOp(N0)) {
15824 return matcher.getNode(
15825 PreferredFusedOpcode, SL, VT,
15828 matcher.getNode(PreferredFusedOpcode, SL, VT,
15834 if (isFusedOp(N00)) {
15848 if (isFusedOp(N1)) {
15869 if (isFusedOp(N10)) {
15886template <
class MatchContextClass>
15890 EVT VT =
N->getValueType(0);
15892 MatchContextClass matcher(DAG, TLI,
N);
15895 bool UseVP = std::is_same_v<MatchContextClass, VPMatchContext>;
15900 bool HasFMAD = !UseVP && (LegalOperations && TLI.
isFMADLegal(DAG,
N));
15905 (!LegalOperations || matcher.isOperationLegalOrCustom(
ISD::FMA, VT));
15908 if (!HasFMAD && !HasFMA)
15913 Options.UnsafeFPMath || HasFMAD);
15916 if (!AllowFusionGlobally && !
N->getFlags().hasAllowContract())
15925 bool NoSignedZero =
Options.NoSignedZerosFPMath ||
Flags.hasNoSignedZeros();
15932 return AllowFusionGlobally ||
N->getFlags().hasAllowContract();
15938 return matcher.getNode(PreferredFusedOpcode, SL, VT, XY.
getOperand(0),
15949 return matcher.getNode(
15950 PreferredFusedOpcode, SL, VT,
15951 matcher.getNode(
ISD::FNEG, SL, VT, YZ.getOperand(0)),
15952 YZ.getOperand(1),
X);
15962 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
15965 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
15969 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
15972 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
15981 return matcher.
getNode(PreferredFusedOpcode, SL, VT,
15982 matcher.getNode(
ISD::FNEG, SL, VT, N00), N01,
15983 matcher.getNode(
ISD::FNEG, SL, VT, N1));
15996 PreferredFusedOpcode, SL, VT,
15999 matcher.getNode(
ISD::FNEG, SL, VT, N1));
16012 PreferredFusedOpcode, SL, VT,
16036 PreferredFusedOpcode, SL, VT,
16060 PreferredFusedOpcode, SL, VT,
16069 return Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
16083 bool CanFuse =
Options.UnsafeFPMath ||
N->getFlags().hasAllowContract();
16086 if (CanFuse && isFusedOp(N0) &&
16087 isContractableAndReassociableFMUL(N0.
getOperand(2)) &&
16089 return matcher.getNode(
16091 matcher.
getNode(PreferredFusedOpcode, SL, VT,
16099 if (CanFuse && isFusedOp(N1) &&
16100 isContractableAndReassociableFMUL(N1.
getOperand(2)) &&
16105 PreferredFusedOpcode, SL, VT,
16108 matcher.
getNode(PreferredFusedOpcode, SL, VT,
16109 matcher.getNode(
ISD::FNEG, SL, VT, N20), N21, N0));
16114 if (isFusedOp(N0) && N0->
hasOneUse()) {
16118 if (isContractableAndReassociableFMUL(N020) &&
16124 PreferredFusedOpcode, SL, VT,
16127 matcher.getNode(
ISD::FNEG, SL, VT, N1)));
16140 if (isFusedOp(N00)) {
16142 if (isContractableAndReassociableFMUL(N002) &&
16146 PreferredFusedOpcode, SL, VT,
16150 PreferredFusedOpcode, SL, VT,
16153 matcher.getNode(
ISD::FNEG, SL, VT, N1)));
16163 if (isContractableAndReassociableFMUL(N120) &&
16169 PreferredFusedOpcode, SL, VT,
16173 PreferredFusedOpcode, SL, VT,
16191 if (isContractableAndReassociableFMUL(N102) &&
16197 PreferredFusedOpcode, SL, VT,
16202 PreferredFusedOpcode, SL, VT,
16216SDValue DAGCombiner::visitFMULForFMADistributiveCombine(
SDNode *
N) {
16219 EVT VT =
N->getValueType(0);
16240 bool HasFMAD =
Options.UnsafeFPMath &&
16244 if (!HasFMAD && !HasFMA)
16256 if (
C->isExactlyValue(+1.0))
16257 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
16259 if (
C->isExactlyValue(-1.0))
16260 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
16267 if (
SDValue FMA = FuseFADD(N0, N1))
16269 if (
SDValue FMA = FuseFADD(N1, N0))
16279 if (C0->isExactlyValue(+1.0))
16280 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
16283 if (C0->isExactlyValue(-1.0))
16284 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
16289 if (C1->isExactlyValue(+1.0))
16290 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
16292 if (C1->isExactlyValue(-1.0))
16293 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
16300 if (
SDValue FMA = FuseFSUB(N0, N1))
16302 if (
SDValue FMA = FuseFSUB(N1, N0))
16312 if (
SDValue Fused = visitFADDForFMACombine<VPMatchContext>(
N)) {
16314 AddToWorklist(Fused.getNode());
16325 EVT VT =
N->getValueType(0);
16339 if (N0CFP && !N1CFP)
16344 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
16349 if (N1C && N1C->
isZero())
16353 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
16359 N1, DAG, LegalOperations, ForCodeSize))
16365 N0, DAG, LegalOperations, ForCodeSize))
16372 return C &&
C->isExactlyValue(-2.0);
16376 if (isFMulNegTwo(N0)) {
16382 if (isFMulNegTwo(N1)) {
16393 if ((
Options.NoNaNsFPMath ||
Flags.hasNoNaNs()) && AllowNewConst) {
16407 (
Flags.hasAllowReassociation() &&
Flags.hasNoSignedZeros())) &&
16427 if (CFP01 && !CFP00 && N0.
getOperand(0) == N1) {
16450 if (CFP11 && !CFP10 && N1.
getOperand(0) == N0) {
16500 VT, N0, N1, Flags))
16505 if (
SDValue Fused = visitFADDForFMACombine<EmptyMatchContext>(
N)) {
16507 AddToWorklist(Fused.getNode());
16517 EVT VT =
N->getValueType(0);
16518 EVT ChainVT =
N->getValueType(1);
16525 N1, DAG, LegalOperations, ForCodeSize)) {
16527 {Chain, N0, NegN1});
16533 N0, DAG, LegalOperations, ForCodeSize)) {
16535 {Chain, N1, NegN0});
16545 EVT VT =
N->getValueType(0);
16560 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
16563 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
16567 if (N1CFP && N1CFP->
isZero()) {
16569 Flags.hasNoSignedZeros()) {
16581 if (N0CFP && N0CFP->
isZero()) {
16583 (
Options.NoSignedZerosFPMath ||
Flags.hasNoSignedZeros())) {
16600 (
Flags.hasAllowReassociation() &&
Flags.hasNoSignedZeros())) &&
16616 if (
SDValue Fused = visitFSUBForFMACombine<EmptyMatchContext>(
N)) {
16617 AddToWorklist(Fused.getNode());
16640 EVT VT =
N->getValueType(0);
16643 std::optional<int> Mantissa;
16644 auto GetConstAndPow2Ops = [&](
unsigned ConstOpIdx) {
16645 if (ConstOpIdx == 1 &&
N->getOpcode() ==
ISD::FDIV)
16649 Pow2Op =
N->getOperand(1 - ConstOpIdx);
16662 if (CFP ==
nullptr)
16665 const APFloat &APF = CFP->getValueAPF();
16673 int CurExp =
ilogb(APF);
16676 N->getOpcode() ==
ISD::FMUL ? CurExp : (CurExp - MaxExpChange);
16679 N->getOpcode() ==
ISD::FDIV ? CurExp : (CurExp + MaxExpChange);
16687 Mantissa = ThisMantissa;
16689 return *Mantissa == ThisMantissa && ThisMantissa > 0;
16696 if (!GetConstAndPow2Ops(0) && !GetConstAndPow2Ops(1))
16725 NewIntVT, DAG.
getBitcast(NewIntVT, ConstOp), Shift);
16734 EVT VT =
N->getValueType(0);
16754 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
16757 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
16760 if (
Options.UnsafeFPMath ||
Flags.hasAllowReassociation()) {
16786 VT, N0, N1, Flags))
16820 if (
Flags.hasNoNaNs() &&
Flags.hasNoSignedZeros() &&
16828 auto TrueOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(1));
16829 auto FalseOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(2));
16831 if (TrueOpnd && FalseOpnd &&
16833 isa<ConstantFPSDNode>(
Cond.getOperand(1)) &&
16834 cast<ConstantFPSDNode>(
Cond.getOperand(1))->isExactlyValue(0.0)) {
16852 if (TrueOpnd->isExactlyValue(-1.0) && FalseOpnd->isExactlyValue(1.0) &&
16856 if (TrueOpnd->isExactlyValue(1.0) && FalseOpnd->isExactlyValue(-1.0))
16865 if (
SDValue Fused = visitFMULForFMADistributiveCombine(
N)) {
16866 AddToWorklist(Fused.getNode());
16872 if (
SDValue R = combineFMulOrFDivWithIntPow2(
N))
16878template <
class MatchContextClass>
SDValue DAGCombiner::visitFMA(
SDNode *
N) {
16884 EVT VT =
N->getValueType(0);
16889 MatchContextClass matcher(DAG, TLI,
N);
16891 bool CanReassociate =
16892 Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
16895 if (isa<ConstantFPSDNode>(N0) &&
16896 isa<ConstantFPSDNode>(N1) &&
16897 isa<ConstantFPSDNode>(N2)) {
16898 return matcher.getNode(
ISD::FMA,
DL, VT, N0, N1, N2);
16919 if (N0CFP && N0CFP->
isZero())
16921 if (N1CFP && N1CFP->
isZero())
16936 if (CanReassociate) {
16941 return matcher.getNode(
16950 return matcher.getNode(
16960 return matcher.getNode(
ISD::FADD,
DL, VT, N0, N2);
16965 AddToWorklist(RHSNeg.
getNode());
16966 return matcher.getNode(
ISD::FADD,
DL, VT, N2, RHSNeg);
16980 if (CanReassociate) {
16982 if (N1CFP && N0 == N2) {
17000 SDValue(
N, 0), DAG, LegalOperations, ForCodeSize))
17009 EVT VT =
N->getValueType(0);
17013 if (isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1) &&
17014 isa<ConstantFPSDNode>(N2))
17033 if (LegalDAG || (!UnsafeMath && !
Flags.hasAllowReciprocal()))
17037 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
17048 unsigned NumElts = 1;
17049 EVT VT =
N->getValueType(0);
17053 if (!MinUses || (N1->
use_size() * NumElts) < MinUses)
17059 for (
auto *U : N1->
uses()) {
17060 if (
U->getOpcode() ==
ISD::FDIV &&
U->getOperand(1) == N1) {
17062 if (
U->getOperand(1).getOpcode() ==
ISD::FSQRT &&
17063 U->getOperand(0) ==
U->getOperand(1).getOperand(0) &&
17064 U->getFlags().hasAllowReassociation() &&
17065 U->getFlags().hasNoSignedZeros())
17070 if (UnsafeMath ||
U->getFlags().hasAllowReciprocal())
17077 if ((
Users.size() * NumElts) < MinUses)
17085 for (
auto *U :
Users) {
17086 SDValue Dividend =
U->getOperand(0);
17087 if (Dividend != FPOne) {
17089 Reciprocal, Flags);
17090 CombineTo(U, NewNode);
17091 }
else if (U != Reciprocal.
getNode()) {
17094 CombineTo(U, Reciprocal);
17103 EVT VT =
N->getValueType(0);
17118 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
17121 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
17127 if (
Options.UnsafeFPMath ||
Flags.hasAllowReciprocal()) {
17129 if (
auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1)) {
17137 (!LegalOperations ||
17186 A =
Y.getOperand(0);
17195 if (
SDValue Rsqrt = buildRsqrtEstimate(AAZ, Flags))
17199 recursivelyDeleteUnusedNodes(AAZ.
getNode());
17207 AddToWorklist(Div.
getNode());
17215 if (
SDValue RV = BuildDivEstimate(N0, N1, Flags))
17220 if ((
Options.NoSignedZerosFPMath ||
Flags.hasNoSignedZeros()) &&
17221 (
Options.UnsafeFPMath ||
Flags.hasAllowReassociation()))
17241 if (
SDValue R = combineFMulOrFDivWithIntPow2(
N))
17250 EVT VT =
N->getValueType(0);
17261 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
17273 if (!
Flags.hasApproximateFuncs() ||
17286 return buildSqrtEstimate(N0, Flags);
17301 if (YTy == MVT::f128)
17320 EVT VT =
N->getValueType(0);
17331 if (!
V.isNegative()) {
17374 EVT VT =
N->getValueType(0);
17384 if (!
Flags.hasNoSignedZeros() || !
Flags.hasNoInfs() || !
Flags.hasNoNaNs() ||
17385 !
Flags.hasApproximateFuncs())
17404 if (ExponentIs025 || ExponentIs075) {
17415 if ((!
Flags.hasNoSignedZeros() && ExponentIs025) || !
Flags.hasNoInfs() ||
17416 !
Flags.hasApproximateFuncs())
17449 EVT VT =
N->getValueType(0);
17470 EVT VT =
N->getValueType(0);
17480 (!LegalOperations ||
17522 EVT VT =
N->getValueType(0);
17532 (!LegalOperations ||
17562 EVT VT =
N->getValueType(0);
17568 EVT SrcVT = Src.getValueType();
17583 unsigned ActualSize = std::min(InputSize, OutputSize);
17603 EVT VT =
N->getValueType(0);
17618 EVT VT =
N->getValueType(0);
17633 EVT VT =
N->getValueType(0);
17649 EVT VT =
N->getValueType(0);
17662 const bool NIsTrunc =
N->getConstantOperandVal(1) == 1;
17702 AddToWorklist(Tmp.
getNode());
17707 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
17715 EVT VT =
N->getValueType(0);
17722 if (
N->hasOneUse() &&
17740 if (
In.getValueType() == VT)
return In;
17755 CombineTo(
N, ExtLoad);
17764 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
17772 EVT VT =
N->getValueType(0);
17783 EVT VT =
N->getValueType(0);
17817 EVT VT =
N->getValueType(0);
17828 EVT VT =
N->getValueType(0);
17845 N->getFlags().hasNoSignedZeros()) && N0.
hasOneUse()) {
17850 if (
SDValue Cast = foldSignChangeInBitcast(
N))
17859 EVT VT =
N->getValueType(0);
17861 unsigned Opc =
N->getOpcode();
17883 return PropagatesNaN ?
N->getOperand(1) :
N->getOperand(0);
17892 if (IsMin == AF.
isNegative() && (!PropagatesNaN ||
Flags.hasNoNaNs()))
17893 return N->getOperand(1);
17899 if (IsMin != AF.
isNegative() && (PropagatesNaN ||
Flags.hasNoNaNs()))
17900 return N->getOperand(0);
17904 if (
SDValue SD = reassociateReduction(
17908 Opc,
SDLoc(
N), VT, N0, N1, Flags))
17916 EVT VT =
N->getValueType(0);
17924 return N->getOperand(0);
17931 if (
SDValue Cast = foldSignChangeInBitcast(
N))
17964 bool Updated =
false;
17976 return True || False;
17980 if (!IsAlwaysTrueOrFalse(
Cond, S1C)) {
17987 S1 =
S1->getOperand(0);
18018 if (
SDValue NewN1 = rebuildSetCC(N1))
18020 ChainHandle.getValue(), NewN1, N2);
18029 (
N.getOperand(0).hasOneUse() &&
18030 N.getOperand(0).getOpcode() ==
ISD::SRL))) {
18033 N =
N.getOperand(0);
18082 SDValue Tmp = visitXOR(
N.getNode());
18089 N = XORHandle.getValue();
18101 bool Equal =
false;
18106 Op0 =
N->getOperand(0);
18107 Op1 =
N->getOperand(1);
18111 EVT SetCCVT =
N.getValueType();
18127 SDValue CondLHS =
N->getOperand(2), CondRHS =
N->getOperand(3);
18137 CondLHS, CondRHS,
CC->get(),
SDLoc(
N),
18155 if (LD->isIndexed())
18157 EVT VT = LD->getMemoryVT();
18160 Ptr = LD->getBasePtr();
18161 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
18162 if (ST->isIndexed())
18164 EVT VT = ST->getMemoryVT();
18167 Ptr = ST->getBasePtr();
18170 if (LD->isIndexed())
18172 EVT VT = LD->getMemoryVT();
18176 Ptr = LD->getBasePtr();
18179 if (ST->isIndexed())
18181 EVT VT = ST->getMemoryVT();
18185 Ptr = ST->getBasePtr();
18199bool DAGCombiner::CombineToPreIndexedLoadStore(
SDNode *
N) {
18203 bool IsLoad =
true;
18204 bool IsMasked =
false;
18226 bool Swapped =
false;
18227 if (isa<ConstantSDNode>(BasePtr)) {
18246 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
18251 SDValue Val = IsMasked ? cast<MaskedStoreSDNode>(
N)->getValue()
18252 : cast<StoreSDNode>(
N)->getValue();
18255 if (Val == BasePtr)
18272 constexpr unsigned int MaxSteps = 8192;
18273 if (isa<ConstantSDNode>(
Offset))
18294 if (!isa<ConstantSDNode>(Op1)) {
18312 bool RealUse =
false;
18348 WorklistRemover DeadNodes(*
this);
18357 deleteAndRecombine(
N);
18363 for (
unsigned i = 0, e = OtherUses.
size(); i != e; ++i) {
18364 unsigned OffsetIdx = 1;
18365 if (OtherUses[i]->getOperand(OffsetIdx).getNode() ==
BasePtr.getNode())
18367 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() ==
18368 BasePtr.getNode() &&
"Expected BasePtr operand");
18381 auto *CN = cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx));
18382 const APInt &Offset0 = CN->getAPIntValue();
18384 int X0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 1) ? -1 : 1;
18385 int Y0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 0) ? -1 : 1;
18391 APInt CNV = Offset0;
18392 if (X0 < 0) CNV = -CNV;
18393 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
18394 else CNV = CNV - Offset1;
18406 deleteAndRecombine(OtherUses[i]);
18411 deleteAndRecombine(
Ptr.getNode());
18412 AddToWorklist(
Result.getNode());
18433 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
18438 if (
Use ==
Ptr.getNode())
18442 if (isa<MemSDNode>(
Use)) {
18443 bool IsLoad =
true;
18444 bool IsMasked =
false;
18447 IsMasked, OtherPtr, TLI)) {
18473 IsMasked,
Ptr, TLI) ||
18491 constexpr unsigned int MaxSteps = 8192;
18507bool DAGCombiner::CombineToPostIndexedLoadStore(
SDNode *
N) {
18511 bool IsLoad =
true;
18512 bool IsMasked =
false;
18533 ++PostIndexedNodes;
18537 WorklistRemover DeadNodes(*
this);
18546 deleteAndRecombine(
N);
18550 Result.getValue(IsLoad ? 1 : 0));
18551 deleteAndRecombine(
Op);
18566 !cast<ConstantSDNode>(Inc)->isOpaque()) &&
18567 "Cannot split out indexing using opaque target constants");
18585 EVT STMemType =
ST->getMemoryVT();
18586 if (STType == STMemType)
18608 EVT LDMemType =
LD->getMemoryVT();
18609 EVT LDType =
LD->getValueType(0);
18611 "Attempting to extend value of non-matching type");
18612 if (LDType == LDMemType)
18615 switch (
LD->getExtensionType()) {
18651 if (!BasePtrST.equalBaseIndex(BasePtrLD, DAG,
Offset))
18654 GatherAllAliases(Store, Chain, Aliases);
18655 if (Aliases.
empty() ||
18656 (Aliases.
size() == 1 && Aliases.
front().getNode() == Store))
18665 if (BasePtrST.equalBaseIndex(BasePtrLD, DAG,
Offset))
18681 if (!ST || !
ST->isSimple() ||
ST->getAddressSpace() !=
LD->getAddressSpace())
18684 EVT LDType =
LD->getValueType(0);
18685 EVT LDMemType =
LD->getMemoryVT();
18686 EVT STMemType =
ST->getMemoryVT();
18687 EVT STType =
ST->getValue().getValueType();
18711 int64_t OrigOffset =
Offset;
18724 STCoversLD = (
Offset == 0) && LdMemSize == StMemSize;
18730 if (
LD->isIndexed()) {
18737 return CombineTo(LD, Ops, 3);
18739 return CombineTo(LD, Val, Chain);
18746 if (
Offset == 0 && LDType == STType && STMemType == LDMemType) {
18749 return ReplaceLd(LD,
ST->getValue(), Chain);
18757 SDLoc(ST), STType);
18759 return ReplaceLd(LD, Val, Chain);
18777 if (
LD->getBasePtr().isUndef() ||
Offset != 0)
18782 if (!getTruncatedStoreValue(ST, Val))
18786 if (STMemType != LDMemType) {
18794 if (!extendLoadedValueToExtension(LD, Val))
18796 return ReplaceLd(LD, Val, Chain);
18801 deleteAndRecombine(Val.
getNode());
18814 if (
LD->isSimple()) {
18815 if (
N->getValueType(1) == MVT::Other) {
18817 if (!
N->hasAnyUseOfValue(0)) {
18825 dbgs() <<
"\nWith chain: "; Chain.
dump(&DAG);
18827 WorklistRemover DeadNodes(*
this);
18829 AddUsersToWorklist(Chain.
getNode());
18830 if (
N->use_empty())
18831 deleteAndRecombine(
N);
18837 assert(
N->getValueType(2) == MVT::Other &&
"Malformed indexed loads?");
18845 if (!
N->hasAnyUseOfValue(0) && (CanSplitIdx || !
N->hasAnyUseOfValue(1))) {
18848 if (
N->hasAnyUseOfValue(1) && CanSplitIdx) {
18849 Index = SplitIndexingFromLoad(LD);
18852 AddUsersToWorklist(
N);
18856 dbgs() <<
"\nWith: ";
Undef.dump(&DAG);
18857 dbgs() <<
" and 2 other values\n");
18858 WorklistRemover DeadNodes(*
this);
18862 deleteAndRecombine(
N);
18870 if (
auto V = ForwardStoreValueToDirectLoad(LD))
18877 if (*Alignment >
LD->getAlign() &&
18878 isAligned(*Alignment,
LD->getSrcValueOffset())) {
18880 LD->getExtensionType(),
SDLoc(
N),
LD->getValueType(0), Chain,
Ptr,
18881 LD->getPointerInfo(),
LD->getMemoryVT(), *Alignment,
18882 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
18890 if (
LD->isUnindexed()) {
18892 SDValue BetterChain = FindBetterChain(LD, Chain);
18895 if (Chain != BetterChain) {
18901 BetterChain,
Ptr,
LD->getMemOperand());
18904 LD->getValueType(0),
18905 BetterChain,
Ptr,
LD->getMemoryVT(),
18906 LD->getMemOperand());
18911 MVT::Other, Chain, ReplLoad.
getValue(1));
18914 return CombineTo(
N, ReplLoad.
getValue(0), Token);
18919 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
18924 if (SliceUpLoad(
N))
18944struct LoadedSlice {
18948 bool ForCodeSize =
false;
18951 unsigned Loads = 0;
18952 unsigned Truncates = 0;
18953 unsigned CrossRegisterBanksCopies = 0;
18954 unsigned ZExts = 0;
18955 unsigned Shift = 0;
18957 explicit Cost(
bool ForCodeSize) : ForCodeSize(ForCodeSize) {}
18960 Cost(
const LoadedSlice &LS,
bool ForCodeSize)
18961 : ForCodeSize(ForCodeSize), Loads(1) {
18962 EVT TruncType =
LS.Inst->getValueType(0);
18963 EVT LoadedType =
LS.getLoadedType();
18964 if (TruncType != LoadedType &&
18965 !
LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
18973 void addSliceGain(
const LoadedSlice &LS) {
18982 if (
LS.canMergeExpensiveCrossRegisterBankCopy())
18983 ++CrossRegisterBanksCopies;
18987 Loads +=
RHS.Loads;
18988 Truncates +=
RHS.Truncates;
18989 CrossRegisterBanksCopies +=
RHS.CrossRegisterBanksCopies;
18990 ZExts +=
RHS.ZExts;
18991 Shift +=
RHS.Shift;
18996 return Loads ==
RHS.Loads && Truncates ==
RHS.Truncates &&
18997 CrossRegisterBanksCopies ==
RHS.CrossRegisterBanksCopies &&
18998 ZExts ==
RHS.ZExts && Shift ==
RHS.Shift;
19006 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
19007 unsigned ExpensiveOpsRHS =
RHS.Loads +
RHS.CrossRegisterBanksCopies;
19010 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
19011 return ExpensiveOpsLHS < ExpensiveOpsRHS;
19012 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
19013 (
RHS.Truncates +
RHS.ZExts +
RHS.Shift + ExpensiveOpsRHS);
19039 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
19044 APInt getUsedBits()
const {
19049 assert(Origin &&
"No original load to compare against.");
19051 assert(Inst &&
"This slice is not bound to an instruction");
19053 "Extracted slice is bigger than the whole type!");
19055 UsedBits.setAllBits();
19056 UsedBits = UsedBits.zext(
BitWidth);
19057 UsedBits <<= Shift;
19062 unsigned getLoadedSize()
const {
19063 unsigned SliceSize = getUsedBits().popcount();
19064 assert(!(SliceSize & 0x7) &&
"Size is not a multiple of a byte.");
19065 return SliceSize / 8;
19070 EVT getLoadedType()
const {
19071 assert(DAG &&
"Missing context");
19086 bool isLegal()
const {
19088 if (!Origin || !Inst || !DAG)
19098 EVT SliceType = getLoadedType();
19109 if (PtrType == MVT::Untyped || PtrType.
isExtended())
19122 if (TruncateType != SliceType &&
19132 uint64_t getOffsetFromBase()
const {
19133 assert(DAG &&
"Missing context.");
19135 assert(!(Shift & 0x7) &&
"Shifts not aligned on Bytes are not supported.");
19139 "The size of the original loaded type is not a multiple of a"
19144 "Invalid shift amount for given loaded size");
19157 assert(Inst && Origin &&
"Unable to replace a non-existing slice.");
19161 int64_t
Offset =
static_cast<int64_t
>(getOffsetFromBase());
19162 assert(
Offset >= 0 &&
"Offset too big to fit in int64_t!");
19172 EVT SliceType = getLoadedType();
19182 if (SliceType != FinalType)
19192 bool canMergeExpensiveCrossRegisterBankCopy()
const {
19198 assert(DAG &&
"Missing context");
19200 EVT ResVT =
Use->getValueType(0);
19205 Use->getOperand(0)->isDivergent());
19214 if (!
TRI ||
TRI->getCommonSubClass(ArgRC, ResRC))
19219 unsigned IsFast = 0;
19260 const LoadedSlice &Second) {
19262 "Unable to match different memory origins.");
19264 assert((UsedBits & Second.getUsedBits()) == 0 &&
19265 "Slices are not supposed to overlap.");
19266 UsedBits |= Second.getUsedBits();
19275 LoadedSlice::Cost &GlobalLSCost) {
19276 unsigned NumberOfSlices = LoadedSlices.
size();
19278 if (NumberOfSlices < 2)
19283 llvm::sort(LoadedSlices, [](
const LoadedSlice &
LHS,
const LoadedSlice &
RHS) {
19284 assert(
LHS.Origin ==
RHS.Origin &&
"Different bases not implemented.");
19285 return LHS.getOffsetFromBase() <
RHS.getOffsetFromBase();
19287 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
19290 const LoadedSlice *
First =
nullptr;
19291 const LoadedSlice *Second =
nullptr;
19292 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
19295 Second = &LoadedSlices[CurrSlice];
19302 EVT LoadedType =
First->getLoadedType();
19305 if (LoadedType != Second->getLoadedType())
19309 Align RequiredAlignment;
19310 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
19316 if (
First->getAlign() < RequiredAlignment)
19323 assert(GlobalLSCost.Loads > 0 &&
"We save more loads than we created!");
19324 --GlobalLSCost.Loads;
19341 const APInt &UsedBits,
bool ForCodeSize) {
19342 unsigned NumberOfSlices = LoadedSlices.
size();
19344 return NumberOfSlices > 1;
19347 if (NumberOfSlices != 2)
19355 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
19357 OrigCost.Loads = 1;
19358 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
19359 const LoadedSlice &LS = LoadedSlices[CurrSlice];
19361 LoadedSlice::Cost SliceCost(LS, ForCodeSize);
19362 GlobalSlicingCost += SliceCost;
19366 OrigCost.addSliceGain(LS);
19371 return OrigCost > GlobalSlicingCost;
19380bool DAGCombiner::SliceUpLoad(
SDNode *
N) {
19386 !
LD->getValueType(0).isInteger())
19392 if (
LD->getValueType(0).isScalableVector())
19397 APInt UsedBits(
LD->getValueSizeInBits(0), 0);
19405 UI != UIEnd; ++UI) {
19407 if (UI.getUse().getResNo() != 0)
19411 unsigned Shift = 0;
19416 Shift =
User->getConstantOperandVal(1);
19429 unsigned Width =
User->getValueSizeInBits(0);
19434 LoadedSlice
LS(
User, LD, Shift, &DAG);
19435 APInt CurrentUsedBits =
LS.getUsedBits();
19438 if ((CurrentUsedBits & UsedBits) != 0)
19441 UsedBits |= CurrentUsedBits;
19462 for (
const LoadedSlice &LS : LoadedSlices) {
19464 CombineTo(
LS.Inst, SliceInst,
true);
19468 "It takes more than a zext to get to the loaded slice!!");
19475 AddToWorklist(Chain.
getNode());
19482static std::pair<unsigned, unsigned>
19484 std::pair<unsigned, unsigned> Result(0, 0);
19488 !isa<ConstantSDNode>(V->getOperand(1)) ||
19493 LoadSDNode *LD = cast<LoadSDNode>(V->getOperand(0));
19494 if (LD->getBasePtr() !=
Ptr)
return Result;
19497 if (V.getValueType() != MVT::i16 &&
19498 V.getValueType() != MVT::i32 &&
19499 V.getValueType() != MVT::i64)
19505 uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue();
19507 if (NotMaskLZ & 7)
return Result;
19509 if (NotMaskTZ & 7)
return Result;
19510 if (NotMaskLZ == 64)
return Result;
19517 if (V.getValueType() != MVT::i64 && NotMaskLZ)
19518 NotMaskLZ -= 64-V.getValueSizeInBits();
19520 unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8;
19521 switch (MaskedBytes) {
19525 default:
return Result;
19530 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes)
return Result;
19539 if (!LD->isOperandOf(Chain.
getNode()))
19544 Result.first = MaskedBytes;
19545 Result.second = NotMaskTZ/8;
19556 unsigned NumBytes = MaskInfo.first;
19557 unsigned ByteShift = MaskInfo.second;
19563 ByteShift*8, (ByteShift+NumBytes)*8);
19572 bool UseTruncStore;
19573 if (DC->isTypeLegal(VT))
19574 UseTruncStore =
false;
19577 UseTruncStore =
true;
19603 StOffset = ByteShift;
19634 if (!
ST->isSimple())
19642 if (
ST->isTruncatingStore() || VT.
isVector())
19645 unsigned Opc =
Value.getOpcode();
19657 std::pair<unsigned, unsigned> MaskedLoad;
19659 if (MaskedLoad.first)
19661 Value.getOperand(1), ST,
this))
19666 if (MaskedLoad.first)
19668 Value.getOperand(0), ST,
this))
19682 if (
LD->getBasePtr() !=
Ptr ||
19683 LD->getPointerInfo().getAddrSpace() !=
19684 ST->getPointerInfo().getAddrSpace())
19693 if (Imm == 0 ||
Imm.isAllOnes())
19695 unsigned ShAmt =
Imm.countr_zero();
19714 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
19716 std::min(
BitWidth, ShAmt + NewBW));
19717 if ((Imm & Mask) == Imm) {
19725 PtrOff = (
BitWidth + 7 - NewBW) / 8 - PtrOff;
19727 unsigned IsFast = 0;
19730 LD->getAddressSpace(), NewAlign,
19731 LD->getMemOperand()->getFlags(), &IsFast) ||
19739 LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
19740 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
19746 ST->getPointerInfo().getWithOffset(PtrOff), NewAlign);
19748 AddToWorklist(NewPtr.
getNode());
19749 AddToWorklist(NewLD.
getNode());
19750 AddToWorklist(NewVal.
getNode());
19751 WorklistRemover DeadNodes(*
this);
19770 EVT VT =
LD->getMemoryVT();
19772 VT !=
ST->getMemoryVT() ||
19773 LD->isNonTemporal() ||
19774 ST->isNonTemporal() ||
19775 LD->getPointerInfo().getAddrSpace() != 0 ||
19776 ST->getPointerInfo().getAddrSpace() != 0)
19786 unsigned FastLD = 0, FastST = 0;
19793 *
LD->getMemOperand(), &FastLD) ||
19795 *
ST->getMemOperand(), &FastST) ||
19796 !FastLD || !FastST)
19801 LD->getPointerInfo(),
LD->getAlign());
19805 ST->getPointerInfo(),
ST->getAlign());
19807 AddToWorklist(NewLD.
getNode());
19808 AddToWorklist(NewST.
getNode());
19809 WorklistRemover DeadNodes(*
this);
19831bool DAGCombiner::isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
19843 if (
Use == MulNode)
19851 if (
Use->getOperand(0) == ConstNode)
19852 OtherOp =
Use->getOperand(1).getNode();
19854 OtherOp =
Use->getOperand(0).getNode();
19866 if (OtherOp == MulVar)
19894 unsigned NumStores) {
19897 SDLoc StoreDL(StoreNodes[0].MemNode);
19899 for (
unsigned i = 0; i < NumStores; ++i) {
19900 Visited.
insert(StoreNodes[i].MemNode);
19904 for (
unsigned i = 0; i < NumStores; ++i) {
19905 if (Visited.
insert(StoreNodes[i].MemNode->getChain().getNode()).second)
19906 Chains.
push_back(StoreNodes[i].MemNode->getChain());
19909 assert(!Chains.
empty() &&
"Chain should have generated a chain");
19914 const Value *UnderlyingObj =
nullptr;
19915 for (
const auto &
MemOp : StoreNodes) {
19927 if (UnderlyingObj && UnderlyingObj != Obj)
19930 if (!UnderlyingObj)
19931 UnderlyingObj = Obj;
19937bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
19939 bool IsConstantSrc,
bool UseVector,
bool UseTrunc) {
19944 assert((!UseTrunc || !UseVector) &&
19945 "This optimization cannot emit a vector truncating store");
19948 SDLoc DL(StoreNodes[0].MemNode);
19951 unsigned SizeInBits = NumStores * ElementSizeBits;
19954 std::optional<MachineMemOperand::Flags>
Flags;
19956 for (
unsigned I = 0;
I != NumStores; ++
I) {
19957 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
I].MemNode);
19972 unsigned Elts = NumStores * NumMemElts;
19980 if (IsConstantSrc) {
19982 for (
unsigned I = 0;
I != NumStores; ++
I) {
19983 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
I].MemNode);
19991 auto *
C = dyn_cast<ConstantSDNode>(Val);
20001 .zextOrTrunc(ElementSizeBits),
20011 DL, StoreTy, BuildVector);
20014 for (
unsigned i = 0; i < NumStores; ++i) {
20015 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
20052 assert(IsConstantSrc &&
"Merged vector elements should use vector store");
20054 APInt StoreInt(SizeInBits, 0);
20059 for (
unsigned i = 0; i < NumStores; ++i) {
20060 unsigned Idx = IsLE ? (NumStores - 1 - i) : i;
20063 SDValue Val = St->getValue();
20065 StoreInt <<= ElementSizeBits;
20067 StoreInt |=
C->getAPIntValue()
20068 .zextOrTrunc(ElementSizeBits)
20069 .zextOrTrunc(SizeInBits);
20071 StoreInt |=
C->getValueAPF()
20073 .zextOrTrunc(ElementSizeBits)
20074 .zextOrTrunc(SizeInBits);
20092 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
20093 bool CanReusePtrInfo = hasSameUnderlyingObj(StoreNodes);
20107 FirstInChain->
getAlign(), *Flags, AAInfo);
20109 EVT LegalizedStoredValTy =
20111 unsigned LegalizedStoreSize = LegalizedStoredValTy.
getSizeInBits();
20114 DAG.
getConstant(
C->getAPIntValue().zextOrTrunc(LegalizedStoreSize),
DL,
20115 LegalizedStoredValTy);
20117 NewChain,
DL, ExtendedStoreVal, FirstInChain->
getBasePtr(),
20126 for (
unsigned i = 0; i < NumStores; ++i)
20127 CombineTo(StoreNodes[i].MemNode, NewStore);
20129 AddToWorklist(NewChain.
getNode());
20133void DAGCombiner::getStoreMergeCandidates(
20140 if (!
BasePtr.getBase().getNode() ||
BasePtr.getBase().isUndef())
20144 StoreSource StoreSrc = getStoreSource(Val);
20145 assert(StoreSrc != StoreSource::Unknown &&
"Expected known source for store");
20151 if (StoreSrc == StoreSource::Load) {
20152 auto *Ld = cast<LoadSDNode>(Val);
20154 LoadVT = Ld->getMemoryVT();
20156 if (MemVT != LoadVT)
20159 if (!Ld->hasNUsesOfValue(1, 0))
20163 if (!Ld->isSimple() || Ld->isIndexed())
20167 int64_t &
Offset) ->
bool {
20170 if (!
Other->isSimple() ||
Other->isIndexed())
20180 :
Other->getMemoryVT() != MemVT;
20181 switch (StoreSrc) {
20182 case StoreSource::Load: {
20186 auto *OtherLd = dyn_cast<LoadSDNode>(OtherBC);
20190 if (LoadVT != OtherLd->getMemoryVT())
20193 if (!OtherLd->hasNUsesOfValue(1, 0))
20197 if (!OtherLd->isSimple() || OtherLd->isIndexed())
20200 if (cast<LoadSDNode>(Val)->isNonTemporal() != OtherLd->isNonTemporal())
20205 if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
20209 case StoreSource::Constant:
20212 if (getStoreSource(OtherBC) != StoreSource::Constant)
20215 case StoreSource::Extract:
20217 if (
Other->isTruncatingStore())
20234 auto OverLimitInDependenceCheck = [&](
SDNode *StoreNode,
20235 SDNode *RootNode) ->
bool {
20236 auto RootCount = StoreRootCountMap.
find(StoreNode);
20237 return RootCount != StoreRootCountMap.
end() &&
20238 RootCount->second.first == RootNode &&
20244 if (UseIter.getOperandNo() != 0)
20246 if (
auto *OtherStore = dyn_cast<StoreSDNode>(*UseIter)) {
20249 if (CandidateMatch(OtherStore,
Ptr, PtrDiff) &&
20250 !OverLimitInDependenceCheck(OtherStore, RootNode))
20251 StoreNodes.
push_back(MemOpLink(OtherStore, PtrDiff));
20273 unsigned NumNodesExplored = 0;
20274 const unsigned MaxSearchNodes = 1024;
20275 if (
auto *Ldn = dyn_cast<LoadSDNode>(RootNode)) {
20276 RootNode = Ldn->getChain().getNode();
20278 I != E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored) {
20279 if (
I.getOperandNo() == 0 && isa<LoadSDNode>(*
I)) {
20280 for (
auto I2 = (*I)->use_begin(), E2 = (*I)->use_end(); I2 != E2; ++I2)
20281 TryToAddCandidate(I2);
20284 if (
I.getOperandNo() == 0 && isa<StoreSDNode>(*
I)) {
20285 TryToAddCandidate(
I);
20290 I != E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored)
20291 TryToAddCandidate(
I);
20298bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
20314 while (!Worklist.
empty()) {
20316 if (!Visited.
insert(
N).second)
20325 unsigned int Max = 1024 + Visited.
size();
20327 for (
unsigned i = 0; i < NumStores; ++i) {
20328 SDNode *
N = StoreNodes[i].MemNode;
20345 for (
unsigned j = 0;
j <
N->getNumOperands(); ++
j)
20346 Worklist.
push_back(
N->getOperand(j).getNode());
20349 for (
unsigned i = 0; i < NumStores; ++i)
20355 if (Visited.
size() >= Max) {
20356 auto &RootCount = StoreRootCountMap[StoreNodes[i].MemNode];
20357 if (RootCount.first == RootNode)
20358 RootCount.second++;
20360 RootCount = {RootNode, 1};
20369 int64_t ElementSizeBytes)
const {
20372 size_t StartIdx = 0;
20373 while ((StartIdx + 1 < StoreNodes.
size()) &&
20374 StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
20375 StoreNodes[StartIdx + 1].OffsetFromBase)
20379 if (StartIdx + 1 >= StoreNodes.
size())
20388 unsigned NumConsecutiveStores = 1;
20389 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
20392 for (
unsigned i = 1, e = StoreNodes.
size(); i < e; ++i) {
20393 int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
20394 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
20396 NumConsecutiveStores = i + 1;
20398 if (NumConsecutiveStores > 1)
20399 return NumConsecutiveStores;
20407bool DAGCombiner::tryStoreMergeOfConstants(
20409 EVT MemVT,
SDNode *RootNode,
bool AllowVectors) {
20414 bool MadeChange =
false;
20417 while (NumConsecutiveStores >= 2) {
20421 unsigned LastLegalType = 1;
20422 unsigned LastLegalVectorType = 1;
20423 bool LastIntegerTrunc =
false;
20424 bool NonZero =
false;
20425 unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
20426 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
20427 StoreSDNode *
ST = cast<StoreSDNode>(StoreNodes[i].MemNode);
20429 bool IsElementZero =
false;
20431 IsElementZero =
C->isZero();
20433 IsElementZero =
C->getConstantFPValue()->isNullValue();
20435 IsElementZero =
true;
20436 if (IsElementZero) {
20437 if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
20438 FirstZeroAfterNonZero = i;
20440 NonZero |= !IsElementZero;
20443 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
20445 unsigned IsFast = 0;
20457 LastIntegerTrunc =
false;
20458 LastLegalType = i + 1;
20462 EVT LegalizedStoredValTy =
20470 LastIntegerTrunc =
true;
20471 LastLegalType = i + 1;
20480 unsigned Elts = (i + 1) * NumMemElts;
20487 LastLegalVectorType = i + 1;
20491 bool UseVector = (LastLegalVectorType > LastLegalType) && AllowVectors;
20492 unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
20493 bool UseTrunc = LastIntegerTrunc && !UseVector;
20505 unsigned NumSkip = 1;
20506 while ((NumSkip < NumConsecutiveStores) &&
20507 (NumSkip < FirstZeroAfterNonZero) &&
20508 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
20512 NumConsecutiveStores -= NumSkip;
20517 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
20520 NumConsecutiveStores -= NumElem;
20524 MadeChange |= mergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
20526 UseVector, UseTrunc);
20530 NumConsecutiveStores -= NumElem;
20535bool DAGCombiner::tryStoreMergeOfExtracts(
20541 bool MadeChange =
false;
20544 while (NumConsecutiveStores >= 2) {
20548 unsigned NumStoresToMerge = 1;
20549 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
20551 unsigned Elts = (i + 1) * NumMemElts;
20553 unsigned IsFast = 0;
20564 NumStoresToMerge = i + 1;
20569 if (NumStoresToMerge < 2) {
20576 unsigned NumSkip = 1;
20577 while ((NumSkip < NumConsecutiveStores) &&
20578 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
20582 NumConsecutiveStores -= NumSkip;
20587 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStoresToMerge,
20590 StoreNodes.
begin() + NumStoresToMerge);
20591 NumConsecutiveStores -= NumStoresToMerge;
20595 MadeChange |= mergeStoresOfConstantsOrVecElts(
20596 StoreNodes, MemVT, NumStoresToMerge,
false,
20599 StoreNodes.
erase(StoreNodes.
begin(), StoreNodes.
begin() + NumStoresToMerge);
20600 NumConsecutiveStores -= NumStoresToMerge;
20606 unsigned NumConsecutiveStores,
EVT MemVT,
20607 SDNode *RootNode,
bool AllowVectors,
20608 bool IsNonTemporalStore,
20609 bool IsNonTemporalLoad) {
20614 bool MadeChange =
false;
20623 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
20624 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
20630 int64_t LdOffset = 0;
20631 if (LdBasePtr.
getBase().getNode()) {
20633 if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
20641 LoadNodes.
push_back(MemOpLink(Ld, LdOffset));
20644 while (NumConsecutiveStores >= 2 && LoadNodes.
size() >= 2) {
20645 Align RequiredAlignment;
20646 bool NeedRotate =
false;
20647 if (LoadNodes.
size() == 2) {
20651 StoreNodes[0].MemNode->getAlign() >= RequiredAlignment) {
20657 int64_t Offset0 = LoadNodes[0].OffsetFromBase;
20658 int64_t Offset1 = LoadNodes[1].OffsetFromBase;
20660 if (Offset0 - Offset1 == ElementSizeBytes &&
20670 LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
20676 unsigned LastConsecutiveLoad = 1;
20679 unsigned LastLegalVectorType = 1;
20680 unsigned LastLegalIntegerType = 1;
20681 bool isDereferenceable =
true;
20682 bool DoIntegerTruncate =
false;
20683 int64_t StartAddress = LoadNodes[0].OffsetFromBase;
20685 for (
unsigned i = 1; i < LoadNodes.
size(); ++i) {
20687 if (LoadNodes[i].MemNode->getChain() != LoadChain)
20690 int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
20691 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
20693 LastConsecutiveLoad = i;
20695 if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
20696 isDereferenceable =
false;
20699 unsigned Elts = (i + 1) * NumMemElts;
20706 unsigned IsFastSt = 0;
20707 unsigned IsFastLd = 0;
20721 LastLegalVectorType = i + 1;
20725 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
20736 LastLegalIntegerType = i + 1;
20737 DoIntegerTruncate =
false;
20754 LastLegalIntegerType = i + 1;
20755 DoIntegerTruncate =
true;
20763 LastLegalVectorType > LastLegalIntegerType && AllowVectors;
20764 unsigned LastLegalType =
20765 std::max(LastLegalVectorType, LastLegalIntegerType);
20769 unsigned NumElem = std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
20770 NumElem = std::min(LastLegalType, NumElem);
20781 unsigned NumSkip = 1;
20782 while ((NumSkip < LoadNodes.
size()) &&
20783 (LoadNodes[NumSkip].MemNode->getAlign() <= FirstLoadAlign) &&
20784 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
20788 NumConsecutiveStores -= NumSkip;
20793 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
20797 NumConsecutiveStores -= NumElem;
20806 unsigned Elts = NumElem * NumMemElts;
20809 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
20813 SDLoc LoadDL(LoadNodes[0].MemNode);
20814 SDLoc StoreDL(StoreNodes[0].MemNode);
20819 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
20820 bool CanReusePtrInfo = hasSameUnderlyingObj(StoreNodes);
20821 AddToWorklist(NewStoreChain.
getNode());
20826 if (IsNonTemporalLoad)
20838 if (UseVectorTy || !DoIntegerTruncate) {
20844 unsigned LoadWidth = ElementSizeBytes * 8 * 2;
20846 "Unexpected type for rotate-able load pair");
20853 NewStoreChain, StoreDL, StoreOp, FirstInChain->
getBasePtr(),
20856 FirstStoreAlign, StMMOFlags);
20863 FirstLoadAlign, LdMMOFlags);
20865 NewStoreChain, StoreDL, NewLoad, FirstInChain->
getBasePtr(),
20868 JointMemOpVT, FirstInChain->
getAlign(),
20873 for (
unsigned i = 0; i < NumElem; ++i) {
20874 LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
20881 for (
unsigned i = 0; i < NumElem; ++i) {
20882 SDValue Val = StoreNodes[i].MemNode->getOperand(1);
20883 CombineTo(StoreNodes[i].MemNode, NewStore);
20885 recursivelyDeleteUnusedNodes(Val.
getNode());
20891 NumConsecutiveStores -= NumElem;
20896bool DAGCombiner::mergeConsecutiveStores(
StoreSDNode *St) {
20912 if (ElementSizeBytes * 8 != (int64_t)MemVT.
getSizeInBits())
20918 const StoreSource StoreSrc = getStoreSource(StoredVal);
20919 if (StoreSrc == StoreSource::Unknown)
20925 getStoreMergeCandidates(St, StoreNodes, RootNode);
20928 if (StoreNodes.
size() < 2)
20933 llvm::sort(StoreNodes, [](MemOpLink LHS, MemOpLink RHS) {
20934 return LHS.OffsetFromBase <
RHS.OffsetFromBase;
20938 Attribute::NoImplicitFloat);
20940 bool IsNonTemporalLoad = StoreSrc == StoreSource::Load &&
20941 cast<LoadSDNode>(StoredVal)->isNonTemporal();
20950 bool MadeChange =
false;
20951 while (StoreNodes.
size() > 1) {
20952 unsigned NumConsecutiveStores =
20953 getConsecutiveStores(StoreNodes, ElementSizeBytes);
20955 if (NumConsecutiveStores == 0)
20959 assert(NumConsecutiveStores >= 2 &&
"Expected at least 2 stores");
20960 switch (StoreSrc) {
20961 case StoreSource::Constant:
20962 MadeChange |= tryStoreMergeOfConstants(StoreNodes, NumConsecutiveStores,
20963 MemVT, RootNode, AllowVectors);
20966 case StoreSource::Extract:
20967 MadeChange |= tryStoreMergeOfExtracts(StoreNodes, NumConsecutiveStores,
20971 case StoreSource::Load:
20972 MadeChange |= tryStoreMergeOfLoads(StoreNodes, NumConsecutiveStores,
20973 MemVT, RootNode, AllowVectors,
20974 IsNonTemporalStore, IsNonTemporalLoad);
20989 if (
ST->isTruncatingStore()) {
20991 ST->getBasePtr(),
ST->getMemoryVT(),
20992 ST->getMemOperand());
20994 ReplStore = DAG.
getStore(BetterChain, SL,
ST->getValue(),
ST->getBasePtr(),
20995 ST->getMemOperand());
21000 MVT::Other,
ST->getChain(), ReplStore);
21003 AddToWorklist(Token.
getNode());
21006 return CombineTo(ST, Token,
false);
21040 if ((
isTypeLegal(MVT::i32) && !LegalOperations &&
ST->isSimple()) ||
21043 bitcastToAPInt().getZExtValue(),
SDLoc(CFP),
21050 if ((TLI.
isTypeLegal(MVT::i64) && !LegalOperations &&
21054 getZExtValue(),
SDLoc(CFP), MVT::i64);
21056 Ptr,
ST->getMemOperand());
21074 ST->getOriginalAlign(), MMOFlags, AAInfo);
21077 ST->getPointerInfo().getWithOffset(4),
21078 ST->getOriginalAlign(), MMOFlags, AAInfo);
21107 EltVT !=
Value.getOperand(0).getValueType().getVectorElementType())
21110 auto *Ld = dyn_cast<LoadSDNode>(
Value.getOperand(0));
21121 ST->getAlign(),
ST->getMemOperand()->getFlags(),
21131 if (
auto *CIdx = dyn_cast<ConstantSDNode>(
Idx)) {
21132 unsigned COffset = CIdx->getSExtValue() * EltVT.
getSizeInBits() / 8;
21134 PointerInfo =
ST->getPointerInfo().getWithOffset(COffset);
21139 return DAG.
getStore(Chain,
DL, Elt, NewPtr, PointerInfo,
ST->getAlign(),
21140 ST->getMemOperand()->getFlags());
21147 EVT MemVT =
ST->getMemoryVT();
21170 ST->isUnindexed()) {
21171 EVT SVT =
Value.getOperand(0).getValueType();
21178 if (((!LegalOperations &&
ST->isSimple()) ||
21181 DAG, *
ST->getMemOperand())) {
21183 ST->getMemOperand());
21188 if (
Value.isUndef() &&
ST->isUnindexed())
21195 if (*Alignment >
ST->getAlign() &&
21196 isAligned(*Alignment,
ST->getSrcValueOffset())) {
21199 ST->getMemoryVT(), *Alignment,
21200 ST->getMemOperand()->getFlags(),
ST->getAAInfo());
21210 if (
SDValue NewST = TransformFPLoadStorePair(
N))
21214 if (
SDValue Store = mergeTruncStores(ST))
21217 if (
ST->isUnindexed()) {
21220 if (findBetterNeighborChains(ST)) {
21225 Chain =
ST->getChain();
21229 if (
ST->isTruncatingStore() &&
ST->isUnindexed() &&
21230 Value.getValueType().isInteger() &&
21231 (!isa<ConstantSDNode>(
Value) ||
21232 !cast<ConstantSDNode>(
Value)->isOpaque())) {
21237 Value.getOperand(0).getValueType() ==
ST->getMemoryVT() &&
21240 ST->getMemOperand());
21242 APInt TruncDemandedBits =
21244 ST->getMemoryVT().getScalarSizeInBits());
21248 AddToWorklist(
Value.getNode());
21265 ST->getMemOperand());
21269 if (
auto *Cst = dyn_cast<ConstantSDNode>(
Value))
21270 if (!Cst->isOpaque()) {
21271 const APInt &CValue = Cst->getAPIntValue();
21272 APInt NewVal = CValue & TruncDemandedBits;
21273 if (NewVal != CValue) {
21277 ST->getMemoryVT(),
ST->getMemOperand());
21289 if (
auto *Ld = dyn_cast<LoadSDNode>(TruncVal)) {
21291 ST->isUnindexed() &&
ST->isSimple() &&
21302 if (
SDValue NewST = replaceStoreOfInsertLoad(ST))
21306 if (
StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
21307 if (
ST->isUnindexed() &&
ST->isSimple() &&
21308 ST1->isUnindexed() && ST1->isSimple()) {
21310 ST1->getValue() ==
Value &&
ST->getMemoryVT() == ST1->getMemoryVT() &&
21311 ST->getAddressSpace() == ST1->getAddressSpace()) {
21318 !ST1->getBasePtr().isUndef() &&
21319 ST->getAddressSpace() == ST1->getAddressSpace()) {
21324 if (
ST->getMemoryVT().isScalableVector() ||
21325 ST1->getMemoryVT().isScalableVector()) {
21326 if (ST1->getBasePtr() ==
Ptr &&
21328 ST->getMemoryVT().getStoreSize())) {
21329 CombineTo(ST1, ST1->getChain());
21339 if (STBase.contains(DAG,
ST->getMemoryVT().getFixedSizeInBits(),
21341 ST1->getMemoryVT().getFixedSizeInBits())) {
21342 CombineTo(ST1, ST1->getChain());
21356 ST->getMemoryVT(), LegalOperations)) {
21358 Ptr,
ST->getMemoryVT(),
ST->getMemOperand());
21369 bool Changed = mergeConsecutiveStores(ST);
21370 if (!Changed)
break;
21379 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
21387 if (isa<ConstantFPSDNode>(
ST->getValue())) {
21388 if (
SDValue NewSt = replaceStoreOfFPConstant(ST))
21395 return ReduceLoadOpStoreWidth(
N);
21399 const auto *LifetimeEnd = cast<LifetimeSDNode>(
N);
21400 if (!LifetimeEnd->hasOffset())
21404 LifetimeEnd->getOffset(),
false);
21408 while (!Chains.
empty()) {
21427 if (!
ST->isSimple() ||
ST->isIndexed())
21429 const TypeSize StoreSize =
ST->getMemoryVT().getStoreSize();
21437 if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
21440 dbgs() <<
"\nwithin LIFETIME_END of : ";
21441 LifetimeEndBase.dump();
dbgs() <<
"\n");
21442 CombineTo(ST,
ST->getChain());
21485 if (!
ST->isSimple())
21518 !
Lo.getOperand(0).getValueType().isScalarInteger() ||
21519 Lo.getOperand(0).getValueSizeInBits() > HalfValBitSize ||
21521 !
Hi.getOperand(0).getValueType().isScalarInteger() ||
21522 Hi.getOperand(0).getValueSizeInBits() > HalfValBitSize)
21528 ?
Lo.getOperand(0).getValueType()
21529 :
Lo.getValueType();
21531 ?
Hi.getOperand(0).getValueType()
21532 :
Hi.getValueType();
21549 ST->getOriginalAlign(), MMOFlags, AAInfo);
21554 St0,
DL,
Hi,
Ptr,
ST->getPointerInfo().getWithOffset(HalfValBitSize / 8),
21555 ST->getOriginalAlign(), MMOFlags, AAInfo);
21566 unsigned InsIndex) {
21575 int ElementOffset = -1;
21584 while (!ArgWorkList.
empty()) {
21587 std::tie(ArgOffset, ArgVal) = ArgWorkList.
pop_back_val();
21589 if (ArgVal == InsertVal0) {
21590 ElementOffset = ArgOffset;
21596 int CurrentArgOffset =
21600 CurrentArgOffset -= Step;
21606 assert(CurrentArgOffset == ArgOffset);
21612 if (ElementOffset == -1) {
21613 if (!
Y.isUndef() || InsertVal0.
getValueType() !=
Y.getValueType())
21615 ElementOffset = Mask.size();
21619 NewMask.
assign(Mask.begin(), Mask.end());
21621 assert(NewMask[InsIndex] < (
int)(2 * Mask.size()) && NewMask[InsIndex] >= 0 &&
21622 "NewMask[InsIndex] is out of bound");
21631SDValue DAGCombiner::mergeInsertEltWithShuffle(
SDNode *
N,
unsigned InsIndex) {
21633 "Expected extract_vector_elt");
21634 SDValue InsertVal =
N->getOperand(1);
21637 auto *SVN = dyn_cast<ShuffleVectorSDNode>(Vec);
21650 return LegalShuffle;
21661SDValue DAGCombiner::combineInsertEltToShuffle(
SDNode *
N,
unsigned InsIndex) {
21663 "Expected extract_vector_elt");
21664 SDValue InsertVal =
N->getOperand(1);
21671 SDValue DestVec =
N->getOperand(0);
21677 if (NumSrcElts == 1)
21680 unsigned NumMaskVals = ExtendRatio * NumSrcElts;
21688 for (
unsigned i = 0; i != NumMaskVals; ++i) {
21689 if (i / NumSrcElts == InsIndex)
21690 Mask[i] = (i % NumSrcElts) + NumMaskVals;
21705 ConcatOps[0] = SubVec;
21711 AddToWorklist(PaddedSubV.
getNode());
21712 AddToWorklist(DestVecBC.
getNode());
21713 AddToWorklist(Shuf.
getNode());
21720SDValue DAGCombiner::combineInsertEltToLoad(
SDNode *
N,
unsigned InsIndex) {
21721 EVT VT =
N->getValueType(0);
21730 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0));
21733 return InsIndex == P.index() || P.value() < 0 ||
21734 (InsIndex == 0 && P.value() == (int)P.index() - 1) ||
21735 (InsIndex == VT.getVectorNumElements() - 1 &&
21736 P.value() == (int)P.index() + 1);
21742 unsigned Extend = 0;
21746 Extend =
Scalar.getOpcode();
21750 auto *ScalarLoad = dyn_cast<LoadSDNode>(Scalar);
21760 auto *
VecLoad = dyn_cast<LoadSDNode>(Vec);
21764 int EltSize = ScalarLoad->getValueType(0).getScalarSizeInBits();
21765 if (EltSize == 0 || EltSize % 8 != 0 || !ScalarLoad->isSimple() ||
21768 ScalarLoad->getAddressSpace() !=
VecLoad->getAddressSpace())
21773 if (InsIndex == 0) {
21784 unsigned IsFast = 0;
21788 NewAlign,
VecLoad->getMemOperand()->getFlags(),
21800 InsIndex == 0 ? ScalarLoad->getPointerInfo()
21801 :
VecLoad->getPointerInfo().getWithOffset(EltSize / 8);
21804 ScalarLoad->getChain(),
Ptr, PtrInfo, NewAlign);
21817 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
21844 unsigned Elt = IndexC->getZExtValue();
21847 if (NumElts == 1) {
21863 && isa<ConstantSDNode>(InVec.
getOperand(2))) {
21865 if (Elt < OtherElt) {
21869 AddToWorklist(NewOp.
getNode());
21875 if (
SDValue Shuf = mergeInsertEltWithShuffle(
N, Elt))
21878 if (
SDValue Shuf = combineInsertEltToShuffle(
N, Elt))
21881 if (
SDValue Shuf = combineInsertEltToLoad(
N, Elt))
21897 EVT EltVT = Elt.getValueType();
21898 MaxEltVT = MaxEltVT.
bitsGE(EltVT) ? MaxEltVT : EltVT;
21906 assert(Ops.
size() == NumElts &&
"Unexpected vector size");
21920 for (
SDValue CurVec = InVec; CurVec;) {
21922 if (CurVec.isUndef())
21923 return CanonicalizeBuildVector(Ops);
21927 for (
unsigned I = 0;
I != NumElts; ++
I)
21928 AddBuildVectorOp(Ops, CurVec.getOperand(
I),
I);
21929 return CanonicalizeBuildVector(Ops);
21934 AddBuildVectorOp(Ops, CurVec.getOperand(0), 0);
21935 return CanonicalizeBuildVector(Ops);
21940 if (
auto *CurIdx = dyn_cast<ConstantSDNode>(CurVec.getOperand(2)))
21941 if (CurIdx->getAPIntValue().ult(NumElts)) {
21942 unsigned Idx = CurIdx->getZExtValue();
21943 AddBuildVectorOp(Ops, CurVec.getOperand(1),
Idx);
21947 return CanonicalizeBuildVector(Ops);
21949 CurVec = CurVec->getOperand(0);
21957 auto *SVN = cast<ShuffleVectorSDNode>(CurVec);
21961 bool Merged =
true;
21970 Mask = std::move(NewMask);
21987 for (
unsigned I = 0;
I != NumElts; ++
I)
22000 for (
unsigned I = 0;
I != NumElts; ++
I)
22007 for (
unsigned I = 0;
I != NumElts; ++
I)
22011 return CanonicalizeBuildVector(Ops);
22018SDValue DAGCombiner::scalarizeExtractedVectorLoad(
SDNode *EVE,
EVT InVecVT,
22041 if (
auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
22042 int Elt = ConstEltNo->getZExtValue();
22053 unsigned IsFast = 0;
22067 if (ResultVT.
bitsGT(VecEltVT)) {
22074 NewPtr, MPI, VecEltVT, Alignment,
22084 if (ResultVT.
bitsLT(VecEltVT))
22096 bool LegalOperations) {
22100 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
22140bool DAGCombiner::refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
22157 auto *IndexC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
22162 "Original ISD::EXTRACT_VECTOR_ELT is undefinend?");
22166 EVT ScalarVT =
N->getValueType(0);
22182 Entry(Entry &&) =
default;
22183 Entry(
SDNode *Producer_,
unsigned BitPos_,
int NumBits_)
22184 :
Producer(Producer_), BitPos(BitPos_), NumBits(NumBits_) {}
22187 Entry(
const Entry &) =
delete;
22188 Entry &operator=(
const Entry &) =
delete;
22189 Entry &operator=(Entry &&) =
delete;
22195 Worklist.
emplace_back(
N, VecEltBitWidth * IndexC->getZExtValue(),
22198 while (!Worklist.
empty()) {
22205 bool ProducerIsLeaf =
false;
22208 switch (
User->getOpcode()) {
22216 User->getValueSizeInBits(0));
22222 if (
auto *ShAmtC = dyn_cast<ConstantSDNode>(
User->
getOperand(1));
22226 unsigned ShAmt = ShAmtC->getZExtValue();
22234 ProducerIsLeaf =
true;
22242 if (ProducerIsLeaf)
22246 unsigned NewVecEltBitWidth = Leafs.
front().NumBits;
22249 if (NewVecEltBitWidth == VecEltBitWidth)
22259 if (!
all_of(Leafs, [NewVecEltBitWidth](
const Entry &E) {
22260 return (
unsigned)E.NumBits == NewVecEltBitWidth &&
22261 E.Producer->getValueSizeInBits(0) == NewVecEltBitWidth &&
22262 E.BitPos % NewVecEltBitWidth == 0;
22274 if (LegalOperations &&
22280 for (
const Entry &E : Leafs) {
22282 unsigned NewIndex = E.BitPos / NewVecEltBitWidth;
22284 "Creating out-of-bounds ISD::EXTRACT_VECTOR_ELT?");
22287 CombineTo(E.Producer, V);
22296 EVT ScalarVT =
N->getValueType(0);
22332 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
22343 "BUILD_VECTOR used for scalable vectors");
22344 unsigned IndexVal =
22352 if (ScalarVT == InEltVT)
22392 unsigned ExtractIndex = IndexC->getZExtValue();
22394 unsigned BCTruncElt = IsLE ? 0 : NumElts - 1;
22405 "Extract element and scalar to vector can't change element type "
22406 "from FP to integer.");
22407 unsigned XBitWidth =
X.getValueSizeInBits();
22408 BCTruncElt = IsLE ? 0 : XBitWidth / VecEltBitWidth - 1;
22413 if (ExtractIndex == BCTruncElt && XBitWidth > VecEltBitWidth) {
22414 assert(XBitWidth % VecEltBitWidth == 0 &&
22415 "Scalar bitwidth must be a multiple of vector element bitwidth");
22428 auto *Shuf = cast<ShuffleVectorSDNode>(VecOp);
22430 int OrigElt = Shuf->getMaskElt(IndexC->getZExtValue());
22438 if (OrigElt < (
int)NumElts) {
22442 OrigElt -= NumElts;
22458 if (!LegalOperations ||
22470 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
22471 Use->getOperand(0) == VecOp &&
22472 isa<ConstantSDNode>(Use->getOperand(1));
22476 auto *CstElt = cast<ConstantSDNode>(
Use->getOperand(1));
22477 if (CstElt->getAPIntValue().ult(NumElts))
22478 DemandedElts.
setBit(CstElt->getZExtValue());
22497 if (refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
N))
22503 bool BCNumEltsChanged =
false;
22518 BCNumEltsChanged =
true;
22524 if (!LegalOperations && !IndexC && VecOp.
hasOneUse() &&
22527 auto *
VecLoad = dyn_cast<LoadSDNode>(VecOp);
22528 if (VecLoad &&
VecLoad->isSimple())
22529 return scalarizeExtractedVectorLoad(
N, VecVT,
Index, VecLoad);
22534 if (!LegalOperations || !IndexC)
22540 int Elt = IndexC->getZExtValue();
22543 LN0 = cast<LoadSDNode>(VecOp);
22551 LN0 = cast<LoadSDNode>(VecOp.
getOperand(0));
22553 if (
auto *Shuf = dyn_cast<ShuffleVectorSDNode>(VecOp)) {
22564 if (BCNumEltsChanged)
22568 int Idx = (Elt > (int)NumElts) ? -1 : Shuf->getMaskElt(Elt);
22579 LN0 = cast<LoadSDNode>(VecOp);
22580 Elt = (
Idx < (int)NumElts) ?
Idx :
Idx - (int)NumElts;
22599 Index.getValueType());
22617 return scalarizeExtractedVectorLoad(
N, VecVT,
Index, LN0);
22631 unsigned NumInScalars =
N->getNumOperands();
22633 EVT VT =
N->getValueType(0);
22640 EVT SourceType = MVT::Other;
22641 bool AllAnyExt =
true;
22643 for (
unsigned i = 0; i != NumInScalars; ++i) {
22646 if (
In.isUndef())
continue;
22652 if (!ZeroExt && !AnyExt) {
22653 SourceType = MVT::Other;
22658 EVT InTy =
In.getOperand(0).getValueType();
22661 if (SourceType == MVT::Other)
22664 else if (InTy != SourceType) {
22666 SourceType = MVT::Other;
22671 AllAnyExt &= AnyExt;
22679 SourceType != MVT::Other &&
22680 llvm::has_single_bit<uint32_t>(OutScalarTy.
getSizeInBits()) &&
22681 llvm::has_single_bit<uint32_t>(SourceType.
getSizeInBits());
22695 assert(ElemRatio > 1 &&
"Invalid element size ratio");
22703 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
22707 Cast.
isUndef()) &&
"Invalid cast opcode");
22713 unsigned Index = isLE ? (i * ElemRatio) :
22714 (i * ElemRatio + (ElemRatio - 1));
22723 "Invalid vector size");
22746 EVT VT =
N->getValueType(0);
22765 unsigned NumInScalars =
N->getNumOperands();
22768 auto PeekThroughBitcast = [](
SDValue Op) {
22770 return Op.getOperand(0);
22776 for (
unsigned i = 0; i != NumInScalars; ++i) {
22777 SDValue In = PeekThroughBitcast(
N->getOperand(i));
22779 if (
In.isUndef())
continue;
22784 In = PeekThroughBitcast(
In.getOperand(0));
22795 SDValue part = PeekThroughBitcast(
In.getOperand(0));
22799 }
else if (Src != part) {
22804 SDValue ShiftAmtVal =
In.getOperand(1);
22805 if (!isa<ConstantSDNode>(ShiftAmtVal))
22808 uint64_t ShiftAmt =
In.getConstantOperandVal(1);
22811 if (ShiftAmt != i * ScalarTypeBitsize)
22817 if (!Src || Src.getValueType().getSizeInBits() != VT.
getSizeInBits())
22826 unsigned LeftIdx,
bool DidSplitVec) {
22829 EVT VT =
N->getValueType(0);
22834 unsigned ShuffleNumElems = NumElems;
22844 assert(InVT2Size <= InVT1Size &&
22845 "Inputs must be sorted to be in non-increasing vector size order.");
22849 if (InVT1 != VT || InVT2 != VT) {
22850 if ((VTSize % InVT1Size == 0) && InVT1 == InVT2) {
22853 unsigned NumConcats = VTSize / InVT1Size;
22854 assert(NumConcats >= 2 &&
"Concat needs at least two inputs!");
22856 ConcatOps[0] = VecIn1;
22857 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.
getUNDEF(InVT1);
22860 }
else if (InVT1Size == VTSize * 2) {
22872 Vec2Offset = NumElems;
22874 assert(InVT2Size <= InVT1Size &&
22875 "Second input is not going to be larger than the first one.");
22881 if (LegalOperations &&
22888 if (InVT1 != InVT2) {
22892 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
22894 ShuffleNumElems = NumElems * 2;
22896 }
else if (InVT2Size * 2 == VTSize && InVT1Size == VTSize) {
22898 ConcatOps[0] = VecIn2;
22900 }
else if (InVT1Size / VTSize > 1 && InVT1Size % VTSize == 0) {
22908 assert(InVT2Size <= InVT1Size &&
22909 "Second input is not going to be larger than the first one.");
22918 if (InVT1 != InVT2) {
22920 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
22922 ShuffleNumElems = InVT1Size / VTSize * NumElems;
22938 for (
unsigned i = 0; i != NumElems; ++i) {
22939 if (VectorMask[i] <= 0)
22942 unsigned ExtIndex =
N->getOperand(i).getConstantOperandVal(1);
22943 if (VectorMask[i] == (
int)LeftIdx) {
22944 Mask[i] = ExtIndex;
22945 }
else if (VectorMask[i] == (
int)LeftIdx + 1) {
22946 Mask[i] = Vec2Offset + ExtIndex;
22959 if (ShuffleNumElems > NumElems)
22972 for (
int i = 0; i != NumBVOps; ++i) {
23004 if (DestSize % SrcSize != 0 ||
23010 int ZextRatio = DestSize / SrcSize;
23011 int NumMaskElts = NumBVOps * ZextRatio;
23013 for (
int i = 0; i != NumMaskElts; ++i) {
23014 if (i / ZextRatio == ZextElt) {
23035 ZeroVec, ShufMask, DAG);
23042template <
typename R,
typename T>
23044 auto I =
find(Range, Val);
23045 if (
I == Range.end())
23046 return static_cast<decltype(std::distance(Range.begin(),
I))
>(-1);
23047 return std::distance(Range.begin(),
I);
23055 EVT VT =
N->getValueType(0);
23068 bool UsesZeroVector =
false;
23069 unsigned NumElems =
N->getNumOperands();
23081 for (
unsigned i = 0; i != NumElems; ++i) {
23091 UsesZeroVector =
true;
23099 !isa<ConstantSDNode>(
Op.getOperand(1)))
23101 SDValue ExtractedFromVec =
Op.getOperand(0);
23106 const APInt &ExtractIdx =
Op.getConstantOperandAPInt(1);
23124 VectorMask[i] =
Idx;
23128 if (VecIn.
size() < 2)
23135 bool DidSplitVec =
false;
23136 if (VecIn.
size() == 2) {
23137 unsigned MaxIndex = 0;
23138 unsigned NearestPow2 = 0;
23143 for (
unsigned i = 0; i < NumElems; i++) {
23144 if (VectorMask[i] <= 0)
23146 unsigned Index =
N->getOperand(i).getConstantOperandVal(1);
23147 IndexVec[i] =
Index;
23148 MaxIndex = std::max(MaxIndex,
Index);
23152 if (InVT.
isSimple() && NearestPow2 > 2 && MaxIndex < NearestPow2 &&
23153 NumElems * 2 < NearestPow2) {
23154 unsigned SplitSize = NearestPow2 / 2;
23167 DidSplitVec =
true;
23169 for (
unsigned i = 0; i < NumElems; i++) {
23170 if (VectorMask[i] <= 0)
23172 VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2;
23185 b.getValueType().getVectorNumElements();
23190 for (
int &SourceVectorIndex : VectorMask) {
23191 if (SourceVectorIndex <= 0)
23195 VecIn[SourceVectorIndex] == SortedVecIn[
Idx] &&
"Remapping failure");
23196 SourceVectorIndex =
Idx;
23199 VecIn = std::move(SortedVecIn);
23217 for (
unsigned In = 0, Len = (VecIn.
size() / 2); In < Len; ++In) {
23218 unsigned LeftIdx = 2 *
In + 1;
23219 SDValue VecLeft = VecIn[LeftIdx];
23221 (LeftIdx + 1) < VecIn.
size() ? VecIn[LeftIdx + 1] :
SDValue();
23223 if (
SDValue Shuffle = createBuildVecShuffle(
DL,
N, VectorMask, VecLeft,
23224 VecRight, LeftIdx, DidSplitVec))
23232 if (UsesZeroVector)
23237 if (Shuffles.
size() == 1)
23238 return Shuffles[0];
23241 for (
int &Vec : VectorMask)
23243 Vec = Shuffles.
size() - 1;
23245 Vec = (Vec - 1) / 2;
23259 if (Shuffles.
size() % 2)
23262 for (
unsigned CurSize = Shuffles.
size(); CurSize > 1; CurSize /= 2) {
23264 Shuffles[CurSize] = DAG.
getUNDEF(VT);
23267 for (
unsigned In = 0, Len = CurSize / 2;
In <
Len; ++
In) {
23274 L.use_empty() &&
L.getOperand(1).isUndef() &&
23275 L.getOperand(0).getValueType() ==
L.getValueType();
23276 if (IsLeftShuffle) {
23277 LMask = cast<ShuffleVectorSDNode>(
L.getNode())->getMask();
23278 L =
L.getOperand(0);
23283 R.use_empty() &&
R.getOperand(1).isUndef() &&
23284 R.getOperand(0).getValueType() ==
R.getValueType();
23285 if (IsRightShuffle) {
23286 RMask = cast<ShuffleVectorSDNode>(
R.getNode())->getMask();
23287 R =
R.getOperand(0);
23289 for (
unsigned I = 0;
I != NumElems; ++
I) {
23290 if (VectorMask[
I] ==
Left) {
23294 VectorMask[
I] =
In;
23295 }
else if (VectorMask[
I] ==
Right) {
23297 if (IsRightShuffle)
23298 Mask[
I] = RMask[
I] + NumElems;
23299 VectorMask[
I] =
In;
23306 return Shuffles[0];
23314 if (LegalOperations)
23317 EVT VT =
N->getValueType(0);
23319 bool FoundZeroExtend =
false;
23321 auto checkElem = [&](
SDValue Op) -> int64_t {
23322 unsigned Opc =
Op.getOpcode();
23327 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op.getOperand(0).getOperand(1)))
23328 return C->getZExtValue();
23336 int64_t
Offset = checkElem(Op0);
23340 unsigned NumElems =
N->getNumOperands();
23342 EVT InSVT =
In.getValueType().getScalarType();
23350 for (
unsigned i = 1; i != NumElems; ++i) {
23351 if ((
Offset + i) != checkElem(
N->getOperand(i)))
23369SDValue DAGCombiner::convertBuildVecZextToBuildVecWithZeros(
SDNode *
N) {
23378 EVT VT =
N->getValueType(0);
23379 EVT OpVT =
N->getOperand(0).getValueType();
23393 unsigned ActiveBits = 0;
23398 if (
auto *Cst = dyn_cast<ConstantSDNode>(
Op)) {
23399 unsigned OpActiveBits =
23400 Cst->getAPIntValue().trunc(EltBitwidth).getActiveBits();
23401 if (OpActiveBits == 0) {
23402 KnownZeroOps.setBit(
I.index());
23410 if (
I.index() != 0)
23416 unsigned CurrActiveBits =
23417 Op.getOperand(0).getValueSizeInBits().getFixedValue();
23418 assert(!ActiveBits &&
"Already encountered non-constant-zero operand?");
23419 ActiveBits = CurrActiveBits;
23421 if (2 * ActiveBits > EltBitwidth)
23426 if (ActiveBits == 0)
23431 EVT NewScalarIntVT, NewIntVT;
23432 std::optional<unsigned> Factor;
23437 assert(2 * ActiveBits <= EltBitwidth &&
23438 "We know that half or less bits of the element are active.");
23439 for (
unsigned Scale = EltBitwidth / ActiveBits; Scale >= 2; --Scale) {
23440 if (EltBitwidth % Scale != 0)
23442 unsigned ChunkBitwidth = EltBitwidth / Scale;
23443 assert(ChunkBitwidth >= ActiveBits &&
"As per starting point.");
23446 Scale *
N->getNumOperands());
23448 (LegalOperations &&
23466 assert(!
Op.isUndef() &&
"FIXME: after allowing UNDEF's, handle them here.");
23467 unsigned SrcOpIdx =
I.index();
23468 if (KnownZeroOps[SrcOpIdx]) {
23469 NewOps.
append(*Factor, ZeroOp);
23475 NewOps.
append(*Factor - 1, ZeroOp);
23484 EVT VT =
N->getValueType(0);
23498 if (!LegalOperations) {
23499 SDValue Splat = cast<BuildVectorSDNode>(
N)->getSplatValue();
23520 if (!LegalTypes && (
N->getNumOperands() > 1)) {
23525 if (
auto CNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
23530 int Offset = checkElem(Op0);
23531 for (
unsigned i = 0; i <
N->getNumOperands(); ++i) {
23532 if (
Offset + i != checkElem(
N->getOperand(i))) {
23542 ((
Offset %
N->getValueType(0).getVectorNumElements()) ==
23548 if (
SDValue V = convertBuildVecZextToZext(
N))
23551 if (
SDValue V = convertBuildVecZextToBuildVecWithZeros(
N))
23554 if (
SDValue V = reduceBuildVecExtToExtBuildVec(
N))
23557 if (
SDValue V = reduceBuildVecTruncToBitCast(
N))
23560 if (
SDValue V = reduceBuildVecToShuffle(
N))
23567 assert(!
V.isUndef() &&
"Splat of undef should have been handled earlier");
23576 EVT OpVT =
N->getOperand(0).getValueType();
23583 EVT VT =
N->getValueType(0);
23588 bool AnyInteger =
false;
23589 bool AnyFP =
false;
23592 !
Op.getOperand(0).getValueType().isVector())
23601 EVT LastOpVT = Ops.
back().getValueType();
23617 if (
Op.getValueType() == SVT)
23637 EVT VT =
N->getValueType(0);
23647 if (!FirstConcat) {
23648 SubVT =
Op.getOperand(0).getValueType();
23654 if (SubVT !=
Op.getOperand(0).getValueType())
23657 assert(FirstConcat &&
"Concat of all-undefs found");
23661 if (
Op.isUndef()) {
23665 ConcatOps.
append(
Op->op_begin(),
Op->op_end());
23675 EVT VT =
N->getValueType(0);
23676 EVT OpVT =
N->getOperand(0).getValueType();
23692 if (
Op.isUndef()) {
23693 Mask.append((
unsigned)NumOpElts, -1);
23702 int ExtIdx =
Op.getConstantOperandVal(1);
23711 Mask.append((
unsigned)NumOpElts, -1);
23722 if (0 == (NumExtElts % NumElts))
23723 ExtIdx /= (NumExtElts / NumElts);
23724 else if (0 == (NumElts % NumExtElts))
23725 ExtIdx *= (NumElts / NumExtElts);
23730 if (SV0.
isUndef() || SV0 == ExtVec) {
23732 for (
int i = 0; i != NumOpElts; ++i)
23733 Mask.push_back(i + ExtIdx);
23734 }
else if (SV1.
isUndef() || SV1 == ExtVec) {
23736 for (
int i = 0; i != NumOpElts; ++i)
23737 Mask.push_back(i + ExtIdx + NumElts);
23749 unsigned CastOpcode =
N->getOperand(0).getOpcode();
23750 switch (CastOpcode) {
23766 EVT SrcVT =
N->getOperand(0).getOperand(0).getValueType();
23774 if (
Op.getOpcode() != CastOpcode || !
Op.hasOneUse() ||
23775 Op.getOperand(0).getValueType() != SrcVT)
23783 EVT VT =
N->getValueType(0);
23788 switch (CastOpcode) {
23808 return DAG.
getNode(CastOpcode,
DL, VT, NewConcat);
23816 bool LegalOperations) {
23817 EVT VT =
N->getValueType(0);
23818 EVT OpVT =
N->getOperand(0).getValueType();
23823 if (
N->getNumOperands() != 2)
23828 (LegalOperations &&
23839 if (
auto *CurSVN = dyn_cast<ShuffleVectorSDNode>(
Op);
23840 CurSVN && CurSVN->getOperand(1).
isUndef() &&
N->isOnlyUserOf(CurSVN) &&
23843 return !Op.isUndef() &&
23844 (Op.getNode() == CurSVN || is_contained(CurSVN->ops(), Op));
23867 std::iota(FirstShufOpIdentityMask.
begin(), FirstShufOpIdentityMask.
end(), 0);
23868 std::iota(SecondShufOpIdentityMask.
begin(), SecondShufOpIdentityMask.
end(),
23875 assert(!
Op.isUndef() &&
"Not expecting to concatenate UNDEF.");
23876 if (
Op.getNode() == SVN) {
23897 std::array<SDValue, 2> ShufOps;
23898 for (
auto I :
zip(SVN->
ops(), ShufOps)) {
23900 SDValue &NewShufOp = std::get<1>(
I);
23906 ShufOpParts[0] = ShufOp;
23916 if (
N->getNumOperands() == 1)
23917 return N->getOperand(0);
23920 EVT VT =
N->getValueType(0);
23926 [](
const SDValue &
Op) { return Op.isUndef(); })) {
23928 assert(
In.getValueType().isVector() &&
"Must concat vectors");
23938 !(LegalDAG &&
In.getValueType().isScalableVector())) {
23939 unsigned NumOps =
N->getNumOperands() *
In.getNumOperands();
23941 Ops.resize(NumOps, DAG.
getUNDEF(Ops[0].getValueType()));
23951 EVT SVT =
Scalar.getValueType().getVectorElementType();
23952 if (SVT ==
Scalar.getOperand(0).getValueType())
23957 if (!
Scalar.getValueType().isVector() &&
In.hasOneUse()) {
23976 if (VNTNumElms < 2)
23992 auto IsBuildVectorOrUndef = [](
const SDValue &
Op) {
24003 bool FoundMinVT =
false;
24006 EVT OpSVT =
Op.getOperand(0).getValueType();
24007 MinVT = (!FoundMinVT || OpSVT.
bitsLE(MinVT)) ? OpSVT : MinVT;
24010 assert(FoundMinVT &&
"Concat vector type mismatch");
24014 EVT OpVT =
Op.getValueType();
24023 Opnds.
append(
Op->op_begin(),
Op->op_begin() + NumElts);
24025 for (
unsigned i = 0; i != NumElts; ++i)
24033 "Concat vector type mismatch");
24056 N, DAG, TLI, LegalTypes, LegalOperations))
24066 unsigned PartNumElem =
24067 N->getOperand(0).getValueType().getVectorMinNumElements();
24069 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
24080 if (SingleSource.
getNode()) {
24081 if (
Op.getOperand(0) != SingleSource)
24084 SingleSource =
Op.getOperand(0);
24094 unsigned IdentityIndex = i * PartNumElem;
24095 if (
Op.getConstantOperandAPInt(1) != IdentityIndex)
24100 return SingleSource;
24109 V.getOperand(1).getValueType() == SubVT && V.getOperand(2) ==
Index) {
24110 return V.getOperand(1);
24112 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
24114 V.getOperand(0).getValueType() == SubVT &&
24117 return V.getOperand(SubIdx);
24124 bool LegalOperations) {
24127 unsigned BinOpcode = BinOp.
getOpcode();
24133 if (VecVT != Bop0.
getValueType() || VecVT != Bop1.getValueType())
24147 if (!Sub0 || !Sub1)
24153 return DAG.
getNode(BinOpcode,
SDLoc(Extract), SubVT, Sub0, Sub1,
24160 bool LegalOperations) {
24168 auto *ExtractIndexC = dyn_cast<ConstantSDNode>(Extract->
getOperand(1));
24169 if (!ExtractIndexC)
24185 if (
C &&
C->getValueAPF().isNegZero())
24198 unsigned ExtractIndex = ExtractIndexC->getZExtValue();
24200 "Extract index is not a multiple of the vector length.");
24205 if (WideWidth % NarrowWidth != 0)
24210 unsigned NarrowingRatio = WideWidth / NarrowWidth;
24212 if (WideNumElts % NarrowingRatio != 0)
24217 WideNumElts / NarrowingRatio);
24244 if (NarrowingRatio != 2)
24259 return V.getOperand(ConcatOpNum);
24265 if (SubVecL || SubVecR) {
24296 auto *Ld = dyn_cast<LoadSDNode>(Extract->
getOperand(0));
24316 assert(
Index % NumElts == 0 &&
"The extract subvector index is not a "
24317 "multiple of the result's element count");
24336 if (
Offset.isScalable()) {
24359 bool LegalOperations) {
24361 "Must only be called on EXTRACT_SUBVECTOR's");
24366 EVT NarrowVT =
N->getValueType(0);
24372 auto *WideShuffleVector = dyn_cast<ShuffleVectorSDNode>(N0);
24373 if (!WideShuffleVector)
24377 if (!WideShuffleVector->hasOneUse())
24381 if (LegalOperations &&
24385 uint64_t FirstExtractedEltIdx =
N->getConstantOperandVal(1);
24387 assert((FirstExtractedEltIdx % NumEltsExtracted) == 0 &&
24388 "Extract index is not a multiple of the output vector length.");
24393 NewMask.
reserve(NumEltsExtracted);
24395 DemandedSubvectors;
24398 for (
int M : WideShuffleVector->getMask().slice(FirstExtractedEltIdx,
24399 NumEltsExtracted)) {
24400 assert((M >= -1) && (M < (2 * WideNumElts)) &&
24401 "Out-of-bounds shuffle mask?");
24410 int WideShufOpIdx = M / WideNumElts;
24412 int OpEltIdx = M % WideNumElts;
24414 assert((OpEltIdx + WideShufOpIdx * WideNumElts) == M &&
24415 "Shuffle mask vector decomposition failure.");
24418 int OpSubvecIdx = OpEltIdx / NumEltsExtracted;
24420 int OpEltIdxInSubvec = OpEltIdx % NumEltsExtracted;
24422 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted) == OpEltIdx &&
24423 "Shuffle mask subvector decomposition failure.");
24425 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted +
24426 WideShufOpIdx * WideNumElts) == M &&
24427 "Shuffle mask full decomposition failure.");
24429 SDValue Op = WideShuffleVector->getOperand(WideShufOpIdx);
24431 if (
Op.isUndef()) {
24437 const std::pair<SDValue, int> DemandedSubvector =
24438 std::make_pair(
Op, OpSubvecIdx);
24440 if (DemandedSubvectors.insert(DemandedSubvector)) {
24441 if (DemandedSubvectors.size() > 2)
24444 int Index = NumEltsExtracted * OpSubvecIdx;
24452 getFirstIndexOf(DemandedSubvectors.getArrayRef(), DemandedSubvector);
24453 assert((NewOpIdx == 0 || NewOpIdx == 1) &&
"Unexpected operand index.");
24455 int AdjM = OpEltIdxInSubvec + NewOpIdx * NumEltsExtracted;
24458 assert(NewMask.
size() == (
unsigned)NumEltsExtracted &&
"Produced bad mask.");
24459 assert(DemandedSubvectors.size() <= 2 &&
24460 "Should have ended up demanding at most two subvectors.");
24463 if (DemandedSubvectors.empty())
24469 any_of(NewMask, [](
int M) {
return M < 0; }))
24470 for (
auto &DemandedSubvector : DemandedSubvectors)
24471 if (DemandedSubvector.second != 0)
24485 for (
const std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>
24486 &DemandedSubvector : DemandedSubvectors) {
24488 int Index = NumEltsExtracted * DemandedSubvector.second;
24491 DemandedSubvector.first, IndexC));
24494 "Should end up with either one or two ops");
24497 if (NewOps.
size() == 1)
24504 EVT NVT =
N->getValueType(0);
24506 uint64_t ExtIdx =
N->getConstantOperandVal(1);
24521 V.getConstantOperandVal(1)) &&
24541 unsigned InsIdx =
V.getConstantOperandVal(2);
24543 if (InsIdx <= ExtIdx && (ExtIdx + NumSubElts) <= (InsIdx + NumInsElts) &&
24546 V.getValueType().isFixedLengthVector())
24554 V.getOperand(0).getValueType().isVector() &&
24559 unsigned DestNumElts =
V.getValueType().getVectorMinNumElements();
24560 if ((SrcNumElts % DestNumElts) == 0) {
24561 unsigned SrcDestRatio = SrcNumElts / DestNumElts;
24568 V.getOperand(0), NewIndex);
24572 if ((DestNumElts % SrcNumElts) == 0) {
24573 unsigned DestSrcRatio = DestNumElts / SrcNumElts;
24578 if ((ExtIdx % DestSrcRatio) == 0) {
24579 unsigned IndexValScaled = ExtIdx / DestSrcRatio;
24586 V.getOperand(0), NewIndex);
24594 V.getOperand(0), NewIndex);
24604 EVT ConcatSrcVT =
V.getOperand(0).getValueType();
24606 "Concat and extract subvector do not change element type");
24607 assert((ExtIdx % ExtNumElts) == 0 &&
24608 "Extract index is not a multiple of the input vector length.");
24611 unsigned ConcatOpIdx = ExtIdx / ConcatSrcNumElts;
24617 return V.getOperand(ConcatOpIdx);
24625 ConcatSrcNumElts % ExtNumElts == 0) {
24626 unsigned NewExtIdx = ExtIdx - ConcatOpIdx * ConcatSrcNumElts;
24627 assert(NewExtIdx + ExtNumElts <= ConcatSrcNumElts &&
24628 "Trying to extract from >1 concat operand?");
24629 assert(NewExtIdx % ExtNumElts == 0 &&
24630 "Extract index is not a multiple of the input vector length.");
24633 V.getOperand(ConcatOpIdx), NewIndexC);
24645 EVT InVT =
V.getValueType();
24649 if (ExtractSize % EltSize == 0) {
24650 unsigned NumElems = ExtractSize / EltSize;
24653 NumElems == 1 ? EltVT
24661 if (NumElems == 1) {
24662 SDValue Src =
V->getOperand(IdxVal);
24663 if (EltVT != Src.getValueType())
24679 EVT SmallVT =
V.getOperand(1).getValueType();
24680 if (!NVT.
bitsEq(SmallVT))
24688 uint64_t InsIdx =
V.getConstantOperandVal(2);
24698 DAG.
getBitcast(
N->getOperand(0).getValueType(),
V.getOperand(0)),
24728 unsigned HalfNumElts = NumElts / 2;
24731 for (
unsigned i = 0; i != NumElts; ++i) {
24735 if ((Mask[i] % NumElts) >= HalfNumElts)
24737 int M = Mask[i] < (int)NumElts ? Mask[i] : Mask[i] - (
int)HalfNumElts;
24738 if (i < HalfNumElts)
24741 Mask1[i - HalfNumElts] = M;
24764 EVT VT =
N->getValueType(0);
24775 unsigned NumConcats = NumElts / NumElemsPerConcat;
24777 auto IsUndefMaskElt = [](
int i) {
return i == -1; };
24782 if (NumElemsPerConcat * 2 == NumElts && N1.
isUndef() &&
24783 llvm::all_of(Mask.slice(NumElemsPerConcat, NumElemsPerConcat),
24787 Mask.slice(0, NumElemsPerConcat));
24794 for (
unsigned I = 0;
I != NumConcats; ++
I) {
24795 unsigned Begin =
I * NumElemsPerConcat;
24796 ArrayRef<int> SubMask = Mask.slice(Begin, NumElemsPerConcat);
24805 for (
int i = 0; i != (int)NumElemsPerConcat; ++i) {
24806 if (IsUndefMaskElt(SubMask[i]))
24808 if ((SubMask[i] % (
int)NumElemsPerConcat) != i)
24810 int EltOpIdx = SubMask[i] / NumElemsPerConcat;
24811 if (0 <= OpIdx && EltOpIdx != OpIdx)
24815 assert(0 <= OpIdx &&
"Unknown concat_vectors op");
24869 bool IsSplat =
false;
24870 auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
24871 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
24873 if (
SDValue Splat0 = BV0->getSplatValue())
24874 IsSplat = (Splat0 == BV1->getSplatValue());
24878 for (
int M : SVN->
getMask()) {
24881 int Idx = M < (int)NumElts ? M : M - NumElts;
24882 SDValue &S = (M < (int)NumElts ? N0 : N1);
24884 Op = S.getOperand(
Idx);
24899 if (!IsSplat && !DuplicateOps.
insert(
Op).second)
24910 SVT = (SVT.
bitsLT(
Op.getValueType()) ?
Op.getValueType() : SVT);
24926 unsigned Opcode,
EVT VT, std::function<
bool(
unsigned)>
Match,
24928 bool LegalOperations) {
24933 return std::nullopt;
24941 for (
unsigned Scale = 2; Scale < NumElts; Scale *= 2) {
24943 if (NumElts % Scale != 0)
24957 return std::nullopt;
24966 bool LegalOperations) {
24976 Mask = SVN->
getMask()](
unsigned Scale) {
24977 for (
unsigned i = 0; i != NumElts; ++i) {
24980 if ((i % Scale) == 0 && Mask[i] == (int)(i / Scale))
24992 Opcode, VT, isAnyExtend, DAG, TLI,
true, LegalOperations);
25004 bool LegalOperations) {
25005 bool LegalTypes =
true;
25017 auto ForEachDecomposedIndice = [NumElts, &Mask](
auto Fn) {
25018 for (
int &Indice : Mask) {
25021 int OpIdx = (
unsigned)Indice < NumElts ? 0 : 1;
25022 int OpEltIdx = (
unsigned)Indice < NumElts ? Indice : Indice - NumElts;
25023 Fn(Indice, OpIdx, OpEltIdx);
25028 std::array<APInt, 2> OpsDemandedElts;
25029 for (
APInt &OpDemandedElts : OpsDemandedElts)
25031 ForEachDecomposedIndice(
25032 [&OpsDemandedElts](
int &Indice,
int OpIdx,
int OpEltIdx) {
25033 OpsDemandedElts[OpIdx].setBit(OpEltIdx);
25037 std::array<APInt, 2> OpsKnownZeroElts;
25038 for (
auto I :
zip(SVN->
ops(), OpsDemandedElts, OpsKnownZeroElts))
25046 bool HadZeroableElts =
false;
25047 ForEachDecomposedIndice([&OpsKnownZeroElts, &HadZeroableElts](
25048 int &Indice,
int OpIdx,
int OpEltIdx) {
25049 if (OpsKnownZeroElts[OpIdx][OpEltIdx]) {
25051 HadZeroableElts =
true;
25059 if (!HadZeroableElts)
25067 Mask.size() % ScaledMask.
size() == 0 &&
"Unexpected mask widening.");
25068 int Prescale = Mask.size() / ScaledMask.
size();
25070 NumElts = ScaledMask.
size();
25071 EltSizeInBits *= Prescale;
25083 auto isZeroExtend = [NumElts, &ScaledMask](
unsigned Scale) {
25084 assert(Scale >= 2 && Scale <= NumElts && NumElts % Scale == 0 &&
25085 "Unexpected mask scaling factor.");
25087 for (
unsigned SrcElt = 0, NumSrcElts = NumElts / Scale;
25088 SrcElt != NumSrcElts; ++SrcElt) {
25091 assert(MaskChunk.
size() == Scale &&
"Unexpected mask size.");
25092 Mask = Mask.drop_front(MaskChunk.
size());
25095 if (
int FirstIndice = MaskChunk[0]; (
unsigned)FirstIndice != SrcElt)
25100 [](
int Indice) { return Indice == -2; }))
25103 assert(Mask.empty() &&
"Did not process the whole mask?");
25108 for (
bool Commuted : {
false,
true}) {
25113 Opcode, PrescaledVT, isZeroExtend, DAG, TLI, LegalTypes,
25149 if (ExtDstSizeInBits % ExtSrcSizeInBits != 0)
25151 unsigned ExtScale = ExtDstSizeInBits / ExtSrcSizeInBits;
25156 auto isTruncate = [&Mask, &NumElts](
unsigned Scale) {
25157 for (
unsigned i = 0; i != NumElts; ++i) {
25160 if ((i * Scale) < NumElts && Mask[i] == (int)(i * Scale))
25170 if (EltSizeInBits != ExtSrcSizeInBits)
25175 if (isTruncate(ExtScale))
25198 APInt DemandedElts(NumElts, 0);
25202 assert((
unsigned)
Idx < NumElts &&
"Out-of-bounds shuffle indice?");
25205 assert(DemandedElts.
popcount() > 1 &&
"Is a splat shuffle already?");
25210 std::optional<unsigned> MinNonUndefIdx;
25212 if (
Idx < 0 || UndefElts[
Idx])
25214 MinNonUndefIdx = std::min<unsigned>(
Idx, MinNonUndefIdx.value_or(~0U));
25216 if (!MinNonUndefIdx)
25218 assert(*MinNonUndefIdx < NumElts &&
"Expected valid element index.");
25221 for (
int &
Idx : SplatMask) {
25226 Idx = UndefElts[
Idx] ? -1 : *MinNonUndefIdx;
25228 assert(SplatMask != Shuf->
getMask() &&
"Expected mask to change!");
25246 assert(ShufMask.
size() == SplatMask.
size() &&
"Mask length mismatch");
25264 auto CanSimplifyToExistingSplat = [](
ArrayRef<int> UserMask,
25266 for (
unsigned i = 0, e = UserMask.
size(); i != e; ++i)
25267 if (UserMask[i] != -1 && SplatMask[i] == -1 &&
25268 SplatMask[UserMask[i]] != -1)
25272 if (CanSimplifyToExistingSplat(ShufMask, SplatMask))
25278 for (
int Idx : ShufMask)
25291 bool LegalOperations) {
25308 if (VTLanes <= InLanes || VTLanes % InLanes != 0 ||
25309 (LegalOperations &&
25312 int Factor = VTLanes / InLanes;
25339 auto *InnerShuf = dyn_cast<ShuffleVectorSDNode>(OuterShuf->
getOperand(0));
25340 if (!InnerShuf || !InnerShuf->getOperand(1).isUndef())
25345 unsigned NumElts = OuterMask.
size();
25346 assert(NumElts == InnerMask.
size() &&
"Mask length mismatch");
25348 int SplatIndex = -1;
25349 for (
unsigned i = 0; i != NumElts; ++i) {
25351 int OuterMaskElt = OuterMask[i];
25352 if (OuterMaskElt == -1)
25356 int InnerMaskElt = InnerMask[OuterMaskElt];
25357 if (InnerMaskElt == -1)
25361 if (SplatIndex == -1)
25362 SplatIndex = InnerMaskElt;
25365 if (SplatIndex != InnerMaskElt)
25368 CombinedMask[i] = InnerMaskElt;
25370 assert((
all_of(CombinedMask, [](
int M) {
return M == -1; }) ||
25372 "Expected a splat mask");
25376 assert(VT == InnerShuf->getValueType(0) &&
"Expected matching shuffle types");
25381 InnerShuf->getOperand(1), CombinedMask);
25389 int MaskSize = Mask.size();
25390 int EltFromOp0 = -1;
25395 for (
int i = 0; i != MaskSize; ++i) {
25396 if (Mask[i] >= 0 && Mask[i] < MaskSize) {
25398 if (EltFromOp0 != -1)
25401 }
else if (Mask[i] != i + MaskSize) {
25421 if (ShufOp0Index == -1) {
25425 if (ShufOp0Index == -1)
25429 Mask = CommutedMask;
25437 assert(Mask[ShufOp0Index] >= 0 && Mask[ShufOp0Index] < (
int)Mask.size() &&
25438 "Shuffle mask value must be from operand 0");
25442 auto *InsIndexC = dyn_cast<ConstantSDNode>(Op0.
getOperand(2));
25443 if (!InsIndexC || InsIndexC->getSExtValue() != Mask[ShufOp0Index])
25467 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Shuf->
getOperand(0));
25473 for (
int i = 0, e = (
int)Mask.size(); i != e; ++i) {
25477 assert(Mask[i] >= 0 && Mask[i] < e &&
"Unexpected shuffle mask value");
25481 if (Mask0[Mask[i]] != Mask0[i])
25490 EVT VT =
N->getValueType(0);
25515 bool Changed =
false;
25517 for (
unsigned i = 0; i != NumElts; ++i) {
25519 if (
Idx >= (
int)NumElts) {
25568 if (
auto *
Idx = dyn_cast<ConstantSDNode>(N0.
getOperand(2)))
25569 if (
Idx->getAPIntValue() == SplatIndex)
25595 SDValue ConvInput =
V->getOperand(0);
25602 assert(
V->getNumOperands() == NumElts &&
25603 "BUILD_VECTOR has wrong number of operands");
25605 bool AllSame =
true;
25606 for (
unsigned i = 0; i != NumElts; ++i) {
25607 if (!
V->getOperand(i).isUndef()) {
25608 Base =
V->getOperand(i);
25613 if (!
Base.getNode())
25615 for (
unsigned i = 0; i != NumElts; ++i) {
25616 if (
V->getOperand(i) !=
Base) {
25626 SDValue Splatted =
V->getOperand(SplatIndex);
25632 if (
V->getValueType(0) != VT)
25671 int HalfNumElts = (int)NumElts / 2;
25673 for (
unsigned i = 0; i != NumElts; ++i) {
25675 if (
Idx >= HalfNumElts) {
25676 assert(
Idx < (
int)NumElts &&
"Shuffle mask chooses undef op");
25677 Idx -= HalfNumElts;
25698 EVT SubVT =
RHS.getOperand(0).getValueType();
25699 int NumSubVecs =
RHS.getNumOperands();
25701 assert((NumElts % NumSubElts) == 0 &&
"Subvector mismatch");
25706 if (
all_of(Mask, [NumElts](
int M) {
return M < (int)NumElts; }))
25712 for (
int SubVec = 0; SubVec != NumSubVecs; ++SubVec) {
25713 for (
int SubIdx = 0; SubIdx != (int)NumElts; SubIdx += NumSubElts) {
25715 std::iota(InsertionMask.begin(), InsertionMask.end(), 0);
25718 std::iota(InsertionMask.begin() + SubIdx,
25719 InsertionMask.begin() + SubIdx + NumSubElts,
25720 NumElts + (SubVec * NumSubElts));
25723 bool MatchingShuffle =
true;
25724 for (
int i = 0; i != (int)NumElts; ++i) {
25725 int ExpectIdx = InsertionMask[i];
25726 int ActualIdx =
Mask[i];
25727 if (0 <= ActualIdx && ExpectIdx != ActualIdx) {
25728 MatchingShuffle =
false;
25733 if (MatchingShuffle)
25735 RHS.getOperand(SubVec),
25743 if (
SDValue InsertN1 = ShuffleToInsert(N0, N1, Mask))
25748 if (
SDValue InsertN0 = ShuffleToInsert(N1, N0, CommuteMask))
25756 bool IsInLaneMask =
true;
25761 for (
int I = 0;
I != (int)NumElts; ++
I) {
25765 ClearMask[
I] =
M ==
I ?
I : (
I + NumElts);
25766 IsInLaneMask &= (
M ==
I) || (M == (
int)(
I + NumElts));
25768 APInt &Demanded =
M < (int)NumElts ? DemandedLHS : DemandedRHS;
25769 Demanded.
setBit(M % NumElts);
25773 if (!IsInLaneMask && (!DemandedLHS.
isZero() || !DemandedRHS.
isZero()) &&
25788 for (
int I = 0;
I != (int)NumElts; ++
I)
25790 AndMask[
I] =
Mask[
I] ==
I ? AllOnesElt : ZeroElt;
25827 EVT ScaleVT = SVT.
bitsLT(InnerSVT) ? VT : InnerVT;
25845 for (
int M : OuterMask)
25846 NewMask.
push_back(M < 0 ? -1 : InnerMask[M]);
25877 auto MergeInnerShuffle =
25884 if (OtherSVN->isSplat())
25890 for (
unsigned i = 0; i != NumElts; ++i) {
25891 int Idx = SVN->getMaskElt(i);
25899 Idx = (
Idx < (int)NumElts) ? (
Idx + NumElts) : (
Idx - NumElts);
25902 if (
Idx < (
int)NumElts) {
25905 Idx = OtherSVN->getMaskElt(
Idx);
25911 CurrentVec = (
Idx < (int)NumElts) ? OtherSVN->
getOperand(0)
25920 Mask.push_back(-1);
25927 if (!SV0.getNode() || SV0 == CurrentVec) {
25934 if (!SV1.getNode() || SV1 == CurrentVec) {
25938 Mask.push_back(
Idx + NumElts);
25944 if (
auto *CurrentSVN = dyn_cast<ShuffleVectorSDNode>(CurrentVec)) {
25945 int InnerIdx = CurrentSVN->getMaskElt(
Idx);
25946 if (InnerIdx < 0) {
25947 Mask.push_back(-1);
25950 SDValue InnerVec = (InnerIdx < (int)NumElts)
25954 Mask.push_back(-1);
25957 InnerIdx %= NumElts;
25958 if (InnerVec == SV0) {
25959 Mask.push_back(InnerIdx);
25962 if (InnerVec == SV1) {
25963 Mask.push_back(InnerIdx + NumElts);
25982 if (TLI.isShuffleMaskLegal(Mask, VT))
25987 return TLI.isShuffleMaskLegal(Mask, VT);
25999 assert(N1->getOperand(0).getValueType() == VT &&
26000 "Shuffle types don't match");
26004 bool HasSameOp0 = N0 == SV0;
26005 bool IsSV1Undef = SV1.
isUndef();
26006 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
26015 cast<ShuffleVectorSDNode>(N0)->isSplat() &&
26016 !cast<ShuffleVectorSDNode>(N1)->isSplat()) {
26027 for (
int i = 0; i != 2; ++i) {
26029 N->isOnlyUserOf(
N->getOperand(i).getNode())) {
26032 auto *OtherSV = cast<ShuffleVectorSDNode>(
N->getOperand(i));
26033 assert(OtherSV->getOperand(0).getValueType() == VT &&
26034 "Shuffle types don't match");
26038 if (MergeInnerShuffle(i != 0, SVN, OtherSV,
N->getOperand(1 - i), TLI,
26046 SV1 ? SV1 : DAG.
getUNDEF(VT), Mask);
26056 if (TLI.isBinOp(SrcOpcode) &&
N->isOnlyUserOf(N0.
getNode()) &&
26058 (SrcOpcode == N1.getOpcode() &&
N->isOnlyUserOf(N1.getNode())))) {
26076 SDValue InnerN = Commute ? N1 : N0;
26077 SDValue Op0 = LeftOp ? Op00 : Op01;
26078 SDValue Op1 = LeftOp ? Op10 : Op11;
26083 auto *SVN0 = dyn_cast<ShuffleVectorSDNode>(Op0);
26085 MergeInnerShuffle(Commute, SVN, SVN0, Op1, TLI, SV0, SV1,
26087 (
llvm::any_of(SVN0->getMask(), [](
int M) { return M < 0; }) ||
26093 bool MergedLeft =
false;
26096 if (CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask,
true,
false) ||
26097 CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask,
true,
true)) {
26100 LeftMask.
assign(SVN->getMask().begin(), SVN->getMask().end());
26101 LeftSV0 = Op00, LeftSV1 = Op10;
26104 bool MergedRight =
false;
26107 if (CanMergeInnerShuffle(RightSV0, RightSV1, RightMask,
false,
false) ||
26108 CanMergeInnerShuffle(RightSV0, RightSV1, RightMask,
false,
true)) {
26109 MergedRight =
true;
26111 RightMask.
assign(SVN->getMask().begin(), SVN->getMask().end());
26112 RightSV0 = Op01, RightSV1 = Op11;
26115 if (MergedLeft || MergedRight) {
26118 VT,
DL, LeftSV0 ? LeftSV0 : DAG.
getUNDEF(VT),
26119 LeftSV1 ? LeftSV1 : DAG.
getUNDEF(VT), LeftMask);
26121 VT,
DL, RightSV0 ? RightSV0 : DAG.
getUNDEF(VT),
26122 RightSV1 ? RightSV1 : DAG.
getUNDEF(VT), RightMask);
26123 return DAG.
getNode(SrcOpcode,
DL, VT, LHS, RHS);
26144 EVT VT =
N->getValueType(0);
26154 unsigned Opcode =
Scalar.getOpcode();
26156 if (
Scalar.hasOneUse() &&
Scalar->getNumValues() == 1 &&
26157 TLI.isBinOp(Opcode) &&
Scalar.getValueType() == VecEltVT &&
26158 Scalar.getOperand(0).getValueType() == VecEltVT &&
26159 Scalar.getOperand(1).getValueType() == VecEltVT &&
26160 Scalar->isOnlyUserOf(
Scalar.getOperand(0).getNode()) &&
26161 Scalar->isOnlyUserOf(
Scalar.getOperand(1).getNode()) &&
26166 for (
int i : {0, 1}) {
26170 auto *
C = dyn_cast<ConstantSDNode>(
Scalar.getOperand(i ? 0 : 1));
26177 if (TLI.isShuffleMaskLegal(ShufMask, VT)) {
26192 !
Scalar.getOperand(0).getValueType().isFixedLengthVector())
26196 if (VecEltVT !=
Scalar.getValueType() &&
26202 auto *ExtIndexC = dyn_cast<ConstantSDNode>(
Scalar.getOperand(1));
26210 if (VecEltVT == SrcVT.
getScalarType() && VTNumElts <= SrcNumElts) {
26213 Mask[0] = ExtIndexC->getZExtValue();
26214 SDValue LegalShuffle = TLI.buildLegalVectorShuffle(
26221 return LegalShuffle;
26224 if (VTNumElts != SrcNumElts) {
26237 EVT VT =
N->getValueType(0);
26241 uint64_t InsIdx =
N->getConstantOperandVal(2);
26340 if ((N0.
isUndef() || N0SrcSVT == N1SrcSVT) &&
26352 }
else if ((N1SrcSVT.
getSizeInBits() % EltSizeInBits) == 0) {
26375 if (InsIdx < OtherIdx) {
26379 AddToWorklist(NewOp.
getNode());
26393 Ops[InsIdx / Factor] = N1;
26415 auto Op =
N->getOpcode();
26417 "opcode should be FP16_TO_FP or BF16_TO_FP.");
26444 return visitFP16_TO_FP(
N);
26450 unsigned Opcode =
N->getOpcode();
26468 if (!TLI.isOperationLegalOrCustom(Opcode, VT) &&
26469 TLI.isOperationLegalOrCustom(NewOpcode, VT) &&
26471 return DAG.
getNode(NewOpcode,
SDLoc(
N),
N->getValueType(0), N0);
26484 return DAG.
getNode(Opcode,
SDLoc(
N),
N->getValueType(0), Subvec);
26494 if (
SDValue Fused = visitFSUBForFMACombine<VPMatchContext>(
N)) {
26495 AddToWorklist(Fused.getNode());
26503 if (
N->getOpcode() == ISD::VP_GATHER)
26504 if (
SDValue SD = visitVPGATHER(
N))
26507 if (
N->getOpcode() == ISD::VP_SCATTER)
26508 if (
SDValue SD = visitVPSCATTER(
N))
26511 if (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD)
26512 if (
SDValue SD = visitVP_STRIDED_LOAD(
N))
26515 if (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE)
26516 if (
SDValue SD = visitVP_STRIDED_STORE(
N))
26522 bool AreAllEltsDisabled =
false;
26526 AreAllEltsDisabled |=
26530 if (!AreAllEltsDisabled) {
26531 switch (
N->getOpcode()) {
26533 return visitVP_FADD(
N);
26535 return visitVP_FSUB(
N);
26537 return visitFMA<VPMatchContext>(
N);
26538 case ISD::VP_SELECT:
26539 return visitVP_SELECT(
N);
26546 return DAG.
getUNDEF(
N->getValueType(0));
26550 if (
const auto *MemSD = dyn_cast<MemSDNode>(
N)) {
26551 if (MemSD->writeMem())
26552 return MemSD->getChain();
26553 return CombineTo(
N, DAG.
getUNDEF(
N->getValueType(0)), MemSD->getChain());
26558 return N->getOperand(0);
26566 EVT MemVT = cast<FPStateAccessSDNode>(
N)->getMemoryVT();
26571 for (
auto *U :
Ptr->uses()) {
26574 if (
auto *Ld = dyn_cast<LoadSDNode>(U)) {
26575 if (LdNode && LdNode != Ld)
26591 if (
U.getResNo() == 0) {
26592 if (
auto *St = dyn_cast<StoreSDNode>(
U.getUser())) {
26610 CombineTo(StNode, Res,
false);
26617 EVT MemVT = cast<FPStateAccessSDNode>(
N)->getMemoryVT();
26621 for (
auto *U :
Ptr->uses()) {
26624 if (
auto *St = dyn_cast<StoreSDNode>(U)) {
26625 if (StNode && StNode != St)
26640 auto *LdNode = dyn_cast<LoadSDNode>(StValue);
26661 EVT VT =
N->getValueType(0);
26668 if (LegalOperations)
26674 EVT RVT =
RHS.getValueType();
26675 unsigned NumElts =
RHS.getNumOperands();
26680 auto BuildClearMask = [&](
int Split) {
26681 int NumSubElts = NumElts *
Split;
26685 for (
int i = 0; i != NumSubElts; ++i) {
26686 int EltIdx = i /
Split;
26687 int SubIdx = i %
Split;
26697 if (
auto *Cst = dyn_cast<ConstantSDNode>(Elt))
26698 Bits = Cst->getAPIntValue();
26699 else if (
auto *CstFP = dyn_cast<ConstantFPSDNode>(Elt))
26706 Bits =
Bits.extractBits(NumSubBits, (Split - SubIdx - 1) * NumSubBits);
26708 Bits =
Bits.extractBits(NumSubBits, SubIdx * NumSubBits);
26710 if (
Bits.isAllOnes())
26712 else if (Bits == 0)
26721 if (!TLI.isVectorClearMaskLegal(Indices, ClearVT))
26735 for (
int Split = 1;
Split <= MaxSplit; ++
Split)
26737 if (
SDValue S = BuildClearMask(Split))
26749 unsigned Opcode =
N->getOpcode();
26750 EVT VT =
N->getValueType(0);
26757 int Index0, Index1;
26764 if (!Src0 || !Src1 || Index0 != Index1 ||
26784 Ops[Index0] = ScalarBO;
26794 EVT VT =
N->getValueType(0);
26795 assert(VT.
isVector() &&
"SimplifyVCastOp only works on vectors!");
26797 unsigned Opcode =
N->getOpcode();
26827 EVT VT =
N->getValueType(0);
26828 assert(VT.
isVector() &&
"SimplifyVBinOp only works on vectors!");
26832 unsigned Opcode =
N->getOpcode();
26843 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(LHS);
26844 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(RHS);
26845 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
26846 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
26849 RHS.getOperand(0), Flags);
26861 Shuf0->hasOneUse() && Shuf0->getOperand(1).isUndef() &&
26870 Shuf1->hasOneUse() && Shuf1->getOperand(1).isUndef() &&
26886 LHS.getOperand(2) ==
RHS.getOperand(2) &&
26891 EVT NarrowVT =
X.getValueType();
26892 if (NarrowVT ==
Y.getValueType() &&
26894 LegalOperations)) {
26907 return Op.isUndef() ||
26908 ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
26917 if (ConcatWithConstantOrUndef(LHS) && ConcatWithConstantOrUndef(RHS) &&
26919 EVT NarrowVT =
LHS.getOperand(0).getValueType();
26920 if (NarrowVT ==
RHS.getOperand(0).getValueType() &&
26922 unsigned NumOperands =
LHS.getNumOperands();
26924 for (
unsigned i = 0; i != NumOperands; ++i) {
26927 RHS.getOperand(i)));
26943 "First argument must be a SetCC node!");
26946 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
26951 if (
SCC.getNode()) {
26958 SCC.getOperand(0),
SCC.getOperand(1),
26959 SCC.getOperand(4), Flags);
26960 AddToWorklist(
SETCC.getNode());
26962 SCC.getOperand(2),
SCC.getOperand(3));
26977bool DAGCombiner::SimplifySelectOps(
SDNode *TheSelect,
SDValue LHS,
26990 CC = cast<CondCodeSDNode>(TheSelect->
getOperand(4))->get();
26997 CC = cast<CondCodeSDNode>(
Cmp.getOperand(2))->get();
26998 CmpLHS =
Cmp.getOperand(0);
27002 if (Zero &&
Zero->isZero() &&
27006 CombineTo(TheSelect, Sqrt);
27016 if (
LHS.getOpcode() !=
RHS.getOpcode() ||
27029 if (
LHS.getOperand(0) !=
RHS.getOperand(0) ||
27073 Visited.
insert(TheSelect);
27152 CombineTo(TheSelect, Load);
27156 CombineTo(
LHS.getNode(),
Load.getValue(0),
Load.getValue(1));
27157 CombineTo(
RHS.getNode(),
Load.getValue(0),
Load.getValue(1));
27198 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
27199 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
27200 unsigned ShCt = XType.
getSizeInBits() - N2C->getAPIntValue().logBase2() - 1;
27204 AddToWorklist(Shift.
getNode());
27206 if (XType.
bitsGT(AType)) {
27208 AddToWorklist(Shift.
getNode());
27212 Shift = DAG.
getNOT(
DL, Shift, AType);
27224 AddToWorklist(Shift.
getNode());
27226 if (XType.
bitsGT(AType)) {
27228 AddToWorklist(Shift.
getNode());
27232 Shift = DAG.
getNOT(
DL, Shift, AType);
27295 EVT VT =
N->getValueType(0);
27303 EVT IntVT =
Int.getValueType();
27317 SignMask = ~SignMask;
27323 SignMask = ~SignMask;
27328 AddToWorklist(
Int.getNode());
27336SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
27344 auto *TV = dyn_cast<ConstantFPSDNode>(N2);
27345 auto *FV = dyn_cast<ConstantFPSDNode>(N3);
27352 TLI.
isFPImmLegal(TV->getValueAPF(), TV->getValueType(0), ForCodeSize) ||
27353 TLI.
isFPImmLegal(FV->getValueAPF(), FV->getValueType(0), ForCodeSize))
27358 if (!TV->hasOneUse() && !FV->hasOneUse())
27362 const_cast<ConstantFP*
>(TV->getConstantFPValue()) };
27370 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
27379 AddToWorklist(
Cond.getNode());
27381 AddToWorklist(CstOffset.
getNode());
27383 AddToWorklist(CPIdx.
getNode());
27393 bool NotExtCompare) {
27395 if (N2 == N3)
return N2;
27400 auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode());
27401 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
27402 auto *N3C = dyn_cast<ConstantSDNode>(N3.
getNode());
27406 AddToWorklist(
SCC.getNode());
27407 if (
auto *SCCC = dyn_cast<ConstantSDNode>(SCC)) {
27410 return !(SCCC->isZero()) ? N2 : N3;
27415 convertSelectOfFPConstantsToLoadOffset(
DL, N0, N1, N2, N3,
CC))
27418 if (
SDValue V = foldSelectCCToShiftAnd(
DL, N0, N1, N2, N3,
CC))
27430 auto *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
27431 if (ConstAndRHS && ConstAndRHS->getAPIntValue().popcount() == 1) {
27433 const APInt &AndMask = ConstAndRHS->getAPIntValue();
27454 bool Fold = N2C &&
isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2();
27455 bool Swap = N3C &&
isNullConstant(N2) && N3C->getAPIntValue().isPowerOf2();
27457 if ((Fold || Swap) &&
27469 if (NotExtCompare && N2C->isOne())
27482 AddToWorklist(
SCC.getNode());
27483 AddToWorklist(Temp.
getNode());
27488 unsigned ShCt = N2C->getAPIntValue().logBase2();
27513 if (
auto *ValueOnZeroC = dyn_cast<ConstantSDNode>(ValueOnZero)) {
27535 if (!NotExtCompare && N1C && N2C && N3C &&
27536 N2C->getAPIntValue() == ~N3C->getAPIntValue() &&
27558 bool foldBooleans) {
27560 DagCombineInfo(DAG, Level,
false,
this);
27658 bool AssumeNonZero) {
27661 auto PeekThroughCastsAndTrunc = [](
SDValue V) {
27663 switch (V.getOpcode()) {
27666 V = V.getOperand(0);
27677 Op = PeekThroughCastsAndTrunc(
Op);
27683 if (
C->isZero() ||
C->isOpaque())
27686 if (
C->getAPIntValue().isPowerOf2()) {
27698 for (
const APInt &Pow2 : Pow2Constants)
27707 auto CastToVT = [&](
EVT NewVT,
SDValue ToCast) {
27708 ToCast = PeekThroughCastsAndTrunc(ToCast);
27709 EVT CurVT = ToCast.getValueType();
27710 if (NewVT == CurVT)
27722 if (AssumeNonZero ||
Op->getFlags().hasNoUnsignedWrap() ||
27725 Depth + 1, AssumeNonZero))
27727 CastToVT(VT,
Op.getOperand(1)));
27734 Depth + 1, AssumeNonZero))
27736 Depth + 1, AssumeNonZero))
27737 return DAG.
getSelect(
DL, VT,
Op.getOperand(0), LogX, LogY);
27752 return DAG.
getNode(
Op.getOpcode(),
DL, VT, LogX, LogY);
27761 bool KnownNonZero,
bool InexpensiveOnly,
27762 std::optional<EVT> OutVT) {
27763 EVT VT = OutVT ? *OutVT :
V.getValueType();
27764 SDValue InexpensiveLogBase2 =
27767 return InexpensiveLogBase2;
27789 EVT VT =
Op.getValueType();
27797 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
27804 AddToWorklist(Est.getNode());
27812 for (
int i = 0; i < Iterations; ++i) {
27815 if (i == Iterations - 1) {
27817 AddToWorklist(MulEst.
getNode());
27821 AddToWorklist(NewEst.
getNode());
27824 (i == Iterations - 1 ?
N : FPOne), NewEst, Flags);
27825 AddToWorklist(NewEst.
getNode());
27828 AddToWorklist(NewEst.
getNode());
27831 AddToWorklist(Est.getNode());
27836 AddToWorklist(Est.getNode());
27852 unsigned Iterations,
27864 for (
unsigned i = 0; i < Iterations; ++i) {
27884 unsigned Iterations,
27897 for (
unsigned i = 0; i < Iterations; ++i) {
27906 if (Reciprocal || (i + 1) < Iterations) {
27929 EVT VT =
Op.getValueType();
27937 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
27944 bool UseOneConstNR =
false;
27948 AddToWorklist(Est.
getNode());
27950 if (Iterations > 0)
27951 Est = UseOneConstNR
27952 ? buildSqrtNROneConst(
Op, Est, Iterations, Flags, Reciprocal)
27953 : buildSqrtNRTwoConst(
Op, Est, Iterations,
Flags, Reciprocal);
27973 return buildSqrtEstimateImpl(
Op, Flags,
true);
27977 return buildSqrtEstimateImpl(
Op, Flags,
false);
27981bool DAGCombiner::mayAlias(
SDNode *Op0,
SDNode *Op1)
const {
27983 struct MemUseCharacteristics {
27992 auto getCharacteristics = [](
SDNode *
N) -> MemUseCharacteristics {
27993 if (
const auto *LSN = dyn_cast<LSBaseSDNode>(
N)) {
27995 if (
auto *
C = dyn_cast<ConstantSDNode>(LSN->getOffset()))
27998 ? -1 *
C->getSExtValue()
28001 return {LSN->isVolatile(), LSN->isAtomic(),
28002 LSN->getBasePtr(),
Offset ,
28005 if (
const auto *LN = cast<LifetimeSDNode>(
N))
28009 (LN->hasOffset()) ? LN->getOffset() : 0,
28022 MemUseCharacteristics MUC0 = getCharacteristics(Op0),
28023 MUC1 = getCharacteristics(Op1);
28026 if (MUC0.BasePtr.getNode() && MUC0.BasePtr == MUC1.BasePtr &&
28027 MUC0.Offset == MUC1.Offset)
28031 if (MUC0.IsVolatile && MUC1.IsVolatile)
28036 if (MUC0.IsAtomic && MUC1.IsAtomic)
28039 if (MUC0.MMO && MUC1.MMO) {
28040 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
28041 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
28047 if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() &&
28048 MUC0.Offset != 0) ||
28049 (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() &&
28061 if (!MUC0.MMO || !MUC1.MMO)
28067 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
28068 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
28076 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
28077 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
28078 Align OrigAlignment0 = MUC0.MMO->getBaseAlign();
28079 Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
28083 if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
28089 int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.
value();
28090 int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.
value();
28094 if ((OffAlign0 +
static_cast<int64_t
>(
28096 (OffAlign1 +
static_cast<int64_t
>(
28110 if (
UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
28113 (!Size0.
isScalable() || SrcValOffset0 == 0) &&
28114 (!Size1.
isScalable() || SrcValOffset1 == 0)) {
28116 int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
28139void DAGCombiner::GatherAllAliases(
SDNode *
N,
SDValue OriginalChain,
28146 const bool IsLoad = isa<LoadSDNode>(
N) && cast<LoadSDNode>(
N)->isSimple();
28150 unsigned Depth = 0;
28153 auto ImproveChain = [&](
SDValue &
C) ->
bool {
28154 switch (
C.getOpcode()) {
28163 bool IsOpLoad = isa<LoadSDNode>(
C.getNode()) &&
28164 cast<LSBaseSDNode>(
C.getNode())->isSimple();
28165 if ((IsLoad && IsOpLoad) || !
mayAlias(
N,
C.getNode())) {
28167 C =
C.getOperand(0);
28176 C =
C.getOperand(0);
28185 C =
C.getOperand(0);
28198 while (!Chains.
empty()) {
28232 if (ImproveChain(Chain)) {
28254 GatherAllAliases(
N, OldChain, Aliases);
28257 if (Aliases.
empty())
28261 if (Aliases.
size() == 1)
28281bool DAGCombiner::parallelizeChainedStores(
StoreSDNode *St) {
28298 if (!
BasePtr.getBase().getNode())
28302 if (
BasePtr.getBase().isUndef())
28320 if (Chain->getMemoryVT().isScalableVector())
28324 if (!
SDValue(Chain, 0)->hasOneUse())
28327 if (!Chain->isSimple() || Chain->isIndexed())
28336 int64_t
Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8;
28339 auto I = Intervals.find(
Offset);
28344 if (
I != Intervals.begin() && (--
I).stop() <=
Offset)
28353 if (ChainedStores.
empty())
28360 for (
unsigned I = ChainedStores.
size();
I;) {
28362 SDValue BetterChain = FindBetterChain(S, NewChain);
28366 ChainedStores[
I] = S;
28370 SDValue BetterChain = FindBetterChain(St, NewChain);
28385 auto hasImprovedChain = [&](
SDValue ST) ->
bool {
28386 return ST->getOperand(0) != NewChain;
28388 bool AddNewChain =
llvm::all_of(TFOps, hasImprovedChain);
28398 AddToWorklist(
Op.getNode());
28399 AddToWorklist(STChain);
28403bool DAGCombiner::findBetterNeighborChains(
StoreSDNode *St) {
28410 if (!
BasePtr.getBase().getNode())
28414 if (
BasePtr.getBase().isUndef())
28418 if (parallelizeChainedStores(St))
28423 if (St->
getChain() != BetterChain) {
28424 replaceStoreChain(St, BetterChain);
28434 DAGCombiner(*
this, AA, OptLevel).Run(Level);
static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< bool > UseAA("aarch64-use-aa", cl::init(true), cl::desc("Enable the use of AA during codegen."))
amdgpu AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI)
For the instruction sequence of store below, F and I values are bundled together as an i64 value befo...
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I)
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI, MachineRegisterInfo &MRI)
Return true if 'MI' is a load or a store that may be fold it's address operand into the load / store ...
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I)
static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques=false)
static cl::opt< bool > EnableShrinkLoadReplaceStoreWithStore("combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable load/<replace bytes>/store with " "a narrower store"))
static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, unsigned ExtOpc, SmallVectorImpl< SDNode * > &ExtendNodes, const TargetLowering &TLI)
static cl::opt< unsigned > TokenFactorInlineLimit("combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048), cl::desc("Limit the number of operands to inline for Token Factors"))
static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc, bool NonNegZExt=false)
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG)
static SDNode * getBuildPairElt(SDNode *N, unsigned i)
static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG, CombineLevel Level)
Fold (sext (select c, load x, load y)) -> (select c, sextload x, sextload y) (zext (select c,...
static SDValue foldBitOrderCrossLogicOp(SDNode *N, SelectionDAG &DAG)
static SDValue tryToFoldExtendOfConstant(SDNode *N, const SDLoc &DL, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalTypes)
Try to fold a sext/zext/aext dag node into a ConstantSDNode or a build_vector of constants.
static SDValue foldBoolSelectToLogic(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
If a vector binop is performed on splat values, it may be profitable to extract, scalarize,...
static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift, SDValue ExtractFrom, SDValue &Mask, const SDLoc &DL)
Helper function for visitOR to extract the needed side of a rotate idiom from a shl/srl/mul/udiv.
static bool getCombineLoadStoreParts(SDNode *N, unsigned Inc, unsigned Dec, bool &IsLoad, bool &IsMasked, SDValue &Ptr, const TargetLowering &TLI)
bool refineUniformBase(SDValue &BasePtr, SDValue &Index, bool IndexIsScaled, SelectionDAG &DAG, const SDLoc &DL)
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, const TargetLowering &TLI)
Return true if divmod libcall is available.
static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG)
static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1, SelectionDAG &DAG, const SDLoc &DL)
Given the operands of an add/sub operation, see if the 2nd operand is a masked 0/1 whose source opera...
static bool mergeEltWithShuffle(SDValue &X, SDValue &Y, ArrayRef< int > Mask, SmallVectorImpl< int > &NewMask, SDValue Elt, unsigned InsIndex)
static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf)
If we have a unary shuffle of a shuffle, see if it can be folded away completely.
static bool canSplitIdx(LoadSDNode *LD)
static SDValue ShrinkLoadReplaceStoreWithStore(const std::pair< unsigned, unsigned > &MaskInfo, SDValue IVal, StoreSDNode *St, DAGCombiner *DC)
Check to see if IVal is something that provides a value as specified by MaskInfo.
static cl::opt< bool > StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden, cl::desc("Bypass the profitability model of load slicing"), cl::init(false))
Hidden option to stress test load slicing, i.e., when this option is enabled, load slicing bypasses m...
static cl::opt< bool > UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true), cl::desc("Enable DAG combiner's use of TBAA"))
static void adjustCostForPairing(SmallVectorImpl< LoadedSlice > &LoadedSlices, LoadedSlice::Cost &GlobalLSCost)
Adjust the GlobalLSCost according to the target paring capabilities and the layout of the slices.
static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, SDValue N0, SDValue N1, SDNode *N)
static SDValue foldExtendVectorInregToExtendOfSubvector(SDNode *N, const SDLoc &DL, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalOperations)
static bool isCompatibleLoad(SDValue N, unsigned ExtOpcode)
Check if N satisfies: N is used once.
static SDValue foldLogicTreeOfShifts(SDNode *N, SDValue LeftHand, SDValue RightHand, SelectionDAG &DAG)
Given a tree of logic operations with shape like (LOGIC (LOGIC (X, Y), LOGIC (Z, Y))) try to match an...
static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG)
static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
If we are extracting a subvector produced by a wide binary operator try to use a narrow binary operat...
static SDValue takeInexpensiveLog2(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op, unsigned Depth, bool AssumeNonZero)
static SDValue combineSelectAsExtAnd(SDValue Cond, SDValue T, SDValue F, const SDLoc &DL, SelectionDAG &DAG)
static bool areUsedBitsDense(const APInt &UsedBits)
Check that all bits set in UsedBits form a dense region, i.e., UsedBits looks like 0....
static SDValue getInputChainForNode(SDNode *N)
Given a node, return its input chain if it has one, otherwise return a null sd operand.
static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG)
If we are extracting a subvector from a wide vector load, convert to a narrow load to eliminate the e...
static ElementCount numVectorEltsOrZero(EVT T)
static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG, bool ShouldCommuteOperands)
This inverts a canonicalization in IR that replaces a variable select arm with an identity constant.
static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG)
Given an extending node with a pop-count operand, if the target does not support a pop-count in the n...
static SDValue foldAndOrOfSETCC(SDNode *LogicOp, SelectionDAG &DAG)
static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
If a shuffle inserts exactly one element from a source vector operand into another vector operand and...
static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType)
static SDValue foldAndToUsubsat(SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
For targets that support usubsat, match a bit-hack form of that operation that ends in 'and' and conv...
static cl::opt< bool > CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden, cl::desc("Enable DAG combiner's use of IR alias analysis"))
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy)
static SDValue formSplatFromShuffles(ShuffleVectorSDNode *OuterShuf, SelectionDAG &DAG)
Combine shuffle of shuffle of the form: shuf (shuf X, undef, InnerMask), undef, OuterMask --> splat X...
static bool isDivisorPowerOfTwo(SDValue Divisor)
static bool matchRotateHalf(const SelectionDAG &DAG, SDValue Op, SDValue &Shift, SDValue &Mask)
Match "(X shl/srl V1) & V2" where V2 may not be present.
static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
Given EXTRACT_SUBVECTOR(VECTOR_SHUFFLE(Op0, Op1, Mask)), try to produce VECTOR_SHUFFLE(EXTRACT_SUBVEC...
static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG, bool LegalOperations)
Transform a vector binary operation into a scalar binary operation by moving the math/logic after an ...
static bool hasNoInfs(const TargetOptions &Options, SDValue N)
static SDValue combineShuffleOfBitcast(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static std::optional< EVT > canCombineShuffleToExtendVectorInreg(unsigned Opcode, EVT VT, std::function< bool(unsigned)> Match, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalTypes, bool LegalOperations)
static SDValue PerformUMinFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, SelectionDAG &DAG)
static SDValue combineShuffleToAnyExtendVectorInreg(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static SDValue foldAddSubOfSignBit(SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
Try to fold a 'not' shifted sign-bit with add/sub with constant operand into a shift and add with a d...
static SDValue stripTruncAndExt(SDValue Value)
static SDValue combineUADDO_CARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG, SDValue X, SDValue Carry0, SDValue Carry1, SDNode *N)
If we are facing some sort of diamond carry propagation pattern try to break it up to generate someth...
static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
Try to convert a wide shuffle of concatenated vectors into 2 narrow shuffles followed by concatenatio...
static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
static auto getFirstIndexOf(R &&Range, const T &Val)
static std::pair< unsigned, unsigned > CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain)
Check to see if V is (and load (ptr), imm), where the load is having specific bytes cleared out.
static int getShuffleMaskIndexOfOneElementFromOp0IntoOp1(ArrayRef< int > Mask)
If the shuffle mask is taking exactly one element from the first vector operand and passing through a...
static bool shouldConvertSelectOfConstantsToMath(const SDValue &Cond, EVT VT, const TargetLowering &TLI)
static cl::opt< bool > EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable merging multiple stores " "into a wider store"))
static bool isContractableFMUL(const TargetOptions &Options, SDValue N)
static cl::opt< bool > MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true), cl::desc("DAG combiner may split indexing from loads"))
static bool areSlicesNextToEachOther(const LoadedSlice &First, const LoadedSlice &Second)
Check whether or not First and Second are next to each other in memory.
static SDValue stripConstantMask(const SelectionDAG &DAG, SDValue Op, SDValue &Mask)
static bool arebothOperandsNotSNan(SDValue Operand1, SDValue Operand2, SelectionDAG &DAG)
static bool isBSwapHWordPair(SDValue N, MutableArrayRef< SDNode * > Parts)
static bool CanCombineFCOPYSIGN_EXTEND_ROUND(EVT XTy, EVT YTy)
copysign(x, fp_extend(y)) -> copysign(x, y) copysign(x, fp_round(y)) -> copysign(x,...
static unsigned getMinMaxOpcodeForFP(SDValue Operand1, SDValue Operand2, ISD::CondCode CC, unsigned OrAndOpcode, SelectionDAG &DAG, bool isFMAXNUMFMINNUM_IEEE, bool isFMAXNUMFMINNUM)
static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getTruncatedUSUBSAT(EVT DstVT, EVT SrcVT, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &DL)
static SDNode * getPostIndexedLoadStoreOp(SDNode *N, bool &IsLoad, bool &IsMasked, SDValue &Ptr, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS, const TargetLowering &TLI)
static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG, const TargetLowering &TLI, bool Force)
Flips a boolean if it is cheaper to compute.
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, KnownBits &Known)
static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT)
static SDValue combineConcatVectorOfShuffleAndItsOperands(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalTypes, bool LegalOperations)
bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT, SelectionDAG &DAG)
static cl::opt< bool > EnableVectorFCopySignExtendRound("combiner-vector-fcopysign-extend-round", cl::Hidden, cl::init(false), cl::desc("Enable merging extends and rounds into FCOPYSIGN on vector types"))
static SDValue combineMinNumMaxNumImpl(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode CC, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG)
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
static SDValue widenAbs(SDNode *Extend, SelectionDAG &DAG)
static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset=0)
static SDValue combineShiftToMULH(SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const TargetLowering &TLI)
static ConstantSDNode * getAsNonOpaqueConstant(SDValue N)
If N is a ConstantSDNode with isOpaque() == false return it casted to a ConstantSDNode pointer else n...
static bool arebothOperandsNotNan(SDValue Operand1, SDValue Operand2, SelectionDAG &DAG)
static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, SelectionDAG &DAG)
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize, SelectionDAG &DAG, bool IsRotate)
static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N)
OR combines for which the commuted variant will be tried as well.
static SDValue combineShuffleToZeroExtendVectorInReg(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static cl::opt< bool > EnableReduceLoadOpStoreWidth("combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable reducing the width of load/op/store " "sequence"))
static bool shouldCombineToPostInc(SDNode *N, SDValue Ptr, SDNode *PtrUse, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineConcatVectorOfCasts(SDNode *N, SelectionDAG &DAG)
static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG)
Try to replace shift/logic that tests if a bit is clear with mask + setcc.
static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI, SelectionDAG &DAG, SDNode *N, SDValue N0, SDValue N1, EVT VT, EVT ShiftAmountTy)
static bool areBitwiseNotOfEachother(SDValue Op0, SDValue Op1)
static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG)
static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG)
static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue combineConcatVectorOfConcatVectors(SDNode *N, SelectionDAG &DAG)
static SDValue tryToFoldExtOfAtomicLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, SDValue N0, ISD::LoadExtType ExtLoadType)
static SDValue getAsCarry(const TargetLowering &TLI, SDValue V, bool ForceCarryReconstruction=false)
static SDValue foldSelectOfConstantsUsingSra(SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
If a (v)select has a condition value that is a sign-bit test, try to smear the condition operand sign...
static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG)
static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG)
static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, SelectionDAG &DAG, bool LegalOperations)
static cl::opt< unsigned > StoreMergeDependenceLimit("combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10), cl::desc("Limit the number of times for the same StoreNode and RootNode " "to bail out in store merging dependence check"))
static cl::opt< std::string > CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden, cl::desc("Only use DAG-combiner alias analysis in this" " function"))
static SDValue foldLogicOfShifts(SDNode *N, SDValue LogicOp, SDValue ShiftOp, SelectionDAG &DAG)
Given a bitwise logic operation N with a matching bitwise logic operand, fold a pattern where 2 of th...
static bool isSlicingProfitable(SmallVectorImpl< LoadedSlice > &LoadedSlices, const APInt &UsedBits, bool ForCodeSize)
Check the profitability of all involved LoadedSlice.
static bool isBSwapHWordElement(SDValue N, MutableArrayRef< SDNode * > Parts)
Return true if the specified node is an element that makes up a 32-bit packed halfword byteswap.
static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, unsigned &BW, bool &Unsigned, SelectionDAG &DAG)
static std::optional< SDByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, std::optional< uint64_t > VectorIndex, unsigned StartingIndex=0)
static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
static bool isUndef(ArrayRef< int > Mask)
static MaybeAlign getAlign(Value *Ptr)
iv Induction Variable Users
static Value * simplifyDivRem(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse)
Check for common or similar folds of integer division or integer remainder.
This file implements a coalescing interval map for small objects.
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const SmallVectorImpl< MachineOperand > & Cond
Contains matchers for matching SelectionDAG nodes and values.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
static cl::opt< bool > UseTBAA("use-tbaa-in-sched-mi", cl::Hidden, cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction"))
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
const fltSemantics & getSemantics() const
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt umul_ov(const APInt &RHS, bool &Overflow) const
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
APInt getLoBits(unsigned numBits) const
Compute an APInt containing numBits lowbits from this APInt.
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
APInt abs() const
Get the absolute value.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
bool isNegative() const
Determine sign of this APInt.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
int32_t exactLogBase2() const
APInt uadd_ov(const APInt &RHS, bool &Overflow) const
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
unsigned countLeadingZeros() const
unsigned logBase2() const
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool getBoolValue() const
Convert APInt to a boolean value.
APInt smul_ov(const APInt &RHS, bool &Overflow) const
bool isMask(unsigned numBits) const
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
size_t size() const
size - Get the array size.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
This is an SDNode representing atomic operations.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
static bool computeAliasing(const SDNode *Op0, const LocationSize NumBytes0, const SDNode *Op1, const LocationSize NumBytes1, const SelectionDAG &DAG, bool &IsAlias)
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Represents known origin of an individual byte in combine pattern.
static ByteProvider getConstantZero()
static ByteProvider getSrc(std::optional< ISelOp > Val, int64_t ByteOffset, int64_t VectorOffset)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
bool isNegative() const
Return true if the value is negative.
bool isZero() const
Return true if the value is positive or negative zero.
ConstantFP - Floating Point Values [float, double].
bool isMinSignedValue() const
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
static bool shouldExecute(unsigned CounterName)
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
static constexpr ElementCount getFixed(ScalarTy MinVal)
constexpr bool isScalar() const
Exactly one element.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
AttributeList getAttributes() const
Return the attribute list for this Function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Helper struct to store a base, index and offset that forms an address.
This class is used to form a handle around another node that is persistent and is updated across invo...
This is an important class for using LLVM in a threaded context.
Base class for LoadSDNode and StoreSDNode.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
TypeSize getValue() const
static auto all_valuetypes()
SimpleValueType Iteration.
static MVT getIntegerVT(unsigned BitWidth)
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MONonTemporal
The memory access is non-temporal.
Flags getFlags() const
Return the raw flags of the source value,.
const Value * getValue() const
Return the base address of the memory access.
This class is used to represent an MGATHER node.
const SDValue & getPassThru() const
ISD::LoadExtType getExtensionType() const
const SDValue & getIndex() const
bool isIndexScaled() const
const SDValue & getScale() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
This class is used to represent an MLOAD node.
const SDValue & getBasePtr() const
bool isExpandingLoad() const
ISD::LoadExtType getExtensionType() const
const SDValue & getMask() const
const SDValue & getPassThru() const
const SDValue & getOffset() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
This class is used to represent an MSCATTER node.
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MSTORE node.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getOffset() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getBasePtr() const
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
bool isDereferenceable() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
MutableArrayRef< T > take_back(size_t N=1) const
Return a copy of *this with only the last N elements.
MutableArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
size_t use_size() const
Return the number of uses of this node.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
TypeSize getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual bool disableGenericCombines(CodeGenOptLevel OptLevel) const
Help to insert SDNodeFlags automatically in transforming.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool willNotOverflowAdd(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 nodes can never overflow.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const TargetSubtargetInfo & getSubtarget() const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
bool willNotOverflowSub(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the sub of 2 nodes can never overflow.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
void Combine(CombineLevel Level, AAResults *AA, CodeGenOptLevel OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together,...
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
bool willNotOverflowMul(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the mul of 2 nodes can never overflow.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
bool isConstantValueOfAnyType(SDValue N) const
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool PoisonOnly=false, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
bool isSafeToSpeculativelyExecuteNode(const SDNode *N) const
Check if the provided node is save to speculatively executed given its current arguments.
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
static unsigned getOpcode_EXTEND(unsigned Opcode)
Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode.
bool isADDLike(SDValue Op, bool NoWrap=false) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A vector that has set insertion semantics.
bool remove(const value_type &X)
Remove an item from the set vector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
value_type pop_back_val()
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
bool has(LibFunc F) const
Tests whether a library function is available.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const
virtual bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool enableAggressiveFMAFusion(EVT VT) const
Return true if target always benefits from combining into FMA for a given value type.
bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
SDValue promoteTargetBoolean(SelectionDAG &DAG, SDValue Bool, EVT ValVT) const
Promote the given target boolean to a target boolean of the given type.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool canCombineTruncStore(EVT ValVT, EVT MemVT, bool LegalOnly) const
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const
Use bitwise logic to make pairs of compares more efficient.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode, LLT DestTy, LLT SrcTy) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
virtual bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
virtual bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst, SDValue IntPow2) const
virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT) const
Returns true if we should normalize select(N0&N1, X, Y) => select(N0, select(N1, X,...
virtual bool preferScalarizeSplat(SDNode *N) const
bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const
Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes.
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to fold a pair of shifts into a mask.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const
Return true if pulling a binary operation into a select with an identity constant is profitable.
virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
virtual bool preferIncOfAddToSubOfNot(EVT VT) const
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
virtual bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual bool isFNegFree(EVT VT) const
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool mergeStoresAfterLegalization(EVT MemVT) const
Allow store merging for the specified type after legalization in addition to before legalization.
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned getGatherAllAliasesMaxDepth() const
virtual bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal or custom on this target.
bool isAtomicLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified atomic load with extension is legal on this target.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const
Returns if it's reasonable to merge stores to MemVT size.
virtual bool preferABDSToABSWithNSW(EVT VT) const
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
AndOrSETCCFoldKind
Enum of different potentially desirable ways to fold (and/or (setcc ...), (setcc ....
virtual bool shouldScalarizeBinop(SDValue VecOp) const
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x,...
bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool isVectorClearMaskLegal(ArrayRef< int >, EVT) const
Similar to isShuffleMaskLegal.
bool hasTargetDAGCombine(ISD::NodeType NT) const
If true, the target has custom DAG combine transformations that it can perform for the specified node...
virtual bool shouldSplatInsEltVarIndex(EVT) const
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
virtual unsigned preferedOpcodeForCmpEqPiecesOfOperand(EVT VT, unsigned ShiftOpc, bool MayTransformRotate, const APInt &ShiftOrRotateAmt, const std::optional< APInt > &AndMask) const
virtual bool isFMADLegal(const MachineInstr &MI, LLT Ty) const
Returns true if MI can be combined with another instruction to form TargetOpcode::G_FMAD.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual bool generateFMAsInMachineCombiner(EVT VT, CodeGenOptLevel OptLevel) const
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool hasPairedLoad(EVT, Align &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
virtual bool convertSelectOfConstantsToMath(EVT VT) const
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const
Hooks for building estimates in place of slower divisions and square roots.
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
SDValue getCheaperOrNeutralNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, const NegatibleCost CostThreshold=NegatibleCost::Neutral, unsigned Depth=0) const
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
virtual bool IsDesirableToPromoteOp(SDValue, EVT &) const
This method query the target whether it is beneficial for dag combiner to promote the specified node.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG, const DenormalMode &Mode) const
Return a target-dependent comparison result if the input operand is suitable for use with a square ro...
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const
Return a reciprocal estimate value for the input operand.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool isConstFalseVal(SDValue N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue getSqrtResultForDenormInput(SDValue Operand, SelectionDAG &DAG) const
Return a target-dependent result if the input operand is not suitable for use with a square root esti...
virtual bool getPostIndexedAddressParts(SDNode *, SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount through its operand,...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
virtual AndOrSETCCFoldKind isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const
virtual bool getPreIndexedAddressParts(SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SREM lowering for power-of-2 denominators.
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned NoSignedZerosFPMath
NoSignedZerosFPMath - This flag is enabled when the -enable-no-signed-zeros-fp-math is specified on t...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine,...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
const fltSemantics & getFltSemantics() const
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
This class is used to represent an VP_GATHER node.
const SDValue & getScale() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
const SDValue & getVectorLength() const
const SDValue & getIndex() const
const SDValue & getBasePtr() const
bool isIndexScaled() const
const SDValue & getMask() const
This class is used to represent an VP_SCATTER node.
const SDValue & getValue() const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
iterator_range< use_iterator > uses()
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ CARRY_FALSE
CARRY_FALSE - This node is used when folding other nodes, like ADDC/SUBC, which indicate the carry re...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ SET_FPENV_MEM
Sets the current floating point environment.
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isIndexTypeSigned(MemIndexType IndexType)
bool isExtVecInRegOpcode(unsigned Opcode)
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool matchUnaryFpPredicate(SDValue Op, std::function< bool(ConstantFPSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantFPSDNode predicate.
bool isFPEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with floati...
bool isExtOpcode(unsigned Opcode)
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
CastInst_match< OpTy, ZExtInst > m_ZExt(const OpTy &Op)
Matches ZExt.
MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty > m_UMax(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
BinaryOp_match< cst_pred_ty< is_all_ones >, ValTy, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Undef
Value of the register doesn't matter.
BinaryOpc_match< LHS, RHS, false > m_Sra(const LHS &L, const RHS &R)
BinaryOpc_match< LHS, RHS, false > m_Srl(const LHS &L, const RHS &R)
UnaryOpc_match< Opnd > m_AnyExt(const Opnd &Op)
Or< Preds... > m_AnyOf(Preds &&...preds)
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
NUses_match< 1, Value_match > m_OneUse()
initializer< Ty > init(const Ty &Val)
int ilogb(const IEEEFloat &Arg)
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
bool operator<(int64_t V1, const APSInt &V2)
void stable_sort(R &&Range)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool operator!=(uint64_t V1, const APInt &V2)
bool operator>=(int64_t V1, const APSInt &V2)
std::string & operator+=(std::string &buffer, StringRef string)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
unsigned M1(unsigned Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isConstantOrConstantVector(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowFP=true, bool AllowOpaqueConstants=true)
Return true if the specified instruction is known to be a constant, or a vector of constants.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool operator>(int64_t V1, const APSInt &V2)
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
auto reverse(ContainerTy &&C)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
detail::ValueMatchesPoly< M > HasValue(M Matcher)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
CodeGenOptLevel
Code generation optimization level.
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
void getShuffleMaskWithWidestElts(ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Repetitively apply widenShuffleMaskElts() for as long as it succeeds, to get the shuffle mask with wi...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
unsigned Log2(Align A)
Returns the log2 of the alignment.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool operator<=(int64_t V1, const APSInt &V2)
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
AAMDNodes concat(const AAMDNodes &Other) const
Determine the best AAMDNodes after concatenating two different locations together.
static ExponentType semanticsMinExponent(const fltSemantics &)
static constexpr roundingMode rmNearestTiesToEven
static ExponentType semanticsMaxExponent(const fltSemantics &)
static unsigned int semanticsPrecision(const fltSemantics &)
opStatus
IEEE-754R 7: Default exception handling.
static unsigned int semanticsIntSizeInBits(const fltSemantics &, bool)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getIEEE()
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalableVT() const
Return true if the type is a scalable type.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isZeroSized() const
Test if the given EVT has zero size, this will fail if called on a scalable type.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isNonNegative() const
Returns true if this value is known to be non-negative.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
bool isConstant() const
Returns true if we know the value of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
bool isAllOnes() const
Returns true if value is all one bits.
const APInt & getConstant() const
Returns the value when all bits have a known value.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasNoSignedWrap() const
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
void AddToWorklist(SDNode *N)
bool recursivelyDeleteUnusedNodes(SDNode *N)
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...