85#define DEBUG_TYPE "dagcombine"
87STATISTIC(NodesCombined ,
"Number of dag nodes combined");
88STATISTIC(PreIndexedNodes ,
"Number of pre-indexed nodes created");
89STATISTIC(PostIndexedNodes,
"Number of post-indexed nodes created");
90STATISTIC(OpsNarrowed ,
"Number of load/op/store narrowed");
91STATISTIC(LdStFP2Int ,
"Number of fp load/store pairs transformed to int");
93STATISTIC(NumFPLogicOpsConv,
"Number of logic ops converted to fp ops");
96 "Controls whether a DAG combine is performed for a node");
100 cl::desc(
"Enable DAG combiner's use of IR alias analysis"));
104 cl::desc(
"Enable DAG combiner's use of TBAA"));
109 cl::desc(
"Only use DAG-combiner alias analysis in this"
117 cl::desc(
"Bypass the profitability model of load slicing"),
122 cl::desc(
"DAG combiner may split indexing from loads"));
126 cl::desc(
"DAG combiner enable merging multiple stores "
127 "into a wider store"));
131 cl::desc(
"Limit the number of operands to inline for Token Factors"));
135 cl::desc(
"Limit the number of times for the same StoreNode and RootNode "
136 "to bail out in store merging dependence check"));
140 cl::desc(
"DAG combiner enable reducing the width of load/op/store "
145 cl::desc(
"DAG combiner enable load/<replace bytes>/store with "
146 "a narrower store"));
151 "Enable merging extends and rounds into FCOPYSIGN on vector types"));
161 bool LegalDAG =
false;
162 bool LegalOperations =
false;
163 bool LegalTypes =
false;
165 bool DisableGenericCombines;
207 void AddUsersToWorklist(
SDNode *
N) {
213 void AddToWorklistWithUsers(
SDNode *
N) {
214 AddUsersToWorklist(
N);
221 void clearAddedDanglingWorklistEntries() {
223 while (!PruningList.
empty()) {
226 recursivelyDeleteUnusedNodes(
N);
230 SDNode *getNextWorklistEntry() {
232 clearAddedDanglingWorklistEntries();
236 while (!
N && !Worklist.
empty()) {
241 bool GoodWorklistEntry = WorklistMap.
erase(
N);
242 (void)GoodWorklistEntry;
243 assert(GoodWorklistEntry &&
244 "Found a worklist entry without a corresponding map entry!");
254 : DAG(
D), TLI(
D.getTargetLoweringInfo()),
255 STI(
D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) {
259 MaximumLegalStoreInBits = 0;
265 VT.getSizeInBits().getKnownMinValue() >= MaximumLegalStoreInBits)
266 MaximumLegalStoreInBits = VT.getSizeInBits().getKnownMinValue();
269 void ConsiderForPruning(
SDNode *
N) {
276 void AddToWorklist(
SDNode *
N,
bool IsCandidateForPruning =
true) {
278 "Deleted Node added to Worklist");
285 if (IsCandidateForPruning)
286 ConsiderForPruning(
N);
288 if (WorklistMap.
insert(std::make_pair(
N, Worklist.
size())).second)
293 void removeFromWorklist(
SDNode *
N) {
296 StoreRootCountMap.
erase(
N);
298 auto It = WorklistMap.
find(
N);
299 if (It == WorklistMap.
end())
303 Worklist[It->second] =
nullptr;
304 WorklistMap.
erase(It);
307 void deleteAndRecombine(
SDNode *
N);
308 bool recursivelyDeleteUnusedNodes(
SDNode *
N);
316 return CombineTo(
N, &Res, 1, AddTo);
323 return CombineTo(
N, To, 2, AddTo);
329 unsigned MaximumLegalStoreInBits;
335 unsigned BitWidth =
Op.getScalarValueSizeInBits();
341 EVT VT =
Op.getValueType();
345 return SimplifyDemandedBits(
Op,
DemandedBits, DemandedElts,
false);
351 bool SimplifyDemandedVectorElts(
SDValue Op) {
353 if (
Op.getValueType().isScalableVector())
356 unsigned NumElts =
Op.getValueType().getVectorNumElements();
358 return SimplifyDemandedVectorElts(
Op, DemandedElts);
362 const APInt &DemandedElts,
363 bool AssumeSingleUse =
false);
364 bool SimplifyDemandedVectorElts(
SDValue Op,
const APInt &DemandedElts,
365 bool AssumeSingleUse =
false);
367 bool CombineToPreIndexedLoadStore(
SDNode *
N);
368 bool CombineToPostIndexedLoadStore(
SDNode *
N);
393 void ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad);
530 bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
SDNode *
N);
559 template <
class MatchContextClass>
561 template <
class MatchContextClass>
566 bool reassociationCanBreakAddressingModePattern(
unsigned Opc,
575 SDValue reassociateReduction(
unsigned RedOpc,
unsigned Opc,
const SDLoc &
DL,
589 bool NotExtCompare =
false);
590 SDValue convertSelectOfFPConstantsToLoadOffset(
605 const SDLoc &
DL,
bool foldBooleans);
609 SDValue &
CC,
bool MatchStrict =
false)
const;
610 bool isOneUseSetCC(
SDValue N)
const;
635 bool KnownNeverZero =
false,
636 bool InexpensiveOnly =
false,
637 std::optional<EVT> OutVT = std::nullopt);
647 bool DemandHighBits =
true);
651 unsigned PosOpcode,
unsigned NegOpcode,
655 unsigned PosOpcode,
unsigned NegOpcode,
671 SDValue VecIn2,
unsigned LeftIdx,
706 int64_t OffsetFromBase;
709 : MemNode(
N), OffsetFromBase(
Offset) {}
714 StoreSource getStoreSource(
SDValue StoreVal) {
718 return StoreSource::Constant;
722 return StoreSource::Constant;
723 return StoreSource::Unknown;
726 return StoreSource::Extract;
728 return StoreSource::Load;
730 return StoreSource::Unknown;
738 bool isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
745 EVT LoadResultTy,
EVT &ExtVT);
750 EVT &MemVT,
unsigned ShAmt = 0);
758 bool BackwardsPropagateMask(
SDNode *
N);
775 EVT MemVT,
unsigned NumStores,
776 bool IsConstantSrc,
bool UseVector,
790 bool checkMergeStoreCandidatesForDependencies(
798 int64_t ElementSizeBytes)
const;
803 unsigned NumConsecutiveStores,
804 EVT MemVT,
SDNode *Root,
bool AllowVectors);
811 unsigned NumConsecutiveStores,
EVT MemVT,
817 unsigned NumConsecutiveStores,
EVT MemVT,
818 SDNode *Root,
bool AllowVectors,
819 bool IsNonTemporalStore,
bool IsNonTemporalLoad);
838 bool hasOperation(
unsigned Opcode,
EVT VT) {
850 EVT getShiftAmountTy(
EVT LHSTy) {
857 bool isTypeLegal(
const EVT &VT) {
858 if (!LegalTypes)
return true;
863 EVT getSetCCResultType(
EVT VT)
const {
878 explicit WorklistRemover(DAGCombiner &dc)
879 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
882 DC.removeFromWorklist(
N);
890 explicit WorklistInserter(DAGCombiner &dc)
891 :
SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
905 ((DAGCombiner*)
DC)->AddToWorklist(
N);
910 return ((DAGCombiner*)DC)->CombineTo(
N, &To[0], To.
size(), AddTo);
915 return ((DAGCombiner*)DC)->CombineTo(
N, Res, AddTo);
920 return ((DAGCombiner*)DC)->CombineTo(
N, Res0, Res1, AddTo);
925 return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(
N);
930 return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
937void DAGCombiner::deleteAndRecombine(
SDNode *
N) {
938 removeFromWorklist(
N);
946 if (
Op->hasOneUse() ||
Op->getNumValues() > 1)
947 AddToWorklist(
Op.getNode());
956 unsigned Bits =
Offset + std::max(
LHS.getBitWidth(),
RHS.getBitWidth());
969 LHS =
N.getOperand(0);
970 RHS =
N.getOperand(1);
971 CC =
N.getOperand(2);
978 LHS =
N.getOperand(1);
979 RHS =
N.getOperand(2);
980 CC =
N.getOperand(3);
992 LHS =
N.getOperand(0);
993 RHS =
N.getOperand(1);
994 CC =
N.getOperand(4);
1001bool DAGCombiner::isOneUseSetCC(
SDValue N)
const {
1003 if (isSetCCEquivalent(
N, N0, N1, N2) &&
N->hasOneUse())
1015 MaskForTy = 0xFFULL;
1018 MaskForTy = 0xFFFFULL;
1021 MaskForTy = 0xFFFFFFFFULL;
1040 return !(Const->isOpaque() && NoOpaques);
1043 unsigned BitWidth =
N.getScalarValueSizeInBits();
1048 if (!Const || Const->getAPIntValue().getBitWidth() !=
BitWidth ||
1049 (Const->isOpaque() && NoOpaques))
1068 !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
1071bool DAGCombiner::reassociationCanBreakAddressingModePattern(
unsigned Opc,
1088 auto *C2 = dyn_cast<ConstantSDNode>(N1);
1092 const APInt &C2APIntVal = C2->getAPIntValue();
1096 if (
auto *C1 = dyn_cast<ConstantSDNode>(N0.
getOperand(1))) {
1100 const APInt &C1APIntVal = C1->getAPIntValue();
1101 const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
1104 const int64_t CombinedValue = CombinedValueIntVal.
getSExtValue();
1107 if (
auto *LoadStore = dyn_cast<MemSDNode>(
Node)) {
1113 AM.
BaseOffs = C2APIntVal.getSExtValue();
1115 unsigned AS =
LoadStore->getAddressSpace();
1127 if (
auto *GA = dyn_cast<GlobalAddressSDNode>(N0.
getOperand(1)))
1140 AM.
BaseOffs = C2APIntVal.getSExtValue();
1142 unsigned AS =
LoadStore->getAddressSpace();
1155SDValue DAGCombiner::reassociateOpsCommutative(
unsigned Opc,
const SDLoc &
DL,
1169 Flags.hasNoUnsignedWrap())
1175 return DAG.
getNode(Opc,
DL, VT, N00, OpNode, NewFlags);
1182 return DAG.
getNode(Opc,
DL, VT, OpNode, N01, NewFlags);
1192 if (N1 == N00 || N1 == N01)
1238 if (CC1 == CC00 && CC1 != CC01) {
1240 return DAG.
getNode(Opc,
DL, VT, OpNode, N01, Flags);
1242 if (CC1 == CC01 && CC1 != CC00) {
1244 return DAG.
getNode(Opc,
DL, VT, OpNode, N00, Flags);
1262 if (!
Flags.hasAllowReassociation() || !
Flags.hasNoSignedZeros())
1265 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N0, N1, Flags))
1267 if (
SDValue Combined = reassociateOpsCommutative(Opc,
DL, N1, N0, Flags))
1275SDValue DAGCombiner::reassociateReduction(
unsigned RedOpc,
unsigned Opc,
1293 assert(
N->getNumValues() == NumTo &&
"Broken CombineTo call!");
1297 dbgs() <<
" and " << NumTo - 1 <<
" other values\n");
1298 for (
unsigned i = 0, e = NumTo; i !=
e; ++i)
1299 assert((!To[i].getNode() ||
1300 N->getValueType(i) == To[i].getValueType()) &&
1301 "Cannot combine value to value of different type!");
1303 WorklistRemover DeadNodes(*
this);
1307 for (
unsigned i = 0, e = NumTo; i !=
e; ++i) {
1308 if (To[i].getNode())
1309 AddToWorklistWithUsers(To[i].getNode());
1317 deleteAndRecombine(
N);
1335 recursivelyDeleteUnusedNodes(TLO.
Old.
getNode());
1341 const APInt &DemandedElts,
1342 bool AssumeSingleUse) {
1350 AddToWorklist(
Op.getNode());
1352 CommitTargetLoweringOpt(TLO);
1359bool DAGCombiner::SimplifyDemandedVectorElts(
SDValue Op,
1360 const APInt &DemandedElts,
1361 bool AssumeSingleUse) {
1363 APInt KnownUndef, KnownZero;
1365 TLO, 0, AssumeSingleUse))
1369 AddToWorklist(
Op.getNode());
1371 CommitTargetLoweringOpt(TLO);
1375void DAGCombiner::ReplaceLoadWithPromotedLoad(
SDNode *Load,
SDNode *ExtLoad) {
1377 EVT VT =
Load->getValueType(0);
1386 AddToWorklist(Trunc.
getNode());
1387 recursivelyDeleteUnusedNodes(Load);
1395 EVT MemVT =
LD->getMemoryVT();
1397 :
LD->getExtensionType();
1400 LD->getChain(),
LD->getBasePtr(),
1401 MemVT,
LD->getMemOperand());
1404 unsigned Opc =
Op.getOpcode();
1408 if (
SDValue Op0 = SExtPromoteOperand(
Op.getOperand(0), PVT))
1412 if (
SDValue Op0 = ZExtPromoteOperand(
Op.getOperand(0), PVT))
1430 EVT OldVT =
Op.getValueType();
1432 bool Replace =
false;
1433 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1436 AddToWorklist(NewOp.
getNode());
1439 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1445 EVT OldVT =
Op.getValueType();
1447 bool Replace =
false;
1448 SDValue NewOp = PromoteOperand(
Op, PVT, Replace);
1451 AddToWorklist(NewOp.
getNode());
1454 ReplaceLoadWithPromotedLoad(
Op.getNode(), NewOp.
getNode());
1462 if (!LegalOperations)
1465 EVT VT =
Op.getValueType();
1471 unsigned Opc =
Op.getOpcode();
1479 assert(PVT != VT &&
"Don't know what type to promote to!");
1483 bool Replace0 =
false;
1485 SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
1487 bool Replace1 =
false;
1489 SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
1501 Replace1 &= (N0 != N1) && !N1->
hasOneUse();
1504 CombineTo(
Op.getNode(), RV);
1530 if (!LegalOperations)
1533 EVT VT =
Op.getValueType();
1539 unsigned Opc =
Op.getOpcode();
1547 assert(PVT != VT &&
"Don't know what type to promote to!");
1551 bool Replace =
false;
1554 N0 = SExtPromoteOperand(N0, PVT);
1556 N0 = ZExtPromoteOperand(N0, PVT);
1558 N0 = PromoteOperand(N0, PVT, Replace);
1569 ReplaceLoadWithPromotedLoad(
Op.getOperand(0).getNode(), N0.
getNode());
1579 if (!LegalOperations)
1582 EVT VT =
Op.getValueType();
1588 unsigned Opc =
Op.getOpcode();
1596 assert(PVT != VT &&
"Don't know what type to promote to!");
1606bool DAGCombiner::PromoteLoad(
SDValue Op) {
1607 if (!LegalOperations)
1613 EVT VT =
Op.getValueType();
1619 unsigned Opc =
Op.getOpcode();
1627 assert(PVT != VT &&
"Don't know what type to promote to!");
1632 EVT MemVT =
LD->getMemoryVT();
1634 :
LD->getExtensionType();
1636 LD->getChain(),
LD->getBasePtr(),
1637 MemVT,
LD->getMemOperand());
1646 AddToWorklist(
Result.getNode());
1647 recursivelyDeleteUnusedNodes(
N);
1660bool DAGCombiner::recursivelyDeleteUnusedNodes(
SDNode *
N) {
1661 if (!
N->use_empty())
1671 if (
N->use_empty()) {
1672 for (
const SDValue &ChildN :
N->op_values())
1673 Nodes.
insert(ChildN.getNode());
1675 removeFromWorklist(
N);
1680 }
while (!Nodes.
empty());
1695 WorklistInserter AddNodes(*
this);
1704 AddToWorklist(&
Node,
Node.use_empty());
1712 while (
SDNode *
N = getNextWorklistEntry()) {
1716 if (recursivelyDeleteUnusedNodes(
N))
1719 WorklistRemover DeadNodes(*
this);
1727 for (
SDNode *LN : UpdatedNodes)
1728 AddToWorklistWithUsers(LN);
1739 for (
const SDValue &ChildN :
N->op_values())
1740 if (!CombinedNodes.
count(ChildN.getNode()))
1741 AddToWorklist(ChildN.getNode());
1760 "Node was deleted but visit returned new node!");
1768 N->getNumValues() == 1 &&
"Type mismatch");
1778 AddToWorklistWithUsers(RV.
getNode());
1784 recursivelyDeleteUnusedNodes(
N);
1794 switch (
N->getOpcode()) {
1883 case ISD::FMA:
return visitFMA<EmptyMatchContext>(
N);
1947#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
1948#include "llvm/IR/VPIntrinsics.def"
1949 return visitVPOp(
N);
1960 if (!DisableGenericCombines)
1966 "Node was deleted but visit returned NULL!");
1973 DagCombineInfo(DAG, Level,
false,
this);
1981 switch (
N->getOpcode()) {
1989 RV = PromoteIntBinOp(
SDValue(
N, 0));
1994 RV = PromoteIntShiftOp(
SDValue(
N, 0));
2015 if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
2030 if (
unsigned NumOps =
N->getNumOperands()) {
2031 if (
N->getOperand(0).getValueType() == MVT::Other)
2032 return N->getOperand(0);
2033 if (
N->getOperand(NumOps-1).getValueType() == MVT::Other)
2034 return N->getOperand(NumOps-1);
2035 for (
unsigned i = 1; i < NumOps-1; ++i)
2036 if (
N->getOperand(i).getValueType() == MVT::Other)
2037 return N->getOperand(i);
2045 if (
N->getNumOperands() == 2) {
2047 return N->getOperand(0);
2049 return N->getOperand(1);
2064 AddToWorklist(*(
N->use_begin()));
2069 bool Changed =
false;
2076 for (
unsigned i = 0; i < TFs.
size(); ++i) {
2081 for (
unsigned j = i;
j < TFs.
size();
j++)
2092 switch (
Op.getOpcode()) {
2110 if (SeenOps.
insert(
Op.getNode()).second)
2121 for (
unsigned i = 1, e = TFs.
size(); i < e; i++)
2122 AddToWorklist(TFs[i]);
2134 bool DidPruneOps =
false;
2136 unsigned NumLeftToConsider = 0;
2138 Worklist.
push_back(std::make_pair(
Op.getNode(), NumLeftToConsider++));
2142 auto AddToWorklist = [&](
unsigned CurIdx,
SDNode *
Op,
unsigned OpNumber) {
2148 unsigned OrigOpNumber = 0;
2149 while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() !=
Op)
2151 assert((OrigOpNumber != Ops.size()) &&
2152 "expected to find TokenFactor Operand");
2154 for (
unsigned i = CurIdx + 1; i < Worklist.
size(); ++i) {
2155 if (Worklist[i].second == OrigOpNumber) {
2156 Worklist[i].second = OpNumber;
2159 OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
2160 OpWorkCount[OrigOpNumber] = 0;
2161 NumLeftToConsider--;
2164 if (SeenChains.
insert(
Op).second) {
2165 OpWorkCount[OpNumber]++;
2170 for (
unsigned i = 0; i < Worklist.
size() && i < 1024; ++i) {
2172 if (NumLeftToConsider <= 1)
2174 auto CurNode = Worklist[i].first;
2175 auto CurOpNumber = Worklist[i].second;
2176 assert((OpWorkCount[CurOpNumber] > 0) &&
2177 "Node should not appear in worklist");
2178 switch (CurNode->getOpcode()) {
2184 NumLeftToConsider++;
2187 for (
const SDValue &
Op : CurNode->op_values())
2188 AddToWorklist(i,
Op.getNode(), CurOpNumber);
2194 AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
2197 if (
auto *MemNode = dyn_cast<MemSDNode>(CurNode))
2198 AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
2201 OpWorkCount[CurOpNumber]--;
2202 if (OpWorkCount[CurOpNumber] == 0)
2203 NumLeftToConsider--;
2217 if (SeenChains.
count(
Op.getNode()) == 0)
2232 WorklistRemover DeadNodes(*
this);
2238 AddUsersToWorklist(
N);
2242 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i)
2245 }
while (!
N->use_empty());
2246 deleteAndRecombine(
N);
2254 return Const !=
nullptr && !Const->isOpaque() ? Const :
nullptr;
2264 Op =
N->getOperand(0);
2270 N.getValueType().getScalarType() != MVT::i1 ||
2271 cast<CondCodeSDNode>(
N.getOperand(2))->get() !=
ISD::SETNE)
2298 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2300 VT = LD->getMemoryVT();
2301 AS = LD->getAddressSpace();
2303 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2305 VT = ST->getMemoryVT();
2306 AS = ST->getAddressSpace();
2308 if (LD->isIndexed() || LD->getBasePtr().getNode() !=
N)
2310 VT = LD->getMemoryVT();
2311 AS = LD->getAddressSpace();
2313 if (ST->isIndexed() || ST->getBasePtr().getNode() !=
N)
2315 VT = ST->getMemoryVT();
2316 AS = ST->getAddressSpace();
2331 }
else if (
N->getOpcode() ==
ISD::SUB) {
2353 bool ShouldCommuteOperands) {
2358 if (ShouldCommuteOperands)
2370 unsigned Opcode =
N->getOpcode();
2371 EVT VT =
N->getValueType(0);
2378 unsigned OpNo = ShouldCommuteOperands ? 0 : 1;
2396 "Unexpected binary operator");
2413 unsigned SelOpNo = 0;
2449 bool CanFoldNonConst =
2455 if (!CanFoldNonConst &&
2463 if (CanFoldNonConst) {
2500 "Expecting add or sub");
2505 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2506 SDValue C = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2507 SDValue Z = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2508 auto *CN = dyn_cast<ConstantSDNode>(
C);
2513 if (Z.getOperand(0).getOpcode() !=
ISD::SETCC ||
2514 Z.getOperand(0).getValueType() != MVT::i1)
2518 SDValue SetCC = Z.getOperand(0);
2529 EVT VT =
C.getValueType();
2562 "Expecting add or sub");
2566 bool IsAdd =
N->getOpcode() ==
ISD::ADD;
2567 SDValue ConstantOp = IsAdd ?
N->getOperand(1) :
N->getOperand(0);
2568 SDValue ShiftOp = IsAdd ?
N->getOperand(0) :
N->getOperand(1);
2590 {ConstantOp, DAG.getConstant(1, DL, VT)})) {
2635 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2667 if ((!LegalOperations ||
2670 X.getScalarValueSizeInBits() == 1) {
2686 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
2690 if (!reassociationCanBreakAddressingModePattern(
ISD::ADD,
DL,
N, N0, N1)) {
2784 return (!Max && !
Op) ||
2785 (
Max &&
Op &&
Max->getAPIntValue() == (-
Op->getAPIntValue()));
2826 !
N->getFlags().hasNoSignedWrap()))) {
2839 if (
SDValue Combined = visitADDLikeCommutative(N0, N1,
N))
2842 if (
SDValue Combined = visitADDLikeCommutative(N1, N0,
N))
2876 if (
SDValue Combined = visitADDLike(
N))
2893 Flags.setDisjoint(
true);
2919 APInt NewStep = C0 + C1;
2929 APInt NewStep = SV0 + SV1;
2938 unsigned Opcode =
N->getOpcode();
2956 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
2960 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
2980 bool ForceCarryReconstruction =
false) {
2981 bool Masked =
false;
2986 V = V.getOperand(0);
2991 if (ForceCarryReconstruction)
2995 V = V.getOperand(0);
2999 if (ForceCarryReconstruction && V.getValueType() == MVT::i1)
3006 if (V.getResNo() != 1)
3013 EVT VT = V->getValueType(0);
3121 if (TN->
getVT() == MVT::i1) {
3138 DAG.
getVTList(VT, Carry.getValueType()), N0,
3151 if (!
N->hasAnyUseOfValue(1))
3184 if (Force && isa<ConstantSDNode>(V))
3194 EVT VT = V.getValueType();
3196 bool IsFlip =
false;
3199 IsFlip = Const->isOne();
3202 IsFlip = Const->isAllOnes();
3205 IsFlip = (Const->getAPIntValue() & 0x01) == 1;
3210 return V.getOperand(0);
3222 EVT CarryVT =
N->getValueType(1);
3226 if (!
N->hasAnyUseOfValue(1))
3233 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
3258 if (
SDValue Combined = visitUADDOLike(N0, N1,
N))
3261 if (
SDValue Combined = visitUADDOLike(N1, N0,
N))
3295 SDValue CarryIn =
N->getOperand(2);
3314 SDValue CarryIn =
N->getOperand(2);
3325 if (!LegalOperations ||
3335 AddToWorklist(CarryExt.
getNode());
3341 if (
SDValue Combined = visitUADDO_CARRYLike(N0, N1, CarryIn,
N))
3344 if (
SDValue Combined = visitUADDO_CARRYLike(N1, N0, CarryIn,
N))
3352 SDValue Ops[] = {N1, N0, CarryIn};
3491 EVT CarryOutType =
N->getValueType(0);
3507 unsigned CarryInOperandNum =
3509 if (Opcode ==
ISD::USUBO && CarryInOperandNum != 1)
3603 SDValue CarryIn =
N->getOperand(2);
3614 if (!LegalOperations ||
3619 if (
SDValue Combined = visitSADDO_CARRYLike(N0, N1, CarryIn,
N))
3622 if (
SDValue Combined = visitSADDO_CARRYLike(N1, N0, CarryIn,
N))
3634 "Illegal truncation");
3660 !(!LegalOperations || hasOperation(
ISD::USUBSAT, DstVT)))
3663 EVT SubVT =
N->getValueType(0);
3722 auto PeekThroughFreeze = [](
SDValue N) {
3724 return N->getOperand(0);
3730 if (PeekThroughFreeze(N0) == PeekThroughFreeze(N1))
3739 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
3747 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
3770 if (
N->getFlags().hasNoUnsignedWrap())
3776 if (
N->getFlags().hasNoSignedWrap())
3953 if ((X0 == S0 && X1 == N1) || (X0 == N1 && X1 == S0))
3955 if (
C->getAPIntValue() == (
BitWidth - 1))
3965 if (GA->getGlobal() == GB->getGlobal())
3973 if (TN->
getVT() == MVT::i1) {
4026 DAG.
getVTList(VT, Carry.getValueType()), NegX, Zero,
4034 if (!C0->isOpaque()) {
4035 const APInt &C0Val = C0->getAPIntValue();
4036 const APInt &MaybeOnes = ~DAG.computeKnownBits(N1).Zero;
4037 if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))
4058 unsigned Opcode =
N->getOpcode();
4079 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4105 if (!
N->hasAnyUseOfValue(1))
4132 EVT CarryVT =
N->getValueType(1);
4136 if (!
N->hasAnyUseOfValue(1))
4171 SDValue CarryIn =
N->getOperand(2);
4183 SDValue CarryIn =
N->getOperand(2);
4187 if (!LegalOperations ||
4198 SDValue CarryIn =
N->getOperand(2);
4202 if (!LegalOperations ||
4253 bool N1IsConst =
false;
4254 bool N1IsOpaqueConst =
false;
4259 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4265 "Splat APInt should be element width");
4267 N1IsConst = isa<ConstantSDNode>(N1);
4270 N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
4275 if (N1IsConst && ConstValue1.
isZero())
4279 if (N1IsConst && ConstValue1.
isOne())
4282 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4286 if (N1IsConst && ConstValue1.
isAllOnes())
4292 if (
SDValue LogBase2 = BuildLogBase2(N1,
DL)) {
4301 unsigned Log2Val = (-ConstValue1).logBase2();
4319 if (LoHi->hasAnyUseOfValue(1))
4322 if (LoHi->hasAnyUseOfValue(1))
4349 unsigned TZeros = MulC == 2 ? 0 : MulC.
countr_zero();
4351 if ((MulC - 1).isPowerOf2())
4353 else if ((MulC + 1).isPowerOf2())
4358 MathOp ==
ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
4361 "multiply-by-constant generated out of bounds shift");
4365 TZeros ? DAG.
getNode(MathOp,
DL, VT, Shl,
4426 APInt NewStep = C0 * MulVal;
4439 if (!V ||
V->isZero()) {
4453 for (
unsigned I = 0;
I != NumElts; ++
I)
4480 EVT NodeType =
Node->getValueType(0);
4481 if (!NodeType.isSimple())
4483 switch (NodeType.getSimpleVT().SimpleTy) {
4484 default:
return false;
4485 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
4486 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
4487 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
4488 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
4489 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
4497 if (
Node->use_empty())
4500 unsigned Opcode =
Node->getOpcode();
4505 EVT VT =
Node->getValueType(0);
4519 unsigned OtherOpcode = 0;
4540 unsigned UserOpc =
User->getOpcode();
4541 if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
4545 if (UserOpc == OtherOpcode) {
4548 }
else if (UserOpc == DivRemOpc) {
4551 assert(UserOpc == Opcode);
4556 CombineTo(
User, combined);
4567 EVT VT =
N->getValueType(0);
4570 unsigned Opc =
N->getOpcode();
4579 if (DAG.
isUndef(Opc, {N0, N1}))
4590 if (N0C && N0C->
isZero())
4613 EVT VT =
N->getValueType(0);
4623 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4640 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4648 if (
SDValue V = visitSDIVLike(N0, N1,
N)) {
4655 AddToWorklist(
Mul.getNode());
4657 CombineTo(RemNode, Sub);
4677 if (
C->isZero() ||
C->isOpaque())
4679 if (
C->getAPIntValue().isPowerOf2())
4681 if (
C->getAPIntValue().isNegatedPowerOf2())
4691 EVT VT =
N->getValueType(0);
4716 AddToWorklist(Sign.
getNode());
4722 AddToWorklist(
Add.getNode());
4733 Sra = DAG.
getSelect(
DL, VT, IsOneOrAllOnes, N0, Sra);
4761 EVT VT =
N->getValueType(0);
4771 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
4785 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4788 if (
SDValue V = visitUDIVLike(N0, N1,
N)) {
4795 AddToWorklist(
Mul.getNode());
4797 CombineTo(RemNode, Sub);
4815 EVT VT =
N->getValueType(0);
4819 if (
SDValue LogBase2 = BuildLogBase2(N1,
DL)) {
4820 AddToWorklist(LogBase2.getNode());
4824 AddToWorklist(Trunc.
getNode());
4833 if (
SDValue LogBase2 = BuildLogBase2(N10,
DL)) {
4834 AddToWorklist(LogBase2.getNode());
4838 AddToWorklist(Trunc.
getNode());
4840 AddToWorklist(
Add.getNode());
4868 unsigned Opcode =
N->getOpcode();
4871 EVT VT =
N->getValueType(0);
4893 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
4906 AddToWorklist(
Add.getNode());
4917 AddToWorklist(
Add.getNode());
4934 if (
SDValue OptimizedRem = buildOptimizedSREM(N0, N1,
N))
4935 return OptimizedRem;
4939 isSigned ? visitSDIVLike(N0, N1,
N) : visitUDIVLike(N0, N1,
N);
4945 CombineTo(DivNode, OptimizedDiv);
4948 AddToWorklist(OptimizedDiv.
getNode());
4949 AddToWorklist(
Mul.getNode());
4956 return DivRem.getValue(1);
4964 EVT VT =
N->getValueType(0);
4977 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5005 unsigned SimpleSize =
Simple.getSizeInBits();
5024 EVT VT =
N->getValueType(0);
5037 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5061 if (
SDValue LogBase2 = BuildLogBase2(N1,
DL)) {
5076 unsigned SimpleSize =
Simple.getSizeInBits();
5099 unsigned Opcode =
N->getOpcode();
5102 EVT VT =
N->getValueType(0);
5112 return DAG.
getNode(Opcode,
DL,
N->getVTList(), N1, N0);
5115 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5143 unsigned Opcode =
N->getOpcode();
5146 EVT VT =
N->getValueType(0);
5156 return DAG.
getNode(Opcode,
DL,
N->getVTList(), N1, N0);
5159 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5187SDValue DAGCombiner::SimplifyNodeWithTwoResults(
SDNode *
N,
unsigned LoOp,
5190 bool HiExists =
N->hasAnyUseOfValue(1);
5191 if (!HiExists && (!LegalOperations ||
5194 return CombineTo(
N, Res, Res);
5198 bool LoExists =
N->hasAnyUseOfValue(0);
5199 if (!LoExists && (!LegalOperations ||
5202 return CombineTo(
N, Res, Res);
5206 if (LoExists && HiExists)
5212 AddToWorklist(
Lo.getNode());
5215 (!LegalOperations ||
5217 return CombineTo(
N, LoOpt, LoOpt);
5222 AddToWorklist(
Hi.getNode());
5225 (!LegalOperations ||
5227 return CombineTo(
N, HiOpt, HiOpt);
5239 EVT VT =
N->getValueType(0);
5243 if (isa<ConstantSDNode>(N0) && isa<ConstantSDNode>(N1))
5255 unsigned SimpleSize =
Simple.getSizeInBits();
5268 return CombineTo(
N,
Lo,
Hi);
5281 EVT VT =
N->getValueType(0);
5285 if (isa<ConstantSDNode>(N0) && isa<ConstantSDNode>(N1))
5296 return CombineTo(
N, Zero, Zero);
5302 return CombineTo(
N, N0, Zero);
5309 unsigned SimpleSize =
Simple.getSizeInBits();
5322 return CombineTo(
N,
Lo,
Hi);
5335 EVT CarryVT =
N->getValueType(1);
5356 return DAG.
getNode(
N->getOpcode(),
DL,
N->getVTList(), N1, N0);
5368 N->getVTList(), N0, N0);
5375 return CombineTo(
N,
And, Cmp);
5413 unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3,
CC);
5451 N0CC = cast<CondCodeSDNode>(N0.
getOperand(4))->get();
5467 unsigned Opcode1 = isSignedMinMax(N00, N01, N02, N03, N0CC);
5468 if (!Opcode1 || Opcode0 == Opcode1)
5478 APInt MinCPlus1 = MinC + 1;
5479 if (-MaxC == MinCPlus1 && MinCPlus1.
isPowerOf2()) {
5536 unsigned BW = (C1 + 1).exactLogBase2();
5556 unsigned Opcode =
N->getOpcode();
5570 return DAG.
getNode(Opcode,
DL, VT, N1, N0);
5574 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
5578 if (
SDValue RMINMAX = reassociateOps(Opcode,
DL, N0, N1,
N->getFlags()))
5598 return DAG.
getNode(AltOpcode,
DL, VT, N0, N1);
5610 auto ReductionOpcode = [](
unsigned Opcode) {
5624 if (
SDValue SD = reassociateReduction(ReductionOpcode(Opcode), Opcode,
5638 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
5640 unsigned LogicOpcode =
N->getOpcode();
5655 EVT XVT =
X.getValueType();
5665 if (XVT !=
Y.getValueType())
5669 if ((VT.
isVector() || LegalOperations) &&
5682 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5692 if (XVT !=
Y.getValueType())
5704 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5725 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5740 return DAG.
getNode(HandOpcode,
DL, VT, Logic0, Logic1, S);
5753 if (XVT.
isInteger() && XVT ==
Y.getValueType() &&
5757 return DAG.
getNode(HandOpcode,
DL, VT, Logic);
5774 auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
5775 auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
5776 assert(
X.getValueType() ==
Y.getValueType() &&
5777 "Inputs to shuffles are not the same type");
5783 if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
5784 !SVN0->getMask().equals(SVN1->getMask()))
5820 SDValue LL, LR, RL, RR, N0CC, N1CC;
5821 if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
5822 !isSetCCEquivalent(N1, RL, RR, N1CC))
5826 "Unexpected operand types for bitwise logic op");
5829 "Unexpected operand types for setcc");
5845 if (LR == RR && CC0 == CC1 && IsInteger) {
5850 bool AndEqZero = IsAnd && CC1 ==
ISD::SETEQ && IsZero;
5852 bool AndGtNeg1 = IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
5854 bool OrNeZero = !IsAnd && CC1 ==
ISD::SETNE && IsZero;
5856 bool OrLtZero = !IsAnd && CC1 ==
ISD::SETLT && IsZero;
5862 if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
5864 AddToWorklist(
Or.getNode());
5869 bool AndEqNeg1 = IsAnd && CC1 ==
ISD::SETEQ && IsNeg1;
5871 bool AndLtZero = IsAnd && CC1 ==
ISD::SETLT && IsZero;
5873 bool OrNeNeg1 = !IsAnd && CC1 ==
ISD::SETNE && IsNeg1;
5875 bool OrGtNeg1 = !IsAnd && CC1 ==
ISD::SETGT && IsNeg1;
5881 if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
5883 AddToWorklist(
And.getNode());
5897 AddToWorklist(
Add.getNode());
5924 return !C0->
isOpaque() && !C1->isOpaque() && (CMax - CMin).isPowerOf2();
5942 if (LL == RR && LR == RL) {
5949 if (LL == RL && LR == RR) {
5953 (!LegalOperations ||
5975 bool isFMAXNUMFMINNUM_IEEE,
5976 bool isFMAXNUMFMINNUM) {
5987 isFMAXNUMFMINNUM_IEEE
5995 isFMAXNUMFMINNUM_IEEE
6013 isFMAXNUMFMINNUM_IEEE
6022 isFMAXNUMFMINNUM_IEEE
6032 "Invalid Op to combine SETCC with");
6043 LogicOp,
LHS.getNode(),
RHS.getNode());
6078 (isFMAXNUMFMINNUM_IEEE || isFMAXNUMFMINNUM))) &&
6084 SDValue CommonValue, Operand1, Operand2;
6092 }
else if (LHS1 == RHS1) {
6105 }
else if (RHS0 == LHS1) {
6122 bool IsSigned = isSignedIntSetCC(
CC);
6134 DAG, isFMAXNUMFMINNUM_IEEE, isFMAXNUMFMINNUM);
6138 DAG.
getNode(NewOpcode,
DL, OpVT, Operand1, Operand2);
6139 return DAG.
getSetCC(
DL, VT, MinMaxValue, CommonValue,
CC);
6149 LHS0 == RHS0 && LHS1C && RHS1C && OpVT.
isInteger()) {
6150 const APInt &APLhs = LHS1C->getAPIntValue();
6151 const APInt &APRhs = RHS1C->getAPIntValue();
6155 if (APLhs == (-APRhs) &&
6166 }
else if (TargetPreference &
6187 APInt Dif = MaxC - MinC;
6221 EVT CondVT =
Cond.getValueType();
6232 EVT OpVT =
T.getValueType();
6251 if (
SDValue V = foldLogicOfSetCCs(
true, N0, N1,
DL))
6269 APInt SRLC = SRLI->getAPIntValue();
6281 CombineTo(N0.
getNode(), NewAdd);
6295 EVT LoadResultTy,
EVT &ExtVT) {
6304 if (ExtVT == LoadedVT &&
6305 (!LegalOperations ||
6321 if (LegalOperations &&
6357 if (LdStMemVT.
bitsLT(MemVT))
6362 assert(ShAmt % 8 == 0 &&
"ShAmt is byte offset");
6363 const unsigned ByteShAmt = ShAmt / 8;
6374 if (PtrType == MVT::Untyped || PtrType.
isExtended())
6377 if (isa<LoadSDNode>(LDST)) {
6381 if (!
SDValue(Load, 0).hasOneUse())
6384 if (LegalOperations &&
6393 if (
Load->getNumValues() > 2)
6406 assert(isa<StoreSDNode>(LDST) &&
"It is not a Load nor a Store SDNode");
6412 if (LegalOperations &&
6419bool DAGCombiner::SearchForAndLoads(
SDNode *
N,
6427 if (
Op.getValueType().isVector())
6431 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
6433 (
Mask->getAPIntValue() &
C->getAPIntValue()) !=
C->getAPIntValue())
6438 if (!
Op.hasOneUse())
6441 switch(
Op.getOpcode()) {
6443 auto *
Load = cast<LoadSDNode>(
Op);
6445 if (isAndLoadExtLoad(Mask, Load,
Load->getValueType(0), ExtVT) &&
6463 unsigned ActiveBits =
Mask->getAPIntValue().countr_one();
6466 cast<VTSDNode>(
Op.getOperand(1))->getVT() :
6467 Op.getOperand(0).getValueType();
6478 if (!SearchForAndLoads(
Op.getNode(), Loads, NodesWithConsts, Mask,
6489 NodeToMask =
Op.getNode();
6492 for (
unsigned i = 0, e = NodeToMask->
getNumValues(); i < e; ++i) {
6494 if (VT != MVT::Glue && VT != MVT::Other) {
6496 NodeToMask =
nullptr;
6508bool DAGCombiner::BackwardsPropagateMask(
SDNode *
N) {
6509 auto *
Mask = dyn_cast<ConstantSDNode>(
N->getOperand(1));
6513 if (!
Mask->getAPIntValue().isMask())
6517 if (isa<LoadSDNode>(
N->getOperand(0)))
6522 SDNode *FixupNode =
nullptr;
6523 if (SearchForAndLoads(
N, Loads, NodesWithConsts, Mask, FixupNode)) {
6536 SDValue(FixupNode, 0), MaskOp);
6538 if (
And.getOpcode() == ISD ::AND)
6543 for (
auto *LogicN : NodesWithConsts) {
6547 if (isa<ConstantSDNode>(Op0))
6551 if (isa<ConstantSDNode>(Op1))
6555 if (isa<ConstantSDNode>(Op0) && !isa<ConstantSDNode>(Op1))
6562 for (
auto *Load : Loads) {
6567 if (
And.getOpcode() == ISD ::AND)
6570 SDValue NewLoad = reduceLoadWidth(
And.getNode());
6572 "Shouldn't be masking the load if it can't be narrowed");
6573 CombineTo(Load, NewLoad, NewLoad.
getValue(1));
6586SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(
SDNode *
N) {
6597 unsigned OuterShift;
6598 unsigned InnerShift;
6600 auto matchMask = [&OuterShift, &InnerShift, &
Y](
SDValue M) ->
bool {
6603 OuterShift =
M->getOpcode();
6612 Y =
M->getOperand(1);
6619 else if (matchMask(N0))
6625 EVT VT =
N->getValueType(0);
6642 SDValue And0 =
And->getOperand(0), And1 =
And->getOperand(1);
6652 bool FoundNot =
false;
6655 Src = Src.getOperand(0);
6661 Src = Src.getOperand(0);
6665 if (Src.getOpcode() !=
ISD::SRL || !Src.hasOneUse())
6669 EVT SrcVT = Src.getValueType();
6676 SDValue ShiftAmt = Src.getOperand(1);
6677 auto *ShiftAmtC = dyn_cast<ConstantSDNode>(ShiftAmt);
6678 if (!ShiftAmtC || !ShiftAmtC->getAPIntValue().ult(
BitWidth))
6682 Src = Src.getOperand(0);
6689 Src = Src.getOperand(0);
6713 EVT VT =
N->getValueType(0);
6739 unsigned LogicOpcode =
N->getOpcode();
6741 "Expected bitwise logic operation");
6747 unsigned ShiftOpcode = ShiftOp.
getOpcode();
6748 if (LogicOp.
getOpcode() != LogicOpcode ||
6772 EVT VT =
N->getValueType(0);
6776 return DAG.
getNode(LogicOpcode,
DL, VT, NewShift, Z);
6787 unsigned LogicOpcode =
N->getOpcode();
6789 "Expected bitwise logic operation");
6790 if (LeftHand.
getOpcode() != LogicOpcode ||
6811 EVT VT =
N->getValueType(0);
6813 return DAG.
getNode(LogicOpcode,
DL, VT, CombinedShifts, W);
6840 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
6854 auto *MLoad = dyn_cast<MaskedLoadSDNode>(N0);
6858 EVT LoadVT = MLoad->getMemoryVT();
6866 if (
Splat->getAPIntValue().isMask(ElementSize)) {
6868 ExtVT,
DL, MLoad->getChain(), MLoad->getBasePtr(),
6869 MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(),
6870 LoadVT, MLoad->getMemOperand(), MLoad->getAddressingMode(),
6872 bool LoadHasOtherUsers = !N0.
hasOneUse();
6873 CombineTo(
N, NewLoad);
6874 if (LoadHasOtherUsers)
6895 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
6909 return RHS->getAPIntValue().isSubsetOf(
LHS->getAPIntValue());
6976 unsigned EltBitWidth =
Vector->getValueType(0).getScalarSizeInBits();
6977 APInt SplatValue, SplatUndef;
6978 unsigned SplatBitSize;
6985 const bool IsBigEndian =
false;
6987 Vector->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
6988 HasAnyUndefs, EltBitWidth, IsBigEndian);
6992 if (IsSplat && (SplatBitSize % EltBitWidth) == 0) {
6995 SplatValue |= SplatUndef;
7002 for (
unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
7011 Load->getValueType(0),
7012 Load->getMemoryVT());
7020 switch (
Load->getExtensionType()) {
7021 default:
B =
false;
break;
7033 CombineTo(
N, (N0.
getNode() == Load) ? NewLoad : N0);
7038 Load->getChain(),
Load->getBasePtr(),
7039 Load->getOffset(),
Load->getMemoryVT(),
7040 Load->getMemOperand());
7042 if (
Load->getNumValues() == 3) {
7046 CombineTo(Load, To, 3,
true);
7048 CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
7058 if (
SDValue Shuffle = XformToShuffleWithZero(
N))
7067 EVT ExtVT =
Ext->getValueType(0);
7084 if (
auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
7085 EVT MemVT = GN0->getMemoryVT();
7088 if (
SDValue(GN0, 0).hasOneUse() &&
7091 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
7092 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
7095 DAG.
getVTList(VT, MVT::Other), MemVT,
DL, Ops, GN0->getMemOperand(),
7098 CombineTo(
N, ZExtLoad);
7099 AddToWorklist(ZExtLoad.
getNode());
7108 if (
SDValue Res = reduceLoadWidth(
N))
7116 if (BackwardsPropagateMask(
N))
7120 if (
SDValue Combined = visitANDLike(N0, N1,
N))
7125 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
7162 auto *LN0 = cast<LoadSDNode>(N0);
7163 EVT MemVT = LN0->getMemoryVT();
7170 ((!LegalOperations && LN0->isSimple()) ||
7174 LN0->getBasePtr(), MemVT, LN0->getMemOperand());
7188 if (
SDValue Shifts = unfoldExtremeBitClearingToShifts(
N))
7204 auto *
C = dyn_cast<ConstantSDNode>(RHS);
7208 if (!
C->getAPIntValue().isMask(
7209 LHS.getOperand(0).getValueType().getFixedSizeInBits()))
7216 if (IsAndZeroExtMask(N0, N1))
7225 if (LegalOperations || VT.
isVector())
7234 bool DemandHighBits) {
7235 if (!LegalOperations)
7238 EVT VT =
N->getValueType(0);
7239 if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
7245 bool LookPassAnd0 =
false;
7246 bool LookPassAnd1 =
false;
7261 LookPassAnd0 =
true;
7271 LookPassAnd1 =
true;
7297 LookPassAnd0 =
true;
7311 LookPassAnd1 =
true;
7320 if (OpSizeInBits > 16) {
7324 if (DemandHighBits && !LookPassAnd0)
7331 if (!LookPassAnd1) {
7332 unsigned HighBit = DemandHighBits ? OpSizeInBits : 24;
7340 if (OpSizeInBits > 16) {
7356 if (!
N->hasOneUse())
7359 unsigned Opc =
N.getOpcode();
7371 N1C = dyn_cast<ConstantSDNode>(
N.getOperand(1));
7373 N1C = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
7377 unsigned MaskByteOffset;
7381 case 0xFF: MaskByteOffset = 0;
break;
7382 case 0xFF00: MaskByteOffset = 1;
break;
7391 case 0xFF0000: MaskByteOffset = 2;
break;
7392 case 0xFF000000: MaskByteOffset = 3;
break;
7397 if (MaskByteOffset == 0 || MaskByteOffset == 2) {
7403 if (!
C ||
C->getZExtValue() != 8)
7411 if (!
C ||
C->getZExtValue() != 8)
7417 if (MaskByteOffset != 0 && MaskByteOffset != 2)
7420 if (!
C ||
C->getZExtValue() != 8)
7425 if (MaskByteOffset != 1 && MaskByteOffset != 3)
7428 if (!
C ||
C->getZExtValue() != 8)
7432 if (Parts[MaskByteOffset])
7447 if (!
C ||
C->getAPIntValue() != 16)
7449 Parts[0] = Parts[1] =
N.getOperand(0).getOperand(0).getNode();
7464 "MatchBSwapHWordOrAndAnd: expecting i32");
7474 if (!Mask0 || !Mask1)
7485 if (!ShiftAmt0 || !ShiftAmt1)
7505 if (!LegalOperations)
7508 EVT VT =
N->getValueType(0);
7548 if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
7576 if (
SDValue V = foldLogicOfSetCCs(
false, N0, N1,
DL))
7591 const APInt &LHSMask = N0O1C->getAPIntValue();
7592 const APInt &RHSMask = N1O1C->getAPIntValue();
7625 auto peekThroughResize = [](
SDValue V) {
7627 return V->getOperand(0);
7631 SDValue N0Resized = peekThroughResize(N0);
7633 SDValue N1Resized = peekThroughResize(N1);
7638 if (N00 == N1Resized || N01 == N1Resized)
7645 if (peekThroughResize(NotOperand) == N1Resized)
7653 if (peekThroughResize(NotOperand) == N1Resized)
7674 auto peekThroughZext = [](
SDValue V) {
7676 return V->getOperand(0);
7716 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
7730 auto *SV0 = dyn_cast<ShuffleVectorSDNode>(N0);
7731 auto *SV1 = dyn_cast<ShuffleVectorSDNode>(N1);
7738 if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
7739 assert((!ZeroN00 || !ZeroN01) &&
"Both inputs zero!");
7740 assert((!ZeroN10 || !ZeroN11) &&
"Both inputs zero!");
7741 bool CanFold =
true;
7745 for (
int i = 0; i != NumElts; ++i) {
7746 int M0 = SV0->getMaskElt(i);
7747 int M1 = SV1->getMaskElt(i);
7750 bool M0Zero =
M0 < 0 || (ZeroN00 == (
M0 < NumElts));
7751 bool M1Zero =
M1 < 0 || (ZeroN10 == (
M1 < NumElts));
7755 if ((M0Zero &&
M1 < 0) || (M1Zero &&
M0 < 0))
7759 if (M0Zero == M1Zero) {
7764 assert((
M0 >= 0 ||
M1 >= 0) &&
"Undef index!");
7770 Mask[i] = M1Zero ?
M0 % NumElts : (
M1 % NumElts) + NumElts;
7779 return LegalShuffle;
7793 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
7804 if (
SDValue Combined = visitORLike(N0, N1,
DL))
7811 if (
SDValue BSwap = MatchBSwapHWord(
N, N0, N1))
7813 if (
SDValue BSwap = MatchBSwapHWordLow(
N, N0, N1))
7847 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
7851 if (
SDValue Rot = MatchRotate(N0, N1,
DL))
7854 if (
SDValue Load = MatchLoadCombine(
N))
7864 if (
SDValue Combined = visitADDLike(
N))
7869 if (LegalOperations || VT.
isVector())
7880 Mask =
Op.getOperand(1);
7881 return Op.getOperand(0);
7924 assert(OppShift && ExtractFrom &&
"Empty SDValue");
7952 bool IsMulOrDiv =
false;
7955 auto SelectOpcode = [&](
unsigned NeededShift,
unsigned MulOrDivVariant) {
7956 IsMulOrDiv = ExtractFrom.
getOpcode() == MulOrDivVariant;
7957 if (!IsMulOrDiv && ExtractFrom.
getOpcode() != NeededShift)
7959 Opcode = NeededShift;
8009 if (Rem != 0 || ResultAmt != OppLHSAmt)
8015 if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.
zextOrTrunc(
8024 return DAG.
getNode(Opcode,
DL, ResVT, OppShiftLHS, NewShiftNode);
8078 unsigned MaskLoBits = 0;
8080 unsigned Bits =
Log2_64(EltSize);
8082 if (NegBits >= Bits) {
8105 if (PosBits >= MaskLoBits) {
8127 if ((Pos == NegOp1) ||
8151 return Width.
getLoBits(MaskLoBits) == 0;
8152 return Width == EltSize;
8162 SDValue InnerNeg,
bool HasPos,
8163 unsigned PosOpcode,
unsigned NegOpcode,
8175 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, Shifted,
8176 HasPos ? Pos : Neg);
8190 SDValue InnerNeg,
bool HasPos,
8191 unsigned PosOpcode,
unsigned NegOpcode,
8203 if (
matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, N0 == N1)) {
8204 return DAG.
getNode(HasPos ? PosOpcode : NegOpcode,
DL, VT, N0, N1,
8205 HasPos ? Pos : Neg);
8212 auto IsBinOpImm = [](
SDValue Op,
unsigned BinOpc,
unsigned Imm) {
8213 if (
Op.getOpcode() != BinOpc)
8222 IsBinOpImm(InnerNeg,
ISD::XOR, EltBits - 1) &&
8231 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
8241 IsBinOpImm(InnerPos,
ISD::XOR, EltBits - 1) &&
8256 EVT VT =
LHS.getValueType();
8261 bool HasROTL = hasOperation(
ISD::ROTL, VT);
8262 bool HasROTR = hasOperation(
ISD::ROTR, VT);
8263 bool HasFSHL = hasOperation(
ISD::FSHL, VT);
8264 bool HasFSHR = hasOperation(
ISD::FSHR, VT);
8275 if (LegalOperations && !HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
8280 LHS.getOperand(0).getValueType() ==
RHS.getOperand(0).getValueType()) {
8282 if (
SDValue Rot = MatchRotate(
LHS.getOperand(0),
RHS.getOperand(0),
DL)) {
8297 if (!LHSShift && !RHSShift)
8312 RHSShift = NewRHSShift;
8317 LHSShift = NewLHSShift;
8320 if (!RHSShift || !LHSShift)
8347 return (
LHS->getAPIntValue() +
RHS->getAPIntValue()) == EltSizeInBits;
8350 auto ApplyMasks = [&](
SDValue Res) {
8374 bool IsRotate = LHSShiftArg == RHSShiftArg;
8375 if (!IsRotate && !(HasFSHL || HasFSHR)) {
8384 if (CommonOp ==
Or.getOperand(0)) {
8386 Y =
Or.getOperand(1);
8389 if (CommonOp ==
Or.getOperand(1)) {
8391 Y =
Or.getOperand(0);
8398 if (matchOr(LHSShiftArg, RHSShiftArg)) {
8403 }
else if (matchOr(RHSShiftArg, LHSShiftArg)) {
8412 return ApplyMasks(Res);
8425 if (IsRotate && (HasROTL || HasROTR || !(HasFSHL || HasFSHR))) {
8426 bool UseROTL = !LegalOperations || HasROTL;
8428 UseROTL ? LHSShiftAmt : RHSShiftAmt);
8430 bool UseFSHL = !LegalOperations || HasFSHL;
8432 RHSShiftArg, UseFSHL ? LHSShiftAmt : RHSShiftAmt);
8435 return ApplyMasks(Res);
8440 if (!HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
8449 SDValue LExtOp0 = LHSShiftAmt;
8450 SDValue RExtOp0 = RHSShiftAmt;
8463 if (IsRotate && (HasROTL || HasROTR)) {
8465 MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt, LExtOp0,
8471 MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt, RExtOp0,
8478 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt, RHSShiftAmt,
8484 MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
8535static std::optional<SDByteProvider>
8537 std::optional<uint64_t> VectorIndex,
8538 unsigned StartingIndex = 0) {
8542 return std::nullopt;
8546 if (
Depth && !
Op.hasOneUse() &&
8547 (
Op.getOpcode() !=
ISD::LOAD || !
Op.getValueType().isVector()))
8548 return std::nullopt;
8552 if (
Op.getOpcode() !=
ISD::LOAD && VectorIndex.has_value())
8553 return std::nullopt;
8557 return std::nullopt;
8559 assert(
Index < ByteWidth &&
"invalid index requested");
8562 switch (
Op.getOpcode()) {
8567 return std::nullopt;
8571 return std::nullopt;
8573 if (
LHS->isConstantZero())
8575 if (
RHS->isConstantZero())
8577 return std::nullopt;
8580 auto ShiftOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
8582 return std::nullopt;
8584 uint64_t BitShift = ShiftOp->getZExtValue();
8586 if (BitShift % 8 != 0)
8587 return std::nullopt;
8593 return Index < ByteShift
8603 if (NarrowBitWidth % 8 != 0)
8604 return std::nullopt;
8605 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
8607 if (
Index >= NarrowByteWidth)
8609 ? std::optional<SDByteProvider>(
8617 Depth + 1, VectorIndex, StartingIndex);
8619 auto OffsetOp = dyn_cast<ConstantSDNode>(
Op->getOperand(1));
8621 return std::nullopt;
8623 VectorIndex = OffsetOp->getZExtValue();
8627 if (NarrowBitWidth % 8 != 0)
8628 return std::nullopt;
8629 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
8632 if (
Index >= NarrowByteWidth)
8633 return std::nullopt;
8641 if (*VectorIndex * NarrowByteWidth > StartingIndex)
8642 return std::nullopt;
8643 if ((*VectorIndex + 1) * NarrowByteWidth <= StartingIndex)
8644 return std::nullopt;
8647 VectorIndex, StartingIndex);
8650 auto L = cast<LoadSDNode>(
Op.getNode());
8651 if (!L->isSimple() || L->isIndexed())
8652 return std::nullopt;
8654 unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
8655 if (NarrowBitWidth % 8 != 0)
8656 return std::nullopt;
8657 uint64_t NarrowByteWidth = NarrowBitWidth / 8;
8662 if (
Index >= NarrowByteWidth)
8664 ? std::optional<SDByteProvider>(
8668 unsigned BPVectorIndex = VectorIndex.value_or(0U);
8673 return std::nullopt;
8688 int64_t FirstOffset) {
8690 unsigned Width = ByteOffsets.
size();
8692 return std::nullopt;
8694 bool BigEndian =
true, LittleEndian =
true;
8695 for (
unsigned i = 0; i < Width; i++) {
8696 int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
8699 if (!BigEndian && !LittleEndian)
8700 return std::nullopt;
8703 assert((BigEndian != LittleEndian) &&
"It should be either big endian or"
8709 switch (
Value.getOpcode()) {
8752 EVT MemVT =
N->getMemoryVT();
8753 if (!(MemVT == MVT::i8 || MemVT == MVT::i16 || MemVT == MVT::i32) ||
8754 !
N->isSimple() ||
N->isIndexed())
8761 unsigned MaxWideNumBits = 64;
8762 unsigned MaxStores = MaxWideNumBits / NarrowNumBits;
8763 while (
auto *Store = dyn_cast<StoreSDNode>(Chain)) {
8771 if (
Store->getMemoryVT() != MemVT || !
Store->isSimple() ||
8775 Chain =
Store->getChain();
8776 if (MaxStores < Stores.
size())
8780 if (Stores.
size() < 2)
8785 unsigned NumStores = Stores.
size();
8786 unsigned WideNumBits = NumStores * NarrowNumBits;
8788 if (WideVT != MVT::i16 && WideVT != MVT::i32 && WideVT != MVT::i64)
8797 std::optional<BaseIndexOffset>
Base;
8798 for (
auto *Store : Stores) {
8809 isa<ConstantSDNode>(WideVal.
getOperand(1))) {
8817 if (ShiftAmtC % NarrowNumBits != 0)
8820 Offset = ShiftAmtC / NarrowNumBits;
8827 SourceValue = WideVal;
8834 SourceValue = WideVal;
8842 int64_t ByteOffsetFromBase = 0;
8845 else if (!
Base->equalBaseIndex(
Ptr, DAG, ByteOffsetFromBase))
8849 if (ByteOffsetFromBase < FirstOffset) {
8851 FirstOffset = ByteOffsetFromBase;
8855 if (Offset < 0 || Offset >= NumStores || OffsetMap[
Offset] !=
INT64_MAX)
8857 OffsetMap[
Offset] = ByteOffsetFromBase;
8861 assert(FirstStore &&
"First store must be set");
8868 if (!Allowed || !
Fast)
8873 auto checkOffsets = [&](
bool MatchLittleEndian) {
8874 if (MatchLittleEndian) {
8875 for (
unsigned i = 0; i != NumStores; ++i)
8876 if (OffsetMap[i] != i * (NarrowNumBits / 8) + FirstOffset)
8879 for (
unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --
j)
8880 if (OffsetMap[j] != i * (NarrowNumBits / 8) + FirstOffset)
8887 bool NeedBswap =
false;
8888 bool NeedRotate =
false;
8891 if (NarrowNumBits == 8 && checkOffsets(Layout.
isBigEndian()))
8893 else if (NumStores == 2 && checkOffsets(Layout.
isBigEndian()))
8902 "Unexpected store value to merge");
8911 }
else if (NeedRotate) {
8912 assert(WideNumBits % 2 == 0 &&
"Unexpected type for rotate");
8958 "Can only match load combining against OR nodes");
8961 EVT VT =
N->getValueType(0);
8962 if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
8968 assert(
P.hasSrc() &&
"Must be a memory byte provider");
8969 auto *
Load = cast<LoadSDNode>(
P.Src.value());
8971 unsigned LoadBitWidth =
Load->getMemoryVT().getScalarSizeInBits();
8973 assert(LoadBitWidth % 8 == 0 &&
8974 "can only analyze providers for individual bytes not bit");
8975 unsigned LoadByteWidth = LoadBitWidth / 8;
8980 std::optional<BaseIndexOffset>
Base;
8984 std::optional<SDByteProvider> FirstByteProvider;
8990 unsigned ZeroExtendedBytes = 0;
8991 for (
int i = ByteWidth - 1; i >= 0; --i) {
8998 if (
P->isConstantZero()) {
9001 if (++ZeroExtendedBytes != (ByteWidth -
static_cast<unsigned>(i)))
9005 assert(
P->hasSrc() &&
"provenance should either be memory or zero");
9006 auto *
L = cast<LoadSDNode>(
P->Src.value());
9012 else if (Chain != LChain)
9017 int64_t ByteOffsetFromBase = 0;
9026 if (
L->getMemoryVT().isVector()) {
9027 unsigned LoadWidthInBit =
L->getMemoryVT().getScalarSizeInBits();
9028 if (LoadWidthInBit % 8 != 0)
9030 unsigned ByteOffsetFromVector =
P->SrcOffset * LoadWidthInBit / 8;
9031 Ptr.addToOffset(ByteOffsetFromVector);
9037 else if (!
Base->equalBaseIndex(
Ptr, DAG, ByteOffsetFromBase))
9041 ByteOffsetFromBase += MemoryByteOffset(*
P);
9042 ByteOffsets[i] = ByteOffsetFromBase;
9045 if (ByteOffsetFromBase < FirstOffset) {
9046 FirstByteProvider =
P;
9047 FirstOffset = ByteOffsetFromBase;
9053 assert(!Loads.
empty() &&
"All the bytes of the value must be loaded from "
9054 "memory, so there must be at least one load which produces the value");
9055 assert(
Base &&
"Base address of the accessed memory location must be set");
9058 bool NeedsZext = ZeroExtendedBytes > 0;
9069 if (LegalOperations &&
9077 ArrayRef(ByteOffsets).drop_back(ZeroExtendedBytes), FirstOffset);
9081 assert(FirstByteProvider &&
"must be set");
9085 if (MemoryByteOffset(*FirstByteProvider) != 0)
9087 auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value());
9094 bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
9101 if (NeedsBswap && (LegalOperations || NeedsZext) &&
9107 if (NeedsBswap && NeedsZext && LegalOperations &&
9115 *FirstLoad->getMemOperand(), &
Fast);
9116 if (!Allowed || !
Fast)
9121 Chain, FirstLoad->getBasePtr(),
9122 FirstLoad->getPointerInfo(), MemVT, FirstLoad->getAlign());
9135 SDLoc(
N), LegalOperations))
9163 EVT VT =
N->getValueType(0);
9185 M =
And.getOperand(XorIdx ? 0 : 1);
9191 if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
9192 !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
9198 if (isa<ConstantSDNode>(
M.getNode()))
9266 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
9278 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
9294 Flags.setDisjoint(
true);
9302 if (
SDValue Combined = visitADDLike(
N))
9309 isSetCCEquivalent(N0, LHS, RHS,
CC,
true)) {
9311 LHS.getValueType());
9312 if (!LegalOperations ||
9330 CombineTo(
N, SetCC);
9332 recursivelyDeleteUnusedNodes(N0.
getNode());
9348 AddToWorklist(
V.getNode());
9356 if (isOneUseSetCC(N01) || isOneUseSetCC(N00)) {
9361 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
9368 if (isa<ConstantSDNode>(N01) || isa<ConstantSDNode>(N00)) {
9373 return DAG.
getNode(NewOpcode,
DL, VT, N00, N01);
9396 AddToWorklist(NotX.
getNode());
9405 SDValue A0 =
A.getOperand(0), A1 =
A.getOperand(1);
9407 if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
9444 if (
SDValue V = hoistLogicOpWithSameOpcodeHands(
N))
9455 if (
SDValue MM = unfoldMaskedMerge(
N))
9478 unsigned LogicOpcode = LogicOp.
getOpcode();
9484 unsigned ShiftOpcode = Shift->
getOpcode();
9487 assert(C1Node &&
"Expected a shift with constant operand");
9490 const APInt *&ShiftAmtVal) {
9491 if (V.getOpcode() != ShiftOpcode || !V.hasOneUse())
9499 ShiftOp = V.getOperand(0);
9504 if (ShiftAmtVal->getBitWidth() != C1Val.
getBitWidth())
9509 bool Overflow =
false;
9510 APInt NewShiftAmt = C1Val.
uadd_ov(*ShiftAmtVal, Overflow);
9515 if (NewShiftAmt.
uge(V.getScalarValueSizeInBits()))
9524 if (matchFirstShift(LogicOp.
getOperand(0),
X, C0Val))
9526 else if (matchFirstShift(LogicOp.
getOperand(1),
X, C0Val))
9538 return DAG.
getNode(LogicOpcode,
DL, VT, NewShift1, NewShift2,
9568 switch (
LHS.getOpcode()) {
9588 isa<ConstantSDNode>(BinOpLHSVal.
getOperand(1));
9592 if (!IsShiftByConstant && !IsCopyOrSelect)
9595 if (IsCopyOrSelect &&
N->hasOneUse())
9600 EVT VT =
N->getValueType(0);
9602 N->getOpcode(),
DL, VT, {LHS.getOperand(1), N->getOperand(1)})) {
9605 return DAG.
getNode(
LHS.getOpcode(),
DL, VT, NewShift, NewRHS);
9616 EVT TruncVT =
N->getValueType(0);
9617 if (
N->hasOneUse() &&
N->getOperand(0).hasOneUse() &&
9619 SDValue N01 =
N->getOperand(0).getOperand(1);
9622 SDValue N00 =
N->getOperand(0).getOperand(0);
9625 AddToWorklist(Trunc00.
getNode());
9626 AddToWorklist(Trunc01.
getNode());
9638 EVT VT =
N->getValueType(0);
9653 bool OutOfRange =
false;
9655 OutOfRange |=
C->getAPIntValue().uge(Bitsize);
9663 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, Amt);
9668 if (RotAmtC && RotAmtC->getAPIntValue() == 8 &&
9680 return DAG.
getNode(
N->getOpcode(), dl, VT, N0, NewOp1);
9692 bool SameSide = (
N->getOpcode() == NextOp);
9701 CombineOp, dl, ShiftVT, {Norm1, Norm2})) {
9703 {CombinedShift, BitsizeC});
9705 ISD::UREM, dl, ShiftVT, {CombinedShift, BitsizeC});
9753 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
9774 return (c1 + c2).uge(OpSizeInBits);
9784 return (c1 + c2).ult(OpSizeInBits);
9812 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
9813 (c1 + c2).uge(OpSizeInBits);
9825 return c2.
uge(OpSizeInBits - InnerBitwidth) &&
9826 (c1 + c2).ult(OpSizeInBits);
9860 AddToWorklist(NewSHL.
getNode());
9868 const APInt &LHSC =
LHS->getAPIntValue();
9869 const APInt &RHSC =
RHS->getAPIntValue();
9870 return LHSC.
ult(OpSizeInBits) && RHSC.
ult(OpSizeInBits) &&
9945 AddToWorklist(Shl0.
getNode());
9949 Flags.setDisjoint(
true);
9965 {Add.getOperand(1)})) {
9985 if (
SDValue NewSHL = visitShiftByConstant(
N))
10004 APInt NewStep = C0 << ShlVal;
10019 "SRL or SRA node is required here!");
10030 SDValue ShiftOperand =
N->getOperand(0);
10041 if (!IsSignExt && !IsZeroExt)
10048 auto UserOfLowerBits = [NarrowVTSize](
SDNode *U) {
10053 if (!UShiftAmtSrc) {
10057 return UShiftAmt < NarrowVTSize;
10071 unsigned ActiveBits = IsSignExt
10072 ?
Constant->getAPIntValue().getSignificantBits()
10073 :
Constant->getAPIntValue().getActiveBits();
10074 if (ActiveBits > NarrowVTSize)
10091 "Cannot have a multiply node with two different operand types.");
10102 if (ShiftAmt != NarrowVTSize)
10124 bool IsSigned =
N->getOpcode() ==
ISD::SRA;
10131 unsigned Opcode =
N->getOpcode();
10136 EVT VT =
N->getValueType(0);
10188 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
10205 APInt Sum = c1 + c2;
10206 unsigned ShiftSum =
10217 "Expected matchBinaryPredicate to return one element for "
10221 ShiftValue = ShiftValues[0];
10249 if ((ShiftAmt > 0) &&
10261 N->getValueType(0), Trunc);
10297 DAG.
getConstant(AddC->getAPIntValue().lshr(ShiftAmt).trunc(
10314 if (
SDValue NewOp1 = distributeTruncateThroughAnd(N1.
getNode()))
10331 if (LargeShift->getAPIntValue() == TruncBits) {
10353 if (
SDValue NewSRA = visitShiftByConstant(
N))
10362 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
10387 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
10403 return (c1 + c2).uge(OpSizeInBits);
10413 return (c1 + c2).ult(OpSizeInBits);
10434 if (c1 + OpSizeInBits == InnerShiftSize) {
10436 if (c1 + c2 >= InnerShiftSize)
10446 c1 + c2 < InnerShiftSize) {
10452 OpSizeInBits - c2),
10467 const APInt &LHSC =
LHS->getAPIntValue();
10468 const APInt &RHSC =
RHS->getAPIntValue();
10469 return LHSC.
ult(OpSizeInBits) && RHSC.
ult(OpSizeInBits) &&
10513 AddToWorklist(SmallShift.
getNode());
10542 APInt UnknownBits = ~Known.Zero;
10559 AddToWorklist(
Op.getNode());
10571 if (
SDValue NewOp1 = distributeTruncateThroughAnd(N1.
getNode()))
10581 if (
SDValue NewSRL = visitShiftByConstant(
N))
10585 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
10612 if (
N->hasOneUse()) {
10621 AddToWorklist(
Use);
10633 EVT VT =
N->getValueType(0);
10645 return IsFSHL ? N0 : N1;
10647 auto IsUndefOrZero = [](
SDValue V) {
10664 return IsFSHL ? N0 : N1;
10670 if (IsUndefOrZero(N0))
10674 if (IsUndefOrZero(N1))
10686 auto *
LHS = dyn_cast<LoadSDNode>(N0);
10687 auto *
RHS = dyn_cast<LoadSDNode>(N1);
10688 if (LHS && RHS &&
LHS->isSimple() &&
RHS->isSimple() &&
10689 LHS->getAddressSpace() ==
RHS->getAddressSpace() &&
10699 RHS->getAddressSpace(), NewAlign,
10700 RHS->getMemOperand()->getFlags(), &
Fast) &&
10704 AddToWorklist(NewPtr.
getNode());
10706 VT,
DL,
RHS->getChain(), NewPtr,
10707 RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
10708 RHS->getMemOperand()->getFlags(),
RHS->getAAInfo());
10710 WorklistRemover DeadNodes(*
this);
10736 if (N0 == N1 && hasOperation(RotOpc, VT))
10782 EVT SrcVT =
N->getValueType(0);
10785 N =
N->getOperand(0).getNode();
10790 EVT VT =
N->getValueType(0);
10791 SDValue AbsOp1 =
N->getOperand(0);
10818 VT0 = cast<VTSDNode>(Op0.
getOperand(1))->getVT();
10819 VT1 = cast<VTSDNode>(Op1.
getOperand(1))->getVT();
10828 EVT MaxVT = VT0.
bitsGT(VT1) ? VT0 : VT1;
10829 if ((VT0 == MaxVT || Op0->
hasOneUse()) &&
10830 (VT1 == MaxVT || Op1->
hasOneUse()) && hasOperation(ABDOpcode, MaxVT)) {
10840 if (hasOperation(ABDOpcode, VT)) {
10850 EVT VT =
N->getValueType(0);
10885 EVT VT =
N->getValueType(0);
10908 auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
10910 if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
10911 ShAmt->getZExtValue() >= (BW / 2) &&
10912 (ShAmt->getZExtValue() % 16) == 0 && TLI.
isTypeLegal(HalfVT) &&
10914 (!LegalOperations || hasOperation(
ISD::BSWAP, HalfVT))) {
10916 if (
uint64_t NewShAmt = (ShAmt->getZExtValue() - (BW / 2)))
10931 auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
10932 if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
10933 ShAmt->getZExtValue() % 8 == 0) {
10948 EVT VT =
N->getValueType(0);
10962 EVT VT =
N->getValueType(0);
10979 EVT VT =
N->getValueType(0);
10991 EVT VT =
N->getValueType(0);
11008 EVT VT =
N->getValueType(0);
11020 EVT VT =
N->getValueType(0);
11032 const APInt &Amt = AmtC->getAPIntValue();
11033 if (Amt.
ult(NumBits)) {
11070 EVT VT =
LHS.getValueType();
11126 if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
11132 True, DAG, LegalOperations, ForCodeSize);
11144 if (LHS == NegTrue) {
11148 RHS, DAG, LegalOperations, ForCodeSize);
11151 if (NegRHS == False) {
11153 False,
CC, TLI, DAG);
11172 EVT VT =
N->getValueType(0);
11174 VT !=
Cond.getOperand(0).getValueType())
11223 EVT VT =
N->getValueType(0);
11224 EVT CondVT =
Cond.getValueType();
11230 auto *C1 = dyn_cast<ConstantSDNode>(N1);
11231 auto *C2 = dyn_cast<ConstantSDNode>(N2);
11235 if (CondVT != MVT::i1 || LegalOperations) {
11250 C1->
isZero() && C2->isOne()) {
11265 assert(CondVT == MVT::i1 && !LegalOperations);
11268 if (C1->
isOne() && C2->isZero())
11276 if (C1->
isZero() && C2->isOne()) {
11283 if (C1->
isZero() && C2->isAllOnes()) {
11296 const APInt &C1Val = C1->getAPIntValue();
11297 const APInt &C2Val = C2->getAPIntValue();
11300 if (C1Val - 1 == C2Val) {
11306 if (C1Val + 1 == C2Val) {
11326 if (C2->isAllOnes()) {
11338template <
class MatchContextClass>
11341 N->getOpcode() == ISD::VP_SELECT) &&
11342 "Expected a (v)(vp.)select");
11344 SDValue T =
N->getOperand(1),
F =
N->getOperand(2);
11345 EVT VT =
N->getValueType(0);
11347 MatchContextClass matcher(DAG, TLI,
N);
11383 EVT VT =
N->getValueType(0);
11441 EVT VT =
N->getValueType(0);
11449 if (
SDValue V = foldBoolSelectToLogic<EmptyMatchContext>(
N, DAG))
11459 if (
SDValue V = foldSelectOfConstants(
N))
11463 if (SimplifySelectOps(
N, N1, N2))
11466 if (VT0 == MVT::i1) {
11475 bool normalizeToSequence =
11484 if (normalizeToSequence || !InnerSelect.
use_empty())
11486 InnerSelect, N2, Flags);
11489 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
11496 Cond1, N1, N2, Flags);
11497 if (normalizeToSequence || !InnerSelect.
use_empty())
11499 InnerSelect, Flags);
11502 recursivelyDeleteUnusedNodes(InnerSelect.
getNode());
11512 if (!normalizeToSequence) {
11518 if (
SDValue Combined = visitANDLike(N0, N1_0,
N)) {
11531 if (!normalizeToSequence) {
11537 if (
SDValue Combined = visitORLike(N0, N2_0,
DL))
11555 combineMinNumMaxNum(
DL, VT, Cond0, Cond1, N1, N2,
CC))
11566 auto *
C = dyn_cast<ConstantSDNode>(N2.
getOperand(1));
11567 auto *NotC = dyn_cast<ConstantSDNode>(Cond1);
11568 if (
C && NotC &&
C->getAPIntValue() == ~NotC->getAPIntValue()) {
11588 (!LegalOperations &&
11599 if (
SDValue NewSel = SimplifySelect(
DL, N0, N1, N2))
11604 if (
SDValue BinOp = foldSelectOfBinops(
N))
11620 EVT VT =
N->getValueType(0);
11628 if (
LHS->getNumOperands() != 2 ||
RHS->getNumOperands() != 2)
11637 for (
int i = 0; i < NumElems / 2; ++i) {
11638 if (
Cond->getOperand(i)->isUndef())
11641 if (BottomHalf ==
nullptr)
11642 BottomHalf = cast<ConstantSDNode>(
Cond.getOperand(i));
11643 else if (
Cond->getOperand(i).getNode() != BottomHalf)
11649 for (
int i = NumElems / 2; i < NumElems; ++i) {
11650 if (
Cond->getOperand(i)->isUndef())
11653 if (TopHalf ==
nullptr)
11654 TopHalf = cast<ConstantSDNode>(
Cond.getOperand(i));
11655 else if (
Cond->getOperand(i).getNode() != TopHalf)
11659 assert(TopHalf && BottomHalf &&
11660 "One half of the selector was all UNDEFs and the other was all the "
11661 "same value. This should have been addressed before this function.");
11664 BottomHalf->
isZero() ?
RHS->getOperand(0) :
LHS->getOperand(0),
11665 TopHalf->
isZero() ?
RHS->getOperand(1) :
LHS->getOperand(1));
11678 EVT VT = BasePtr.getValueType();
11682 SplatVal.getValueType() == VT) {
11813 MST1->isSimple() && MST1->getBasePtr() ==
Ptr &&
11816 MST1->getMemoryVT().getStoreSize()) ||
11820 CombineTo(MST1, MST1->getChain());
11837 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
11841 Value.getValueType().isInteger() &&
11842 (!isa<ConstantSDNode>(
Value) ||
11843 !cast<ConstantSDNode>(
Value)->isOpaque())) {
11844 APInt TruncDemandedBits =
11870 Value.getOperand(0).getValueType());
11881 auto *SST = cast<VPStridedStoreSDNode>(
N);
11884 if (
auto *CStride = dyn_cast<ConstantSDNode>(SST->getStride());
11885 CStride && CStride->getZExtValue() == EltVT.
getStoreSize()) {
11887 SST->getBasePtr(), SST->getOffset(), SST->getMask(),
11888 SST->getVectorLength(), SST->getMemoryVT(),
11889 SST->getMemOperand(), SST->getAddressingMode(),
11890 SST->isTruncatingStore(), SST->isCompressingStore());
11936 return CombineTo(
N, PassThru, MGT->
getChain());
11972 return CombineTo(
N, NewLd, NewLd.
getValue(1));
11976 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
11983 auto *SLD = cast<VPStridedLoadSDNode>(
N);
11986 if (
auto *CStride = dyn_cast<ConstantSDNode>(SLD->getStride());
11987 CStride && CStride->getZExtValue() == EltVT.
getStoreSize()) {
11989 SLD->getAddressingMode(), SLD->getExtensionType(), SLD->getValueType(0),
11990 SDLoc(
N), SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(),
11991 SLD->getMask(), SLD->getVectorLength(), SLD->getMemoryVT(),
11992 SLD->getMemOperand(), SLD->isExpandingLoad());
11993 return CombineTo(
N, NewLd, NewLd.
getValue(1));
12004 EVT VT =
N->getValueType(0);
12005 if (!
Cond.hasOneUse() ||
Cond.getScalarValueSizeInBits() != 1 ||
12014 bool AllAddOne =
true;
12015 bool AllSubOne =
true;
12017 for (
unsigned i = 0; i != Elts; ++i) {
12036 if (AllAddOne || AllSubOne) {
12071 if (
SDValue V = foldBoolSelectToLogic<VPMatchContext>(
N, DAG))
12081 EVT VT =
N->getValueType(0);
12087 if (
SDValue V = foldBoolSelectToLogic<EmptyMatchContext>(
N, DAG))
12113 bool isAbs =
false;
12132 AddToWorklist(Shift.
getNode());
12133 AddToWorklist(
Add.getNode());
12144 if (
SDValue FMinMax = combineMinNumMaxNum(
DL, VT, LHS, RHS, N1, N2,
CC))
12159 EVT NarrowVT =
LHS.getValueType();
12167 SetCCWidth != 1 && SetCCWidth < WideWidth &&
12193 if (hasOperation(ABDOpc, VT)) {
12200 return DAG.
getNode(ABDOpc,
DL, VT, LHS, RHS);
12207 return DAG.
getNode(ABDOpc,
DL, VT, LHS, RHS);
12242 (OpLHS == CondLHS || OpRHS == CondLHS))
12245 if (OpRHS.getOpcode() == CondRHS.getOpcode() &&
12248 CondLHS == OpLHS) {
12253 return Cond->getAPIntValue() ==
~Op->getAPIntValue();
12294 if (OpLHS == LHS) {
12310 return (!
Op && !
Cond) ||
12312 Cond->getAPIntValue() == (-
Op->getAPIntValue() - 1));
12343 if (SimplifySelectOps(
N, N1, N2))
12363 if (
SDValue V = foldVSelectOfConstants(
N))
12396 AddToWorklist(
SCC.getNode());
12400 if (
auto *SCCC = dyn_cast<ConstantSDNode>(
SCC.getNode()))
12401 return SCCC->isZero() ? N3 : N2;
12405 if (
SCC->isUndef())
12412 SCC.getOperand(1), N2, N3,
SCC.getOperand(2));
12419 if (SimplifySelectOps(
N, N2, N3))
12423 return SimplifySelectCC(
SDLoc(
N), N0, N1, N2, N3,
CC);
12431 N->hasOneUse() &&
N->use_begin()->getOpcode() ==
ISD::BRCOND;
12434 EVT VT =
N->getValueType(0);
12435 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
12443 SDValue NewSetCC = rebuildSetCC(Combined);
12471 A.getOperand(0) ==
B.getOperand(0);
12475 B.getOperand(0) ==
A;
12478 bool IsRotate =
false;
12481 if (IsAndWithShift(N0, N1)) {
12483 ShiftOrRotate = N1;
12484 }
else if (IsAndWithShift(N1, N0)) {
12486 ShiftOrRotate = N0;
12487 }
else if (IsRotateWithOp(N0, N1)) {
12490 ShiftOrRotate = N1;
12491 }
else if (IsRotateWithOp(N1, N0)) {
12494 ShiftOrRotate = N0;
12497 if (AndOrOp && ShiftOrRotate && ShiftOrRotate.hasOneUse() &&
12502 auto GetAPIntValue = [](
SDValue Op) -> std::optional<APInt> {
12505 if (CNode ==
nullptr)
12506 return std::nullopt;
12509 std::optional<APInt> AndCMask =
12510 IsRotate ? std::nullopt : GetAPIntValue(AndOrOp.
getOperand(1));
12511 std::optional<APInt> ShiftCAmt =
12512 GetAPIntValue(ShiftOrRotate.getOperand(1));
12516 if (ShiftCAmt && (IsRotate || AndCMask) && ShiftCAmt->ult(NumBits)) {
12517 unsigned ShiftOpc = ShiftOrRotate.getOpcode();
12519 bool CanTransform = IsRotate;
12520 if (!CanTransform) {
12522 CanTransform = *ShiftCAmt == (~*AndCMask).
popcount();
12524 CanTransform &= (*ShiftCAmt + AndCMask->popcount()) == NumBits;
12532 OpVT, ShiftOpc, ShiftCAmt->isPowerOf2(), *ShiftCAmt, AndCMask);
12534 if (CanTransform && NewShiftOpc != ShiftOpc) {
12537 DAG.
getNode(NewShiftOpc,
DL, OpVT, ShiftOrRotate.getOperand(0),
12538 ShiftOrRotate.getOperand(1));
12545 NumBits - ShiftCAmt->getZExtValue())
12546 :
APInt::getLowBitsSet(NumBits,
12547 NumBits - ShiftCAmt->getZExtValue());
12555 return DAG.
getSetCC(
DL, VT, NewAndOrOp, NewShiftOrRotate,
Cond);
12584 if (!
N.hasOneUse())
12587 if (!isa<LoadSDNode>(
N))
12613 unsigned Opcode =
N->getOpcode();
12615 EVT VT =
N->getValueType(0);
12620 "Expected EXTEND dag node in input!");
12661 unsigned Opcode =
N->getOpcode();
12663 EVT VT =
N->getValueType(0);
12666 "Expected EXTEND dag node in input!");
12671 if (isa<ConstantSDNode>(N0))
12672 return DAG.
getNode(Opcode,
DL, VT, N0);
12680 if (isa<ConstantSDNode>(Op1) && isa<ConstantSDNode>(Op2) &&
12691 unsigned FoldOpc = Opcode;
12714 for (
unsigned i = 0; i != NumElts; ++i) {
12716 if (
Op.isUndef()) {
12727 APInt C =
Op->getAsAPIntVal().zextOrTrunc(EVTBits);
12745 bool HasCopyToRegUses =
false;
12752 if (UI.getUse().getResNo() != N0.
getResNo())
12761 for (
unsigned i = 0; i != 2; ++i) {
12765 if (!isa<ConstantSDNode>(UseOp))
12779 HasCopyToRegUses =
true;
12782 if (HasCopyToRegUses) {
12783 bool BothLiveOut =
false;
12788 BothLiveOut =
true;
12795 return !ExtendNodes.
empty();
12805 for (
SDNode *SetCC : SetCCs) {
12808 for (
unsigned j = 0;
j != 2; ++
j) {
12810 if (SOp == OrigLoad)
12824 EVT DstVT =
N->getValueType(0);
12829 "Unexpected node type (not an extend)!");
12867 EVT SplitSrcVT = SrcVT;
12868 EVT SplitDstVT = DstVT;
12881 const unsigned NumSplits =
12888 for (
unsigned Idx = 0;
Idx < NumSplits;
Idx++) {
12907 AddToWorklist(NewChain.
getNode());
12909 CombineTo(
N, NewValue);
12915 ExtendSetCCUses(SetCCs, N0, NewValue, (
ISD::NodeType)
N->getOpcode());
12916 CombineTo(N0.
getNode(), Trunc, NewChain);
12924 EVT VT =
N->getValueType(0);
12925 EVT OrigVT =
N->getOperand(0).getValueType();
12947 EVT MemVT =
Load->getMemoryVT();
12968 Load->getChain(),
Load->getBasePtr(),
12969 Load->getMemoryVT(),
Load->getMemOperand());
12982 if (
SDValue(Load, 0).hasOneUse()) {
12986 Load->getValueType(0), ExtLoad);
12987 CombineTo(Load, Trunc, ExtLoad.
getValue(1));
12991 recursivelyDeleteUnusedNodes(N0.
getNode());
13000SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(
SDNode *Cast) {
13001 unsigned CastOpcode = Cast->
getOpcode();
13005 "Unexpected opcode for vector select narrowing/widening");
13045 bool LegalOperations,
SDNode *
N,
13056 if ((LegalOperations || !LN0->
isSimple() ||
13067 Combiner.recursivelyDeleteUnusedNodes(LN0);
13080 bool NonNegZExt =
false) {
13087 "Unexpected load type or opcode");
13104 !cast<LoadSDNode>(N0)->isSimple()) &&
13108 bool DoXform =
true;
13121 Combiner.ExtendSetCCUses(SetCCs, N0, ExtLoad, ExtOpc);
13125 if (NoReplaceTrunc) {
13127 Combiner.recursivelyDeleteUnusedNodes(LN0);
13147 if ((LegalOperations || !cast<MaskedLoadSDNode>(N0)->
isSimple()) &&
13169 auto *ALoad = dyn_cast<AtomicSDNode>(N0);
13172 EVT MemoryVT = ALoad->getMemoryVT();
13181 EVT OrigVT = ALoad->getValueType(0);
13183 auto *NewALoad = cast<AtomicSDNode>(DAG.
getAtomic(
13185 ALoad->getBasePtr(), ALoad->getMemOperand()));
13186 NewALoad->setExtensionType(ExtLoadType);
13196 bool LegalOperations) {
13208 EVT VT =
N->getValueType(0);
13209 EVT XVT =
X.getValueType();
13225 return DAG.
getNode(ShiftOpcode,
DL, VT, NotX, ShiftAmount);
13239 EVT VT =
N->getValueType(0);
13249 if (VT.
isVector() && !LegalOperations &&
13268 if (SVT == MatchingVecType) {
13284 auto IsFreeToExtend = [&](
SDValue V) {
13292 cast<LoadSDNode>(V)->isSimple() &&
13302 if (UI.getUse().getResNo() != 0 ||
User == N0.
getNode())
13307 if (
User->getOpcode() != ExtOpcode ||
User->getValueType(0) != VT)
13313 if (IsFreeToExtend(N00) && IsFreeToExtend(N01)) {
13333 SDValue ExtTrueVal = (SetCCWidth == 1)
13337 if (
SDValue SCC = SimplifySelectCC(
DL, N00, N01, ExtTrueVal, Zero,
CC,
true))
13349 return DAG.
getSelect(
DL, VT, SetCC, ExtTrueVal, Zero);
13358 EVT VT =
N->getValueType(0);
13362 if (
SDValue FoldedVOp = SimplifyVCastOp(
N,
DL))
13400 if (NarrowLoad.getNode() != N0.
getNode()) {
13401 CombineTo(N0.
getNode(), NarrowLoad);
13403 AddToWorklist(oye);
13411 unsigned OpBits =
Op.getScalarValueSizeInBits();
13416 if (OpBits == DestBits) {
13419 if (NumSignBits > DestBits-MidBits)
13421 }
else if (OpBits < DestBits) {
13424 if (NumSignBits > OpBits-MidBits)
13429 if (NumSignBits > OpBits-MidBits)
13436 if (OpBits < DestBits)
13438 else if (OpBits > DestBits)
13458 if (
SDValue ExtLoad = CombineExtLoad(
N))
13493 bool NoReplaceTruncAnd = !N0.
hasOneUse();
13497 if (NoReplaceTruncAnd) {
13500 CombineTo(N0.
getNode(), TruncAnd);
13502 if (NoReplaceTrunc) {
13507 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
13525 Flags.setNonNeg(
true);
13529 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
13560 if (NewXor.getNode() == N0.
getNode()) {
13632 EVT VT =
N->getValueType(0);
13636 if (
SDValue FoldedVOp = SimplifyVCastOp(
N,
DL))
13668 APInt TruncatedBits =
13670 APInt(
Op.getScalarValueSizeInBits(), 0) :
13673 std::min(
Op.getScalarValueSizeInBits(),
13679 return ZExtOrTrunc;
13689 if (NarrowLoad.getNode() != N0.
getNode()) {
13690 CombineTo(N0.
getNode(), NarrowLoad);
13692 AddToWorklist(oye);
13700 if (
N->getFlags().hasNonNeg()) {
13707 if (OpBits == DestBits) {
13710 if (NumSignBits > DestBits - MidBits)
13712 }
else if (OpBits < DestBits) {
13716 if (NumSignBits > OpBits - MidBits)
13721 if (NumSignBits > OpBits - MidBits)
13733 AddToWorklist(
Op.getNode());
13737 return ZExtOrTrunc;
13743 AddToWorklist(
Op.getNode());
13779 if (
SDValue ExtLoad = CombineExtLoad(
N))
13799 bool DoXform =
true;
13803 auto *AndC = cast<ConstantSDNode>(N0.
getOperand(1));
13806 if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
13822 bool NoReplaceTruncAnd = !N0.
hasOneUse();
13826 if (NoReplaceTruncAnd) {
13829 CombineTo(N0.
getNode(), TruncAnd);
13831 if (NoReplaceTrunc) {
13836 CombineTo(LN00, Trunc, ExtLoad.
getValue(1));
13845 if (
SDValue ZExtLoad = CombineZExtLogicopShiftLoad(
N))
13861 if (!LegalOperations && VT.
isVector() &&
13893 if (
SDValue SCC = SimplifySelectCC(
13897 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
13906 if (
auto *ShAmtC = dyn_cast<ConstantSDNode>(ShAmt)) {
13913 if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) {
13917 APInt ShiftOutMask =
13919 ShAmtC->getAPIntValue().getZExtValue());
13937 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
13961 EVT VT =
N->getValueType(0);
13995 if (NarrowLoad.getNode() != N0.
getNode()) {
13996 CombineTo(N0.
getNode(), NarrowLoad);
13998 AddToWorklist(oye);
14016 assert(isa<ConstantSDNode>(
Y) &&
"Expected constant to be folded!");
14032 bool DoXform =
true;
14045 CombineTo(
N, ExtLoad);
14046 if (NoReplaceTrunc) {
14048 recursivelyDeleteUnusedNodes(LN0);
14052 CombineTo(LN0, Trunc, ExtLoad.
getValue(1));
14066 if (!LegalOperations || TLI.
isLoadExtLegal(ExtType, VT, MemVT)) {
14070 CombineTo(
N, ExtLoad);
14072 recursivelyDeleteUnusedNodes(LN0);
14086 if (VT.
isVector() && !LegalOperations) {
14098 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
14106 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
14111 if (
SDValue SCC = SimplifySelectCC(
14114 cast<CondCodeSDNode>(N0.
getOperand(2))->get(),
true))
14128 unsigned Opcode =
N->getOpcode();
14131 EVT AssertVT = cast<VTSDNode>(N1)->getVT();
14135 AssertVT == cast<VTSDNode>(N0.
getOperand(1))->getVT())
14147 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
14148 EVT MinAssertVT = AssertVT.
bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
14162 EVT BigA_AssertVT = cast<VTSDNode>(BigA.
getOperand(1))->getVT();
14163 if (AssertVT.
bitsLT(BigA_AssertVT)) {
14177 Align AL = cast<AssertAlignSDNode>(
N)->getAlign();
14182 if (
auto *AAN = dyn_cast<AssertAlignSDNode>(N0))
14184 std::max(AL, AAN->getAlign()));
14194 unsigned AlignShift =
Log2(AL);
14199 if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
14200 if (LHSAlignShift < AlignShift)
14202 if (RHSAlignShift < AlignShift)
14217 unsigned Opc =
N->getOpcode();
14221 EVT VT =
N->getValueType(0);
14231 unsigned ShAmt = 0;
14236 unsigned ShiftedOffset = 0;
14241 ExtVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
14249 auto *LN = dyn_cast<LoadSDNode>(N0);
14250 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
14256 uint64_t MemoryWidth = LN->getMemoryVT().getScalarSizeInBits();
14257 if (MemoryWidth <= ShAmt)
14268 LN->getExtensionType() != ExtType)
14272 auto AndC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
14277 unsigned ActiveBits = 0;
14278 if (
Mask.isMask()) {
14279 ActiveBits =
Mask.countr_one();
14280 }
else if (
Mask.isShiftedMask(ShAmt, ActiveBits)) {
14281 ShiftedOffset = ShAmt;
14302 if (!
SRL.hasOneUse())
14307 auto *LN = dyn_cast<LoadSDNode>(
SRL.getOperand(0));
14308 auto *SRL1C = dyn_cast<ConstantSDNode>(
SRL.getOperand(1));
14315 ShAmt = SRL1C->getZExtValue();
14316 uint64_t MemoryWidth = LN->getMemoryVT().getSizeInBits();
14317 if (ShAmt >= MemoryWidth)
14344 isa<ConstantSDNode>(
Mask->getOperand(1))) {
14345 unsigned Offset, ActiveBits;
14346 const APInt& ShiftMask =
Mask->getConstantOperandAPInt(1);
14347 if (ShiftMask.
isMask()) {
14369 N0 =
SRL.getOperand(0);
14377 unsigned ShLeftAmt = 0;
14381 ShLeftAmt = N01->getZExtValue();
14387 if (!isa<LoadSDNode>(N0))
14394 !isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
14397 auto AdjustBigEndianShift = [&](
unsigned ShAmt) {
14398 unsigned LVTStoreBits =
14401 return LVTStoreBits - EVTStoreBits - ShAmt;
14406 unsigned PtrAdjustmentInBits =
14409 uint64_t PtrOff = PtrAdjustmentInBits / 8;
14413 Flags.setNoUnsignedWrap(
true);
14416 AddToWorklist(NewPtr.
getNode());
14431 WorklistRemover DeadNodes(*
this);
14436 if (ShLeftAmt != 0) {
14451 if (ShiftedOffset != 0) {
14468 EVT VT =
N->getValueType(0);
14469 EVT ExtVT = cast<VTSDNode>(N1)->getVT();
14498 if ((N00Bits <= ExtVTBits ||
14514 if ((N00Bits == ExtVTBits ||
14515 (!IsZext && (N00Bits < ExtVTBits ||
14517 (!LegalOperations ||
14542 if (
SDValue NarrowLoad = reduceLoadWidth(
N))
14549 if (
auto *ShAmt = dyn_cast<ConstantSDNode>(N0.
getOperand(1)))
14550 if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) {
14554 if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
14566 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
14567 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple() &&
14575 CombineTo(
N, ExtLoad);
14577 AddToWorklist(ExtLoad.
getNode());
14584 ExtVT == cast<LoadSDNode>(N0)->getMemoryVT() &&
14585 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) &&
14592 CombineTo(
N, ExtLoad);
14600 if (ExtVT == Ld->getMemoryVT() && N0.
hasOneUse() &&
14604 VT,
SDLoc(
N), Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
14605 Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
14606 Ld->getAddressingMode(),
ISD::SEXTLOAD, Ld->isExpandingLoad());
14607 CombineTo(
N, ExtMaskedLoad);
14614 if (
auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
14615 if (
SDValue(GN0, 0).hasOneUse() &&
14616 ExtVT == GN0->getMemoryVT() &&
14618 SDValue Ops[] = {GN0->getChain(), GN0->getPassThru(), GN0->getMask(),
14619 GN0->getBasePtr(), GN0->getIndex(), GN0->getScale()};
14625 CombineTo(
N, ExtLoad);
14627 AddToWorklist(ExtLoad.
getNode());
14650 (!LegalOperations ||
14664 bool LegalOperations) {
14665 unsigned InregOpcode =
N->getOpcode();
14669 EVT VT =
N->getValueType(0);
14671 Src.getValueType().getVectorElementType(),
14675 "Expected EXTEND_VECTOR_INREG dag node in input!");
14684 Src = Src.getOperand(0);
14685 if (Src.getValueType() != SrcVT)
14691 return DAG.
getNode(Opcode,
DL, VT, Src);
14696 EVT VT =
N->getValueType(0);
14722 EVT VT =
N->getValueType(0);
14760 EVT ExtVT = cast<VTSDNode>(ExtVal)->getVT();
14782 LegalTypes && !LegalOperations && N0->
hasOneUse() && VT != MVT::i1) {
14785 EVT TrTy =
N->getValueType(0);
14789 auto NewEltCnt = EltCnt * SizeRatio;
14795 if (isa<ConstantSDNode>(EltNo) &&
isTypeLegal(NVT)) {
14797 int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
14828 AddToWorklist(Amt.
getNode());
14877 if (BuildVectEltTy == TruncVecEltTy) {
14881 unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
14883 assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
14884 "Invalid number of elements");
14887 for (
unsigned i = 0, e = BuildVecNumElts; i !=
e; i += TruncEltOffset)
14897 if (
SDValue Reduced = reduceLoadWidth(
N))
14903 auto *LN0 = cast<LoadSDNode>(N0);
14920 unsigned NumDefs = 0;
14924 if (!
X.isUndef()) {
14935 X.getValueType().getVectorElementCount()));
14941 if (NumDefs == 1) {
14942 assert(
V.getNode() &&
"The single defined operand is empty!");
14944 for (
unsigned i = 0, e = VTs.
size(); i != e; ++i) {
14950 AddToWorklist(
NV.getNode());
14965 (!LegalOperations ||
14993 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
15007 if (!LegalOperations && N0.
hasOneUse() &&
15040 if (!LegalOperations && N0.
hasOneUse() &&
15076 !LD1->hasOneUse() || !LD2->hasOneUse() ||
15077 LD1->getAddressSpace() != LD2->getAddressSpace())
15080 unsigned LD1Fast = 0;
15081 EVT LD1VT = LD1->getValueType(0);
15086 *LD1->getMemOperand(), &LD1Fast) && LD1Fast)
15087 return DAG.
getLoad(VT,
SDLoc(
N), LD1->getChain(), LD1->getBasePtr(),
15088 LD1->getPointerInfo(), LD1->getAlign());
15103 EVT VT =
N->getValueType(0);
15140 auto IsBitCastOrFree = [&TLI, FPOpcode](
SDValue Op,
EVT VT) {
15141 if (
Op.getOpcode() ==
ISD::BITCAST &&
Op.getOperand(0).getValueType() == VT)
15154 IsBitCastOrFree(LogicOp0, VT)) {
15157 NumFPLogicOpsConv++;
15168 EVT VT =
N->getValueType(0);
15184 cast<BuildVectorSDNode>(N0)->isConstant())
15185 return ConstantFoldBITCASTofBUILD_VECTOR(N0.
getNode(),
15193 if (!LegalOperations ||
15199 if (
C.getNode() !=
N)
15212 auto IsFreeBitcast = [VT](
SDValue V) {
15214 V.getOperand(0).getValueType() == VT) ||
15235 ((!LegalOperations && cast<LoadSDNode>(N0)->isSimple()) ||
15249 if (
SDValue V = foldBitcastedFPLogic(
N, DAG, TLI))
15269 AddToWorklist(NewConv.
getNode());
15272 if (N0.
getValueType() == MVT::ppcf128 && !LegalTypes) {
15279 AddToWorklist(FlipBit.
getNode());
15286 AddToWorklist(
Hi.getNode());
15288 AddToWorklist(FlipBit.
getNode());
15292 AddToWorklist(FlipBits.
getNode());
15322 AddToWorklist(
X.getNode());
15326 if (OrigXWidth < VTWidth) {
15328 AddToWorklist(
X.getNode());
15329 }
else if (OrigXWidth > VTWidth) {
15334 X.getValueType(),
X,
15336 X.getValueType()));
15337 AddToWorklist(
X.getNode());
15339 AddToWorklist(
X.getNode());
15342 if (N0.
getValueType() == MVT::ppcf128 && !LegalTypes) {
15345 AddToWorklist(Cst.
getNode());
15347 AddToWorklist(
X.getNode());
15349 AddToWorklist(XorResult.
getNode());
15353 SDLoc(XorResult)));
15354 AddToWorklist(XorResult64.
getNode());
15358 AddToWorklist(FlipBit.
getNode());
15361 AddToWorklist(FlipBits.
getNode());
15367 AddToWorklist(
X.getNode());
15372 AddToWorklist(Cst.
getNode());
15380 if (
SDValue CombineLD = CombineConsecutiveLoads(N0.
getNode(), VT))
15395 auto PeekThroughBitcast = [&](
SDValue Op) {
15397 Op.getOperand(0).getValueType() == VT)
15416 for (
int i = 0; i != MaskScale; ++i)
15417 NewMask.
push_back(M < 0 ? -1 : M * MaskScale + i);
15422 return LegalShuffle;
15429 EVT VT =
N->getValueType(0);
15430 return CombineConsecutiveLoads(
N, VT);
15460 bool HadMaybePoisonOperands = !MaybePoisonOperands.
empty();
15461 bool IsNewMaybePoisonOperand = MaybePoisonOperands.
insert(
Op);
15462 if (!HadMaybePoisonOperands)
15464 if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) {
15473 for (
SDValue MaybePoisonOperand : MaybePoisonOperands) {
15475 if (MaybePoisonOperand.getOpcode() ==
ISD::UNDEF)
15482 FrozenMaybePoisonOperand.
getOperand(0) == FrozenMaybePoisonOperand) {
15486 MaybePoisonOperand);
15496 N0 =
N->getOperand(0);
15509 "Can't create node that may be undef/poison!");
15516ConstantFoldBITCASTofBUILD_VECTOR(
SDNode *BV,
EVT DstEltVT) {
15520 if (SrcEltVT == DstEltVT)
return SDValue(BV, 0);
15527 if (SrcBitSize == DstBitSize) {
15532 if (
Op.getValueType() != SrcEltVT)
15535 AddToWorklist(Ops.
back().getNode());
15549 BV = ConstantFoldBITCASTofBUILD_VECTOR(BV, IntVT).getNode();
15557 SDNode *Tmp = ConstantFoldBITCASTofBUILD_VECTOR(BV, TmpVT).getNode();
15560 return ConstantFoldBITCASTofBUILD_VECTOR(Tmp, DstEltVT);
15568 auto *BVN = cast<BuildVectorSDNode>(BV);
15574 if (!BVN->getConstantRawBits(IsLE, DstBitSize, RawBits, UndefElements))
15579 for (
unsigned I = 0, E = RawBits.
size();
I != E; ++
I) {
15580 if (UndefElements[
I])
15596 N->getFlags().hasAllowContract();
15601 return Options.NoInfsFPMath ||
N->getFlags().hasNoInfs();
15605template <
class MatchContextClass>
15609 EVT VT =
N->getValueType(0);
15611 MatchContextClass matcher(DAG, TLI,
N);
15614 bool UseVP = std::is_same_v<MatchContextClass, VPMatchContext>;
15619 bool HasFMAD = !UseVP && (LegalOperations && TLI.
isFMADLegal(DAG,
N));
15624 (!LegalOperations || matcher.isOperationLegalOrCustom(
ISD::FMA, VT));
15627 if (!HasFMAD && !HasFMA)
15630 bool CanReassociate =
15631 Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
15633 Options.UnsafeFPMath || HasFMAD);
15635 if (!AllowFusionGlobally && !
N->getFlags().hasAllowContract())
15661 return AllowFusionGlobally ||
N->getFlags().hasAllowContract();
15672 return matcher.getNode(PreferredFusedOpcode, SL, VT, N0.
getOperand(0),
15679 return matcher.getNode(PreferredFusedOpcode, SL, VT, N1.
getOperand(0),
15691 if (CanReassociate) {
15696 }
else if (isFusedOp(N1) && N1.
hasOneUse()) {
15702 while (E && isFusedOp(TmpFMA) && TmpFMA.
hasOneUse()) {
15727 PreferredFusedOpcode, SL, VT,
15741 PreferredFusedOpcode, SL, VT,
15753 return matcher.getNode(
15754 PreferredFusedOpcode, SL, VT,
X,
Y,
15755 matcher.getNode(PreferredFusedOpcode, SL, VT,
15759 if (isFusedOp(N0)) {
15780 return matcher.getNode(
15781 PreferredFusedOpcode, SL, VT,
15784 matcher.getNode(PreferredFusedOpcode, SL, VT,
15790 if (isFusedOp(N00)) {
15804 if (isFusedOp(N1)) {
15825 if (isFusedOp(N10)) {
15842template <
class MatchContextClass>
15846 EVT VT =
N->getValueType(0);
15848 MatchContextClass matcher(DAG, TLI,
N);
15851 bool UseVP = std::is_same_v<MatchContextClass, VPMatchContext>;
15856 bool HasFMAD = !UseVP && (LegalOperations && TLI.
isFMADLegal(DAG,
N));
15861 (!LegalOperations || matcher.isOperationLegalOrCustom(
ISD::FMA, VT));
15864 if (!HasFMAD && !HasFMA)
15869 Options.UnsafeFPMath || HasFMAD);
15872 if (!AllowFusionGlobally && !
N->getFlags().hasAllowContract())
15881 bool NoSignedZero =
Options.NoSignedZerosFPMath ||
Flags.hasNoSignedZeros();
15888 return AllowFusionGlobally ||
N->getFlags().hasAllowContract();
15894 return matcher.getNode(PreferredFusedOpcode, SL, VT, XY.
getOperand(0),
15905 return matcher.getNode(
15906 PreferredFusedOpcode, SL, VT,
15907 matcher.getNode(
ISD::FNEG, SL, VT, YZ.getOperand(0)),
15908 YZ.getOperand(1),
X);
15918 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
15921 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
15925 if (
SDValue V = tryToFoldXYSubZ(N0, N1))
15928 if (
SDValue V = tryToFoldXSubYZ(N0, N1))
15937 return matcher.
getNode(PreferredFusedOpcode, SL, VT,
15938 matcher.getNode(
ISD::FNEG, SL, VT, N00), N01,
15939 matcher.getNode(
ISD::FNEG, SL, VT, N1));
15952 PreferredFusedOpcode, SL, VT,
15955 matcher.getNode(
ISD::FNEG, SL, VT, N1));
15968 PreferredFusedOpcode, SL, VT,
15992 PreferredFusedOpcode, SL, VT,
16016 PreferredFusedOpcode, SL, VT,
16025 return Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
16039 bool CanFuse =
Options.UnsafeFPMath ||
N->getFlags().hasAllowContract();
16042 if (CanFuse && isFusedOp(N0) &&
16043 isContractableAndReassociableFMUL(N0.
getOperand(2)) &&
16045 return matcher.getNode(
16047 matcher.
getNode(PreferredFusedOpcode, SL, VT,
16055 if (CanFuse && isFusedOp(N1) &&
16056 isContractableAndReassociableFMUL(N1.
getOperand(2)) &&
16061 PreferredFusedOpcode, SL, VT,
16064 matcher.
getNode(PreferredFusedOpcode, SL, VT,
16065 matcher.getNode(
ISD::FNEG, SL, VT, N20), N21, N0));
16070 if (isFusedOp(N0) && N0->
hasOneUse()) {
16074 if (isContractableAndReassociableFMUL(N020) &&
16080 PreferredFusedOpcode, SL, VT,
16083 matcher.getNode(
ISD::FNEG, SL, VT, N1)));
16096 if (isFusedOp(N00)) {
16098 if (isContractableAndReassociableFMUL(N002) &&
16102 PreferredFusedOpcode, SL, VT,
16106 PreferredFusedOpcode, SL, VT,
16109 matcher.getNode(
ISD::FNEG, SL, VT, N1)));
16119 if (isContractableAndReassociableFMUL(N120) &&
16125 PreferredFusedOpcode, SL, VT,
16129 PreferredFusedOpcode, SL, VT,
16147 if (isContractableAndReassociableFMUL(N102) &&
16153 PreferredFusedOpcode, SL, VT,
16158 PreferredFusedOpcode, SL, VT,
16172SDValue DAGCombiner::visitFMULForFMADistributiveCombine(
SDNode *
N) {
16175 EVT VT =
N->getValueType(0);
16196 bool HasFMAD =
Options.UnsafeFPMath &&
16200 if (!HasFMAD && !HasFMA)
16212 if (
C->isExactlyValue(+1.0))
16213 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
16215 if (
C->isExactlyValue(-1.0))
16216 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
16223 if (
SDValue FMA = FuseFADD(N0, N1))
16225 if (
SDValue FMA = FuseFADD(N1, N0))
16235 if (C0->isExactlyValue(+1.0))
16236 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
16239 if (C0->isExactlyValue(-1.0))
16240 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
16245 if (C1->isExactlyValue(+1.0))
16246 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
16248 if (C1->isExactlyValue(-1.0))
16249 return DAG.
getNode(PreferredFusedOpcode, SL, VT,
X.getOperand(0),
Y,
16256 if (
SDValue FMA = FuseFSUB(N0, N1))
16258 if (
SDValue FMA = FuseFSUB(N1, N0))
16268 if (
SDValue Fused = visitFADDForFMACombine<VPMatchContext>(
N)) {
16270 AddToWorklist(Fused.getNode());
16281 EVT VT =
N->getValueType(0);
16295 if (N0CFP && !N1CFP)
16300 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
16305 if (N1C && N1C->
isZero())
16309 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
16315 N1, DAG, LegalOperations, ForCodeSize))
16321 N0, DAG, LegalOperations, ForCodeSize))
16328 return C &&
C->isExactlyValue(-2.0);
16332 if (isFMulNegTwo(N0)) {
16338 if (isFMulNegTwo(N1)) {
16349 if ((
Options.NoNaNsFPMath ||
Flags.hasNoNaNs()) && AllowNewConst) {
16363 (
Flags.hasAllowReassociation() &&
Flags.hasNoSignedZeros())) &&
16383 if (CFP01 && !CFP00 && N0.
getOperand(0) == N1) {
16406 if (CFP11 && !CFP10 && N1.
getOperand(0) == N0) {
16456 VT, N0, N1, Flags))
16461 if (
SDValue Fused = visitFADDForFMACombine<EmptyMatchContext>(
N)) {
16463 AddToWorklist(Fused.getNode());
16473 EVT VT =
N->getValueType(0);
16474 EVT ChainVT =
N->getValueType(1);
16481 N1, DAG, LegalOperations, ForCodeSize)) {
16483 {Chain, N0, NegN1});
16489 N0, DAG, LegalOperations, ForCodeSize)) {
16491 {Chain, N1, NegN0});
16501 EVT VT =
N->getValueType(0);
16516 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
16519 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
16523 if (N1CFP && N1CFP->
isZero()) {
16525 Flags.hasNoSignedZeros()) {
16537 if (N0CFP && N0CFP->
isZero()) {
16539 (
Options.NoSignedZerosFPMath ||
Flags.hasNoSignedZeros())) {
16556 (
Flags.hasAllowReassociation() &&
Flags.hasNoSignedZeros())) &&
16572 if (
SDValue Fused = visitFSUBForFMACombine<EmptyMatchContext>(
N)) {
16573 AddToWorklist(Fused.getNode());
16596 EVT VT =
N->getValueType(0);
16599 std::optional<int> Mantissa;
16600 auto GetConstAndPow2Ops = [&](
unsigned ConstOpIdx) {
16601 if (ConstOpIdx == 1 &&
N->getOpcode() ==
ISD::FDIV)
16605 Pow2Op =
N->getOperand(1 - ConstOpIdx);
16618 if (CFP ==
nullptr)
16621 const APFloat &APF = CFP->getValueAPF();
16629 int CurExp =
ilogb(APF);
16632 N->getOpcode() ==
ISD::FMUL ? CurExp : (CurExp - MaxExpChange);
16635 N->getOpcode() ==
ISD::FDIV ? CurExp : (CurExp + MaxExpChange);
16643 Mantissa = ThisMantissa;
16645 return *Mantissa == ThisMantissa && ThisMantissa > 0;
16652 if (!GetConstAndPow2Ops(0) && !GetConstAndPow2Ops(1))
16681 NewIntVT, DAG.
getBitcast(NewIntVT, ConstOp), Shift);
16690 EVT VT =
N->getValueType(0);
16710 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
16713 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
16716 if (
Options.UnsafeFPMath ||
Flags.hasAllowReassociation()) {
16742 VT, N0, N1, Flags))
16776 if (
Flags.hasNoNaNs() &&
Flags.hasNoSignedZeros() &&
16784 auto TrueOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(1));
16785 auto FalseOpnd = dyn_cast<ConstantFPSDNode>(
Select.getOperand(2));
16787 if (TrueOpnd && FalseOpnd &&
16789 isa<ConstantFPSDNode>(
Cond.getOperand(1)) &&
16790 cast<ConstantFPSDNode>(
Cond.getOperand(1))->isExactlyValue(0.0)) {
16808 if (TrueOpnd->isExactlyValue(-1.0) && FalseOpnd->isExactlyValue(1.0) &&
16812 if (TrueOpnd->isExactlyValue(1.0) && FalseOpnd->isExactlyValue(-1.0))
16821 if (
SDValue Fused = visitFMULForFMADistributiveCombine(
N)) {
16822 AddToWorklist(Fused.getNode());
16828 if (
SDValue R = combineFMulOrFDivWithIntPow2(
N))
16834template <
class MatchContextClass>
SDValue DAGCombiner::visitFMA(
SDNode *
N) {
16840 EVT VT =
N->getValueType(0);
16845 MatchContextClass matcher(DAG, TLI,
N);
16847 bool CanReassociate =
16848 Options.UnsafeFPMath ||
N->getFlags().hasAllowReassociation();
16851 if (isa<ConstantFPSDNode>(N0) &&
16852 isa<ConstantFPSDNode>(N1) &&
16853 isa<ConstantFPSDNode>(N2)) {
16854 return matcher.getNode(
ISD::FMA,
DL, VT, N0, N1, N2);
16875 if (N0CFP && N0CFP->
isZero())
16877 if (N1CFP && N1CFP->
isZero())
16892 if (CanReassociate) {
16897 return matcher.getNode(
16906 return matcher.getNode(
16916 return matcher.getNode(
ISD::FADD,
DL, VT, N0, N2);
16921 AddToWorklist(RHSNeg.
getNode());
16922 return matcher.getNode(
ISD::FADD,
DL, VT, N2, RHSNeg);
16936 if (CanReassociate) {
16938 if (N1CFP && N0 == N2) {
16956 SDValue(
N, 0), DAG, LegalOperations, ForCodeSize))
16965 EVT VT =
N->getValueType(0);
16969 if (isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1) &&
16970 isa<ConstantFPSDNode>(N2))
16989 if (LegalDAG || (!UnsafeMath && !
Flags.hasAllowReciprocal()))
16993 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
17004 unsigned NumElts = 1;
17005 EVT VT =
N->getValueType(0);
17009 if (!MinUses || (N1->
use_size() * NumElts) < MinUses)
17015 for (
auto *U : N1->
uses()) {
17016 if (
U->getOpcode() ==
ISD::FDIV &&
U->getOperand(1) == N1) {
17018 if (
U->getOperand(1).getOpcode() ==
ISD::FSQRT &&
17019 U->getOperand(0) ==
U->getOperand(1).getOperand(0) &&
17020 U->getFlags().hasAllowReassociation() &&
17021 U->getFlags().hasNoSignedZeros())
17026 if (UnsafeMath ||
U->getFlags().hasAllowReciprocal())
17033 if ((
Users.size() * NumElts) < MinUses)
17041 for (
auto *U :
Users) {
17042 SDValue Dividend =
U->getOperand(0);
17043 if (Dividend != FPOne) {
17045 Reciprocal, Flags);
17046 CombineTo(U, NewNode);
17047 }
else if (U != Reciprocal.
getNode()) {
17050 CombineTo(U, Reciprocal);
17059 EVT VT =
N->getValueType(0);
17074 if (
SDValue FoldedVOp = SimplifyVBinOp(
N,
DL))
17077 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
17083 if (
Options.UnsafeFPMath ||
Flags.hasAllowReciprocal()) {
17085 if (
auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1)) {
17093 (!LegalOperations ||
17142 A =
Y.getOperand(0);
17151 if (
SDValue Rsqrt = buildRsqrtEstimate(AAZ, Flags))
17155 recursivelyDeleteUnusedNodes(AAZ.
getNode());
17163 AddToWorklist(Div.
getNode());
17171 if (
SDValue RV = BuildDivEstimate(N0, N1, Flags))
17176 if ((
Options.NoSignedZerosFPMath ||
Flags.hasNoSignedZeros()) &&
17177 (
Options.UnsafeFPMath ||
Flags.hasAllowReassociation()))
17197 if (
SDValue R = combineFMulOrFDivWithIntPow2(
N))
17206 EVT VT =
N->getValueType(0);
17217 if (
SDValue NewSel = foldBinOpIntoSelect(
N))
17229 if (!
Flags.hasApproximateFuncs() ||
17242 return buildSqrtEstimate(N0, Flags);
17257 if (YTy == MVT::f128)
17276 EVT VT =
N->getValueType(0);
17287 if (!
V.isNegative()) {
17330 EVT VT =
N->getValueType(0);
17340 if (!
Flags.hasNoSignedZeros() || !
Flags.hasNoInfs() || !
Flags.hasNoNaNs() ||
17341 !
Flags.hasApproximateFuncs())
17360 if (ExponentIs025 || ExponentIs075) {
17371 if ((!
Flags.hasNoSignedZeros() && ExponentIs025) || !
Flags.hasNoInfs() ||
17372 !
Flags.hasApproximateFuncs())
17405 EVT VT =
N->getValueType(0);
17426 EVT VT =
N->getValueType(0);
17436 (!LegalOperations ||
17478 EVT VT =
N->getValueType(0);
17488 (!LegalOperations ||
17518 EVT VT =
N->getValueType(0);
17524 EVT SrcVT = Src.getValueType();
17539 unsigned ActualSize = std::min(InputSize, OutputSize);
17559 EVT VT =
N->getValueType(0);
17574 EVT VT =
N->getValueType(0);
17589 EVT VT =
N->getValueType(0);
17605 EVT VT =
N->getValueType(0);
17618 const bool NIsTrunc =
N->getConstantOperandVal(1) == 1;
17658 AddToWorklist(Tmp.
getNode());
17663 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
17671 EVT VT =
N->getValueType(0);
17678 if (
N->hasOneUse() &&
17696 if (
In.getValueType() == VT)
return In;
17711 CombineTo(
N, ExtLoad);
17720 if (
SDValue NewVSel = matchVSelectOpSizesWithSetCC(
N))
17728 EVT VT =
N->getValueType(0);
17739 EVT VT =
N->getValueType(0);
17773 EVT VT =
N->getValueType(0);
17784 EVT VT =
N->getValueType(0);
17801 N->getFlags().hasNoSignedZeros()) && N0.
hasOneUse()) {
17806 if (
SDValue Cast = foldSignChangeInBitcast(
N))
17815 EVT VT =
N->getValueType(0);
17817 unsigned Opc =
N->getOpcode();
17839 return PropagatesNaN ?
N->getOperand(1) :
N->getOperand(0);
17848 if (IsMin == AF.
isNegative() && (!PropagatesNaN ||
Flags.hasNoNaNs()))
17849 return N->getOperand(1);
17855 if (IsMin != AF.
isNegative() && (PropagatesNaN ||
Flags.hasNoNaNs()))
17856 return N->getOperand(0);
17860 if (
SDValue SD = reassociateReduction(
17864 Opc,
SDLoc(
N), VT, N0, N1, Flags))
17872 EVT VT =
N->getValueType(0);
17880 return N->getOperand(0);
17887 if (
SDValue Cast = foldSignChangeInBitcast(
N))
17920 bool Updated =
false;
17932 return True || False;
17936 if (!IsAlwaysTrueOrFalse(
Cond, S1C)) {
17943 S1 =
S1->getOperand(0);
17974 if (
SDValue NewN1 = rebuildSetCC(N1))
17976 ChainHandle.getValue(), NewN1, N2);
17985 (
N.getOperand(0).hasOneUse() &&
17986 N.getOperand(0).getOpcode() ==
ISD::SRL))) {
17989 N =
N.getOperand(0);
18038 SDValue Tmp = visitXOR(
N.getNode());
18045 N = XORHandle.getValue();
18057 bool Equal =
false;
18062 Op0 =
N->getOperand(0);
18063 Op1 =
N->getOperand(1);
18067 EVT SetCCVT =
N.getValueType();
18083 SDValue CondLHS =
N->getOperand(2), CondRHS =
N->getOperand(3);
18093 CondLHS, CondRHS,
CC->get(),
SDLoc(
N),
18111 if (LD->isIndexed())
18113 EVT VT = LD->getMemoryVT();
18116 Ptr = LD->getBasePtr();
18117 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
18118 if (ST->isIndexed())
18120 EVT VT = ST->getMemoryVT();
18123 Ptr = ST->getBasePtr();
18126 if (LD->isIndexed())
18128 EVT VT = LD->getMemoryVT();
18132 Ptr = LD->getBasePtr();
18135 if (ST->isIndexed())
18137 EVT VT = ST->getMemoryVT();
18141 Ptr = ST->getBasePtr();
18155bool DAGCombiner::CombineToPreIndexedLoadStore(
SDNode *
N) {
18159 bool IsLoad =
true;
18160 bool IsMasked =
false;
18182 bool Swapped =
false;
18183 if (isa<ConstantSDNode>(BasePtr)) {
18202 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
18207 SDValue Val = IsMasked ? cast<MaskedStoreSDNode>(
N)->getValue()
18208 : cast<StoreSDNode>(
N)->getValue();
18211 if (Val == BasePtr)
18228 constexpr unsigned int MaxSteps = 8192;
18229 if (isa<ConstantSDNode>(
Offset))
18250 if (!isa<ConstantSDNode>(Op1)) {
18268 bool RealUse =
false;
18304 WorklistRemover DeadNodes(*
this);
18313 deleteAndRecombine(
N);
18319 for (
unsigned i = 0, e = OtherUses.
size(); i != e; ++i) {
18320 unsigned OffsetIdx = 1;
18321 if (OtherUses[i]->getOperand(OffsetIdx).getNode() ==
BasePtr.getNode())
18323 assert(OtherUses[i]->getOperand(!OffsetIdx).getNode() ==
18324 BasePtr.getNode() &&
"Expected BasePtr operand");
18337 auto *CN = cast<ConstantSDNode>(OtherUses[i]->getOperand(OffsetIdx));
18338 const APInt &Offset0 = CN->getAPIntValue();
18340 int X0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 1) ? -1 : 1;
18341 int Y0 = (OtherUses[i]->getOpcode() ==
ISD::SUB && OffsetIdx == 0) ? -1 : 1;
18347 APInt CNV = Offset0;
18348 if (X0 < 0) CNV = -CNV;
18349 if (X1 * Y0 * Y1 < 0) CNV = CNV + Offset1;
18350 else CNV = CNV - Offset1;
18362 deleteAndRecombine(OtherUses[i]);
18367 deleteAndRecombine(
Ptr.getNode());
18368 AddToWorklist(
Result.getNode());
18389 if (isa<FrameIndexSDNode>(BasePtr) || isa<RegisterSDNode>(BasePtr))
18394 if (
Use ==
Ptr.getNode())
18398 if (isa<MemSDNode>(
Use)) {
18399 bool IsLoad =
true;
18400 bool IsMasked =
false;
18403 IsMasked, OtherPtr, TLI)) {
18429 IsMasked,
Ptr, TLI) ||
18447 constexpr unsigned int MaxSteps = 8192;
18463bool DAGCombiner::CombineToPostIndexedLoadStore(
SDNode *
N) {
18467 bool IsLoad =
true;
18468 bool IsMasked =
false;
18489 ++PostIndexedNodes;
18493 WorklistRemover DeadNodes(*
this);
18502 deleteAndRecombine(
N);
18506 Result.getValue(IsLoad ? 1 : 0));
18507 deleteAndRecombine(
Op);
18522 !cast<ConstantSDNode>(Inc)->isOpaque()) &&
18523 "Cannot split out indexing using opaque target constants");
18541 EVT STMemType =
ST->getMemoryVT();
18542 if (STType == STMemType)
18564 EVT LDMemType =
LD->getMemoryVT();
18565 EVT LDType =
LD->getValueType(0);
18567 "Attempting to extend value of non-matching type");
18568 if (LDType == LDMemType)
18571 switch (
LD->getExtensionType()) {
18607 if (!BasePtrST.equalBaseIndex(BasePtrLD, DAG,
Offset))
18610 GatherAllAliases(Store, Chain, Aliases);
18611 if (Aliases.
empty() ||
18612 (Aliases.
size() == 1 && Aliases.
front().getNode() == Store))
18621 if (BasePtrST.equalBaseIndex(BasePtrLD, DAG,
Offset))
18637 if (!ST || !
ST->isSimple() ||
ST->getAddressSpace() !=
LD->getAddressSpace())
18640 EVT LDType =
LD->getValueType(0);
18641 EVT LDMemType =
LD->getMemoryVT();
18642 EVT STMemType =
ST->getMemoryVT();
18643 EVT STType =
ST->getValue().getValueType();
18667 int64_t OrigOffset =
Offset;
18680 STCoversLD = (
Offset == 0) && LdMemSize == StMemSize;
18686 if (
LD->isIndexed()) {
18693 return CombineTo(LD, Ops, 3);
18695 return CombineTo(LD, Val, Chain);
18702 if (
Offset == 0 && LDType == STType && STMemType == LDMemType) {
18705 return ReplaceLd(LD,
ST->getValue(), Chain);
18713 SDLoc(ST), STType);
18715 return ReplaceLd(LD, Val, Chain);
18733 if (
LD->getBasePtr().isUndef() ||
Offset != 0)
18738 if (!getTruncatedStoreValue(ST, Val))
18742 if (STMemType != LDMemType) {
18750 if (!extendLoadedValueToExtension(LD, Val))
18752 return ReplaceLd(LD, Val, Chain);
18757 deleteAndRecombine(Val.
getNode());
18770 if (
LD->isSimple()) {
18771 if (
N->getValueType(1) == MVT::Other) {
18773 if (!
N->hasAnyUseOfValue(0)) {
18781 dbgs() <<
"\nWith chain: "; Chain.
dump(&DAG);
18783 WorklistRemover DeadNodes(*
this);
18785 AddUsersToWorklist(Chain.
getNode());
18786 if (
N->use_empty())
18787 deleteAndRecombine(
N);
18793 assert(
N->getValueType(2) == MVT::Other &&
"Malformed indexed loads?");
18801 if (!
N->hasAnyUseOfValue(0) && (CanSplitIdx || !
N->hasAnyUseOfValue(1))) {
18804 if (
N->hasAnyUseOfValue(1) && CanSplitIdx) {
18805 Index = SplitIndexingFromLoad(LD);
18808 AddUsersToWorklist(
N);
18812 dbgs() <<
"\nWith: ";
Undef.dump(&DAG);
18813 dbgs() <<
" and 2 other values\n");
18814 WorklistRemover DeadNodes(*
this);
18818 deleteAndRecombine(
N);
18826 if (
auto V = ForwardStoreValueToDirectLoad(LD))
18833 if (*Alignment >
LD->getAlign() &&
18834 isAligned(*Alignment,
LD->getSrcValueOffset())) {
18836 LD->getExtensionType(),
SDLoc(
N),
LD->getValueType(0), Chain,
Ptr,
18837 LD->getPointerInfo(),
LD->getMemoryVT(), *Alignment,
18838 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
18846 if (
LD->isUnindexed()) {
18848 SDValue BetterChain = FindBetterChain(LD, Chain);
18851 if (Chain != BetterChain) {
18857 BetterChain,
Ptr,
LD->getMemOperand());
18860 LD->getValueType(0),
18861 BetterChain,
Ptr,
LD->getMemoryVT(),
18862 LD->getMemOperand());
18867 MVT::Other, Chain, ReplLoad.
getValue(1));
18870 return CombineTo(
N, ReplLoad.
getValue(0), Token);
18875 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
18880 if (SliceUpLoad(
N))
18900struct LoadedSlice {
18904 bool ForCodeSize =
false;
18907 unsigned Loads = 0;
18908 unsigned Truncates = 0;
18909 unsigned CrossRegisterBanksCopies = 0;
18910 unsigned ZExts = 0;
18911 unsigned Shift = 0;
18913 explicit Cost(
bool ForCodeSize) : ForCodeSize(ForCodeSize) {}
18916 Cost(
const LoadedSlice &LS,
bool ForCodeSize)
18917 : ForCodeSize(ForCodeSize), Loads(1) {
18918 EVT TruncType =
LS.Inst->getValueType(0);
18919 EVT LoadedType =
LS.getLoadedType();
18920 if (TruncType != LoadedType &&
18921 !
LS.DAG->getTargetLoweringInfo().isZExtFree(LoadedType, TruncType))
18929 void addSliceGain(
const LoadedSlice &LS) {
18938 if (
LS.canMergeExpensiveCrossRegisterBankCopy())
18939 ++CrossRegisterBanksCopies;
18943 Loads +=
RHS.Loads;
18944 Truncates +=
RHS.Truncates;
18945 CrossRegisterBanksCopies +=
RHS.CrossRegisterBanksCopies;
18946 ZExts +=
RHS.ZExts;
18947 Shift +=
RHS.Shift;
18952 return Loads ==
RHS.Loads && Truncates ==
RHS.Truncates &&
18953 CrossRegisterBanksCopies ==
RHS.CrossRegisterBanksCopies &&
18954 ZExts ==
RHS.ZExts && Shift ==
RHS.Shift;
18962 unsigned ExpensiveOpsLHS = Loads + CrossRegisterBanksCopies;
18963 unsigned ExpensiveOpsRHS =
RHS.Loads +
RHS.CrossRegisterBanksCopies;
18966 if (!ForCodeSize && ExpensiveOpsLHS != ExpensiveOpsRHS)
18967 return ExpensiveOpsLHS < ExpensiveOpsRHS;
18968 return (Truncates + ZExts + Shift + ExpensiveOpsLHS) <
18969 (
RHS.Truncates +
RHS.ZExts +
RHS.Shift + ExpensiveOpsRHS);
18995 : Inst(Inst), Origin(Origin), Shift(Shift), DAG(DAG) {}
19000 APInt getUsedBits()
const {
19005 assert(Origin &&
"No original load to compare against.");
19007 assert(Inst &&
"This slice is not bound to an instruction");
19009 "Extracted slice is bigger than the whole type!");
19011 UsedBits.setAllBits();
19012 UsedBits = UsedBits.zext(
BitWidth);
19013 UsedBits <<= Shift;
19018 unsigned getLoadedSize()
const {
19019 unsigned SliceSize = getUsedBits().popcount();
19020 assert(!(SliceSize & 0x7) &&
"Size is not a multiple of a byte.");
19021 return SliceSize / 8;
19026 EVT getLoadedType()
const {
19027 assert(DAG &&
"Missing context");
19042 bool isLegal()
const {
19044 if (!Origin || !Inst || !DAG)
19054 EVT SliceType = getLoadedType();
19065 if (PtrType == MVT::Untyped || PtrType.
isExtended())
19078 if (TruncateType != SliceType &&
19088 uint64_t getOffsetFromBase()
const {
19089 assert(DAG &&
"Missing context.");
19091 assert(!(Shift & 0x7) &&
"Shifts not aligned on Bytes are not supported.");
19095 "The size of the original loaded type is not a multiple of a"
19100 "Invalid shift amount for given loaded size");
19113 assert(Inst && Origin &&
"Unable to replace a non-existing slice.");
19117 int64_t
Offset =
static_cast<int64_t
>(getOffsetFromBase());
19118 assert(
Offset >= 0 &&
"Offset too big to fit in int64_t!");
19128 EVT SliceType = getLoadedType();
19138 if (SliceType != FinalType)
19148 bool canMergeExpensiveCrossRegisterBankCopy()
const {
19154 assert(DAG &&
"Missing context");
19156 EVT ResVT =
Use->getValueType(0);
19161 Use->getOperand(0)->isDivergent());
19170 if (!
TRI ||
TRI->getCommonSubClass(ArgRC, ResRC))
19175 unsigned IsFast = 0;
19216 const LoadedSlice &Second) {
19218 "Unable to match different memory origins.");
19220 assert((UsedBits & Second.getUsedBits()) == 0 &&
19221 "Slices are not supposed to overlap.");
19222 UsedBits |= Second.getUsedBits();
19231 LoadedSlice::Cost &GlobalLSCost) {
19232 unsigned NumberOfSlices = LoadedSlices.
size();
19234 if (NumberOfSlices < 2)
19239 llvm::sort(LoadedSlices, [](
const LoadedSlice &
LHS,
const LoadedSlice &
RHS) {
19240 assert(
LHS.Origin ==
RHS.Origin &&
"Different bases not implemented.");
19241 return LHS.getOffsetFromBase() <
RHS.getOffsetFromBase();
19243 const TargetLowering &TLI = LoadedSlices[0].DAG->getTargetLoweringInfo();
19246 const LoadedSlice *
First =
nullptr;
19247 const LoadedSlice *Second =
nullptr;
19248 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice,
19251 Second = &LoadedSlices[CurrSlice];
19258 EVT LoadedType =
First->getLoadedType();
19261 if (LoadedType != Second->getLoadedType())
19265 Align RequiredAlignment;
19266 if (!TLI.hasPairedLoad(LoadedType, RequiredAlignment)) {
19272 if (
First->getAlign() < RequiredAlignment)
19279 assert(GlobalLSCost.Loads > 0 &&
"We save more loads than we created!");
19280 --GlobalLSCost.Loads;
19297 const APInt &UsedBits,
bool ForCodeSize) {
19298 unsigned NumberOfSlices = LoadedSlices.
size();
19300 return NumberOfSlices > 1;
19303 if (NumberOfSlices != 2)
19311 LoadedSlice::Cost OrigCost(ForCodeSize), GlobalSlicingCost(ForCodeSize);
19313 OrigCost.Loads = 1;
19314 for (
unsigned CurrSlice = 0; CurrSlice < NumberOfSlices; ++CurrSlice) {
19315 const LoadedSlice &LS = LoadedSlices[CurrSlice];
19317 LoadedSlice::Cost SliceCost(LS, ForCodeSize);
19318 GlobalSlicingCost += SliceCost;
19322 OrigCost.addSliceGain(LS);
19327 return OrigCost > GlobalSlicingCost;
19336bool DAGCombiner::SliceUpLoad(
SDNode *
N) {
19342 !
LD->getValueType(0).isInteger())
19348 if (
LD->getValueType(0).isScalableVector())
19353 APInt UsedBits(
LD->getValueSizeInBits(0), 0);
19361 UI != UIEnd; ++UI) {
19363 if (UI.getUse().getResNo() != 0)
19367 unsigned Shift = 0;
19372 Shift =
User->getConstantOperandVal(1);
19385 unsigned Width =
User->getValueSizeInBits(0);
19390 LoadedSlice
LS(
User, LD, Shift, &DAG);
19391 APInt CurrentUsedBits =
LS.getUsedBits();
19394 if ((CurrentUsedBits & UsedBits) != 0)
19397 UsedBits |= CurrentUsedBits;
19418 for (
const LoadedSlice &LS : LoadedSlices) {
19420 CombineTo(
LS.Inst, SliceInst,
true);
19424 "It takes more than a zext to get to the loaded slice!!");
19431 AddToWorklist(Chain.
getNode());
19438static std::pair<unsigned, unsigned>
19440 std::pair<unsigned, unsigned> Result(0, 0);
19444 !isa<ConstantSDNode>(V->getOperand(1)) ||
19449 LoadSDNode *LD = cast<LoadSDNode>(V->getOperand(0));
19450 if (LD->getBasePtr() !=
Ptr)
return Result;
19453 if (V.getValueType() != MVT::i16 &&
19454 V.getValueType() != MVT::i32 &&
19455 V.getValueType() != MVT::i64)
19461 uint64_t NotMask = ~cast<ConstantSDNode>(V->getOperand(1))->getSExtValue();
19463 if (NotMaskLZ & 7)
return Result;
19465 if (NotMaskTZ & 7)
return Result;
19466 if (NotMaskLZ == 64)
return Result;
19473 if (V.getValueType() != MVT::i64 && NotMaskLZ)
19474 NotMaskLZ -= 64-V.getValueSizeInBits();
19476 unsigned MaskedBytes = (V.getValueSizeInBits()-NotMaskLZ-NotMaskTZ)/8;
19477 switch (MaskedBytes) {
19481 default:
return Result;
19486 if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes)
return Result;
19495 if (!LD->isOperandOf(Chain.
getNode()))
19500 Result.first = MaskedBytes;
19501 Result.second = NotMaskTZ/8;
19512 unsigned NumBytes = MaskInfo.first;
19513 unsigned ByteShift = MaskInfo.second;
19519 ByteShift*8, (ByteShift+NumBytes)*8);
19528 bool UseTruncStore;
19529 if (DC->isTypeLegal(VT))
19530 UseTruncStore =
false;
19533 UseTruncStore =
true;
19559 StOffset = ByteShift;
19590 if (!
ST->isSimple())
19598 if (
ST->isTruncatingStore() || VT.
isVector())
19601 unsigned Opc =
Value.getOpcode();
19613 std::pair<unsigned, unsigned> MaskedLoad;
19615 if (MaskedLoad.first)
19617 Value.getOperand(1), ST,
this))
19622 if (MaskedLoad.first)
19624 Value.getOperand(0), ST,
this))
19638 if (
LD->getBasePtr() !=
Ptr ||
19639 LD->getPointerInfo().getAddrSpace() !=
19640 ST->getPointerInfo().getAddrSpace())
19649 if (Imm == 0 ||
Imm.isAllOnes())
19651 unsigned ShAmt =
Imm.countr_zero();
19670 ShAmt = (((ShAmt + NewBW - 1) / NewBW) * NewBW) - NewBW;
19672 std::min(
BitWidth, ShAmt + NewBW));
19673 if ((Imm & Mask) == Imm) {
19681 PtrOff = (
BitWidth + 7 - NewBW) / 8 - PtrOff;
19683 unsigned IsFast = 0;
19686 LD->getAddressSpace(), NewAlign,
19687 LD->getMemOperand()->getFlags(), &IsFast) ||
19695 LD->getPointerInfo().getWithOffset(PtrOff), NewAlign,
19696 LD->getMemOperand()->getFlags(),
LD->getAAInfo());
19702 ST->getPointerInfo().getWithOffset(PtrOff), NewAlign);
19704 AddToWorklist(NewPtr.
getNode());
19705 AddToWorklist(NewLD.
getNode());
19706 AddToWorklist(NewVal.
getNode());
19707 WorklistRemover DeadNodes(*
this);
19726 EVT VT =
LD->getMemoryVT();
19728 VT !=
ST->getMemoryVT() ||
19729 LD->isNonTemporal() ||
19730 ST->isNonTemporal() ||
19731 LD->getPointerInfo().getAddrSpace() != 0 ||
19732 ST->getPointerInfo().getAddrSpace() != 0)
19742 unsigned FastLD = 0, FastST = 0;
19749 *
LD->getMemOperand(), &FastLD) ||
19751 *
ST->getMemOperand(), &FastST) ||
19752 !FastLD || !FastST)
19757 LD->getPointerInfo(),
LD->getAlign());
19761 ST->getPointerInfo(),
ST->getAlign());
19763 AddToWorklist(NewLD.
getNode());
19764 AddToWorklist(NewST.
getNode());
19765 WorklistRemover DeadNodes(*
this);
19787bool DAGCombiner::isMulAddWithConstProfitable(
SDNode *MulNode,
SDValue AddNode,
19799 if (
Use == MulNode)
19807 if (
Use->getOperand(0) == ConstNode)
19808 OtherOp =
Use->getOperand(1).getNode();
19810 OtherOp =
Use->getOperand(0).getNode();
19822 if (OtherOp == MulVar)
19850 unsigned NumStores) {
19853 SDLoc StoreDL(StoreNodes[0].MemNode);
19855 for (
unsigned i = 0; i < NumStores; ++i) {
19856 Visited.
insert(StoreNodes[i].MemNode);
19860 for (
unsigned i = 0; i < NumStores; ++i) {
19861 if (Visited.
insert(StoreNodes[i].MemNode->getChain().getNode()).second)
19862 Chains.
push_back(StoreNodes[i].MemNode->getChain());
19865 assert(!Chains.
empty() &&
"Chain should have generated a chain");
19870 const Value *UnderlyingObj =
nullptr;
19871 for (
const auto &
MemOp : StoreNodes) {
19883 if (UnderlyingObj && UnderlyingObj != Obj)
19886 if (!UnderlyingObj)
19887 UnderlyingObj = Obj;
19893bool DAGCombiner::mergeStoresOfConstantsOrVecElts(
19895 bool IsConstantSrc,
bool UseVector,
bool UseTrunc) {
19900 assert((!UseTrunc || !UseVector) &&
19901 "This optimization cannot emit a vector truncating store");
19904 SDLoc DL(StoreNodes[0].MemNode);
19907 unsigned SizeInBits = NumStores * ElementSizeBits;
19910 std::optional<MachineMemOperand::Flags>
Flags;
19912 for (
unsigned I = 0;
I != NumStores; ++
I) {
19913 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
I].MemNode);
19928 unsigned Elts = NumStores * NumMemElts;
19936 if (IsConstantSrc) {
19938 for (
unsigned I = 0;
I != NumStores; ++
I) {
19939 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[
I].MemNode);
19947 auto *
C = dyn_cast<ConstantSDNode>(Val);
19957 .zextOrTrunc(ElementSizeBits),
19967 DL, StoreTy, BuildVector);
19970 for (
unsigned i = 0; i < NumStores; ++i) {
19971 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
20008 assert(IsConstantSrc &&
"Merged vector elements should use vector store");
20010 APInt StoreInt(SizeInBits, 0);
20015 for (
unsigned i = 0; i < NumStores; ++i) {
20016 unsigned Idx = IsLE ? (NumStores - 1 - i) : i;
20019 SDValue Val = St->getValue();
20021 StoreInt <<= ElementSizeBits;
20023 StoreInt |=
C->getAPIntValue()
20024 .zextOrTrunc(ElementSizeBits)
20025 .zextOrTrunc(SizeInBits);
20027 StoreInt |=
C->getValueAPF()
20029 .zextOrTrunc(ElementSizeBits)
20030 .zextOrTrunc(SizeInBits);
20048 SDValue NewChain = getMergeStoreChains(StoreNodes, NumStores);
20049 bool CanReusePtrInfo = hasSameUnderlyingObj(StoreNodes);
20063 FirstInChain->
getAlign(), *Flags, AAInfo);
20065 EVT LegalizedStoredValTy =
20067 unsigned LegalizedStoreSize = LegalizedStoredValTy.
getSizeInBits();
20070 DAG.
getConstant(
C->getAPIntValue().zextOrTrunc(LegalizedStoreSize),
DL,
20071 LegalizedStoredValTy);
20073 NewChain,
DL, ExtendedStoreVal, FirstInChain->
getBasePtr(),
20082 for (
unsigned i = 0; i < NumStores; ++i)
20083 CombineTo(StoreNodes[i].MemNode, NewStore);
20085 AddToWorklist(NewChain.
getNode());
20089void DAGCombiner::getStoreMergeCandidates(
20096 if (!
BasePtr.getBase().getNode() ||
BasePtr.getBase().isUndef())
20100 StoreSource StoreSrc = getStoreSource(Val);
20101 assert(StoreSrc != StoreSource::Unknown &&
"Expected known source for store");
20107 if (StoreSrc == StoreSource::Load) {
20108 auto *Ld = cast<LoadSDNode>(Val);
20110 LoadVT = Ld->getMemoryVT();
20112 if (MemVT != LoadVT)
20115 if (!Ld->hasNUsesOfValue(1, 0))
20119 if (!Ld->isSimple() || Ld->isIndexed())
20123 int64_t &
Offset) ->
bool {
20126 if (!
Other->isSimple() ||
Other->isIndexed())
20136 :
Other->getMemoryVT() != MemVT;
20137 switch (StoreSrc) {
20138 case StoreSource::Load: {
20142 auto *OtherLd = dyn_cast<LoadSDNode>(OtherBC);
20146 if (LoadVT != OtherLd->getMemoryVT())
20149 if (!OtherLd->hasNUsesOfValue(1, 0))
20153 if (!OtherLd->isSimple() || OtherLd->isIndexed())
20156 if (cast<LoadSDNode>(Val)->isNonTemporal() != OtherLd->isNonTemporal())
20161 if (!(LBasePtr.equalBaseIndex(LPtr, DAG)))
20165 case StoreSource::Constant:
20168 if (getStoreSource(OtherBC) != StoreSource::Constant)
20171 case StoreSource::Extract:
20173 if (
Other->isTruncatingStore())
20190 auto OverLimitInDependenceCheck = [&](
SDNode *StoreNode,
20191 SDNode *RootNode) ->
bool {
20192 auto RootCount = StoreRootCountMap.
find(StoreNode);
20193 return RootCount != StoreRootCountMap.
end() &&
20194 RootCount->second.first == RootNode &&
20200 if (UseIter.getOperandNo() != 0)
20202 if (
auto *OtherStore = dyn_cast<StoreSDNode>(*UseIter)) {
20205 if (CandidateMatch(OtherStore,
Ptr, PtrDiff) &&
20206 !OverLimitInDependenceCheck(OtherStore, RootNode))
20207 StoreNodes.
push_back(MemOpLink(OtherStore, PtrDiff));
20229 unsigned NumNodesExplored = 0;
20230 const unsigned MaxSearchNodes = 1024;
20231 if (
auto *Ldn = dyn_cast<LoadSDNode>(RootNode)) {
20232 RootNode = Ldn->getChain().getNode();
20234 I != E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored) {
20235 if (
I.getOperandNo() == 0 && isa<LoadSDNode>(*
I)) {
20236 for (
auto I2 = (*I)->use_begin(), E2 = (*I)->use_end(); I2 != E2; ++I2)
20237 TryToAddCandidate(I2);
20240 if (
I.getOperandNo() == 0 && isa<StoreSDNode>(*
I)) {
20241 TryToAddCandidate(
I);
20246 I != E && NumNodesExplored < MaxSearchNodes; ++
I, ++NumNodesExplored)
20247 TryToAddCandidate(
I);
20254bool DAGCombiner::checkMergeStoreCandidatesForDependencies(
20270 while (!Worklist.
empty()) {
20272 if (!Visited.
insert(
N).second)
20281 unsigned int Max = 1024 + Visited.
size();
20283 for (
unsigned i = 0; i < NumStores; ++i) {
20284 SDNode *
N = StoreNodes[i].MemNode;
20301 for (
unsigned j = 0;
j <
N->getNumOperands(); ++
j)
20302 Worklist.
push_back(
N->getOperand(j).getNode());
20305 for (
unsigned i = 0; i < NumStores; ++i)
20311 if (Visited.
size() >= Max) {
20312 auto &RootCount = StoreRootCountMap[StoreNodes[i].MemNode];
20313 if (RootCount.first == RootNode)
20314 RootCount.second++;
20316 RootCount = {RootNode, 1};
20325 int64_t ElementSizeBytes)
const {
20328 size_t StartIdx = 0;
20329 while ((StartIdx + 1 < StoreNodes.
size()) &&
20330 StoreNodes[StartIdx].OffsetFromBase + ElementSizeBytes !=
20331 StoreNodes[StartIdx + 1].OffsetFromBase)
20335 if (StartIdx + 1 >= StoreNodes.
size())
20344 unsigned NumConsecutiveStores = 1;
20345 int64_t StartAddress = StoreNodes[0].OffsetFromBase;
20348 for (
unsigned i = 1, e = StoreNodes.
size(); i < e; ++i) {
20349 int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
20350 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
20352 NumConsecutiveStores = i + 1;
20354 if (NumConsecutiveStores > 1)
20355 return NumConsecutiveStores;
20363bool DAGCombiner::tryStoreMergeOfConstants(
20365 EVT MemVT,
SDNode *RootNode,
bool AllowVectors) {
20370 bool MadeChange =
false;
20373 while (NumConsecutiveStores >= 2) {
20377 unsigned LastLegalType = 1;
20378 unsigned LastLegalVectorType = 1;
20379 bool LastIntegerTrunc =
false;
20380 bool NonZero =
false;
20381 unsigned FirstZeroAfterNonZero = NumConsecutiveStores;
20382 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
20383 StoreSDNode *
ST = cast<StoreSDNode>(StoreNodes[i].MemNode);
20385 bool IsElementZero =
false;
20387 IsElementZero =
C->isZero();
20389 IsElementZero =
C->getConstantFPValue()->isNullValue();
20391 IsElementZero =
true;
20392 if (IsElementZero) {
20393 if (NonZero && FirstZeroAfterNonZero == NumConsecutiveStores)
20394 FirstZeroAfterNonZero = i;
20396 NonZero |= !IsElementZero;
20399 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
20401 unsigned IsFast = 0;
20413 LastIntegerTrunc =
false;
20414 LastLegalType = i + 1;
20418 EVT LegalizedStoredValTy =
20426 LastIntegerTrunc =
true;
20427 LastLegalType = i + 1;
20436 unsigned Elts = (i + 1) * NumMemElts;
20443 LastLegalVectorType = i + 1;
20447 bool UseVector = (LastLegalVectorType > LastLegalType) && AllowVectors;
20448 unsigned NumElem = (UseVector) ? LastLegalVectorType : LastLegalType;
20449 bool UseTrunc = LastIntegerTrunc && !UseVector;
20461 unsigned NumSkip = 1;
20462 while ((NumSkip < NumConsecutiveStores) &&
20463 (NumSkip < FirstZeroAfterNonZero) &&
20464 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
20468 NumConsecutiveStores -= NumSkip;
20473 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
20476 NumConsecutiveStores -= NumElem;
20480 MadeChange |= mergeStoresOfConstantsOrVecElts(StoreNodes, MemVT, NumElem,
20482 UseVector, UseTrunc);
20486 NumConsecutiveStores -= NumElem;
20491bool DAGCombiner::tryStoreMergeOfExtracts(
20497 bool MadeChange =
false;
20500 while (NumConsecutiveStores >= 2) {
20504 unsigned NumStoresToMerge = 1;
20505 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
20507 unsigned Elts = (i + 1) * NumMemElts;
20509 unsigned IsFast = 0;
20520 NumStoresToMerge = i + 1;
20525 if (NumStoresToMerge < 2) {
20532 unsigned NumSkip = 1;
20533 while ((NumSkip < NumConsecutiveStores) &&
20534 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
20538 NumConsecutiveStores -= NumSkip;
20543 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumStoresToMerge,
20546 StoreNodes.
begin() + NumStoresToMerge);
20547 NumConsecutiveStores -= NumStoresToMerge;
20551 MadeChange |= mergeStoresOfConstantsOrVecElts(
20552 StoreNodes, MemVT, NumStoresToMerge,
false,
20555 StoreNodes.
erase(StoreNodes.
begin(), StoreNodes.
begin() + NumStoresToMerge);
20556 NumConsecutiveStores -= NumStoresToMerge;
20562 unsigned NumConsecutiveStores,
EVT MemVT,
20563 SDNode *RootNode,
bool AllowVectors,
20564 bool IsNonTemporalStore,
20565 bool IsNonTemporalLoad) {
20570 bool MadeChange =
false;
20579 for (
unsigned i = 0; i < NumConsecutiveStores; ++i) {
20580 StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
20586 int64_t LdOffset = 0;
20587 if (LdBasePtr.getBase().getNode()) {
20589 if (!LdBasePtr.equalBaseIndex(LdPtr, DAG, LdOffset))
20597 LoadNodes.
push_back(MemOpLink(Ld, LdOffset));
20600 while (NumConsecutiveStores >= 2 && LoadNodes.
size() >= 2) {
20601 Align RequiredAlignment;
20602 bool NeedRotate =
false;
20603 if (LoadNodes.
size() == 2) {
20607 StoreNodes[0].MemNode->getAlign() >= RequiredAlignment) {
20613 int64_t Offset0 = LoadNodes[0].OffsetFromBase;
20614 int64_t Offset1 = LoadNodes[1].OffsetFromBase;
20616 if (Offset0 - Offset1 == ElementSizeBytes &&
20626 LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
20632 unsigned LastConsecutiveLoad = 1;
20635 unsigned LastLegalVectorType = 1;
20636 unsigned LastLegalIntegerType = 1;
20637 bool isDereferenceable =
true;
20638 bool DoIntegerTruncate =
false;
20639 int64_t StartAddress = LoadNodes[0].OffsetFromBase;
20641 for (
unsigned i = 1; i < LoadNodes.
size(); ++i) {
20643 if (LoadNodes[i].MemNode->getChain() != LoadChain)
20646 int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
20647 if (CurrAddress - StartAddress != (ElementSizeBytes * i))
20649 LastConsecutiveLoad = i;
20651 if (isDereferenceable && !LoadNodes[i].MemNode->isDereferenceable())
20652 isDereferenceable =
false;
20655 unsigned Elts = (i + 1) * NumMemElts;
20662 unsigned IsFastSt = 0;
20663 unsigned IsFastLd = 0;
20677 LastLegalVectorType = i + 1;
20681 unsigned SizeInBits = (i + 1) * ElementSizeBytes * 8;
20692 LastLegalIntegerType = i + 1;
20693 DoIntegerTruncate =
false;
20710 LastLegalIntegerType = i + 1;
20711 DoIntegerTruncate =
true;
20719 LastLegalVectorType > LastLegalIntegerType && AllowVectors;
20720 unsigned LastLegalType =
20721 std::max(LastLegalVectorType, LastLegalIntegerType);
20725 unsigned NumElem = std::min(NumConsecutiveStores, LastConsecutiveLoad + 1);
20726 NumElem = std::min(LastLegalType, NumElem);
20737 unsigned NumSkip = 1;
20738 while ((NumSkip < LoadNodes.
size()) &&
20739 (LoadNodes[NumSkip].MemNode->getAlign() <= FirstLoadAlign) &&
20740 (StoreNodes[NumSkip].MemNode->getAlign() <= FirstStoreAlign))
20744 NumConsecutiveStores -= NumSkip;
20749 if (!checkMergeStoreCandidatesForDependencies(StoreNodes, NumElem,
20753 NumConsecutiveStores -= NumElem;
20762 unsigned Elts = NumElem * NumMemElts;
20765 unsigned SizeInBits = NumElem * ElementSizeBytes * 8;
20769 SDLoc LoadDL(LoadNodes[0].MemNode);
20770 SDLoc StoreDL(StoreNodes[0].MemNode);
20775 SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem);
20776 bool CanReusePtrInfo = hasSameUnderlyingObj(StoreNodes);
20777 AddToWorklist(NewStoreChain.
getNode());
20782 if (IsNonTemporalLoad)
20794 if (UseVectorTy || !DoIntegerTruncate) {
20800 unsigned LoadWidth = ElementSizeBytes * 8 * 2;
20802 "Unexpected type for rotate-able load pair");
20809 NewStoreChain, StoreDL, StoreOp, FirstInChain->
getBasePtr(),
20812 FirstStoreAlign, StMMOFlags);
20819 FirstLoadAlign, LdMMOFlags);
20821 NewStoreChain, StoreDL, NewLoad, FirstInChain->
getBasePtr(),
20824 JointMemOpVT, FirstInChain->
getAlign(),
20829 for (
unsigned i = 0; i < NumElem; ++i) {
20830 LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
20837 for (
unsigned i = 0; i < NumElem; ++i) {
20838 SDValue Val = StoreNodes[i].MemNode->getOperand(1);
20839 CombineTo(StoreNodes[i].MemNode, NewStore);
20841 recursivelyDeleteUnusedNodes(Val.
getNode());
20847 NumConsecutiveStores -= NumElem;
20852bool DAGCombiner::mergeConsecutiveStores(
StoreSDNode *St) {
20868 if (ElementSizeBytes * 8 != (int64_t)MemVT.
getSizeInBits())
20874 const StoreSource StoreSrc = getStoreSource(StoredVal);
20875 if (StoreSrc == StoreSource::Unknown)
20881 getStoreMergeCandidates(St, StoreNodes, RootNode);
20884 if (StoreNodes.
size() < 2)
20889 llvm::sort(StoreNodes, [](MemOpLink LHS, MemOpLink RHS) {
20890 return LHS.OffsetFromBase <
RHS.OffsetFromBase;
20894 Attribute::NoImplicitFloat);
20896 bool IsNonTemporalLoad = StoreSrc == StoreSource::Load &&
20897 cast<LoadSDNode>(StoredVal)->isNonTemporal();
20906 bool MadeChange =
false;
20907 while (StoreNodes.
size() > 1) {
20908 unsigned NumConsecutiveStores =
20909 getConsecutiveStores(StoreNodes, ElementSizeBytes);
20911 if (NumConsecutiveStores == 0)
20915 assert(NumConsecutiveStores >= 2 &&
"Expected at least 2 stores");
20916 switch (StoreSrc) {
20917 case StoreSource::Constant:
20918 MadeChange |= tryStoreMergeOfConstants(StoreNodes, NumConsecutiveStores,
20919 MemVT, RootNode, AllowVectors);
20922 case StoreSource::Extract:
20923 MadeChange |= tryStoreMergeOfExtracts(StoreNodes, NumConsecutiveStores,
20927 case StoreSource::Load:
20928 MadeChange |= tryStoreMergeOfLoads(StoreNodes, NumConsecutiveStores,
20929 MemVT, RootNode, AllowVectors,
20930 IsNonTemporalStore, IsNonTemporalLoad);
20945 if (
ST->isTruncatingStore()) {
20947 ST->getBasePtr(),
ST->getMemoryVT(),
20948 ST->getMemOperand());
20950 ReplStore = DAG.
getStore(BetterChain, SL,
ST->getValue(),
ST->getBasePtr(),
20951 ST->getMemOperand());
20956 MVT::Other,
ST->getChain(), ReplStore);
20959 AddToWorklist(Token.
getNode());
20962 return CombineTo(ST, Token,
false);
20996 if ((
isTypeLegal(MVT::i32) && !LegalOperations &&
ST->isSimple()) ||
20999 bitcastToAPInt().getZExtValue(),
SDLoc(CFP),
21006 if ((TLI.
isTypeLegal(MVT::i64) && !LegalOperations &&
21010 getZExtValue(),
SDLoc(CFP), MVT::i64);
21012 Ptr,
ST->getMemOperand());
21030 ST->getOriginalAlign(), MMOFlags, AAInfo);
21033 ST->getPointerInfo().getWithOffset(4),
21034 ST->getOriginalAlign(), MMOFlags, AAInfo);
21063 EltVT !=
Value.getOperand(0).getValueType().getVectorElementType())
21066 auto *Ld = dyn_cast<LoadSDNode>(
Value.getOperand(0));
21077 ST->getAlign(),
ST->getMemOperand()->getFlags(),
21087 if (
auto *CIdx = dyn_cast<ConstantSDNode>(
Idx)) {
21088 unsigned COffset = CIdx->getSExtValue() * EltVT.
getSizeInBits() / 8;
21090 PointerInfo =
ST->getPointerInfo().getWithOffset(COffset);
21095 return DAG.
getStore(Chain,
DL, Elt, NewPtr, PointerInfo,
ST->getAlign(),
21096 ST->getMemOperand()->getFlags());
21108 ST->isUnindexed()) {
21109 EVT SVT =
Value.getOperand(0).getValueType();
21116 if (((!LegalOperations &&
ST->isSimple()) ||
21119 DAG, *
ST->getMemOperand())) {
21121 ST->getMemOperand());
21126 if (
Value.isUndef() &&
ST->isUnindexed())
21133 if (*Alignment >
ST->getAlign() &&
21134 isAligned(*Alignment,
ST->getSrcValueOffset())) {
21137 ST->getMemoryVT(), *Alignment,
21138 ST->getMemOperand()->getFlags(),
ST->getAAInfo());
21148 if (
SDValue NewST = TransformFPLoadStorePair(
N))
21152 if (
SDValue Store = mergeTruncStores(ST))
21155 if (
ST->isUnindexed()) {
21158 if (findBetterNeighborChains(ST)) {
21163 Chain =
ST->getChain();
21167 if (
ST->isTruncatingStore() &&
ST->isUnindexed() &&
21168 Value.getValueType().isInteger() &&
21169 (!isa<ConstantSDNode>(
Value) ||
21170 !cast<ConstantSDNode>(
Value)->isOpaque())) {
21175 Value.getOperand(0).getValueType() ==
ST->getMemoryVT() &&
21178 ST->getMemOperand());
21180 APInt TruncDemandedBits =
21182 ST->getMemoryVT().getScalarSizeInBits());
21186 AddToWorklist(
Value.getNode());
21203 ST->getMemOperand());
21207 if (
auto *Cst = dyn_cast<ConstantSDNode>(
Value))
21208 if (!Cst->isOpaque()) {
21209 const APInt &CValue = Cst->getAPIntValue();
21210 APInt NewVal = CValue & TruncDemandedBits;
21211 if (NewVal != CValue) {
21215 ST->getMemoryVT(),
ST->getMemOperand());
21227 if (
auto *Ld = dyn_cast<LoadSDNode>(TruncVal)) {
21229 ST->isUnindexed() &&
ST->isSimple() &&
21240 if (
SDValue NewST = replaceStoreOfInsertLoad(ST))
21244 if (
StoreSDNode *ST1 = dyn_cast<StoreSDNode>(Chain)) {
21245 if (
ST->isUnindexed() &&
ST->isSimple() &&
21246 ST1->isUnindexed() && ST1->isSimple()) {
21248 ST1->getValue() ==
Value &&
ST->getMemoryVT() == ST1->getMemoryVT() &&
21249 ST->getAddressSpace() == ST1->getAddressSpace()) {
21256 !ST1->getBasePtr().isUndef() &&
21257 ST->getAddressSpace() == ST1->getAddressSpace()) {
21262 if (
ST->getMemoryVT().isScalableVector() ||
21263 ST1->getMemoryVT().isScalableVector()) {
21264 if (ST1->getBasePtr() ==
Ptr &&
21266 ST->getMemoryVT().getStoreSize())) {
21267 CombineTo(ST1, ST1->getChain());
21277 if (STBase.contains(DAG,
ST->getMemoryVT().getFixedSizeInBits(),
21279 ST1->getMemoryVT().getFixedSizeInBits())) {
21280 CombineTo(ST1, ST1->getChain());
21294 ST->getMemoryVT(), LegalOperations)) {
21296 Ptr,
ST->getMemoryVT(),
ST->getMemOperand());
21307 bool Changed = mergeConsecutiveStores(ST);
21308 if (!Changed)
break;
21317 if (CombineToPreIndexedLoadStore(
N) || CombineToPostIndexedLoadStore(
N))
21325 if (isa<ConstantFPSDNode>(
ST->getValue())) {
21326 if (
SDValue NewSt = replaceStoreOfFPConstant(ST))
21333 return ReduceLoadOpStoreWidth(
N);
21337 const auto *LifetimeEnd = cast<LifetimeSDNode>(
N);
21338 if (!LifetimeEnd->hasOffset())
21342 LifetimeEnd->getOffset(),
false);
21346 while (!Chains.
empty()) {
21365 if (!
ST->isSimple() ||
ST->isIndexed())
21367 const TypeSize StoreSize =
ST->getMemoryVT().getStoreSize();
21375 if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase,
21378 dbgs() <<
"\nwithin LIFETIME_END of : ";
21379 LifetimeEndBase.dump();
dbgs() <<
"\n");
21380 CombineTo(ST,
ST->getChain());
21423 if (!
ST->isSimple())
21456 !
Lo.getOperand(0).getValueType().isScalarInteger() ||
21457 Lo.getOperand(0).getValueSizeInBits() > HalfValBitSize ||
21459 !
Hi.getOperand(0).getValueType().isScalarInteger() ||
21460 Hi.getOperand(0).getValueSizeInBits() > HalfValBitSize)
21466 ?
Lo.getOperand(0).getValueType()
21467 :
Lo.getValueType();
21469 ?
Hi.getOperand(0).getValueType()
21470 :
Hi.getValueType();
21487 ST->getOriginalAlign(), MMOFlags, AAInfo);
21492 St0,
DL,
Hi,
Ptr,
ST->getPointerInfo().getWithOffset(HalfValBitSize / 8),
21493 ST->getOriginalAlign(), MMOFlags, AAInfo);
21504 unsigned InsIndex) {
21513 int ElementOffset = -1;
21522 while (!ArgWorkList.
empty()) {
21525 std::tie(ArgOffset, ArgVal) = ArgWorkList.
pop_back_val();
21527 if (ArgVal == InsertVal0) {
21528 ElementOffset = ArgOffset;
21534 int CurrentArgOffset =
21538 CurrentArgOffset -= Step;
21544 assert(CurrentArgOffset == ArgOffset);
21550 if (ElementOffset == -1) {
21551 if (!
Y.isUndef() || InsertVal0.
getValueType() !=
Y.getValueType())
21553 ElementOffset = Mask.size();
21557 NewMask.
assign(Mask.begin(), Mask.end());
21559 assert(NewMask[InsIndex] < (
int)(2 * Mask.size()) && NewMask[InsIndex] >= 0 &&
21560 "NewMask[InsIndex] is out of bound");
21569SDValue DAGCombiner::mergeInsertEltWithShuffle(
SDNode *
N,
unsigned InsIndex) {
21571 "Expected extract_vector_elt");
21572 SDValue InsertVal =
N->getOperand(1);
21575 auto *SVN = dyn_cast<ShuffleVectorSDNode>(Vec);
21588 return LegalShuffle;
21599SDValue DAGCombiner::combineInsertEltToShuffle(
SDNode *
N,
unsigned InsIndex) {
21601 "Expected extract_vector_elt");
21602 SDValue InsertVal =
N->getOperand(1);
21609 SDValue DestVec =
N->getOperand(0);
21615 if (NumSrcElts == 1)
21618 unsigned NumMaskVals = ExtendRatio * NumSrcElts;
21626 for (
unsigned i = 0; i != NumMaskVals; ++i) {
21627 if (i / NumSrcElts == InsIndex)
21628 Mask[i] = (i % NumSrcElts) + NumMaskVals;
21643 ConcatOps[0] = SubVec;
21649 AddToWorklist(PaddedSubV.
getNode());
21650 AddToWorklist(DestVecBC.
getNode());
21651 AddToWorklist(Shuf.
getNode());
21658SDValue DAGCombiner::combineInsertEltToLoad(
SDNode *
N,
unsigned InsIndex) {
21659 EVT VT =
N->getValueType(0);
21668 auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(
N->getOperand(0));
21671 return InsIndex == P.index() || P.value() < 0 ||
21672 (InsIndex == 0 && P.value() == (int)P.index() - 1) ||
21673 (InsIndex == VT.getVectorNumElements() - 1 &&
21674 P.value() == (int)P.index() + 1);
21680 unsigned Extend = 0;
21684 Extend =
Scalar.getOpcode();
21688 auto *ScalarLoad = dyn_cast<LoadSDNode>(Scalar);
21698 auto *
VecLoad = dyn_cast<LoadSDNode>(Vec);
21702 int EltSize = ScalarLoad->getValueType(0).getScalarSizeInBits();
21703 if (EltSize == 0 || EltSize % 8 != 0 || !ScalarLoad->isSimple() ||
21706 ScalarLoad->getAddressSpace() !=
VecLoad->getAddressSpace())
21711 if (InsIndex == 0) {
21722 unsigned IsFast = 0;
21726 NewAlign,
VecLoad->getMemOperand()->getFlags(),
21738 InsIndex == 0 ? ScalarLoad->getPointerInfo()
21739 :
VecLoad->getPointerInfo().getWithOffset(EltSize / 8);
21742 ScalarLoad->getChain(),
Ptr, PtrInfo, NewAlign);
21755 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
21782 unsigned Elt = IndexC->getZExtValue();
21785 if (NumElts == 1) {
21801 && isa<ConstantSDNode>(InVec.
getOperand(2))) {
21803 if (Elt < OtherElt) {
21807 AddToWorklist(NewOp.
getNode());
21813 if (
SDValue Shuf = mergeInsertEltWithShuffle(
N, Elt))
21816 if (
SDValue Shuf = combineInsertEltToShuffle(
N, Elt))
21819 if (
SDValue Shuf = combineInsertEltToLoad(
N, Elt))
21835 EVT EltVT = Elt.getValueType();
21836 MaxEltVT = MaxEltVT.
bitsGE(EltVT) ? MaxEltVT : EltVT;
21844 assert(Ops.
size() == NumElts &&
"Unexpected vector size");
21858 for (
SDValue CurVec = InVec; CurVec;) {
21860 if (CurVec.isUndef())
21861 return CanonicalizeBuildVector(Ops);
21865 for (
unsigned I = 0;
I != NumElts; ++
I)
21866 AddBuildVectorOp(Ops, CurVec.getOperand(
I),
I);
21867 return CanonicalizeBuildVector(Ops);
21872 AddBuildVectorOp(Ops, CurVec.getOperand(0), 0);
21873 return CanonicalizeBuildVector(Ops);
21878 if (
auto *CurIdx = dyn_cast<ConstantSDNode>(CurVec.getOperand(2)))
21879 if (CurIdx->getAPIntValue().ult(NumElts)) {
21880 unsigned Idx = CurIdx->getZExtValue();
21881 AddBuildVectorOp(Ops, CurVec.getOperand(1),
Idx);
21885 return CanonicalizeBuildVector(Ops);
21887 CurVec = CurVec->getOperand(0);
21895 auto *SVN = cast<ShuffleVectorSDNode>(CurVec);
21899 bool Merged =
true;
21908 Mask = std::move(NewMask);
21925 for (
unsigned I = 0;
I != NumElts; ++
I)
21938 for (
unsigned I = 0;
I != NumElts; ++
I)
21945 for (
unsigned I = 0;
I != NumElts; ++
I)
21949 return CanonicalizeBuildVector(Ops);
21956SDValue DAGCombiner::scalarizeExtractedVectorLoad(
SDNode *EVE,
EVT InVecVT,
21979 if (
auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo)) {
21980 int Elt = ConstEltNo->getZExtValue();
21991 unsigned IsFast = 0;
22005 if (ResultVT.
bitsGT(VecEltVT)) {
22012 NewPtr, MPI, VecEltVT, Alignment,
22022 if (ResultVT.
bitsLT(VecEltVT))
22034 bool LegalOperations) {
22038 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
22078bool DAGCombiner::refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
22095 auto *IndexC = dyn_cast<ConstantSDNode>(
N->getOperand(1));
22100 "Original ISD::EXTRACT_VECTOR_ELT is undefinend?");
22104 EVT ScalarVT =
N->getValueType(0);
22120 Entry(Entry &&) =
default;
22121 Entry(
SDNode *Producer_,
unsigned BitPos_,
int NumBits_)
22122 :
Producer(Producer_), BitPos(BitPos_), NumBits(NumBits_) {}
22125 Entry(
const Entry &) =
delete;
22126 Entry &operator=(
const Entry &) =
delete;
22127 Entry &operator=(Entry &&) =
delete;
22133 Worklist.
emplace_back(
N, VecEltBitWidth * IndexC->getZExtValue(),
22136 while (!Worklist.
empty()) {
22143 bool ProducerIsLeaf =
false;
22146 switch (
User->getOpcode()) {
22154 User->getValueSizeInBits(0));
22160 if (
auto *ShAmtC = dyn_cast<ConstantSDNode>(
User->
getOperand(1));
22164 unsigned ShAmt = ShAmtC->getZExtValue();
22172 ProducerIsLeaf =
true;
22180 if (ProducerIsLeaf)
22184 unsigned NewVecEltBitWidth = Leafs.
front().NumBits;
22187 if (NewVecEltBitWidth == VecEltBitWidth)
22197 if (!
all_of(Leafs, [NewVecEltBitWidth](
const Entry &E) {
22198 return (
unsigned)E.NumBits == NewVecEltBitWidth &&
22199 E.Producer->getValueSizeInBits(0) == NewVecEltBitWidth &&
22200 E.BitPos % NewVecEltBitWidth == 0;
22212 if (LegalOperations &&
22218 for (
const Entry &E : Leafs) {
22220 unsigned NewIndex = E.BitPos / NewVecEltBitWidth;
22222 "Creating out-of-bounds ISD::EXTRACT_VECTOR_ELT?");
22225 CombineTo(E.Producer, V);
22234 EVT ScalarVT =
N->getValueType(0);
22270 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
22281 "BUILD_VECTOR used for scalable vectors");
22282 unsigned IndexVal =
22290 if (ScalarVT == InEltVT)
22330 unsigned ExtractIndex = IndexC->getZExtValue();
22332 unsigned BCTruncElt = IsLE ? 0 : NumElts - 1;
22343 "Extract element and scalar to vector can't change element type "
22344 "from FP to integer.");
22345 unsigned XBitWidth =
X.getValueSizeInBits();
22346 BCTruncElt = IsLE ? 0 : XBitWidth / VecEltBitWidth - 1;
22351 if (ExtractIndex == BCTruncElt && XBitWidth > VecEltBitWidth) {
22352 assert(XBitWidth % VecEltBitWidth == 0 &&
22353 "Scalar bitwidth must be a multiple of vector element bitwidth");
22366 auto *Shuf = cast<ShuffleVectorSDNode>(VecOp);
22368 int OrigElt = Shuf->getMaskElt(IndexC->getZExtValue());
22376 if (OrigElt < (
int)NumElts) {
22380 OrigElt -= NumElts;
22396 if (!LegalOperations ||
22408 return Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
22409 Use->getOperand(0) == VecOp &&
22410 isa<ConstantSDNode>(Use->getOperand(1));
22414 auto *CstElt = cast<ConstantSDNode>(
Use->getOperand(1));
22415 if (CstElt->getAPIntValue().ult(NumElts))
22416 DemandedElts.
setBit(CstElt->getZExtValue());
22435 if (refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(
N))
22441 bool BCNumEltsChanged =
false;
22456 BCNumEltsChanged =
true;
22462 if (!LegalOperations && !IndexC && VecOp.
hasOneUse() &&
22465 auto *
VecLoad = dyn_cast<LoadSDNode>(VecOp);
22466 if (VecLoad &&
VecLoad->isSimple())
22467 return scalarizeExtractedVectorLoad(
N, VecVT,
Index, VecLoad);
22472 if (!LegalOperations || !IndexC)
22478 int Elt = IndexC->getZExtValue();
22481 LN0 = cast<LoadSDNode>(VecOp);
22489 LN0 = cast<LoadSDNode>(VecOp.
getOperand(0));
22491 if (
auto *Shuf = dyn_cast<ShuffleVectorSDNode>(VecOp)) {
22502 if (BCNumEltsChanged)
22506 int Idx = (Elt > (int)NumElts) ? -1 : Shuf->getMaskElt(Elt);
22517 LN0 = cast<LoadSDNode>(VecOp);
22518 Elt = (
Idx < (int)NumElts) ?
Idx :
Idx - (int)NumElts;
22537 Index.getValueType());
22555 return scalarizeExtractedVectorLoad(
N, VecVT,
Index, LN0);
22569 unsigned NumInScalars =
N->getNumOperands();
22571 EVT VT =
N->getValueType(0);
22578 EVT SourceType = MVT::Other;
22579 bool AllAnyExt =
true;
22581 for (
unsigned i = 0; i != NumInScalars; ++i) {
22584 if (
In.isUndef())
continue;
22590 if (!ZeroExt && !AnyExt) {
22591 SourceType = MVT::Other;
22596 EVT InTy =
In.getOperand(0).getValueType();
22599 if (SourceType == MVT::Other)
22602 else if (InTy != SourceType) {
22604 SourceType = MVT::Other;
22609 AllAnyExt &= AnyExt;
22617 SourceType != MVT::Other &&
22618 llvm::has_single_bit<uint32_t>(OutScalarTy.
getSizeInBits()) &&
22619 llvm::has_single_bit<uint32_t>(SourceType.
getSizeInBits());
22633 assert(ElemRatio > 1 &&
"Invalid element size ratio");
22641 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
22645 Cast.
isUndef()) &&
"Invalid cast opcode");
22651 unsigned Index = isLE ? (i * ElemRatio) :
22652 (i * ElemRatio + (ElemRatio - 1));
22661 "Invalid vector size");
22684 EVT VT =
N->getValueType(0);
22703 unsigned NumInScalars =
N->getNumOperands();
22706 auto PeekThroughBitcast = [](
SDValue Op) {
22708 return Op.getOperand(0);
22714 for (
unsigned i = 0; i != NumInScalars; ++i) {
22715 SDValue In = PeekThroughBitcast(
N->getOperand(i));
22717 if (
In.isUndef())
continue;
22722 In = PeekThroughBitcast(
In.getOperand(0));
22733 SDValue part = PeekThroughBitcast(
In.getOperand(0));
22737 }
else if (Src != part) {
22742 SDValue ShiftAmtVal =
In.getOperand(1);
22743 if (!isa<ConstantSDNode>(ShiftAmtVal))
22746 uint64_t ShiftAmt =
In.getConstantOperandVal(1);
22749 if (ShiftAmt != i * ScalarTypeBitsize)
22755 if (!Src || Src.getValueType().getSizeInBits() != VT.
getSizeInBits())
22764 unsigned LeftIdx,
bool DidSplitVec) {
22767 EVT VT =
N->getValueType(0);
22772 unsigned ShuffleNumElems = NumElems;
22782 assert(InVT2Size <= InVT1Size &&
22783 "Inputs must be sorted to be in non-increasing vector size order.");
22787 if (InVT1 != VT || InVT2 != VT) {
22788 if ((VTSize % InVT1Size == 0) && InVT1 == InVT2) {
22791 unsigned NumConcats = VTSize / InVT1Size;
22792 assert(NumConcats >= 2 &&
"Concat needs at least two inputs!");
22794 ConcatOps[0] = VecIn1;
22795 ConcatOps[1] = VecIn2 ? VecIn2 : DAG.
getUNDEF(InVT1);
22798 }
else if (InVT1Size == VTSize * 2) {
22810 Vec2Offset = NumElems;
22812 assert(InVT2Size <= InVT1Size &&
22813 "Second input is not going to be larger than the first one.");
22819 if (LegalOperations &&
22826 if (InVT1 != InVT2) {
22830 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
22832 ShuffleNumElems = NumElems * 2;
22834 }
else if (InVT2Size * 2 == VTSize && InVT1Size == VTSize) {
22836 ConcatOps[0] = VecIn2;
22838 }
else if (InVT1Size / VTSize > 1 && InVT1Size % VTSize == 0) {
22846 assert(InVT2Size <= InVT1Size &&
22847 "Second input is not going to be larger than the first one.");
22856 if (InVT1 != InVT2) {
22858 DAG.
getUNDEF(InVT1), VecIn2, ZeroIdx);
22860 ShuffleNumElems = InVT1Size / VTSize * NumElems;
22876 for (
unsigned i = 0; i != NumElems; ++i) {
22877 if (VectorMask[i] <= 0)
22880 unsigned ExtIndex =
N->getOperand(i).getConstantOperandVal(1);
22881 if (VectorMask[i] == (
int)LeftIdx) {
22882 Mask[i] = ExtIndex;
22883 }
else if (VectorMask[i] == (
int)LeftIdx + 1) {
22884 Mask[i] = Vec2Offset + ExtIndex;
22897 if (ShuffleNumElems > NumElems)
22910 for (
int i = 0; i != NumBVOps; ++i) {
22942 if (DestSize % SrcSize != 0 ||
22948 int ZextRatio = DestSize / SrcSize;
22949 int NumMaskElts = NumBVOps * ZextRatio;
22951 for (
int i = 0; i != NumMaskElts; ++i) {
22952 if (i / ZextRatio == ZextElt) {
22973 ZeroVec, ShufMask, DAG);
22980template <
typename R,
typename T>
22982 auto I =
find(Range, Val);
22983 if (
I == Range.end())
22984 return static_cast<decltype(std::distance(Range.begin(),
I))
>(-1);
22985 return std::distance(Range.begin(),
I);
22993 EVT VT =
N->getValueType(0);
23006 bool UsesZeroVector =
false;
23007 unsigned NumElems =
N->getNumOperands();
23019 for (
unsigned i = 0; i != NumElems; ++i) {
23029 UsesZeroVector =
true;
23037 !isa<ConstantSDNode>(
Op.getOperand(1)))
23039 SDValue ExtractedFromVec =
Op.getOperand(0);
23044 const APInt &ExtractIdx =
Op.getConstantOperandAPInt(1);
23062 VectorMask[i] =
Idx;
23066 if (VecIn.
size() < 2)
23073 bool DidSplitVec =
false;
23074 if (VecIn.
size() == 2) {
23075 unsigned MaxIndex = 0;
23076 unsigned NearestPow2 = 0;
23081 for (
unsigned i = 0; i < NumElems; i++) {
23082 if (VectorMask[i] <= 0)
23084 unsigned Index =
N->getOperand(i).getConstantOperandVal(1);
23085 IndexVec[i] =
Index;
23086 MaxIndex = std::max(MaxIndex,
Index);
23090 if (InVT.
isSimple() && NearestPow2 > 2 && MaxIndex < NearestPow2 &&
23091 NumElems * 2 < NearestPow2) {
23092 unsigned SplitSize = NearestPow2 / 2;
23105 DidSplitVec =
true;
23107 for (
unsigned i = 0; i < NumElems; i++) {
23108 if (VectorMask[i] <= 0)
23110 VectorMask[i] = (IndexVec[i] < SplitSize) ? 1 : 2;
23123 b.getValueType().getVectorNumElements();
23128 for (
int &SourceVectorIndex : VectorMask) {
23129 if (SourceVectorIndex <= 0)
23133 VecIn[SourceVectorIndex] == SortedVecIn[
Idx] &&
"Remapping failure");
23134 SourceVectorIndex =
Idx;
23137 VecIn = std::move(SortedVecIn);
23155 for (
unsigned In = 0, Len = (VecIn.
size() / 2); In < Len; ++In) {
23156 unsigned LeftIdx = 2 *
In + 1;
23157 SDValue VecLeft = VecIn[LeftIdx];
23159 (LeftIdx + 1) < VecIn.
size() ? VecIn[LeftIdx + 1] :
SDValue();
23161 if (
SDValue Shuffle = createBuildVecShuffle(
DL,
N, VectorMask, VecLeft,
23162 VecRight, LeftIdx, DidSplitVec))
23170 if (UsesZeroVector)
23175 if (Shuffles.
size() == 1)
23176 return Shuffles[0];
23179 for (
int &Vec : VectorMask)
23181 Vec = Shuffles.
size() - 1;
23183 Vec = (Vec - 1) / 2;
23197 if (Shuffles.
size() % 2)
23200 for (
unsigned CurSize = Shuffles.
size(); CurSize > 1; CurSize /= 2) {
23202 Shuffles[CurSize] = DAG.
getUNDEF(VT);
23205 for (
unsigned In = 0, Len = CurSize / 2;
In <
Len; ++
In) {
23212 L.use_empty() &&
L.getOperand(1).isUndef() &&
23213 L.getOperand(0).getValueType() ==
L.getValueType();
23214 if (IsLeftShuffle) {
23215 LMask = cast<ShuffleVectorSDNode>(
L.getNode())->getMask();
23216 L =
L.getOperand(0);
23221 R.use_empty() &&
R.getOperand(1).isUndef() &&
23222 R.getOperand(0).getValueType() ==
R.getValueType();
23223 if (IsRightShuffle) {
23224 RMask = cast<ShuffleVectorSDNode>(
R.getNode())->getMask();
23225 R =
R.getOperand(0);
23227 for (
unsigned I = 0;
I != NumElems; ++
I) {
23228 if (VectorMask[
I] ==
Left) {
23232 VectorMask[
I] =
In;
23233 }
else if (VectorMask[
I] ==
Right) {
23235 if (IsRightShuffle)
23236 Mask[
I] = RMask[
I] + NumElems;
23237 VectorMask[
I] =
In;
23244 return Shuffles[0];
23252 if (LegalOperations)
23255 EVT VT =
N->getValueType(0);
23257 bool FoundZeroExtend =
false;
23259 auto checkElem = [&](
SDValue Op) -> int64_t {
23260 unsigned Opc =
Op.getOpcode();
23265 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op.getOperand(0).getOperand(1)))
23266 return C->getZExtValue();
23274 int64_t
Offset = checkElem(Op0);
23278 unsigned NumElems =
N->getNumOperands();
23280 EVT InSVT =
In.getValueType().getScalarType();
23288 for (
unsigned i = 1; i != NumElems; ++i) {
23289 if ((
Offset + i) != checkElem(
N->getOperand(i)))
23307SDValue DAGCombiner::convertBuildVecZextToBuildVecWithZeros(
SDNode *
N) {
23316 EVT VT =
N->getValueType(0);
23317 EVT OpVT =
N->getOperand(0).getValueType();
23331 unsigned ActiveBits = 0;
23336 if (
auto *Cst = dyn_cast<ConstantSDNode>(
Op)) {
23337 unsigned OpActiveBits =
23338 Cst->getAPIntValue().trunc(EltBitwidth).getActiveBits();
23339 if (OpActiveBits == 0) {
23340 KnownZeroOps.setBit(
I.index());
23348 if (
I.index() != 0)
23354 unsigned CurrActiveBits =
23355 Op.getOperand(0).getValueSizeInBits().getFixedValue();
23356 assert(!ActiveBits &&
"Already encountered non-constant-zero operand?");
23357 ActiveBits = CurrActiveBits;
23359 if (2 * ActiveBits > EltBitwidth)
23364 if (ActiveBits == 0)
23369 EVT NewScalarIntVT, NewIntVT;
23370 std::optional<unsigned> Factor;
23375 assert(2 * ActiveBits <= EltBitwidth &&
23376 "We know that half or less bits of the element are active.");
23377 for (
unsigned Scale = EltBitwidth / ActiveBits; Scale >= 2; --Scale) {
23378 if (EltBitwidth % Scale != 0)
23380 unsigned ChunkBitwidth = EltBitwidth / Scale;
23381 assert(ChunkBitwidth >= ActiveBits &&
"As per starting point.");
23384 Scale *
N->getNumOperands());
23386 (LegalOperations &&
23404 assert(!
Op.isUndef() &&
"FIXME: after allowing UNDEF's, handle them here.");
23405 unsigned SrcOpIdx =
I.index();
23406 if (KnownZeroOps[SrcOpIdx]) {
23407 NewOps.
append(*Factor, ZeroOp);
23413 NewOps.
append(*Factor - 1, ZeroOp);
23422 EVT VT =
N->getValueType(0);
23436 if (!LegalOperations) {
23437 SDValue Splat = cast<BuildVectorSDNode>(
N)->getSplatValue();
23458 if (!LegalTypes && (
N->getNumOperands() > 1)) {
23463 if (
auto CNode = dyn_cast<ConstantSDNode>(
Op.getOperand(1)))
23468 int Offset = checkElem(Op0);
23469 for (
unsigned i = 0; i <
N->getNumOperands(); ++i) {
23470 if (
Offset + i != checkElem(
N->getOperand(i))) {
23480 ((
Offset %
N->getValueType(0).getVectorNumElements()) ==
23486 if (
SDValue V = convertBuildVecZextToZext(
N))
23489 if (
SDValue V = convertBuildVecZextToBuildVecWithZeros(
N))
23492 if (
SDValue V = reduceBuildVecExtToExtBuildVec(
N))
23495 if (
SDValue V = reduceBuildVecTruncToBitCast(
N))
23498 if (
SDValue V = reduceBuildVecToShuffle(
N))
23505 assert(!
V.isUndef() &&
"Splat of undef should have been handled earlier");
23514 EVT OpVT =
N->getOperand(0).getValueType();
23521 EVT VT =
N->getValueType(0);
23526 bool AnyInteger =
false;
23527 bool AnyFP =
false;
23530 !
Op.getOperand(0).getValueType().isVector())
23539 EVT LastOpVT = Ops.
back().getValueType();
23555 if (
Op.getValueType() == SVT)
23575 EVT VT =
N->getValueType(0);
23585 if (!FirstConcat) {
23586 SubVT =
Op.getOperand(0).getValueType();
23592 if (SubVT !=
Op.getOperand(0).getValueType())
23595 assert(FirstConcat &&
"Concat of all-undefs found");
23599 if (
Op.isUndef()) {
23603 ConcatOps.
append(
Op->op_begin(),
Op->op_end());
23613 EVT VT =
N->getValueType(0);
23614 EVT OpVT =
N->getOperand(0).getValueType();
23630 if (
Op.isUndef()) {
23631 Mask.append((
unsigned)NumOpElts, -1);
23640 int ExtIdx =
Op.getConstantOperandVal(1);
23649 Mask.append((
unsigned)NumOpElts, -1);
23660 if (0 == (NumExtElts % NumElts))
23661 ExtIdx /= (NumExtElts / NumElts);
23662 else if (0 == (NumElts % NumExtElts))
23663 ExtIdx *= (NumElts / NumExtElts);
23668 if (SV0.
isUndef() || SV0 == ExtVec) {
23670 for (
int i = 0; i != NumOpElts; ++i)
23671 Mask.push_back(i + ExtIdx);
23672 }
else if (SV1.
isUndef() || SV1 == ExtVec) {
23674 for (
int i = 0; i != NumOpElts; ++i)
23675 Mask.push_back(i + ExtIdx + NumElts);
23687 unsigned CastOpcode =
N->getOperand(0).getOpcode();
23688 switch (CastOpcode) {
23704 EVT SrcVT =
N->getOperand(0).getOperand(0).getValueType();
23712 if (
Op.getOpcode() != CastOpcode || !
Op.hasOneUse() ||
23713 Op.getOperand(0).getValueType() != SrcVT)
23721 EVT VT =
N->getValueType(0);
23726 switch (CastOpcode) {
23746 return DAG.
getNode(CastOpcode,
DL, VT, NewConcat);
23754 bool LegalOperations) {
23755 EVT VT =
N->getValueType(0);
23756 EVT OpVT =
N->getOperand(0).getValueType();
23761 if (
N->getNumOperands() != 2)
23766 (LegalOperations &&
23777 if (
auto *CurSVN = dyn_cast<ShuffleVectorSDNode>(
Op);
23778 CurSVN && CurSVN->getOperand(1).
isUndef() &&
N->isOnlyUserOf(CurSVN) &&
23781 return !Op.isUndef() &&
23782 (Op.getNode() == CurSVN || is_contained(CurSVN->ops(), Op));
23805 std::iota(FirstShufOpIdentityMask.
begin(), FirstShufOpIdentityMask.
end(), 0);
23806 std::iota(SecondShufOpIdentityMask.
begin(), SecondShufOpIdentityMask.
end(),
23813 assert(!
Op.isUndef() &&
"Not expecting to concatenate UNDEF.");
23814 if (
Op.getNode() == SVN) {
23835 std::array<SDValue, 2> ShufOps;
23836 for (
auto I :
zip(SVN->
ops(), ShufOps)) {
23838 SDValue &NewShufOp = std::get<1>(
I);
23844 ShufOpParts[0] = ShufOp;
23854 if (
N->getNumOperands() == 1)
23855 return N->getOperand(0);
23858 EVT VT =
N->getValueType(0);
23864 [](
const SDValue &
Op) { return Op.isUndef(); })) {
23866 assert(
In.getValueType().isVector() &&
"Must concat vectors");
23876 !(LegalDAG &&
In.getValueType().isScalableVector())) {
23877 unsigned NumOps =
N->getNumOperands() *
In.getNumOperands();
23879 Ops.resize(NumOps, DAG.
getUNDEF(Ops[0].getValueType()));
23889 EVT SVT =
Scalar.getValueType().getVectorElementType();
23890 if (SVT ==
Scalar.getOperand(0).getValueType())
23895 if (!
Scalar.getValueType().isVector() &&
In.hasOneUse()) {
23914 if (VNTNumElms < 2)
23930 auto IsBuildVectorOrUndef = [](
const SDValue &
Op) {
23941 bool FoundMinVT =
false;
23944 EVT OpSVT =
Op.getOperand(0).getValueType();
23945 MinVT = (!FoundMinVT || OpSVT.
bitsLE(MinVT)) ? OpSVT : MinVT;
23948 assert(FoundMinVT &&
"Concat vector type mismatch");
23952 EVT OpVT =
Op.getValueType();
23961 Opnds.
append(
Op->op_begin(),
Op->op_begin() + NumElts);
23963 for (
unsigned i = 0; i != NumElts; ++i)
23971 "Concat vector type mismatch");
23994 N, DAG, TLI, LegalTypes, LegalOperations))
24004 unsigned PartNumElem =
24005 N->getOperand(0).getValueType().getVectorMinNumElements();
24007 for (
unsigned i = 0, e =
N->getNumOperands(); i != e; ++i) {
24018 if (SingleSource.
getNode()) {
24019 if (
Op.getOperand(0) != SingleSource)
24022 SingleSource =
Op.getOperand(0);
24032 unsigned IdentityIndex = i * PartNumElem;
24033 if (
Op.getConstantOperandAPInt(1) != IdentityIndex)
24038 return SingleSource;
24047 V.getOperand(1).getValueType() == SubVT && V.getOperand(2) ==
Index) {
24048 return V.getOperand(1);
24050 auto *IndexC = dyn_cast<ConstantSDNode>(
Index);
24052 V.getOperand(0).getValueType() == SubVT &&
24055 return V.getOperand(SubIdx);
24062 bool LegalOperations) {
24065 unsigned BinOpcode = BinOp.
getOpcode();
24071 if (VecVT != Bop0.
getValueType() || VecVT != Bop1.getValueType())
24085 if (!Sub0 || !Sub1)
24091 return DAG.
getNode(BinOpcode,
SDLoc(Extract), SubVT, Sub0, Sub1,
24098 bool LegalOperations) {
24106 auto *ExtractIndexC = dyn_cast<ConstantSDNode>(Extract->
getOperand(1));
24107 if (!ExtractIndexC)
24123 if (
C &&
C->getValueAPF().isNegZero())
24136 unsigned ExtractIndex = ExtractIndexC->getZExtValue();
24138 "Extract index is not a multiple of the vector length.");
24143 if (WideWidth % NarrowWidth != 0)
24148 unsigned NarrowingRatio = WideWidth / NarrowWidth;
24150 if (WideNumElts % NarrowingRatio != 0)
24155 WideNumElts / NarrowingRatio);
24182 if (NarrowingRatio != 2)
24197 return V.getOperand(ConcatOpNum);
24203 if (SubVecL || SubVecR) {
24234 auto *Ld = dyn_cast<LoadSDNode>(Extract->
getOperand(0));
24254 assert(
Index % NumElts == 0 &&
"The extract subvector index is not a "
24255 "multiple of the result's element count");
24274 if (
Offset.isScalable()) {
24297 bool LegalOperations) {
24299 "Must only be called on EXTRACT_SUBVECTOR's");
24304 EVT NarrowVT =
N->getValueType(0);
24310 auto *WideShuffleVector = dyn_cast<ShuffleVectorSDNode>(N0);
24311 if (!WideShuffleVector)
24315 if (!WideShuffleVector->hasOneUse())
24319 if (LegalOperations &&
24323 uint64_t FirstExtractedEltIdx =
N->getConstantOperandVal(1);
24325 assert((FirstExtractedEltIdx % NumEltsExtracted) == 0 &&
24326 "Extract index is not a multiple of the output vector length.");
24331 NewMask.
reserve(NumEltsExtracted);
24333 DemandedSubvectors;
24336 for (
int M : WideShuffleVector->getMask().slice(FirstExtractedEltIdx,
24337 NumEltsExtracted)) {
24338 assert((M >= -1) && (M < (2 * WideNumElts)) &&
24339 "Out-of-bounds shuffle mask?");
24348 int WideShufOpIdx = M / WideNumElts;
24350 int OpEltIdx = M % WideNumElts;
24352 assert((OpEltIdx + WideShufOpIdx * WideNumElts) == M &&
24353 "Shuffle mask vector decomposition failure.");
24356 int OpSubvecIdx = OpEltIdx / NumEltsExtracted;
24358 int OpEltIdxInSubvec = OpEltIdx % NumEltsExtracted;
24360 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted) == OpEltIdx &&
24361 "Shuffle mask subvector decomposition failure.");
24363 assert((OpEltIdxInSubvec + OpSubvecIdx * NumEltsExtracted +
24364 WideShufOpIdx * WideNumElts) == M &&
24365 "Shuffle mask full decomposition failure.");
24367 SDValue Op = WideShuffleVector->getOperand(WideShufOpIdx);
24369 if (
Op.isUndef()) {
24375 const std::pair<SDValue, int> DemandedSubvector =
24376 std::make_pair(
Op, OpSubvecIdx);
24378 if (DemandedSubvectors.insert(DemandedSubvector)) {
24379 if (DemandedSubvectors.size() > 2)
24382 int Index = NumEltsExtracted * OpSubvecIdx;
24390 getFirstIndexOf(DemandedSubvectors.getArrayRef(), DemandedSubvector);
24391 assert((NewOpIdx == 0 || NewOpIdx == 1) &&
"Unexpected operand index.");
24393 int AdjM = OpEltIdxInSubvec + NewOpIdx * NumEltsExtracted;
24396 assert(NewMask.
size() == (
unsigned)NumEltsExtracted &&
"Produced bad mask.");
24397 assert(DemandedSubvectors.size() <= 2 &&
24398 "Should have ended up demanding at most two subvectors.");
24401 if (DemandedSubvectors.empty())
24407 any_of(NewMask, [](
int M) {
return M < 0; }))
24408 for (
auto &DemandedSubvector : DemandedSubvectors)
24409 if (DemandedSubvector.second != 0)
24423 for (
const std::pair<SDValue /*Op*/, int /*SubvectorIndex*/>
24424 &DemandedSubvector : DemandedSubvectors) {
24426 int Index = NumEltsExtracted * DemandedSubvector.second;
24429 DemandedSubvector.first, IndexC));
24432 "Should end up with either one or two ops");
24435 if (NewOps.
size() == 1)
24442 EVT NVT =
N->getValueType(0);
24444 uint64_t ExtIdx =
N->getConstantOperandVal(1);
24459 V.getConstantOperandVal(1)) &&
24479 unsigned InsIdx =
V.getConstantOperandVal(2);
24481 if (InsIdx <= ExtIdx && (ExtIdx + NumSubElts) <= (InsIdx + NumInsElts) &&
24484 V.getValueType().isFixedLengthVector())
24492 V.getOperand(0).getValueType().isVector() &&
24497 unsigned DestNumElts =
V.getValueType().getVectorMinNumElements();
24498 if ((SrcNumElts % DestNumElts) == 0) {
24499 unsigned SrcDestRatio = SrcNumElts / DestNumElts;
24506 V.getOperand(0), NewIndex);
24510 if ((DestNumElts % SrcNumElts) == 0) {
24511 unsigned DestSrcRatio = DestNumElts / SrcNumElts;
24516 if ((ExtIdx % DestSrcRatio) == 0) {
24517 unsigned IndexValScaled = ExtIdx / DestSrcRatio;
24524 V.getOperand(0), NewIndex);
24532 V.getOperand(0), NewIndex);
24542 EVT ConcatSrcVT =
V.getOperand(0).getValueType();
24544 "Concat and extract subvector do not change element type");
24545 assert((ExtIdx % ExtNumElts) == 0 &&
24546 "Extract index is not a multiple of the input vector length.");
24549 unsigned ConcatOpIdx = ExtIdx / ConcatSrcNumElts;
24555 return V.getOperand(ConcatOpIdx);
24563 ConcatSrcNumElts % ExtNumElts == 0) {
24564 unsigned NewExtIdx = ExtIdx - ConcatOpIdx * ConcatSrcNumElts;
24565 assert(NewExtIdx + ExtNumElts <= ConcatSrcNumElts &&
24566 "Trying to extract from >1 concat operand?");
24567 assert(NewExtIdx % ExtNumElts == 0 &&
24568 "Extract index is not a multiple of the input vector length.");
24571 V.getOperand(ConcatOpIdx), NewIndexC);
24583 EVT InVT =
V.getValueType();
24587 if (ExtractSize % EltSize == 0) {
24588 unsigned NumElems = ExtractSize / EltSize;
24591 NumElems == 1 ? EltVT
24599 if (NumElems == 1) {
24600 SDValue Src =
V->getOperand(IdxVal);
24601 if (EltVT != Src.getValueType())
24617 EVT SmallVT =
V.getOperand(1).getValueType();
24618 if (!NVT.
bitsEq(SmallVT))
24626 uint64_t InsIdx =
V.getConstantOperandVal(2);
24636 DAG.
getBitcast(
N->getOperand(0).getValueType(),
V.getOperand(0)),
24666 unsigned HalfNumElts = NumElts / 2;
24669 for (
unsigned i = 0; i != NumElts; ++i) {
24673 if ((Mask[i] % NumElts) >= HalfNumElts)
24675 int M = Mask[i] < (int)NumElts ? Mask[i] : Mask[i] - (
int)HalfNumElts;
24676 if (i < HalfNumElts)
24679 Mask1[i - HalfNumElts] = M;
24702 EVT VT =
N->getValueType(0);
24713 unsigned NumConcats = NumElts / NumElemsPerConcat;
24715 auto IsUndefMaskElt = [](
int i) {
return i == -1; };
24720 if (NumElemsPerConcat * 2 == NumElts && N1.
isUndef() &&
24721 llvm::all_of(Mask.slice(NumElemsPerConcat, NumElemsPerConcat),
24725 Mask.slice(0, NumElemsPerConcat));
24732 for (
unsigned I = 0;
I != NumConcats; ++
I) {
24733 unsigned Begin =
I * NumElemsPerConcat;
24734 ArrayRef<int> SubMask = Mask.slice(Begin, NumElemsPerConcat);
24743 for (
int i = 0; i != (int)NumElemsPerConcat; ++i) {
24744 if (IsUndefMaskElt(SubMask[i]))
24746 if ((SubMask[i] % (
int)NumElemsPerConcat) != i)
24748 int EltOpIdx = SubMask[i] / NumElemsPerConcat;
24749 if (0 <= OpIdx && EltOpIdx != OpIdx)
24753 assert(0 <= OpIdx &&
"Unknown concat_vectors op");
24807 bool IsSplat =
false;
24808 auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
24809 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
24811 if (
SDValue Splat0 = BV0->getSplatValue())
24812 IsSplat = (Splat0 == BV1->getSplatValue());
24816 for (
int M : SVN->
getMask()) {
24819 int Idx = M < (int)NumElts ? M : M - NumElts;
24820 SDValue &S = (M < (int)NumElts ? N0 : N1);
24822 Op = S.getOperand(
Idx);
24837 if (!IsSplat && !DuplicateOps.
insert(
Op).second)
24848 SVT = (SVT.
bitsLT(
Op.getValueType()) ?
Op.getValueType() : SVT);
24864 unsigned Opcode,
EVT VT, std::function<
bool(
unsigned)>
Match,
24866 bool LegalOperations) {
24871 return std::nullopt;
24879 for (
unsigned Scale = 2; Scale < NumElts; Scale *= 2) {
24881 if (NumElts % Scale != 0)
24895 return std::nullopt;
24904 bool LegalOperations) {
24914 Mask = SVN->
getMask()](
unsigned Scale) {
24915 for (
unsigned i = 0; i != NumElts; ++i) {
24918 if ((i % Scale) == 0 && Mask[i] == (int)(i / Scale))
24930 Opcode, VT, isAnyExtend, DAG, TLI,
true, LegalOperations);
24942 bool LegalOperations) {
24943 bool LegalTypes =
true;
24955 auto ForEachDecomposedIndice = [NumElts, &Mask](
auto Fn) {
24956 for (
int &Indice : Mask) {
24959 int OpIdx = (
unsigned)Indice < NumElts ? 0 : 1;
24960 int OpEltIdx = (
unsigned)Indice < NumElts ? Indice : Indice - NumElts;
24961 Fn(Indice, OpIdx, OpEltIdx);
24966 std::array<APInt, 2> OpsDemandedElts;
24967 for (
APInt &OpDemandedElts : OpsDemandedElts)
24969 ForEachDecomposedIndice(
24970 [&OpsDemandedElts](
int &Indice,
int OpIdx,
int OpEltIdx) {
24971 OpsDemandedElts[OpIdx].setBit(OpEltIdx);
24975 std::array<APInt, 2> OpsKnownZeroElts;
24976 for (
auto I :
zip(SVN->
ops(), OpsDemandedElts, OpsKnownZeroElts))
24984 bool HadZeroableElts =
false;
24985 ForEachDecomposedIndice([&OpsKnownZeroElts, &HadZeroableElts](
24986 int &Indice,
int OpIdx,
int OpEltIdx) {
24987 if (OpsKnownZeroElts[OpIdx][OpEltIdx]) {
24989 HadZeroableElts =
true;
24997 if (!HadZeroableElts)
25005 Mask.size() % ScaledMask.
size() == 0 &&
"Unexpected mask widening.");
25006 int Prescale = Mask.size() / ScaledMask.
size();
25008 NumElts = ScaledMask.
size();
25009 EltSizeInBits *= Prescale;
25021 auto isZeroExtend = [NumElts, &ScaledMask](
unsigned Scale) {
25022 assert(Scale >= 2 && Scale <= NumElts && NumElts % Scale == 0 &&
25023 "Unexpected mask scaling factor.");
25025 for (
unsigned SrcElt = 0, NumSrcElts = NumElts / Scale;
25026 SrcElt != NumSrcElts; ++SrcElt) {
25029 assert(MaskChunk.
size() == Scale &&
"Unexpected mask size.");
25030 Mask = Mask.drop_front(MaskChunk.
size());
25033 if (
int FirstIndice = MaskChunk[0]; (
unsigned)FirstIndice != SrcElt)
25038 [](
int Indice) { return Indice == -2; }))
25041 assert(Mask.empty() &&
"Did not process the whole mask?");
25046 for (
bool Commuted : {
false,
true}) {
25051 Opcode, PrescaledVT, isZeroExtend, DAG, TLI, LegalTypes,
25087 if (ExtDstSizeInBits % ExtSrcSizeInBits != 0)
25089 unsigned ExtScale = ExtDstSizeInBits / ExtSrcSizeInBits;
25094 auto isTruncate = [&Mask, &NumElts](
unsigned Scale) {
25095 for (
unsigned i = 0; i != NumElts; ++i) {
25098 if ((i * Scale) < NumElts && Mask[i] == (int)(i * Scale))
25108 if (EltSizeInBits != ExtSrcSizeInBits)
25113 if (isTruncate(ExtScale))
25136 APInt DemandedElts(NumElts, 0);
25140 assert((
unsigned)
Idx < NumElts &&
"Out-of-bounds shuffle indice?");
25143 assert(DemandedElts.
popcount() > 1 &&
"Is a splat shuffle already?");
25148 std::optional<unsigned> MinNonUndefIdx;
25150 if (
Idx < 0 || UndefElts[
Idx])
25152 MinNonUndefIdx = std::min<unsigned>(
Idx, MinNonUndefIdx.value_or(~0U));
25154 if (!MinNonUndefIdx)
25156 assert(*MinNonUndefIdx < NumElts &&
"Expected valid element index.");
25159 for (
int &
Idx : SplatMask) {
25164 Idx = UndefElts[
Idx] ? -1 : *MinNonUndefIdx;
25166 assert(SplatMask != Shuf->
getMask() &&
"Expected mask to change!");
25184 assert(ShufMask.
size() == SplatMask.
size() &&
"Mask length mismatch");
25202 auto CanSimplifyToExistingSplat = [](
ArrayRef<int> UserMask,
25204 for (
unsigned i = 0, e = UserMask.
size(); i != e; ++i)
25205 if (UserMask[i] != -1 && SplatMask[i] == -1 &&
25206 SplatMask[UserMask[i]] != -1)
25210 if (CanSimplifyToExistingSplat(ShufMask, SplatMask))
25216 for (
int Idx : ShufMask)
25229 bool LegalOperations) {
25246 if (VTLanes <= InLanes || VTLanes % InLanes != 0 ||
25247 (LegalOperations &&
25250 int Factor = VTLanes / InLanes;
25277 auto *InnerShuf = dyn_cast<ShuffleVectorSDNode>(OuterShuf->
getOperand(0));
25278 if (!InnerShuf || !InnerShuf->getOperand(1).isUndef())
25283 unsigned NumElts = OuterMask.
size();
25284 assert(NumElts == InnerMask.
size() &&
"Mask length mismatch");
25286 int SplatIndex = -1;
25287 for (
unsigned i = 0; i != NumElts; ++i) {
25289 int OuterMaskElt = OuterMask[i];
25290 if (OuterMaskElt == -1)
25294 int InnerMaskElt = InnerMask[OuterMaskElt];
25295 if (InnerMaskElt == -1)
25299 if (SplatIndex == -1)
25300 SplatIndex = InnerMaskElt;
25303 if (SplatIndex != InnerMaskElt)
25306 CombinedMask[i] = InnerMaskElt;
25308 assert((
all_of(CombinedMask, [](
int M) {
return M == -1; }) ||
25310 "Expected a splat mask");
25314 assert(VT == InnerShuf->getValueType(0) &&
"Expected matching shuffle types");
25319 InnerShuf->getOperand(1), CombinedMask);
25327 int MaskSize = Mask.size();
25328 int EltFromOp0 = -1;
25333 for (
int i = 0; i != MaskSize; ++i) {
25334 if (Mask[i] >= 0 && Mask[i] < MaskSize) {
25336 if (EltFromOp0 != -1)
25339 }
else if (Mask[i] != i + MaskSize) {
25359 if (ShufOp0Index == -1) {
25363 if (ShufOp0Index == -1)
25367 Mask = CommutedMask;
25375 assert(Mask[ShufOp0Index] >= 0 && Mask[ShufOp0Index] < (
int)Mask.size() &&
25376 "Shuffle mask value must be from operand 0");
25380 auto *InsIndexC = dyn_cast<ConstantSDNode>(Op0.
getOperand(2));
25381 if (!InsIndexC || InsIndexC->getSExtValue() != Mask[ShufOp0Index])
25405 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(Shuf->
getOperand(0));
25411 for (
int i = 0, e = (
int)Mask.size(); i != e; ++i) {
25415 assert(Mask[i] >= 0 && Mask[i] < e &&
"Unexpected shuffle mask value");
25419 if (Mask0[Mask[i]] != Mask0[i])
25428 EVT VT =
N->getValueType(0);
25453 bool Changed =
false;
25455 for (
unsigned i = 0; i != NumElts; ++i) {
25457 if (
Idx >= (
int)NumElts) {
25506 if (
auto *
Idx = dyn_cast<ConstantSDNode>(N0.
getOperand(2)))
25507 if (
Idx->getAPIntValue() == SplatIndex)
25533 SDValue ConvInput =
V->getOperand(0);
25540 assert(
V->getNumOperands() == NumElts &&
25541 "BUILD_VECTOR has wrong number of operands");
25543 bool AllSame =
true;
25544 for (
unsigned i = 0; i != NumElts; ++i) {
25545 if (!
V->getOperand(i).isUndef()) {
25546 Base =
V->getOperand(i);
25551 if (!
Base.getNode())
25553 for (
unsigned i = 0; i != NumElts; ++i) {
25554 if (
V->getOperand(i) !=
Base) {
25564 SDValue Splatted =
V->getOperand(SplatIndex);
25570 if (
V->getValueType(0) != VT)
25609 int HalfNumElts = (int)NumElts / 2;
25611 for (
unsigned i = 0; i != NumElts; ++i) {
25613 if (
Idx >= HalfNumElts) {
25614 assert(
Idx < (
int)NumElts &&
"Shuffle mask chooses undef op");
25615 Idx -= HalfNumElts;
25636 EVT SubVT =
RHS.getOperand(0).getValueType();
25637 int NumSubVecs =
RHS.getNumOperands();
25639 assert((NumElts % NumSubElts) == 0 &&
"Subvector mismatch");
25644 if (
all_of(Mask, [NumElts](
int M) {
return M < (int)NumElts; }))
25650 for (
int SubVec = 0; SubVec != NumSubVecs; ++SubVec) {
25651 for (
int SubIdx = 0; SubIdx != (int)NumElts; SubIdx += NumSubElts) {
25653 std::iota(InsertionMask.begin(), InsertionMask.end(), 0);
25656 std::iota(InsertionMask.begin() + SubIdx,
25657 InsertionMask.begin() + SubIdx + NumSubElts,
25658 NumElts + (SubVec * NumSubElts));
25661 bool MatchingShuffle =
true;
25662 for (
int i = 0; i != (int)NumElts; ++i) {
25663 int ExpectIdx = InsertionMask[i];
25664 int ActualIdx =
Mask[i];
25665 if (0 <= ActualIdx && ExpectIdx != ActualIdx) {
25666 MatchingShuffle =
false;
25671 if (MatchingShuffle)
25673 RHS.getOperand(SubVec),
25681 if (
SDValue InsertN1 = ShuffleToInsert(N0, N1, Mask))
25686 if (
SDValue InsertN0 = ShuffleToInsert(N1, N0, CommuteMask))
25694 bool IsInLaneMask =
true;
25699 for (
int I = 0;
I != (int)NumElts; ++
I) {
25703 ClearMask[
I] =
M ==
I ?
I : (
I + NumElts);
25704 IsInLaneMask &= (
M ==
I) || (M == (
int)(
I + NumElts));
25706 APInt &Demanded =
M < (int)NumElts ? DemandedLHS : DemandedRHS;
25707 Demanded.
setBit(M % NumElts);
25711 if (!IsInLaneMask && (!DemandedLHS.
isZero() || !DemandedRHS.
isZero()) &&
25726 for (
int I = 0;
I != (int)NumElts; ++
I)
25728 AndMask[
I] =
Mask[
I] ==
I ? AllOnesElt : ZeroElt;
25765 EVT ScaleVT = SVT.
bitsLT(InnerSVT) ? VT : InnerVT;
25783 for (
int M : OuterMask)
25784 NewMask.
push_back(M < 0 ? -1 : InnerMask[M]);
25815 auto MergeInnerShuffle =
25822 if (OtherSVN->isSplat())
25828 for (
unsigned i = 0; i != NumElts; ++i) {
25829 int Idx = SVN->getMaskElt(i);
25837 Idx = (
Idx < (int)NumElts) ? (
Idx + NumElts) : (
Idx - NumElts);
25840 if (
Idx < (
int)NumElts) {
25843 Idx = OtherSVN->getMaskElt(
Idx);
25849 CurrentVec = (
Idx < (int)NumElts) ? OtherSVN->
getOperand(0)
25858 Mask.push_back(-1);
25865 if (!SV0.getNode() || SV0 == CurrentVec) {
25872 if (!SV1.getNode() || SV1 == CurrentVec) {
25876 Mask.push_back(
Idx + NumElts);
25882 if (
auto *CurrentSVN = dyn_cast<ShuffleVectorSDNode>(CurrentVec)) {
25883 int InnerIdx = CurrentSVN->getMaskElt(
Idx);
25884 if (InnerIdx < 0) {
25885 Mask.push_back(-1);
25888 SDValue InnerVec = (InnerIdx < (int)NumElts)
25892 Mask.push_back(-1);
25895 InnerIdx %= NumElts;
25896 if (InnerVec == SV0) {
25897 Mask.push_back(InnerIdx);
25900 if (InnerVec == SV1) {
25901 Mask.push_back(InnerIdx + NumElts);
25920 if (TLI.isShuffleMaskLegal(Mask, VT))
25925 return TLI.isShuffleMaskLegal(Mask, VT);
25937 assert(N1->getOperand(0).getValueType() == VT &&
25938 "Shuffle types don't match");
25942 bool HasSameOp0 = N0 == SV0;
25943 bool IsSV1Undef = SV1.
isUndef();
25944 if (HasSameOp0 || IsSV1Undef || N0 == SV1)
25953 cast<ShuffleVectorSDNode>(N0)->isSplat() &&
25954 !cast<ShuffleVectorSDNode>(N1)->isSplat()) {
25965 for (
int i = 0; i != 2; ++i) {
25967 N->isOnlyUserOf(
N->getOperand(i).getNode())) {
25970 auto *OtherSV = cast<ShuffleVectorSDNode>(
N->getOperand(i));
25971 assert(OtherSV->getOperand(0).getValueType() == VT &&
25972 "Shuffle types don't match");
25976 if (MergeInnerShuffle(i != 0, SVN, OtherSV,
N->getOperand(1 - i), TLI,
25984 SV1 ? SV1 : DAG.
getUNDEF(VT), Mask);
25994 if (TLI.isBinOp(SrcOpcode) &&
N->isOnlyUserOf(N0.
getNode()) &&
25996 (SrcOpcode == N1.getOpcode() &&
N->isOnlyUserOf(N1.getNode())))) {
26014 SDValue InnerN = Commute ? N1 : N0;
26015 SDValue Op0 = LeftOp ? Op00 : Op01;
26016 SDValue Op1 = LeftOp ? Op10 : Op11;
26021 auto *SVN0 = dyn_cast<ShuffleVectorSDNode>(Op0);
26023 MergeInnerShuffle(Commute, SVN, SVN0, Op1, TLI, SV0, SV1,
26025 (
llvm::any_of(SVN0->getMask(), [](
int M) { return M < 0; }) ||
26031 bool MergedLeft =
false;
26034 if (CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask,
true,
false) ||
26035 CanMergeInnerShuffle(LeftSV0, LeftSV1, LeftMask,
true,
true)) {
26038 LeftMask.
assign(SVN->getMask().begin(), SVN->getMask().end());
26039 LeftSV0 = Op00, LeftSV1 = Op10;
26042 bool MergedRight =
false;
26045 if (CanMergeInnerShuffle(RightSV0, RightSV1, RightMask,
false,
false) ||
26046 CanMergeInnerShuffle(RightSV0, RightSV1, RightMask,
false,
true)) {
26047 MergedRight =
true;
26049 RightMask.
assign(SVN->getMask().begin(), SVN->getMask().end());
26050 RightSV0 = Op01, RightSV1 = Op11;
26053 if (MergedLeft || MergedRight) {
26056 VT,
DL, LeftSV0 ? LeftSV0 : DAG.
getUNDEF(VT),
26057 LeftSV1 ? LeftSV1 : DAG.
getUNDEF(VT), LeftMask);
26059 VT,
DL, RightSV0 ? RightSV0 : DAG.
getUNDEF(VT),
26060 RightSV1 ? RightSV1 : DAG.
getUNDEF(VT), RightMask);
26061 return DAG.
getNode(SrcOpcode,
DL, VT, LHS, RHS);
26082 EVT VT =
N->getValueType(0);
26092 unsigned Opcode =
Scalar.getOpcode();
26094 if (
Scalar.hasOneUse() &&
Scalar->getNumValues() == 1 &&
26095 TLI.isBinOp(Opcode) &&
Scalar.getValueType() == VecEltVT &&
26096 Scalar.getOperand(0).getValueType() == VecEltVT &&
26097 Scalar.getOperand(1).getValueType() == VecEltVT &&
26098 Scalar->isOnlyUserOf(
Scalar.getOperand(0).getNode()) &&
26099 Scalar->isOnlyUserOf(
Scalar.getOperand(1).getNode()) &&
26104 for (
int i : {0, 1}) {
26108 auto *
C = dyn_cast<ConstantSDNode>(
Scalar.getOperand(i ? 0 : 1));
26115 if (TLI.isShuffleMaskLegal(ShufMask, VT)) {
26130 !
Scalar.getOperand(0).getValueType().isFixedLengthVector())
26134 if (VecEltVT !=
Scalar.getValueType() &&
26140 auto *ExtIndexC = dyn_cast<ConstantSDNode>(
Scalar.getOperand(1));
26148 if (VecEltVT == SrcVT.
getScalarType() && VTNumElts <= SrcNumElts) {
26151 Mask[0] = ExtIndexC->getZExtValue();
26152 SDValue LegalShuffle = TLI.buildLegalVectorShuffle(
26159 return LegalShuffle;
26162 if (VTNumElts != SrcNumElts) {
26175 EVT VT =
N->getValueType(0);
26179 uint64_t InsIdx =
N->getConstantOperandVal(2);
26278 if ((N0.
isUndef() || N0SrcSVT == N1SrcSVT) &&
26290 }
else if ((N1SrcSVT.
getSizeInBits() % EltSizeInBits) == 0) {
26313 if (InsIdx < OtherIdx) {
26317 AddToWorklist(NewOp.
getNode());
26331 Ops[InsIdx / Factor] = N1;
26353 auto Op =
N->getOpcode();
26355 "opcode should be FP16_TO_FP or BF16_TO_FP.");
26382 return visitFP16_TO_FP(
N);
26388 unsigned Opcode =
N->getOpcode();
26406 if (!TLI.isOperationLegalOrCustom(Opcode, VT) &&
26407 TLI.isOperationLegalOrCustom(NewOpcode, VT) &&
26409 return DAG.
getNode(NewOpcode,
SDLoc(
N),
N->getValueType(0), N0);
26422 return DAG.
getNode(Opcode,
SDLoc(
N),
N->getValueType(0), Subvec);
26432 if (
SDValue Fused = visitFSUBForFMACombine<VPMatchContext>(
N)) {
26433 AddToWorklist(Fused.getNode());
26441 if (
N->getOpcode() == ISD::VP_GATHER)
26442 if (
SDValue SD = visitVPGATHER(
N))
26445 if (
N->getOpcode() == ISD::VP_SCATTER)
26446 if (
SDValue SD = visitVPSCATTER(
N))
26449 if (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD)
26450 if (
SDValue SD = visitVP_STRIDED_LOAD(
N))
26453 if (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE)
26454 if (
SDValue SD = visitVP_STRIDED_STORE(
N))
26460 bool AreAllEltsDisabled =
false;
26464 AreAllEltsDisabled |=
26468 if (!AreAllEltsDisabled) {
26469 switch (
N->getOpcode()) {
26471 return visitVP_FADD(
N);
26473 return visitVP_FSUB(
N);
26475 return visitFMA<VPMatchContext>(
N);
26476 case ISD::VP_SELECT:
26477 return visitVP_SELECT(
N);
26484 return DAG.
getUNDEF(
N->getValueType(0));
26488 if (
const auto *MemSD = dyn_cast<MemSDNode>(
N)) {
26489 if (MemSD->writeMem())
26490 return MemSD->getChain();
26491 return CombineTo(
N, DAG.
getUNDEF(
N->getValueType(0)), MemSD->getChain());
26496 return N->getOperand(0);
26504 EVT MemVT = cast<FPStateAccessSDNode>(
N)->getMemoryVT();
26509 for (
auto *U :
Ptr->uses()) {
26512 if (
auto *Ld = dyn_cast<LoadSDNode>(U)) {
26513 if (LdNode && LdNode != Ld)
26529 if (
U.getResNo() == 0) {
26530 if (
auto *St = dyn_cast<StoreSDNode>(
U.getUser())) {
26548 CombineTo(StNode, Res,
false);
26555 EVT MemVT = cast<FPStateAccessSDNode>(
N)->getMemoryVT();
26559 for (
auto *U :
Ptr->uses()) {
26562 if (
auto *St = dyn_cast<StoreSDNode>(U)) {
26563 if (StNode && StNode != St)
26578 auto *LdNode = dyn_cast<LoadSDNode>(StValue);
26599 EVT VT =
N->getValueType(0);
26606 if (LegalOperations)
26612 EVT RVT =
RHS.getValueType();
26613 unsigned NumElts =
RHS.getNumOperands();
26618 auto BuildClearMask = [&](
int Split) {
26619 int NumSubElts = NumElts *
Split;
26623 for (
int i = 0; i != NumSubElts; ++i) {
26624 int EltIdx = i /
Split;
26625 int SubIdx = i %
Split;
26635 if (
auto *Cst = dyn_cast<ConstantSDNode>(Elt))
26636 Bits = Cst->getAPIntValue();
26637 else if (
auto *CstFP = dyn_cast<ConstantFPSDNode>(Elt))
26644 Bits =
Bits.extractBits(NumSubBits, (Split - SubIdx - 1) * NumSubBits);
26646 Bits =
Bits.extractBits(NumSubBits, SubIdx * NumSubBits);
26648 if (
Bits.isAllOnes())
26650 else if (Bits == 0)
26659 if (!TLI.isVectorClearMaskLegal(Indices, ClearVT))
26673 for (
int Split = 1;
Split <= MaxSplit; ++
Split)
26675 if (
SDValue S = BuildClearMask(Split))
26687 unsigned Opcode =
N->getOpcode();
26688 EVT VT =
N->getValueType(0);
26695 int Index0, Index1;
26702 if (!Src0 || !Src1 || Index0 != Index1 ||
26722 Ops[Index0] = ScalarBO;
26732 EVT VT =
N->getValueType(0);
26733 assert(VT.
isVector() &&
"SimplifyVCastOp only works on vectors!");
26735 unsigned Opcode =
N->getOpcode();
26765 EVT VT =
N->getValueType(0);
26766 assert(VT.
isVector() &&
"SimplifyVBinOp only works on vectors!");
26770 unsigned Opcode =
N->getOpcode();
26781 auto *Shuf0 = dyn_cast<ShuffleVectorSDNode>(LHS);
26782 auto *Shuf1 = dyn_cast<ShuffleVectorSDNode>(RHS);
26783 if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) &&
26784 LHS.getOperand(1).isUndef() &&
RHS.getOperand(1).isUndef() &&
26787 RHS.getOperand(0), Flags);
26799 Shuf0->hasOneUse() && Shuf0->getOperand(1).isUndef() &&
26808 Shuf1->hasOneUse() && Shuf1->getOperand(1).isUndef() &&
26824 LHS.getOperand(2) ==
RHS.getOperand(2) &&
26829 EVT NarrowVT =
X.getValueType();
26830 if (NarrowVT ==
Y.getValueType() &&
26832 LegalOperations)) {
26845 return Op.isUndef() ||
26846 ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
26855 if (ConcatWithConstantOrUndef(LHS) && ConcatWithConstantOrUndef(RHS) &&
26857 EVT NarrowVT =
LHS.getOperand(0).getValueType();
26858 if (NarrowVT ==
RHS.getOperand(0).getValueType() &&
26860 unsigned NumOperands =
LHS.getNumOperands();
26862 for (
unsigned i = 0; i != NumOperands; ++i) {
26865 RHS.getOperand(i)));
26881 "First argument must be a SetCC node!");
26884 cast<CondCodeSDNode>(N0.
getOperand(2))->get());
26889 if (
SCC.getNode()) {
26896 SCC.getOperand(0),
SCC.getOperand(1),
26897 SCC.getOperand(4), Flags);
26898 AddToWorklist(
SETCC.getNode());
26900 SCC.getOperand(2),
SCC.getOperand(3));
26915bool DAGCombiner::SimplifySelectOps(
SDNode *TheSelect,
SDValue LHS,
26928 CC = cast<CondCodeSDNode>(TheSelect->
getOperand(4))->get();
26935 CC = cast<CondCodeSDNode>(
Cmp.getOperand(2))->get();
26936 CmpLHS =
Cmp.getOperand(0);
26940 if (Zero &&
Zero->isZero() &&
26944 CombineTo(TheSelect, Sqrt);
26954 if (
LHS.getOpcode() !=
RHS.getOpcode() ||
26967 if (
LHS.getOperand(0) !=
RHS.getOperand(0) ||
27011 Visited.
insert(TheSelect);
27090 CombineTo(TheSelect, Load);
27094 CombineTo(
LHS.getNode(),
Load.getValue(0),
Load.getValue(1));
27095 CombineTo(
RHS.getNode(),
Load.getValue(0),
Load.getValue(1));
27136 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
27137 if (N2C && ((N2C->getAPIntValue() & (N2C->getAPIntValue() - 1)) == 0)) {
27138 unsigned ShCt = XType.
getSizeInBits() - N2C->getAPIntValue().logBase2() - 1;
27142 AddToWorklist(Shift.
getNode());
27144 if (XType.
bitsGT(AType)) {
27146 AddToWorklist(Shift.
getNode());
27150 Shift = DAG.
getNOT(
DL, Shift, AType);
27162 AddToWorklist(Shift.
getNode());
27164 if (XType.
bitsGT(AType)) {
27166 AddToWorklist(Shift.
getNode());
27170 Shift = DAG.
getNOT(
DL, Shift, AType);
27233 EVT VT =
N->getValueType(0);
27241 EVT IntVT =
Int.getValueType();
27255 SignMask = ~SignMask;
27261 SignMask = ~SignMask;
27266 AddToWorklist(
Int.getNode());
27274SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset(
27282 auto *TV = dyn_cast<ConstantFPSDNode>(N2);
27283 auto *FV = dyn_cast<ConstantFPSDNode>(N3);
27290 TLI.
isFPImmLegal(TV->getValueAPF(), TV->getValueType(0), ForCodeSize) ||
27291 TLI.
isFPImmLegal(FV->getValueAPF(), FV->getValueType(0), ForCodeSize))
27296 if (!TV->hasOneUse() && !FV->hasOneUse())
27300 const_cast<ConstantFP*
>(TV->getConstantFPValue()) };
27308 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
27317 AddToWorklist(
Cond.getNode());
27319 AddToWorklist(CstOffset.
getNode());
27321 AddToWorklist(CPIdx.
getNode());
27331 bool NotExtCompare) {
27333 if (N2 == N3)
return N2;
27338 auto *N1C = dyn_cast<ConstantSDNode>(N1.
getNode());
27339 auto *N2C = dyn_cast<ConstantSDNode>(N2.
getNode());
27340 auto *N3C = dyn_cast<ConstantSDNode>(N3.
getNode());
27344 AddToWorklist(
SCC.getNode());
27345 if (
auto *SCCC = dyn_cast<ConstantSDNode>(SCC)) {
27348 return !(SCCC->isZero()) ? N2 : N3;
27353 convertSelectOfFPConstantsToLoadOffset(
DL, N0, N1, N2, N3,
CC))
27356 if (
SDValue V = foldSelectCCToShiftAnd(
DL, N0, N1, N2, N3,
CC))
27368 auto *ConstAndRHS = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
27369 if (ConstAndRHS && ConstAndRHS->getAPIntValue().popcount() == 1) {
27371 const APInt &AndMask = ConstAndRHS->getAPIntValue();
27392 bool Fold = N2C &&
isNullConstant(N3) && N2C->getAPIntValue().isPowerOf2();
27393 bool Swap = N3C &&
isNullConstant(N2) && N3C->getAPIntValue().isPowerOf2();
27395 if ((Fold || Swap) &&
27407 if (NotExtCompare && N2C->isOne())
27420 AddToWorklist(
SCC.getNode());
27421 AddToWorklist(Temp.
getNode());
27426 unsigned ShCt = N2C->getAPIntValue().logBase2();
27451 if (
auto *ValueOnZeroC = dyn_cast<ConstantSDNode>(ValueOnZero)) {
27473 if (!NotExtCompare && N1C && N2C && N3C &&
27474 N2C->getAPIntValue() == ~N3C->getAPIntValue() &&
27496 bool foldBooleans) {
27498 DagCombineInfo(DAG, Level,
false,
this);
27596 bool AssumeNonZero) {
27599 auto PeekThroughCastsAndTrunc = [](
SDValue V) {
27601 switch (V.getOpcode()) {
27604 V = V.getOperand(0);
27615 Op = PeekThroughCastsAndTrunc(
Op);
27621 if (
C->isZero() ||
C->isOpaque())
27624 if (
C->getAPIntValue().isPowerOf2()) {
27636 for (
const APInt &Pow2 : Pow2Constants)
27645 auto CastToVT = [&](
EVT NewVT,
SDValue ToCast) {
27646 ToCast = PeekThroughCastsAndTrunc(ToCast);
27647 EVT CurVT = ToCast.getValueType();
27648 if (NewVT == CurVT)
27660 if (AssumeNonZero ||
Op->getFlags().hasNoUnsignedWrap() ||
27663 Depth + 1, AssumeNonZero))
27665 CastToVT(VT,
Op.getOperand(1)));
27672 Depth + 1, AssumeNonZero))
27674 Depth + 1, AssumeNonZero))
27675 return DAG.
getSelect(
DL, VT,
Op.getOperand(0), LogX, LogY);
27690 return DAG.
getNode(
Op.getOpcode(),
DL, VT, LogX, LogY);
27699 bool KnownNonZero,
bool InexpensiveOnly,
27700 std::optional<EVT> OutVT) {
27701 EVT VT = OutVT ? *OutVT :
V.getValueType();
27702 SDValue InexpensiveLogBase2 =
27705 return InexpensiveLogBase2;
27727 EVT VT =
Op.getValueType();
27735 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
27742 AddToWorklist(Est.getNode());
27750 for (
int i = 0; i < Iterations; ++i) {
27753 if (i == Iterations - 1) {
27755 AddToWorklist(MulEst.
getNode());
27759 AddToWorklist(NewEst.
getNode());
27762 (i == Iterations - 1 ?
N : FPOne), NewEst, Flags);
27763 AddToWorklist(NewEst.
getNode());
27766 AddToWorklist(NewEst.
getNode());
27769 AddToWorklist(Est.getNode());
27774 AddToWorklist(Est.getNode());
27790 unsigned Iterations,
27802 for (
unsigned i = 0; i < Iterations; ++i) {
27822 unsigned Iterations,
27835 for (
unsigned i = 0; i < Iterations; ++i) {
27844 if (Reciprocal || (i + 1) < Iterations) {
27867 EVT VT =
Op.getValueType();
27875 if (
Enabled == TLI.ReciprocalEstimate::Disabled)
27882 bool UseOneConstNR =
false;
27886 AddToWorklist(Est.
getNode());
27888 if (Iterations > 0)
27889 Est = UseOneConstNR
27890 ? buildSqrtNROneConst(
Op, Est, Iterations, Flags, Reciprocal)
27891 : buildSqrtNRTwoConst(
Op, Est, Iterations,
Flags, Reciprocal);
27911 return buildSqrtEstimateImpl(
Op, Flags,
true);
27915 return buildSqrtEstimateImpl(
Op, Flags,
false);
27919bool DAGCombiner::mayAlias(
SDNode *Op0,
SDNode *Op1)
const {
27921 struct MemUseCharacteristics {
27930 auto getCharacteristics = [](
SDNode *
N) -> MemUseCharacteristics {
27931 if (
const auto *LSN = dyn_cast<LSBaseSDNode>(
N)) {
27933 if (
auto *
C = dyn_cast<ConstantSDNode>(LSN->getOffset()))
27936 ? -1 *
C->getSExtValue()
27939 return {LSN->isVolatile(), LSN->isAtomic(),
27940 LSN->getBasePtr(),
Offset ,
27943 if (
const auto *LN = cast<LifetimeSDNode>(
N))
27947 (LN->hasOffset()) ? LN->getOffset() : 0,
27960 MemUseCharacteristics MUC0 = getCharacteristics(Op0),
27961 MUC1 = getCharacteristics(Op1);
27964 if (MUC0.BasePtr.getNode() && MUC0.BasePtr == MUC1.BasePtr &&
27965 MUC0.Offset == MUC1.Offset)
27969 if (MUC0.IsVolatile && MUC1.IsVolatile)
27974 if (MUC0.IsAtomic && MUC1.IsAtomic)
27977 if (MUC0.MMO && MUC1.MMO) {
27978 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
27979 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
27985 if ((MUC0.NumBytes.hasValue() && MUC0.NumBytes.isScalable() &&
27986 MUC0.Offset != 0) ||
27987 (MUC1.NumBytes.hasValue() && MUC1.NumBytes.isScalable() &&
27999 if (!MUC0.MMO || !MUC1.MMO)
28005 if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) ||
28006 (MUC1.MMO->isInvariant() && MUC0.MMO->isStore()))
28014 int64_t SrcValOffset0 = MUC0.MMO->getOffset();
28015 int64_t SrcValOffset1 = MUC1.MMO->getOffset();
28016 Align OrigAlignment0 = MUC0.MMO->getBaseAlign();
28017 Align OrigAlignment1 = MUC1.MMO->getBaseAlign();
28021 if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 &&
28027 int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0.
value();
28028 int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1.
value();
28032 if ((OffAlign0 +
static_cast<int64_t
>(
28034 (OffAlign1 +
static_cast<int64_t
>(
28048 if (
UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() &&
28051 int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1);
28074void DAGCombiner::GatherAllAliases(
SDNode *
N,
SDValue OriginalChain,
28081 const bool IsLoad = isa<LoadSDNode>(
N) && cast<LoadSDNode>(
N)->isSimple();
28085 unsigned Depth = 0;
28088 auto ImproveChain = [&](
SDValue &
C) ->
bool {
28089 switch (
C.getOpcode()) {
28098 bool IsOpLoad = isa<LoadSDNode>(
C.getNode()) &&
28099 cast<LSBaseSDNode>(
C.getNode())->isSimple();
28100 if ((IsLoad && IsOpLoad) || !
mayAlias(
N,
C.getNode())) {
28102 C =
C.getOperand(0);
28111 C =
C.getOperand(0);
28120 C =
C.getOperand(0);
28133 while (!Chains.
empty()) {
28167 if (ImproveChain(Chain)) {
28189 GatherAllAliases(
N, OldChain, Aliases);
28192 if (Aliases.
empty())
28196 if (Aliases.
size() == 1)
28216bool DAGCombiner::parallelizeChainedStores(
StoreSDNode *St) {
28233 if (!
BasePtr.getBase().getNode())
28237 if (
BasePtr.getBase().isUndef())
28255 if (Chain->getMemoryVT().isScalableVector())
28259 if (!
SDValue(Chain, 0)->hasOneUse())
28262 if (!Chain->isSimple() || Chain->isIndexed())
28271 int64_t
Length = (Chain->getMemoryVT().getSizeInBits() + 7) / 8;
28274 auto I = Intervals.find(
Offset);
28279 if (
I != Intervals.begin() && (--
I).stop() <=
Offset)
28288 if (ChainedStores.
empty())
28295 for (
unsigned I = ChainedStores.
size();
I;) {
28297 SDValue BetterChain = FindBetterChain(S, NewChain);
28301 ChainedStores[
I] = S;
28305 SDValue BetterChain = FindBetterChain(St, NewChain);
28320 auto hasImprovedChain = [&](
SDValue ST) ->
bool {
28321 return ST->getOperand(0) != NewChain;
28323 bool AddNewChain =
llvm::all_of(TFOps, hasImprovedChain);
28333 AddToWorklist(
Op.getNode());
28334 AddToWorklist(STChain);
28338bool DAGCombiner::findBetterNeighborChains(
StoreSDNode *St) {
28345 if (!
BasePtr.getBase().getNode())
28349 if (
BasePtr.getBase().isUndef())
28353 if (parallelizeChainedStores(St))
28358 if (St->
getChain() != BetterChain) {
28359 replaceStoreChain(St, BetterChain);
28369 DAGCombiner(*
this, AA, OptLevel).Run(Level);
static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< bool > UseAA("aarch64-use-aa", cl::init(true), cl::desc("Enable the use of AA during codegen."))
amdgpu AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI)
For the instruction sequence of store below, F and I values are bundled together as an i64 value befo...
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I)
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI, MachineRegisterInfo &MRI)
Return true if 'MI' is a load or a store that may be fold it's address operand into the load / store ...
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I)
static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques=false)
static cl::opt< bool > EnableShrinkLoadReplaceStoreWithStore("combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable load/<replace bytes>/store with " "a narrower store"))
static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0, unsigned ExtOpc, SmallVectorImpl< SDNode * > &ExtendNodes, const TargetLowering &TLI)
static cl::opt< unsigned > TokenFactorInlineLimit("combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048), cl::desc("Limit the number of operands to inline for Token Factors"))
static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc, bool NonNegZExt=false)
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG)
static SDNode * getBuildPairElt(SDNode *N, unsigned i)
static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI, SelectionDAG &DAG, CombineLevel Level)
Fold (sext (select c, load x, load y)) -> (select c, sextload x, sextload y) (zext (select c,...
static SDValue foldBitOrderCrossLogicOp(SDNode *N, SelectionDAG &DAG)
static SDValue tryToFoldExtendOfConstant(SDNode *N, const SDLoc &DL, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalTypes)
Try to fold a sext/zext/aext dag node into a ConstantSDNode or a build_vector of constants.
static SDValue foldBoolSelectToLogic(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
If a vector binop is performed on splat values, it may be profitable to extract, scalarize,...
static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift, SDValue ExtractFrom, SDValue &Mask, const SDLoc &DL)
Helper function for visitOR to extract the needed side of a rotate idiom from a shl/srl/mul/udiv.
static bool getCombineLoadStoreParts(SDNode *N, unsigned Inc, unsigned Dec, bool &IsLoad, bool &IsMasked, SDValue &Ptr, const TargetLowering &TLI)
bool refineUniformBase(SDValue &BasePtr, SDValue &Index, bool IndexIsScaled, SelectionDAG &DAG, const SDLoc &DL)
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned, const TargetLowering &TLI)
Return true if divmod libcall is available.
static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG)
static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1, SelectionDAG &DAG, const SDLoc &DL)
Given the operands of an add/sub operation, see if the 2nd operand is a masked 0/1 whose source opera...
static bool mergeEltWithShuffle(SDValue &X, SDValue &Y, ArrayRef< int > Mask, SmallVectorImpl< int > &NewMask, SDValue Elt, unsigned InsIndex)
static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf)
If we have a unary shuffle of a shuffle, see if it can be folded away completely.
static bool canSplitIdx(LoadSDNode *LD)
static SDValue ShrinkLoadReplaceStoreWithStore(const std::pair< unsigned, unsigned > &MaskInfo, SDValue IVal, StoreSDNode *St, DAGCombiner *DC)
Check to see if IVal is something that provides a value as specified by MaskInfo.
static cl::opt< bool > StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden, cl::desc("Bypass the profitability model of load slicing"), cl::init(false))
Hidden option to stress test load slicing, i.e., when this option is enabled, load slicing bypasses m...
static cl::opt< bool > UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true), cl::desc("Enable DAG combiner's use of TBAA"))
static void adjustCostForPairing(SmallVectorImpl< LoadedSlice > &LoadedSlices, LoadedSlice::Cost &GlobalLSCost)
Adjust the GlobalLSCost according to the target paring capabilities and the layout of the slices.
static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI, SDValue N0, SDValue N1, SDNode *N)
static SDValue foldExtendVectorInregToExtendOfSubvector(SDNode *N, const SDLoc &DL, const TargetLowering &TLI, SelectionDAG &DAG, bool LegalOperations)
static bool isCompatibleLoad(SDValue N, unsigned ExtOpcode)
Check if N satisfies: N is used once.
static SDValue foldLogicTreeOfShifts(SDNode *N, SDValue LeftHand, SDValue RightHand, SelectionDAG &DAG)
Given a tree of logic operations with shape like (LOGIC (LOGIC (X, Y), LOGIC (Z, Y))) try to match an...
static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG)
static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG, bool LegalOperations)
If we are extracting a subvector produced by a wide binary operator try to use a narrow binary operat...
static SDValue takeInexpensiveLog2(SelectionDAG &DAG, const SDLoc &DL, EVT VT, SDValue Op, unsigned Depth, bool AssumeNonZero)
static SDValue combineSelectAsExtAnd(SDValue Cond, SDValue T, SDValue F, const SDLoc &DL, SelectionDAG &DAG)
static bool areUsedBitsDense(const APInt &UsedBits)
Check that all bits set in UsedBits form a dense region, i.e., UsedBits looks like 0....
static SDValue getInputChainForNode(SDNode *N)
Given a node, return its input chain if it has one, otherwise return a null sd operand.
static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG)
If we are extracting a subvector from a wide vector load, convert to a narrow load to eliminate the e...
static ElementCount numVectorEltsOrZero(EVT T)
static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG, bool ShouldCommuteOperands)
This inverts a canonicalization in IR that replaces a variable select arm with an identity constant.
static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG)
Given an extending node with a pop-count operand, if the target does not support a pop-count in the n...
static SDValue foldAndOrOfSETCC(SDNode *LogicOp, SelectionDAG &DAG)
static SDValue foldSelectOfConstantsUsingSra(SDNode *N, SelectionDAG &DAG)
If a (v)select has a condition value that is a sign-bit test, try to smear the condition operand sign...
static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
If a shuffle inserts exactly one element from a source vector operand into another vector operand and...
static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType)
static SDValue foldAndToUsubsat(SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
For targets that support usubsat, match a bit-hack form of that operation that ends in 'and' and conv...
static cl::opt< bool > CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden, cl::desc("Enable DAG combiner's use of IR alias analysis"))
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy)
static SDValue formSplatFromShuffles(ShuffleVectorSDNode *OuterShuf, SelectionDAG &DAG)
Combine shuffle of shuffle of the form: shuf (shuf X, undef, InnerMask), undef, OuterMask --> splat X...
static bool isDivisorPowerOfTwo(SDValue Divisor)
static bool matchRotateHalf(const SelectionDAG &DAG, SDValue Op, SDValue &Shift, SDValue &Mask)
Match "(X shl/srl V1) & V2" where V2 may not be present.
static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
Given EXTRACT_SUBVECTOR(VECTOR_SHUFFLE(Op0, Op1, Mask)), try to produce VECTOR_SHUFFLE(EXTRACT_SUBVEC...
static SDValue combineConcatVectorOfExtracts(SDNode *N, SelectionDAG &DAG)
static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG, bool LegalOperations)
Transform a vector binary operation into a scalar binary operation by moving the math/logic after an ...
static bool hasNoInfs(const TargetOptions &Options, SDValue N)
static SDValue combineShuffleOfBitcast(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static std::optional< EVT > canCombineShuffleToExtendVectorInreg(unsigned Opcode, EVT VT, std::function< bool(unsigned)> Match, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalTypes, bool LegalOperations)
static SDValue PerformUMinFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, SelectionDAG &DAG)
static SDValue combineShuffleToAnyExtendVectorInreg(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static SDValue foldAddSubOfSignBit(SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
Try to fold a 'not' shifted sign-bit with add/sub with constant operand into a shift and add with a d...
static SDValue stripTruncAndExt(SDValue Value)
static SDValue combineUADDO_CARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG, SDValue X, SDValue Carry0, SDValue Carry1, SDNode *N)
If we are facing some sort of diamond carry propagation pattern try to break it up to generate someth...
static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
Try to convert a wide shuffle of concatenated vectors into 2 narrow shuffles followed by concatenatio...
static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
static auto getFirstIndexOf(R &&Range, const T &Val)
static std::pair< unsigned, unsigned > CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain)
Check to see if V is (and load (ptr), imm), where the load is having specific bytes cleared out.
static int getShuffleMaskIndexOfOneElementFromOp0IntoOp1(ArrayRef< int > Mask)
If the shuffle mask is taking exactly one element from the first vector operand and passing through a...
static bool shouldConvertSelectOfConstantsToMath(const SDValue &Cond, EVT VT, const TargetLowering &TLI)
static cl::opt< bool > EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable merging multiple stores " "into a wider store"))
static bool isContractableFMUL(const TargetOptions &Options, SDValue N)
static cl::opt< bool > MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true), cl::desc("DAG combiner may split indexing from loads"))
static bool areSlicesNextToEachOther(const LoadedSlice &First, const LoadedSlice &Second)
Check whether or not First and Second are next to each other in memory.
static SDValue stripConstantMask(const SelectionDAG &DAG, SDValue Op, SDValue &Mask)
static bool arebothOperandsNotSNan(SDValue Operand1, SDValue Operand2, SelectionDAG &DAG)
static bool isBSwapHWordPair(SDValue N, MutableArrayRef< SDNode * > Parts)
static bool CanCombineFCOPYSIGN_EXTEND_ROUND(EVT XTy, EVT YTy)
copysign(x, fp_extend(y)) -> copysign(x, y) copysign(x, fp_round(y)) -> copysign(x,...
static unsigned getMinMaxOpcodeForFP(SDValue Operand1, SDValue Operand2, ISD::CondCode CC, unsigned OrAndOpcode, SelectionDAG &DAG, bool isFMAXNUMFMINNUM_IEEE, bool isFMAXNUMFMINNUM)
static SDValue foldFPToIntToFP(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue getTruncatedUSUBSAT(EVT DstVT, EVT SrcVT, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &DL)
static SDNode * getPostIndexedLoadStoreOp(SDNode *N, bool &IsLoad, bool &IsMasked, SDValue &Ptr, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS, const TargetLowering &TLI)
static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG, const TargetLowering &TLI, bool Force)
Flips a boolean if it is cheaper to compute.
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op, KnownBits &Known)
static SDValue tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, bool LegalOperations, SDNode *N, SDValue N0, ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc)
static SDValue getSubVectorSrc(SDValue V, SDValue Index, EVT SubVT)
static SDValue combineConcatVectorOfShuffleAndItsOperands(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalTypes, bool LegalOperations)
bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT, SelectionDAG &DAG)
static cl::opt< bool > EnableVectorFCopySignExtendRound("combiner-vector-fcopysign-extend-round", cl::Hidden, cl::init(false), cl::desc("Enable merging extends and rounds into FCOPYSIGN on vector types"))
static SDValue combineMinNumMaxNumImpl(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode CC, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG)
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
static SDValue widenAbs(SDNode *Extend, SelectionDAG &DAG)
static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset=0)
static ConstantSDNode * getAsNonOpaqueConstant(SDValue N)
If N is a ConstantSDNode with isOpaque() == false return it casted to a ConstantSDNode pointer else n...
static bool arebothOperandsNotNan(SDValue Operand1, SDValue Operand2, SelectionDAG &DAG)
static SDValue combineShiftToMULH(SDNode *N, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, SelectionDAG &DAG)
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize, SelectionDAG &DAG, bool IsRotate)
static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N)
OR combines for which the commuted variant will be tried as well.
static SDValue combineShuffleToZeroExtendVectorInReg(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI, bool LegalOperations)
static cl::opt< bool > EnableReduceLoadOpStoreWidth("combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true), cl::desc("DAG combiner enable reducing the width of load/op/store " "sequence"))
static bool shouldCombineToPostInc(SDNode *N, SDValue Ptr, SDNode *PtrUse, SDValue &BasePtr, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG, bool LegalOperations)
static SDValue combineConcatVectorOfCasts(SDNode *N, SelectionDAG &DAG)
static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG)
Try to replace shift/logic that tests if a bit is clear with mask + setcc.
static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI, SelectionDAG &DAG, SDNode *N, SDValue N0, SDValue N1, EVT VT, EVT ShiftAmountTy)
static bool areBitwiseNotOfEachother(SDValue Op0, SDValue Op1)
static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const TargetLowering &TLI)
static SDValue combineConcatVectorOfScalars(SDNode *N, SelectionDAG &DAG)
static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG)
static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue combineConcatVectorOfConcatVectors(SDNode *N, SelectionDAG &DAG)
static SDValue tryToFoldExtOfAtomicLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT, SDValue N0, ISD::LoadExtType ExtLoadType)
static SDValue getAsCarry(const TargetLowering &TLI, SDValue V, bool ForceCarryReconstruction=false)
static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG)
static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG)
static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT, SelectionDAG &DAG, bool LegalOperations)
static cl::opt< unsigned > StoreMergeDependenceLimit("combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10), cl::desc("Limit the number of times for the same StoreNode and RootNode " "to bail out in store merging dependence check"))
static cl::opt< std::string > CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden, cl::desc("Only use DAG-combiner alias analysis in this" " function"))
static SDValue foldLogicOfShifts(SDNode *N, SDValue LogicOp, SDValue ShiftOp, SelectionDAG &DAG)
Given a bitwise logic operation N with a matching bitwise logic operand, fold a pattern where 2 of th...
static bool isSlicingProfitable(SmallVectorImpl< LoadedSlice > &LoadedSlices, const APInt &UsedBits, bool ForCodeSize)
Check the profitability of all involved LoadedSlice.
static bool isBSwapHWordElement(SDValue N, MutableArrayRef< SDNode * > Parts)
Return true if the specified node is an element that makes up a 32-bit packed halfword byteswap.
static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2, SDValue N3, ISD::CondCode CC, unsigned &BW, bool &Unsigned, SelectionDAG &DAG)
static std::optional< SDByteProvider > calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, std::optional< uint64_t > VectorIndex, unsigned StartingIndex=0)
static SDValue FoldIntToFPToInt(SDNode *N, SelectionDAG &DAG)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
This file defines the DenseMap class.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
static bool isUndef(ArrayRef< int > Mask)
static MaybeAlign getAlign(Value *Ptr)
iv Induction Variable Users
static Value * simplifyDivRem(Instruction::BinaryOps Opcode, Value *Op0, Value *Op1, const SimplifyQuery &Q, unsigned MaxRecurse)
Check for common or similar folds of integer division or integer remainder.
This file implements a coalescing interval map for small objects.
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const SmallVectorImpl< MachineOperand > & Cond
Contains matchers for matching SelectionDAG nodes and values.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isSimple(Instruction *I)
static cl::opt< bool > UseTBAA("use-tbaa-in-sched-mi", cl::Hidden, cl::init(true), cl::desc("Enable use of TBAA during MI DAG construction"))
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
const fltSemantics & getSemantics() const
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt umul_ov(const APInt &RHS, bool &Overflow) const
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
static void udivrem(const APInt &LHS, const APInt &RHS, APInt &Quotient, APInt &Remainder)
Dual division/remainder interface.
APInt getLoBits(unsigned numBits) const
Compute an APInt containing numBits lowbits from this APInt.
bool isNegatedPowerOf2() const
Check if this APInt's negated value is a power of two greater than zero.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
unsigned popcount() const
Count the number of bits set.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
APInt abs() const
Get the absolute value.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
bool isSignMask() const
Check if the APInt's value is returned by getSignMask.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
bool isNegative() const
Determine sign of this APInt.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
int32_t exactLogBase2() const
APInt uadd_ov(const APInt &RHS, bool &Overflow) const
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
unsigned countLeadingZeros() const
unsigned logBase2() const
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
bool getBoolValue() const
Convert APInt to a boolean value.
APInt smul_ov(const APInt &RHS, bool &Overflow) const
bool isMask(unsigned numBits) const
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
bool isOne() const
Determine if this is a value of 1.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > drop_front(size_t N=1) const
Drop the first N elements of the array.
size_t size() const
size - Get the array size.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
static bool computeAliasing(const SDNode *Op0, const LocationSize NumBytes0, const SDNode *Op1, const LocationSize NumBytes1, const SelectionDAG &DAG, bool &IsAlias)
A "pseudo-class" with methods for operating on BUILD_VECTORs.
Represents known origin of an individual byte in combine pattern.
static ByteProvider getConstantZero()
static ByteProvider getSrc(std::optional< ISelOp > Val, int64_t ByteOffset, int64_t VectorOffset)
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
bool isNegative() const
Return true if the value is negative.
bool isZero() const
Return true if the value is positive or negative zero.
ConstantFP - Floating Point Values [float, double].
bool isMinSignedValue() const
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
bool isLittleEndian() const
Layout endianness...
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
static bool shouldExecute(unsigned CounterName)
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
static constexpr ElementCount getFixed(ScalarTy MinVal)
constexpr bool isScalar() const
Exactly one element.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
AttributeList getAttributes() const
Return the attribute list for this Function.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
This class is used to form a handle around another node that is persistent and is updated across invo...
This is an important class for using LLVM in a threaded context.
Base class for LoadSDNode and StoreSDNode.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
TypeSize getValue() const
static auto all_valuetypes()
SimpleValueType Iteration.
static MVT getIntegerVT(unsigned BitWidth)
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Function & getFunction()
Return the LLVM function that this machine code represents.
A description of a memory reference used in the backend.
const PseudoSourceValue * getPseudoValue() const
Flags
Flags values. These may be or'd together.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MONonTemporal
The memory access is non-temporal.
Flags getFlags() const
Return the raw flags of the source value,.
const Value * getValue() const
Return the base address of the memory access.
This class is used to represent an MGATHER node.
const SDValue & getPassThru() const
ISD::LoadExtType getExtensionType() const
const SDValue & getIndex() const
bool isIndexScaled() const
const SDValue & getScale() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
This class is used to represent an MLOAD node.
const SDValue & getBasePtr() const
bool isExpandingLoad() const
ISD::LoadExtType getExtensionType() const
const SDValue & getMask() const
const SDValue & getPassThru() const
const SDValue & getOffset() const
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
This class is used to represent an MSCATTER node.
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
This class is used to represent an MSTORE node.
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
const SDValue & getOffset() const
const SDValue & getBasePtr() const
const SDValue & getMask() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
const MDNode * getRanges() const
Returns the Ranges that describes the dereference.
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getBasePtr() const
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
bool isNonTemporal() const
bool isDereferenceable() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
MutableArrayRef< T > take_back(size_t N=1) const
Return a copy of *this with only the last N elements.
MutableArrayRef< T > take_front(size_t N=1) const
Return a copy of *this with only the first N elements.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
size_t use_size() const
Return the number of uses of this node.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
TypeSize getValueSizeInBits(unsigned ResNo) const
Returns MVT::getSizeInBits(getValueType(ResNo)).
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool isPredecessorOf(const SDNode *N) const
Return true if this node is a predecessor of N.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
Targets can subclass this to parameterize the SelectionDAG lowering and instruction selection process...
virtual bool disableGenericCombines(CodeGenOptLevel OptLevel) const
Help to insert SDNodeFlags automatically in transforming.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool willNotOverflowAdd(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the addition of 2 nodes can never overflow.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const TargetSubtargetInfo & getSubtarget() const
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
bool isADDLike(SDValue Op) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
bool willNotOverflowSub(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the sub of 2 nodes can never overflow.
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
void Combine(CombineLevel Level, AAResults *AA, CodeGenOptLevel OptLevel)
This iterates over the nodes in the SelectionDAG, folding certain types of nodes together,...
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
bool willNotOverflowMul(bool IsSigned, SDValue N0, SDValue N1) const
Determine if the result of the mul of 2 nodes can never overflow.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
bool isConstantValueOfAnyType(SDValue N) const
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool PoisonOnly=false, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
bool isSafeToSpeculativelyExecuteNode(const SDNode *N) const
Check if the provided node is save to speculatively executed given its current arguments.
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
DenormalMode getDenormalMode(EVT VT) const
Return the current function's default denormal handling kind for the given floating point type.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
static unsigned getOpcode_EXTEND(unsigned Opcode)
Convert *_EXTEND_VECTOR_INREG to *_EXTEND opcode.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
A vector that has set insertion semantics.
bool remove(const value_type &X)
Remove an item from the set vector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
value_type pop_back_val()
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
int getMaskElt(unsigned Idx) const
int getSplatIndex() const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
iterator erase(const_iterator CI)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SDValue & getValue() const
bool isTruncatingStore() const
Return true if the op does a truncation before store.
bool has(LibFunc F) const
Tests whether a library function is available.
virtual bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT) const
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isOperationExpand(unsigned Op, EVT VT) const
Return true if the specified operation is illegal on this target or unlikely to be made legal with cu...
virtual bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const
virtual bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
virtual bool hasAndNot(SDValue X) const
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
virtual bool enableAggressiveFMAFusion(EVT VT) const
Return true if target always benefits from combining into FMA for a given value type.
bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
SDValue promoteTargetBoolean(SelectionDAG &DAG, SDValue Bool, EVT ValVT) const
Promote the given target boolean to a target boolean of the given type.
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual bool canCombineTruncStore(EVT ValVT, EVT MemVT, bool LegalOnly) const
virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const
Use bitwise logic to make pairs of compares more efficient.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a square root of the given type based on the function's at...
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual bool isFPExtFoldable(const MachineInstr &MI, unsigned Opcode, LLT DestTy, LLT SrcTy) const
Return true if an fpext operation input to an Opcode operation is free (for instance,...
virtual bool hasBitTest(SDValue X, SDValue Y) const
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const
Return true if the specified store with truncation is legal on this target.
virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
virtual bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual bool isFPImmLegal(const APFloat &, EVT, bool ForCodeSize=false) const
Returns true if the target can instruction select the specified FP immediate natively.
virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
virtual bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst, SDValue IntPow2) const
virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context, EVT VT) const
Returns true if we should normalize select(N0&N1, X, Y) => select(N0, select(N1, X,...
virtual bool preferScalarizeSplat(SDNode *N) const
bool isIndexedMaskedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const
Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
int getDivRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a division of the given type based on the function's attributes.
virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to fold a pair of shifts into a mask.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldAvoidTransformToShift(EVT VT, unsigned Amount) const
Return true if creating a shift of the type by the given amount is not profitable.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
virtual bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const
Return true if pulling a binary operation into a select with an identity constant is profitable.
virtual bool shouldReassociateReduction(unsigned RedOpc, EVT VT) const
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const
Return a ReciprocalEstimate enum value for a division of the given type based on the function's attri...
virtual bool preferIncOfAddToSubOfNot(EVT VT) const
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
virtual bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
virtual bool isLegalAddImmediate(int64_t) const
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const
virtual bool isFNegFree(EVT VT) const
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool mergeStoresAfterLegalization(EVT MemVT) const
Allow store merging for the specified type after legalization in addition to before legalization.
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned getGatherAllAliasesMaxDepth() const
virtual bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AddrSpace) const
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal or custom on this target.
bool isAtomicLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified atomic load with extension is legal on this target.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, const MachineFunction &MF) const
Returns if it's reasonable to merge stores to MemVT size.
virtual bool preferABDSToABSWithNSW(EVT VT) const
bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const
Return true if the specified load with extension is legal on this target.
AndOrSETCCFoldKind
Enum of different potentially desirable ways to fold (and/or (setcc ...), (setcc ....
virtual bool shouldScalarizeBinop(SDValue VecOp) const
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const
Return true if the following transform is beneficial: (store (y (conv x)), y*)) -> (store x,...
bool isIndexedMaskedStoreLegal(unsigned IdxMode, EVT VT) const
Return true if the specified indexed load is legal on this target.
virtual bool isVectorClearMaskLegal(ArrayRef< int >, EVT) const
Similar to isShuffleMaskLegal.
bool hasTargetDAGCombine(ISD::NodeType NT) const
If true, the target has custom DAG combine transformations that it can perform for the specified node...
virtual bool shouldSplatInsEltVarIndex(EVT) const
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
NegatibleCost
Enum that specifies when a float negation is beneficial.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const
Return the refinement step count for a square root of the given type based on the function's attribut...
virtual unsigned preferedOpcodeForCmpEqPiecesOfOperand(EVT VT, unsigned ShiftOpc, bool MayTransformRotate, const APInt &ShiftOrRotateAmt, const std::optional< APInt > &AndMask) const
virtual bool isFMADLegal(const MachineInstr &MI, LLT Ty) const
Returns true if MI can be combined with another instruction to form TargetOpcode::G_FMAD.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const
virtual bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const
virtual bool isFAbsFree(EVT VT) const
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
virtual bool generateFMAsInMachineCombiner(EVT VT, CodeGenOptLevel OptLevel) const
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
virtual bool hasPairedLoad(EVT, Align &) const
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
virtual bool convertSelectOfConstantsToMath(EVT VT) const
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const
Hooks for building estimates in place of slower divisions and square roots.
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
SDValue getCheaperOrNeutralNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, const NegatibleCost CostThreshold=NegatibleCost::Neutral, unsigned Depth=0) const
SDValue getCheaperNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, unsigned Depth=0) const
This is the helper function to return the newly negated expression only when the cost is cheaper.
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
virtual bool IsDesirableToPromoteOp(SDValue, EVT &) const
This method query the target whether it is beneficial for dag combiner to promote the specified node.
virtual bool isTypeDesirableForOp(unsigned, EVT VT) const
Return true if the target has native support for the specified value type and it is 'desirable' to us...
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
virtual SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG, const DenormalMode &Mode) const
Return a target-dependent comparison result if the input operand is suitable for use with a square ro...
SDValue buildLegalVectorShuffle(EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, MutableArrayRef< int > Mask, SelectionDAG &DAG) const
Tries to build a legal vector shuffle using the provided parameters or equivalent variations.
virtual SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const
Return a reciprocal estimate value for the input operand.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool isConstFalseVal(SDValue N) const
Return if the N is a constant or constant vector equal to the false value from getBooleanContents().
SDValue BuildUDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::UDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue getSqrtResultForDenormInput(SDValue Operand, SelectionDAG &DAG) const
Return a target-dependent result if the input operand is not suitable for use with a square root esti...
virtual bool getPostIndexedAddressParts(SDNode *, SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
bool isConstTrueVal(SDValue N) const
Return if the N is a constant or constant vector equal to the true value from getBooleanContents().
SDValue getVectorElementPointer(SelectionDAG &DAG, SDValue VecPtr, EVT VecVT, SDValue Index) const
Get a pointer to vector element Idx located in memory for a vector of type VecVT starting at a base a...
virtual bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const
Return true if it is profitable to move this shift by a constant amount through its operand,...
virtual unsigned combineRepeatedFPDivisors() const
Indicate whether this target prefers to combine FDIVs with the same divisor.
virtual AndOrSETCCFoldKind isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const
virtual bool getPreIndexedAddressParts(SDNode *, SDValue &, SDValue &, ISD::MemIndexedMode &, SelectionDAG &) const
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue BuildSDIV(SDNode *N, SelectionDAG &DAG, bool IsAfterLegalization, SmallVectorImpl< SDNode * > &Created) const
Given an ISD::SDIV node expressing a divide by constant, return a DAG expression to select that will ...
virtual SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SDIV lowering for power-of-2 denominators.
virtual SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Targets may override this function to provide custom SREM lowering for power-of-2 denominators.
virtual bool isDesirableToTransformToIntegerOp(unsigned, EVT) const
Return true if it is profitable for dag combiner to transform a floating point op of specified opcode...
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
unsigned NoSignedZerosFPMath
NoSignedZerosFPMath - This flag is enabled when the -enable-no-signed-zeros-fp-math is specified on t...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual bool useAA() const
Enable use of alias analysis during code generation (during MI scheduling, DAGCombine,...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
const fltSemantics & getFltSemantics() const
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
This class is used to represent an VP_GATHER node.
const SDValue & getScale() const
ISD::MemIndexType getIndexType() const
How is Index applied to BasePtr when computing addresses.
const SDValue & getVectorLength() const
const SDValue & getIndex() const
const SDValue & getBasePtr() const
bool isIndexScaled() const
const SDValue & getMask() const
This class is used to represent an VP_SCATTER node.
const SDValue & getValue() const
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
iterator_range< use_iterator > uses()
constexpr bool isKnownMultipleOf(ScalarTy RHS) const
This function tells the caller whether the element count is known at compile time to be a multiple of...
constexpr ScalarTy getFixedValue() const
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ LIFETIME_START
This corresponds to the llvm.lifetime.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ CARRY_FALSE
CARRY_FALSE - This node is used when folding other nodes, like ADDC/SUBC, which indicate the carry re...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ SET_FPENV_MEM
Sets the current floating point environment.
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isIndexTypeSigned(MemIndexType IndexType)
bool isExtVecInRegOpcode(unsigned Opcode)
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool matchUnaryFpPredicate(SDValue Op, std::function< bool(ConstantFPSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantFPSDNode predicate.
bool isFPEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with floati...
bool isExtOpcode(unsigned Opcode)
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
specific_intval< false > m_SpecificInt(const APInt &V)
Match a specific integer value or vector with all elements equal to the value.
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Mul > m_Mul(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty > m_UMax(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
BinaryOp_match< LHS, RHS, Instruction::Shl > m_Shl(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Or > m_Or(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
@ Undef
Value of the register doesn't matter.
BinaryOpc_match< LHS, RHS, false > m_Sra(const LHS &L, const RHS &R)
BinaryOpc_match< LHS, RHS, false > m_Srl(const LHS &L, const RHS &R)
Or< Preds... > m_AnyOf(Preds &&...preds)
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
NUses_match< 1, Value_match > m_OneUse()
initializer< Ty > init(const Ty &Val)
int ilogb(const IEEEFloat &Arg)
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
bool operator<(int64_t V1, const APSInt &V2)
void stable_sort(R &&Range)
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
int popcount(T Value) noexcept
Count the number of set bits in a value.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool operator!=(uint64_t V1, const APInt &V2)
bool operator>=(int64_t V1, const APSInt &V2)
std::string & operator+=(std::string &buffer, StringRef string)
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
unsigned M1(unsigned Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
bool isConstantOrConstantVector(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowFP=true, bool AllowOpaqueConstants=true)
Return true if the specified instruction is known to be a constant, or a vector of constants.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
bool operator>(int64_t V1, const APSInt &V2)
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
auto reverse(ContainerTy &&C)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void sort(IteratorTy Start, IteratorTy End)
detail::ValueMatchesPoly< M > HasValue(M Matcher)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
CodeGenOptLevel
Code generation optimization level.
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
void getShuffleMaskWithWidestElts(ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Repetitively apply widenShuffleMaskElts() for as long as it succeeds, to get the shuffle mask with wi...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
unsigned Log2(Align A)
Returns the log2 of the alignment.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool operator<=(int64_t V1, const APSInt &V2)
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
AAMDNodes concat(const AAMDNodes &Other) const
Determine the best AAMDNodes after concatenating two different locations together.
static ExponentType semanticsMinExponent(const fltSemantics &)
static constexpr roundingMode rmNearestTiesToEven
static ExponentType semanticsMaxExponent(const fltSemantics &)
static unsigned int semanticsPrecision(const fltSemantics &)
opStatus
IEEE-754R 7: Default exception handling.
static unsigned int semanticsIntSizeInBits(const fltSemantics &, bool)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Represent subnormal handling kind for floating point instruction inputs and outputs.
static constexpr DenormalMode getIEEE()
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isScalableVT() const
Return true if the type is a scalable type.
bool isFixedLengthVector() const
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isRound() const
Return true if the size is a power-of-two number of bytes.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool isZeroSized() const
Test if the given EVT has zero size, this will fail if called on a scalable type.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Helper struct to store a base, index and offset that forms an address.
bool isNonNegative() const
Returns true if this value is known to be non-negative.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
bool isConstant() const
Returns true if we know the value of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
bool isAllOnes() const
Returns true if value is all one bits.
const APInt & getConstant() const
Returns the value when all bits have a known value.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
These are IR-level optimization flags that may be propagated to SDNodes.
bool hasNoUnsignedWrap() const
bool hasNoSignedWrap() const
bool hasAllowReassociation() const
void setNoUnsignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
void AddToWorklist(SDNode *N)
bool recursivelyDeleteUnusedNodes(SDNode *N)
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...