97void SelectionDAG::DAGNodeDeletedListener::anchor() {}
98void SelectionDAG::DAGNodeInsertedListener::anchor() {}
100#define DEBUG_TYPE "selectiondag"
104 cl::desc(
"Gang up loads and stores generated by inlining of memcpy"));
107 cl::desc(
"Number limit for gluing ld/st of memcpy."),
123 return getValueAPF().bitwiseIsEqual(V);
146 N->getValueType(0).getVectorElementType().getSizeInBits();
147 if (
auto *Op0 = dyn_cast<ConstantSDNode>(
N->getOperand(0))) {
148 SplatVal = Op0->getAPIntValue().
trunc(EltSize);
151 if (
auto *Op0 = dyn_cast<ConstantFPSDNode>(
N->getOperand(0))) {
152 SplatVal = Op0->getValueAPF().bitcastToAPInt().
trunc(EltSize);
157 auto *BV = dyn_cast<BuildVectorSDNode>(
N);
162 unsigned SplatBitSize;
164 unsigned EltSize =
N->getValueType(0).getVectorElementType().getSizeInBits();
169 const bool IsBigEndian =
false;
170 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
171 EltSize, IsBigEndian) &&
172 EltSize == SplatBitSize;
181 N =
N->getOperand(0).getNode();
190 unsigned i = 0, e =
N->getNumOperands();
193 while (i != e &&
N->getOperand(i).isUndef())
197 if (i == e)
return false;
208 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
210 if (CN->getAPIntValue().countr_one() < EltSize)
213 if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
221 for (++i; i != e; ++i)
222 if (
N->getOperand(i) != NotZero && !
N->getOperand(i).isUndef())
230 N =
N->getOperand(0).getNode();
239 bool IsAllUndef =
true;
252 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
254 if (CN->getAPIntValue().countr_zero() < EltSize)
257 if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
284 if (!isa<ConstantSDNode>(
Op))
297 if (!isa<ConstantFPSDNode>(
Op))
305 assert(
N->getValueType(0).isVector() &&
"Expected a vector!");
307 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
308 if (EltSize <= NewEltSize)
312 return (
N->getOperand(0).getValueType().getScalarSizeInBits() <=
317 return (
N->getOperand(0).getValueType().getScalarSizeInBits() <=
327 if (!isa<ConstantSDNode>(
Op))
330 APInt C = cast<ConstantSDNode>(
Op)->getAPIntValue().trunc(EltSize);
331 if (
Signed &&
C.trunc(NewEltSize).sext(EltSize) !=
C)
333 if (!
Signed &&
C.trunc(NewEltSize).zext(EltSize) !=
C)
344 if (
N->getNumOperands() == 0)
350 return N->getOpcode() ==
ISD::FREEZE &&
N->getOperand(0).isUndef();
353template <
typename ConstNodeType>
355 std::function<
bool(ConstNodeType *)>
Match,
358 if (
auto *
C = dyn_cast<ConstNodeType>(
Op))
366 EVT SVT =
Op.getValueType().getScalarType();
368 if (AllowUndefs &&
Op.getOperand(i).isUndef()) {
374 auto *Cst = dyn_cast<ConstNodeType>(
Op.getOperand(i));
375 if (!Cst || Cst->getValueType(0) != SVT || !
Match(Cst))
381template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
383template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
389 bool AllowUndefs,
bool AllowTypeMismatch) {
390 if (!AllowTypeMismatch &&
LHS.getValueType() !=
RHS.getValueType())
394 if (
auto *LHSCst = dyn_cast<ConstantSDNode>(
LHS))
395 if (
auto *RHSCst = dyn_cast<ConstantSDNode>(
RHS))
396 return Match(LHSCst, RHSCst);
399 if (
LHS.getOpcode() !=
RHS.getOpcode() ||
404 EVT SVT =
LHS.getValueType().getScalarType();
405 for (
unsigned i = 0, e =
LHS.getNumOperands(); i != e; ++i) {
408 bool LHSUndef = AllowUndefs && LHSOp.
isUndef();
409 bool RHSUndef = AllowUndefs && RHSOp.
isUndef();
410 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
411 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
412 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
414 if (!AllowTypeMismatch && (LHSOp.
getValueType() != SVT ||
417 if (!
Match(LHSCst, RHSCst))
424 switch (VecReduceOpcode) {
429 case ISD::VP_REDUCE_FADD:
430 case ISD::VP_REDUCE_SEQ_FADD:
434 case ISD::VP_REDUCE_FMUL:
435 case ISD::VP_REDUCE_SEQ_FMUL:
438 case ISD::VP_REDUCE_ADD:
441 case ISD::VP_REDUCE_MUL:
444 case ISD::VP_REDUCE_AND:
447 case ISD::VP_REDUCE_OR:
450 case ISD::VP_REDUCE_XOR:
453 case ISD::VP_REDUCE_SMAX:
456 case ISD::VP_REDUCE_SMIN:
459 case ISD::VP_REDUCE_UMAX:
462 case ISD::VP_REDUCE_UMIN:
465 case ISD::VP_REDUCE_FMAX:
468 case ISD::VP_REDUCE_FMIN:
481#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
484#include "llvm/IR/VPIntrinsics.def"
492#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
493#define VP_PROPERTY_BINARYOP return true;
494#define END_REGISTER_VP_SDNODE(VPSD) break;
495#include "llvm/IR/VPIntrinsics.def"
504#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
505#define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
506#define END_REGISTER_VP_SDNODE(VPSD) break;
507#include "llvm/IR/VPIntrinsics.def"
517#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
520#include "llvm/IR/VPIntrinsics.def"
529#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
532#include "llvm/IR/VPIntrinsics.def"
542#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
543#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
544#define END_REGISTER_VP_SDNODE(VPOPC) break;
545#include "llvm/IR/VPIntrinsics.def"
554#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
555#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
556#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
557#include "llvm/IR/VPIntrinsics.def"
604 bool isIntegerLike) {
629 bool IsInteger =
Type.isInteger();
634 unsigned Op = Op1 | Op2;
650 bool IsInteger =
Type.isInteger();
685 ID.AddPointer(VTList.
VTs);
691 for (
const auto &
Op : Ops) {
692 ID.AddPointer(
Op.getNode());
693 ID.AddInteger(
Op.getResNo());
700 for (
const auto &
Op : Ops) {
701 ID.AddPointer(
Op.getNode());
702 ID.AddInteger(
Op.getResNo());
715 switch (
N->getOpcode()) {
724 ID.AddPointer(
C->getConstantIntValue());
725 ID.AddBoolean(
C->isOpaque());
730 ID.AddPointer(cast<ConstantFPSDNode>(
N)->getConstantFPValue());
746 ID.AddInteger(cast<RegisterSDNode>(
N)->
getReg());
749 ID.AddPointer(cast<RegisterMaskSDNode>(
N)->getRegMask());
752 ID.AddPointer(cast<SrcValueSDNode>(
N)->getValue());
756 ID.AddInteger(cast<FrameIndexSDNode>(
N)->getIndex());
760 if (cast<LifetimeSDNode>(
N)->hasOffset()) {
761 ID.AddInteger(cast<LifetimeSDNode>(
N)->
getSize());
766 ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getGuid());
767 ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getIndex());
768 ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getAttributes());
772 ID.AddInteger(cast<JumpTableSDNode>(
N)->getIndex());
773 ID.AddInteger(cast<JumpTableSDNode>(
N)->getTargetFlags());
778 ID.AddInteger(CP->getAlign().value());
779 ID.AddInteger(CP->getOffset());
780 if (CP->isMachineConstantPoolEntry())
781 CP->getMachineCPVal()->addSelectionDAGCSEId(
ID);
783 ID.AddPointer(CP->getConstVal());
784 ID.AddInteger(CP->getTargetFlags());
796 ID.AddInteger(LD->getMemoryVT().getRawBits());
797 ID.AddInteger(LD->getRawSubclassData());
798 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
799 ID.AddInteger(LD->getMemOperand()->getFlags());
804 ID.AddInteger(ST->getMemoryVT().getRawBits());
805 ID.AddInteger(ST->getRawSubclassData());
806 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
807 ID.AddInteger(ST->getMemOperand()->getFlags());
818 case ISD::VP_STORE: {
826 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
833 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
840 case ISD::VP_GATHER: {
848 case ISD::VP_SCATTER: {
913 for (
unsigned i = 0, e =
N->getValueType(0).getVectorNumElements();
938 if (
auto *MN = dyn_cast<MemIntrinsicSDNode>(
N)) {
939 ID.AddInteger(MN->getRawSubclassData());
940 ID.AddInteger(MN->getPointerInfo().getAddrSpace());
941 ID.AddInteger(MN->getMemOperand()->getFlags());
942 ID.AddInteger(MN->getMemoryVT().getRawBits());
965 if (
N->getValueType(0) == MVT::Glue)
968 switch (
N->getOpcode()) {
976 for (
unsigned i = 1, e =
N->getNumValues(); i != e; ++i)
977 if (
N->getValueType(i) == MVT::Glue)
994 if (Node.use_empty())
1009 while (!DeadNodes.
empty()) {
1018 DUL->NodeDeleted(
N,
nullptr);
1021 RemoveNodeFromCSEMaps(
N);
1052 RemoveNodeFromCSEMaps(
N);
1056 DeleteNodeNotInCSEMaps(
N);
1059void SelectionDAG::DeleteNodeNotInCSEMaps(
SDNode *
N) {
1060 assert(
N->getIterator() != AllNodes.begin() &&
1061 "Cannot delete the entry node!");
1062 assert(
N->use_empty() &&
"Cannot delete a node that is not dead!");
1071 assert(!(V->isVariadic() && isParameter));
1073 ByvalParmDbgValues.push_back(V);
1075 DbgValues.push_back(V);
1076 for (
const SDNode *Node : V->getSDNodes())
1078 DbgValMap[Node].push_back(V);
1083 if (
I == DbgValMap.end())
1085 for (
auto &Val:
I->second)
1086 Val->setIsInvalidated();
1090void SelectionDAG::DeallocateNode(
SDNode *
N) {
1114 switch (
N->getOpcode()) {
1118 EVT VT =
N->getValueType(0);
1119 assert(
N->getNumValues() == 1 &&
"Too many results!");
1121 "Wrong return type!");
1122 assert(
N->getNumOperands() == 2 &&
"Wrong number of operands!");
1123 assert(
N->getOperand(0).getValueType() ==
N->getOperand(1).getValueType() &&
1124 "Mismatched operand types!");
1126 "Wrong operand type!");
1128 "Wrong return type size");
1132 assert(
N->getNumValues() == 1 &&
"Too many results!");
1133 assert(
N->getValueType(0).isVector() &&
"Wrong return type!");
1134 assert(
N->getNumOperands() ==
N->getValueType(0).getVectorNumElements() &&
1135 "Wrong number of operands!");
1136 EVT EltVT =
N->getValueType(0).getVectorElementType();
1138 assert((
Op.getValueType() == EltVT ||
1139 (EltVT.
isInteger() &&
Op.getValueType().isInteger() &&
1140 EltVT.
bitsLE(
Op.getValueType()))) &&
1141 "Wrong operand type!");
1142 assert(
Op.getValueType() ==
N->getOperand(0).getValueType() &&
1143 "Operands must all have the same type");
1155void SelectionDAG::InsertNode(
SDNode *
N) {
1156 AllNodes.push_back(
N);
1158 N->PersistentId = NextPersistentId++;
1161 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1162 DUL->NodeInserted(
N);
1169bool SelectionDAG::RemoveNodeFromCSEMaps(
SDNode *
N) {
1170 bool Erased =
false;
1171 switch (
N->getOpcode()) {
1174 assert(CondCodeNodes[cast<CondCodeSDNode>(
N)->
get()] &&
1175 "Cond code doesn't exist!");
1176 Erased = CondCodeNodes[cast<CondCodeSDNode>(
N)->get()] !=
nullptr;
1177 CondCodeNodes[cast<CondCodeSDNode>(
N)->get()] =
nullptr;
1180 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(
N)->getSymbol());
1184 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1189 auto *MCSN = cast<MCSymbolSDNode>(
N);
1190 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1194 EVT VT = cast<VTSDNode>(
N)->getVT();
1196 Erased = ExtendedValueTypeNodes.erase(VT);
1207 Erased = CSEMap.RemoveNode(
N);
1214 if (!Erased &&
N->getValueType(
N->getNumValues()-1) != MVT::Glue &&
1229SelectionDAG::AddModifiedNodeToCSEMaps(
SDNode *
N) {
1233 SDNode *Existing = CSEMap.GetOrInsertNode(
N);
1234 if (Existing !=
N) {
1241 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1242 DUL->NodeDeleted(
N, Existing);
1243 DeleteNodeNotInCSEMaps(
N);
1249 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1250 DUL->NodeUpdated(
N);
1268 Node->intersectFlagsWith(
N->getFlags());
1288 Node->intersectFlagsWith(
N->getFlags());
1306 Node->intersectFlagsWith(
N->getFlags());
1319 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0,
DebugLoc(),
1322 InsertNode(&EntryNode);
1333 SDAGISelPass = PassPtr;
1337 LibInfo = LibraryInfo;
1342 FnVarLocs = VarLocs;
1346 assert(!UpdateListeners &&
"Dangling registered DAGUpdateListeners");
1348 OperandRecycler.clear(OperandAllocator);
1357void SelectionDAG::allnodes_clear() {
1358 assert(&*AllNodes.begin() == &EntryNode);
1359 AllNodes.remove(AllNodes.begin());
1360 while (!AllNodes.empty())
1361 DeallocateNode(&AllNodes.front());
1363 NextPersistentId = 0;
1369 SDNode *
N = CSEMap.FindNodeOrInsertPos(
ID, InsertPos);
1371 switch (
N->getOpcode()) {
1376 "debug location. Use another overload.");
1383 const SDLoc &
DL,
void *&InsertPos) {
1384 SDNode *
N = CSEMap.FindNodeOrInsertPos(
ID, InsertPos);
1386 switch (
N->getOpcode()) {
1392 if (
N->getDebugLoc() !=
DL.getDebugLoc())
1399 if (
DL.getIROrder() &&
DL.getIROrder() <
N->getIROrder())
1400 N->setDebugLoc(
DL.getDebugLoc());
1409 OperandRecycler.clear(OperandAllocator);
1410 OperandAllocator.
Reset();
1413 ExtendedValueTypeNodes.clear();
1414 ExternalSymbols.clear();
1415 TargetExternalSymbols.clear();
1418 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1420 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1421 static_cast<SDNode*
>(
nullptr));
1423 EntryNode.UseList =
nullptr;
1424 InsertNode(&EntryNode);
1430 return VT.
bitsGT(
Op.getValueType())
1436std::pair<SDValue, SDValue>
1440 "Strict no-op FP extend/round not allowed.");
1447 return std::pair<SDValue, SDValue>(Res,
SDValue(Res.
getNode(), 1));
1451 return VT.
bitsGT(
Op.getValueType()) ?
1457 return VT.
bitsGT(
Op.getValueType()) ?
1463 return VT.
bitsGT(
Op.getValueType()) ?
1471 auto Type =
Op.getValueType();
1475 auto Size =
Op.getValueSizeInBits();
1486 auto Type =
Op.getValueType();
1490 auto Size =
Op.getValueSizeInBits();
1501 auto Type =
Op.getValueType();
1505 auto Size =
Op.getValueSizeInBits();
1523 EVT OpVT =
Op.getValueType();
1525 "Cannot getZeroExtendInReg FP types");
1527 "getZeroExtendInReg type should be vector iff the operand "
1531 "Vector element counts must match in getZeroExtendInReg");
1569 return getNode(ISD::VP_XOR,
DL, VT, Val, TrueValue, Mask, EVL);
1580 return getNode(ISD::VP_ZERO_EXTEND,
DL, VT,
Op, Mask, EVL);
1582 return getNode(ISD::VP_TRUNCATE,
DL, VT,
Op, Mask, EVL);
1602 bool isT,
bool isO) {
1606 "getConstant with a uint64_t value that doesn't fit in the type!");
1611 bool isT,
bool isO) {
1616 EVT VT,
bool isT,
bool isO) {
1653 "Can only handle an even split!");
1657 for (
unsigned i = 0; i != Parts; ++i)
1659 NewVal.
extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits),
DL,
1660 ViaEltVT, isT, isO));
1665 unsigned ViaVecNumElts = VT.
getSizeInBits() / ViaEltSizeInBits;
1676 NewVal.
extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits),
DL,
1677 ViaEltVT, isT, isO));
1682 std::reverse(EltParts.
begin(), EltParts.
end());
1701 "APInt size does not match type size!");
1709 if ((
N = FindNodeOrInsertPos(
ID,
DL, IP)))
1714 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1715 CSEMap.InsertNode(
N, IP);
1732 const SDLoc &
DL,
bool LegalTypes) {
1749 EVT VT,
bool isTarget) {
1763 if ((
N = FindNodeOrInsertPos(
ID,
DL, IP)))
1768 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1769 CSEMap.InsertNode(
N, IP);
1783 if (EltVT == MVT::f32)
1785 if (EltVT == MVT::f64)
1787 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1788 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1799 EVT VT, int64_t
Offset,
bool isTargetGA,
1800 unsigned TargetFlags) {
1801 assert((TargetFlags == 0 || isTargetGA) &&
1802 "Cannot set target flags on target-independent globals");
1819 ID.AddInteger(TargetFlags);
1824 auto *
N = newSDNode<GlobalAddressSDNode>(
1825 Opc,
DL.getIROrder(),
DL.getDebugLoc(), GV, VT,
Offset, TargetFlags);
1826 CSEMap.InsertNode(
N, IP);
1837 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1840 auto *
N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1841 CSEMap.InsertNode(
N, IP);
1847 unsigned TargetFlags) {
1848 assert((TargetFlags == 0 || isTarget) &&
1849 "Cannot set target flags on target-independent jump tables");
1854 ID.AddInteger(TargetFlags);
1856 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1859 auto *
N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1860 CSEMap.InsertNode(
N, IP);
1874 bool isTarget,
unsigned TargetFlags) {
1875 assert((TargetFlags == 0 || isTarget) &&
1876 "Cannot set target flags on target-independent globals");
1884 ID.AddInteger(Alignment->value());
1887 ID.AddInteger(TargetFlags);
1889 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1892 auto *
N = newSDNode<ConstantPoolSDNode>(isTarget,
C, VT,
Offset, *Alignment,
1894 CSEMap.InsertNode(
N, IP);
1903 bool isTarget,
unsigned TargetFlags) {
1904 assert((TargetFlags == 0 || isTarget) &&
1905 "Cannot set target flags on target-independent globals");
1911 ID.AddInteger(Alignment->value());
1913 C->addSelectionDAGCSEId(
ID);
1914 ID.AddInteger(TargetFlags);
1916 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1919 auto *
N = newSDNode<ConstantPoolSDNode>(isTarget,
C, VT,
Offset, *Alignment,
1921 CSEMap.InsertNode(
N, IP);
1931 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1934 auto *
N = newSDNode<BasicBlockSDNode>(
MBB);
1935 CSEMap.InsertNode(
N, IP);
1942 ValueTypeNodes.size())
1949 N = newSDNode<VTSDNode>(VT);
1957 N = newSDNode<ExternalSymbolSDNode>(
false,
Sym, 0, VT);
1966 N = newSDNode<MCSymbolSDNode>(
Sym, VT);
1972 unsigned TargetFlags) {
1974 TargetExternalSymbols[std::pair<std::string, unsigned>(
Sym, TargetFlags)];
1976 N = newSDNode<ExternalSymbolSDNode>(
true,
Sym, TargetFlags, VT);
1982 if ((
unsigned)
Cond >= CondCodeNodes.size())
1983 CondCodeNodes.resize(
Cond+1);
1985 if (!CondCodeNodes[
Cond]) {
1986 auto *
N = newSDNode<CondCodeSDNode>(
Cond);
1987 CondCodeNodes[
Cond] =
N;
1997 "APInt size does not match type size!");
2015 if (EC.isScalable())
2051 "Must have the same number of vector elements as mask elements!");
2053 "Invalid VECTOR_SHUFFLE");
2061 int NElts = Mask.size();
2063 [&](
int M) {
return M < (NElts * 2) && M >= -1; }) &&
2064 "Index out of range");
2072 for (
int i = 0; i != NElts; ++i)
2073 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2089 for (
int i = 0; i < NElts; ++i) {
2090 if (MaskVec[i] <
Offset || MaskVec[i] >= (
Offset + NElts))
2094 if (UndefElements[MaskVec[i] -
Offset]) {
2100 if (!UndefElements[i])
2104 if (
auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2105 BlendSplat(N1BV, 0);
2106 if (
auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2107 BlendSplat(N2BV, NElts);
2112 bool AllLHS =
true, AllRHS =
true;
2114 for (
int i = 0; i != NElts; ++i) {
2115 if (MaskVec[i] >= NElts) {
2120 }
else if (MaskVec[i] >= 0) {
2124 if (AllLHS && AllRHS)
2126 if (AllLHS && !N2Undef)
2139 bool Identity =
true, AllSame =
true;
2140 for (
int i = 0; i != NElts; ++i) {
2141 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity =
false;
2142 if (MaskVec[i] != MaskVec[0]) AllSame =
false;
2144 if (Identity && NElts)
2154 V = V->getOperand(0);
2157 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2177 if (AllSame && SameNumElts) {
2178 EVT BuildVT = BV->getValueType(0);
2194 for (
int i = 0; i != NElts; ++i)
2195 ID.AddInteger(MaskVec[i]);
2198 if (
SDNode *
E = FindNodeOrInsertPos(
ID, dl, IP))
2204 int *MaskAlloc = OperandAllocator.
Allocate<
int>(NElts);
2207 auto *
N = newSDNode<ShuffleVectorSDNode>(VT, dl.
getIROrder(),
2209 createOperands(
N, Ops);
2211 CSEMap.InsertNode(
N, IP);
2231 ID.AddInteger(RegNo);
2233 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2236 auto *
N = newSDNode<RegisterSDNode>(RegNo, VT);
2238 CSEMap.InsertNode(
N, IP);
2248 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2251 auto *
N = newSDNode<RegisterMaskSDNode>(
RegMask);
2252 CSEMap.InsertNode(
N, IP);
2267 ID.AddPointer(Label);
2269 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2274 createOperands(
N, Ops);
2276 CSEMap.InsertNode(
N, IP);
2282 int64_t
Offset,
bool isTarget,
2283 unsigned TargetFlags) {
2290 ID.AddInteger(TargetFlags);
2292 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2295 auto *
N = newSDNode<BlockAddressSDNode>(Opc, VT, BA,
Offset, TargetFlags);
2296 CSEMap.InsertNode(
N, IP);
2307 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2310 auto *
N = newSDNode<SrcValueSDNode>(V);
2311 CSEMap.InsertNode(
N, IP);
2322 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2325 auto *
N = newSDNode<MDNodeSDNode>(MD);
2326 CSEMap.InsertNode(
N, IP);
2332 if (VT == V.getValueType())
2339 unsigned SrcAS,
unsigned DestAS) {
2343 ID.AddInteger(SrcAS);
2344 ID.AddInteger(DestAS);
2347 if (
SDNode *
E = FindNodeOrInsertPos(
ID, dl, IP))
2352 createOperands(
N, Ops);
2354 CSEMap.InsertNode(
N, IP);
2366 EVT OpTy =
Op.getValueType();
2368 if (OpTy == ShTy || OpTy.
isVector())
return Op;
2376 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2377 EVT VT = Node->getValueType(0);
2378 SDValue Tmp1 = Node->getOperand(0);
2379 SDValue Tmp2 = Node->getOperand(1);
2380 const MaybeAlign MA(Node->getConstantOperandVal(3));
2412 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2413 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2424 Align RedAlign = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2434 if (RedAlign > StackAlign) {
2437 unsigned NumIntermediates;
2439 NumIntermediates, RegisterVT);
2441 Align RedAlign2 = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2442 if (RedAlign2 < RedAlign)
2443 RedAlign = RedAlign2;
2458 false,
nullptr, StackID);
2473 "Don't know how to choose the maximum size when creating a stack "
2482 Align Align = std::max(
DL.getPrefTypeAlign(Ty1),
DL.getPrefTypeAlign(Ty2));
2490 auto GetUndefBooleanConstant = [&]() {
2529 return GetUndefBooleanConstant();
2534 return GetUndefBooleanConstant();
2543 const APInt &C2 = N2C->getAPIntValue();
2545 const APInt &C1 = N1C->getAPIntValue();
2552 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2553 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2555 if (N1CFP && N2CFP) {
2560 return GetUndefBooleanConstant();
2565 return GetUndefBooleanConstant();
2571 return GetUndefBooleanConstant();
2576 return GetUndefBooleanConstant();
2581 return GetUndefBooleanConstant();
2587 return GetUndefBooleanConstant();
2616 return getSetCC(dl, VT, N2, N1, SwappedCond);
2617 }
else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2632 return GetUndefBooleanConstant();
2643 unsigned BitWidth =
Op.getScalarValueSizeInBits();
2651 unsigned Depth)
const {
2659 const APInt &DemandedElts,
2660 unsigned Depth)
const {
2667 unsigned Depth )
const {
2673 unsigned Depth)
const {
2678 const APInt &DemandedElts,
2679 unsigned Depth)
const {
2680 EVT VT =
Op.getValueType();
2687 for (
unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2688 if (!DemandedElts[EltIdx])
2692 KnownZeroElements.
setBit(EltIdx);
2694 return KnownZeroElements;
2704 unsigned Opcode = V.getOpcode();
2705 EVT VT = V.getValueType();
2708 "scalable demanded bits are ignored");
2720 UndefElts = V.getOperand(0).isUndef()
2729 APInt UndefLHS, UndefRHS;
2734 UndefElts = UndefLHS | UndefRHS;
2764 for (
unsigned i = 0; i != NumElts; ++i) {
2770 if (!DemandedElts[i])
2772 if (Scl && Scl !=
Op)
2782 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2783 for (
int i = 0; i != (int)NumElts; ++i) {
2789 if (!DemandedElts[i])
2791 if (M < (
int)NumElts)
2794 DemandedRHS.
setBit(M - NumElts);
2806 auto CheckSplatSrc = [&](
SDValue Src,
const APInt &SrcElts) {
2808 return (SrcElts.popcount() == 1) ||
2810 (SrcElts & SrcUndefs).
isZero());
2812 if (!DemandedLHS.
isZero())
2813 return CheckSplatSrc(V.getOperand(0), DemandedLHS);
2814 return CheckSplatSrc(V.getOperand(1), DemandedRHS);
2818 SDValue Src = V.getOperand(0);
2820 if (Src.getValueType().isScalableVector())
2823 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2836 SDValue Src = V.getOperand(0);
2838 if (Src.getValueType().isScalableVector())
2840 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2842 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
2844 UndefElts = UndefSrcElts.
trunc(NumElts);
2850 SDValue Src = V.getOperand(0);
2851 EVT SrcVT = Src.getValueType();
2861 if ((
BitWidth % SrcBitWidth) == 0) {
2863 unsigned Scale =
BitWidth / SrcBitWidth;
2865 APInt ScaledDemandedElts =
2867 for (
unsigned I = 0;
I != Scale; ++
I) {
2871 SubDemandedElts &= ScaledDemandedElts;
2875 if (!SubUndefElts.
isZero())
2889 EVT VT = V.getValueType();
2899 (AllowUndefs || !UndefElts);
2905 EVT VT = V.getValueType();
2906 unsigned Opcode = V.getOpcode();
2927 SplatIdx = (UndefElts & DemandedElts).
countr_one();
2941 auto *SVN = cast<ShuffleVectorSDNode>(V);
2942 if (!SVN->isSplat())
2944 int Idx = SVN->getSplatIndex();
2945 int NumElts = V.getValueType().getVectorNumElements();
2946 SplatIdx =
Idx % NumElts;
2947 return V.getOperand(
Idx / NumElts);
2963 if (LegalSVT.
bitsLT(SVT))
2974 const APInt &DemandedElts)
const {
2977 "Unknown shift node");
2978 unsigned BitWidth = V.getScalarValueSizeInBits();
2981 const APInt &ShAmt = SA->getAPIntValue();
2992 "Unknown shift node");
2995 unsigned BitWidth = V.getScalarValueSizeInBits();
2996 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
2999 const APInt *MinShAmt =
nullptr;
3000 for (
unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3001 if (!DemandedElts[i])
3003 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3007 const APInt &ShAmt = SA->getAPIntValue();
3010 if (MinShAmt && MinShAmt->
ule(ShAmt))
3021 "Unknown shift node");
3024 unsigned BitWidth = V.getScalarValueSizeInBits();
3025 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
3028 const APInt *MaxShAmt =
nullptr;
3029 for (
unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3030 if (!DemandedElts[i])
3032 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3036 const APInt &ShAmt = SA->getAPIntValue();
3039 if (MaxShAmt && MaxShAmt->
uge(ShAmt))
3050 EVT VT =
Op.getValueType();
3065 unsigned Depth)
const {
3066 unsigned BitWidth =
Op.getScalarValueSizeInBits();
3070 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
3074 if (
auto *
C = dyn_cast<ConstantFPSDNode>(
Op)) {
3084 assert((!
Op.getValueType().isFixedLengthVector() ||
3085 NumElts ==
Op.getValueType().getVectorNumElements()) &&
3086 "Unexpected vector size");
3099 "Expected SPLAT_VECTOR implicit truncation");
3106 unsigned ScalarSize =
Op.getOperand(0).getScalarValueSizeInBits();
3108 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3115 assert(!
Op.getValueType().isScalableVector());
3119 if (!DemandedElts[i])
3128 "Expected BUILD_VECTOR implicit truncation");
3141 assert(!
Op.getValueType().isScalableVector());
3144 APInt DemandedLHS, DemandedRHS;
3148 DemandedLHS, DemandedRHS))
3153 if (!!DemandedLHS) {
3161 if (!!DemandedRHS) {
3170 const APInt &Multiplier =
Op.getConstantOperandAPInt(0);
3175 if (
Op.getValueType().isScalableVector())
3179 EVT SubVectorVT =
Op.getOperand(0).getValueType();
3182 for (
unsigned i = 0; i != NumSubVectors; ++i) {
3184 DemandedElts.
extractBits(NumSubVectorElts, i * NumSubVectorElts);
3185 if (!!DemandedSub) {
3197 if (
Op.getValueType().isScalableVector())
3206 APInt DemandedSrcElts = DemandedElts;
3211 if (!!DemandedSubElts) {
3216 if (!!DemandedSrcElts) {
3226 if (
Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3229 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3235 if (
Op.getValueType().isScalableVector())
3239 if (DemandedElts != 1)
3250 if (
Op.getValueType().isScalableVector())
3270 if ((
BitWidth % SubBitWidth) == 0) {
3277 unsigned SubScale =
BitWidth / SubBitWidth;
3278 APInt SubDemandedElts(NumElts * SubScale, 0);
3279 for (
unsigned i = 0; i != NumElts; ++i)
3280 if (DemandedElts[i])
3281 SubDemandedElts.
setBit(i * SubScale);
3283 for (
unsigned i = 0; i != SubScale; ++i) {
3286 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3287 Known.
insertBits(Known2, SubBitWidth * Shifts);
3292 if ((SubBitWidth %
BitWidth) == 0) {
3293 assert(
Op.getValueType().isVector() &&
"Expected bitcast to vector");
3298 unsigned SubScale = SubBitWidth /
BitWidth;
3299 APInt SubDemandedElts =
3304 for (
unsigned i = 0; i != NumElts; ++i)
3305 if (DemandedElts[i]) {
3306 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3337 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3341 Op.getOperand(0), DemandedElts,
false,
Depth + 1);
3347 if (
Op->getFlags().hasNoSignedWrap() &&
3348 Op.getOperand(0) ==
Op.getOperand(1) &&
3366 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3369 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3370 if (
Op.getResNo() == 0)
3377 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3380 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3381 if (
Op.getResNo() == 0)
3420 if (
Op.getResNo() != 1)
3435 unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
3449 if (
const APInt *ShMinAmt =
3459 if (
const APInt *ShMinAmt =
3471 unsigned Amt =
C->getAPIntValue().urem(
BitWidth);
3477 DemandedElts,
Depth + 1);
3502 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3505 unsigned LoBits =
Op.getOperand(0).getScalarValueSizeInBits();
3506 unsigned HiBits =
Op.getOperand(1).getScalarValueSizeInBits();
3509 Known = Known2.
concat(Known);
3523 if (
Op.getResNo() == 0)
3531 EVT EVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3572 !
Op.getValueType().isScalableVector()) {
3586 for (
unsigned i = 0; i != NumElts; ++i) {
3587 if (!DemandedElts[i])
3590 if (
auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3596 if (
auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3597 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3608 if (
auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3610 }
else if (
auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3618 EVT VT = LD->getMemoryVT();
3621 }
else if (
const MDNode *Ranges = LD->getRanges()) {
3622 EVT VT = LD->getValueType(0);
3634 EVT MemVT = LD->getMemoryVT();
3646 if (
Op.getValueType().isScalableVector())
3648 EVT InVT =
Op.getOperand(0).getValueType();
3660 if (
Op.getValueType().isScalableVector())
3662 EVT InVT =
Op.getOperand(0).getValueType();
3678 if (
Op.getValueType().isScalableVector())
3680 EVT InVT =
Op.getOperand(0).getValueType();
3697 EVT VT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3700 Known.
Zero |= (~InMask);
3701 Known.
One &= (~Known.Zero);
3705 unsigned LogOfAlign =
Log2(cast<AssertAlignSDNode>(
Op)->
getAlign());
3725 Flags.hasNoSignedWrap(), Known, Known2);
3732 if (
Op.getResNo() == 1) {
3743 "We only compute knownbits for the difference here.");
3750 Borrow = Borrow.
trunc(1);
3764 if (
Op.getResNo() == 1) {
3775 assert(
Op.getResNo() == 0 &&
"We only compute knownbits for the sum here.");
3785 Carry = Carry.
trunc(1);
3821 const unsigned Index =
Op.getConstantOperandVal(1);
3822 const unsigned EltBitWidth =
Op.getValueSizeInBits();
3829 Known = Known.
trunc(EltBitWidth);
3845 Known = Known.
trunc(EltBitWidth);
3850 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3851 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3861 if (
Op.getValueType().isScalableVector())
3870 bool DemandedVal =
true;
3871 APInt DemandedVecElts = DemandedElts;
3872 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3873 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3874 unsigned EltIdx = CEltNo->getZExtValue();
3875 DemandedVal = !!DemandedElts[EltIdx];
3884 if (!!DemandedVecElts) {
3902 Known = Known2.
abs();
3933 if (CstLow && CstHigh) {
3938 const APInt &ValueHigh = CstHigh->getAPIntValue();
3939 if (ValueLow.
sle(ValueHigh)) {
3942 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3965 if (IsMax && CstLow) {
3977 EVT VT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3982 if (
Op.getResNo() == 1) {
4009 cast<AtomicSDNode>(
Op)->getMemoryVT().getScalarSizeInBits();
4011 if (
Op.getResNo() == 0) {
4032 if (
Op.getValueType().isScalableVector())
4179 return C->getAPIntValue().zextOrTrunc(
BitWidth).isPowerOf2();
4187 if (
C &&
C->getAPIntValue() == 1)
4197 if (
C &&
C->getAPIntValue().isSignMask())
4209 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
4210 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4218 if (
C->getAPIntValue().zextOrTrunc(
BitWidth).isPowerOf2())
4243 for (
unsigned OpIdx = 0; OpIdx < 2; ++OpIdx) {
4261 EVT VT =
Op.getValueType();
4273 unsigned Depth)
const {
4274 EVT VT =
Op.getValueType();
4279 unsigned FirstAnswer = 1;
4281 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
4282 const APInt &Val =
C->getAPIntValue();