95 void SelectionDAG::DAGNodeDeletedListener::anchor() {}
96 void SelectionDAG::DAGNodeInsertedListener::anchor() {}
98 #define DEBUG_TYPE "selectiondag"
102 cl::desc(
"Gang up loads and stores generated by inlining of memcpy"));
105 cl::desc(
"Number limit for gluing ld/st of memcpy."),
121 return getValueAPF().bitwiseIsEqual(V);
144 N->getValueType(0).getVectorElementType().getSizeInBits();
145 if (
auto *Op0 = dyn_cast<ConstantSDNode>(
N->getOperand(0))) {
146 SplatVal = Op0->getAPIntValue().
trunc(EltSize);
149 if (
auto *Op0 = dyn_cast<ConstantFPSDNode>(
N->getOperand(0))) {
150 SplatVal = Op0->getValueAPF().bitcastToAPInt().
trunc(EltSize);
155 auto *BV = dyn_cast<BuildVectorSDNode>(
N);
160 unsigned SplatBitSize;
162 unsigned EltSize =
N->getValueType(0).getVectorElementType().getSizeInBits();
163 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
165 EltSize == SplatBitSize;
174 N =
N->getOperand(0).getNode();
183 unsigned i = 0,
e =
N->getNumOperands();
186 while (
i !=
e &&
N->getOperand(
i).isUndef())
190 if (
i ==
e)
return false;
201 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
203 if (CN->getAPIntValue().countTrailingOnes() < EltSize)
206 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
214 for (++
i;
i !=
e; ++
i)
215 if (
N->getOperand(
i) != NotZero && !
N->getOperand(
i).isUndef())
223 N =
N->getOperand(0).getNode();
232 bool IsAllUndef =
true;
245 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
247 if (CN->getAPIntValue().countTrailingZeros() < EltSize)
250 if (CFPN->getValueAPF().bitcastToAPInt().countTrailingZeros() < EltSize)
277 if (!isa<ConstantSDNode>(
Op))
290 if (!isa<ConstantFPSDNode>(
Op))
298 assert(
N->getValueType(0).isVector() &&
"Expected a vector!");
300 unsigned EltSize =
N->getValueType(0).getScalarSizeInBits();
301 if (EltSize <= NewEltSize)
305 return (
N->getOperand(0).getValueType().getScalarSizeInBits() <=
310 return (
N->getOperand(0).getValueType().getScalarSizeInBits() <=
320 if (!isa<ConstantSDNode>(
Op))
323 APInt C = cast<ConstantSDNode>(
Op)->getAPIntValue().
trunc(EltSize);
324 if (
Signed &&
C.trunc(NewEltSize).sext(EltSize) !=
C)
326 if (!
Signed &&
C.trunc(NewEltSize).zext(EltSize) !=
C)
337 if (
N->getNumOperands() == 0)
343 return N->getOpcode() ==
ISD::FREEZE &&
N->getOperand(0).isUndef();
350 if (
auto *Cst = dyn_cast<ConstantSDNode>(
Op))
358 EVT SVT =
Op.getValueType().getScalarType();
359 for (
unsigned i = 0,
e =
Op.getNumOperands();
i !=
e; ++
i) {
360 if (AllowUndefs &&
Op.getOperand(
i).isUndef()) {
366 auto *Cst = dyn_cast<ConstantSDNode>(
Op.getOperand(
i));
367 if (!Cst || Cst->getValueType(0) != SVT || !
Match(Cst))
376 bool AllowUndefs,
bool AllowTypeMismatch) {
377 if (!AllowTypeMismatch &&
LHS.getValueType() !=
RHS.getValueType())
381 if (
auto *LHSCst = dyn_cast<ConstantSDNode>(
LHS))
382 if (
auto *RHSCst = dyn_cast<ConstantSDNode>(
RHS))
383 return Match(LHSCst, RHSCst);
386 if (
LHS.getOpcode() !=
RHS.getOpcode() ||
391 EVT SVT =
LHS.getValueType().getScalarType();
392 for (
unsigned i = 0,
e =
LHS.getNumOperands();
i !=
e; ++
i) {
395 bool LHSUndef = AllowUndefs && LHSOp.
isUndef();
396 bool RHSUndef = AllowUndefs && RHSOp.
isUndef();
397 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
398 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
399 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
401 if (!AllowTypeMismatch && (LHSOp.
getValueType() != SVT ||
404 if (!
Match(LHSCst, RHSCst))
411 switch (VecReduceOpcode) {
416 case ISD::VP_REDUCE_FADD:
417 case ISD::VP_REDUCE_SEQ_FADD:
421 case ISD::VP_REDUCE_FMUL:
422 case ISD::VP_REDUCE_SEQ_FMUL:
425 case ISD::VP_REDUCE_ADD:
428 case ISD::VP_REDUCE_MUL:
431 case ISD::VP_REDUCE_AND:
434 case ISD::VP_REDUCE_OR:
437 case ISD::VP_REDUCE_XOR:
440 case ISD::VP_REDUCE_SMAX:
443 case ISD::VP_REDUCE_SMIN:
446 case ISD::VP_REDUCE_UMAX:
449 case ISD::VP_REDUCE_UMIN:
452 case ISD::VP_REDUCE_FMAX:
455 case ISD::VP_REDUCE_FMIN:
464 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
467 #include "llvm/IR/VPIntrinsics.def"
475 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
476 #define VP_PROPERTY_BINARYOP return true;
477 #define END_REGISTER_VP_SDNODE(VPSD) break;
478 #include "llvm/IR/VPIntrinsics.def"
487 #define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
488 #define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
489 #define END_REGISTER_VP_SDNODE(VPSD) break;
490 #include "llvm/IR/VPIntrinsics.def"
500 #define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
503 #include "llvm/IR/VPIntrinsics.def"
512 #define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
515 #include "llvm/IR/VPIntrinsics.def"
562 bool isIntegerLike) {
587 bool IsInteger =
Type.isInteger();
592 unsigned Op = Op1 | Op2;
608 bool IsInteger =
Type.isInteger();
643 ID.AddPointer(VTList.
VTs);
649 for (
const auto &
Op : Ops) {
650 ID.AddPointer(
Op.getNode());
651 ID.AddInteger(
Op.getResNo());
658 for (
const auto &
Op : Ops) {
659 ID.AddPointer(
Op.getNode());
660 ID.AddInteger(
Op.getResNo());
673 switch (
N->getOpcode()) {
682 ID.AddPointer(
C->getConstantIntValue());
683 ID.AddBoolean(
C->isOpaque());
688 ID.AddPointer(cast<ConstantFPSDNode>(
N)->getConstantFPValue());
704 ID.AddInteger(cast<RegisterSDNode>(
N)->
getReg());
707 ID.AddPointer(cast<RegisterMaskSDNode>(
N)->getRegMask());
710 ID.AddPointer(cast<SrcValueSDNode>(
N)->getValue());
714 ID.AddInteger(cast<FrameIndexSDNode>(
N)->getIndex());
718 if (cast<LifetimeSDNode>(
N)->hasOffset()) {
719 ID.AddInteger(cast<LifetimeSDNode>(
N)->
getSize());
724 ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getGuid());
725 ID.AddInteger(cast<PseudoProbeSDNode>(
N)->getIndex());
730 ID.AddInteger(cast<JumpTableSDNode>(
N)->getIndex());
731 ID.AddInteger(cast<JumpTableSDNode>(
N)->getTargetFlags());
736 ID.AddInteger(
CP->getAlign().value());
737 ID.AddInteger(
CP->getOffset());
738 if (
CP->isMachineConstantPoolEntry())
739 CP->getMachineCPVal()->addSelectionDAGCSEId(
ID);
741 ID.AddPointer(
CP->getConstVal());
742 ID.AddInteger(
CP->getTargetFlags());
754 ID.AddInteger(
LD->getMemoryVT().getRawBits());
755 ID.AddInteger(
LD->getRawSubclassData());
756 ID.AddInteger(
LD->getPointerInfo().getAddrSpace());
757 ID.AddInteger(
LD->getMemOperand()->getFlags());
762 ID.AddInteger(
ST->getMemoryVT().getRawBits());
763 ID.AddInteger(
ST->getRawSubclassData());
764 ID.AddInteger(
ST->getPointerInfo().getAddrSpace());
765 ID.AddInteger(
ST->getMemOperand()->getFlags());
776 case ISD::VP_STORE: {
784 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
791 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
798 case ISD::VP_GATHER: {
806 case ISD::VP_SCATTER: {
877 for (
unsigned i = 0,
e =
N->getValueType(0).getVectorNumElements();
891 ID.AddInteger(cast<AssertAlignSDNode>(
N)->
getAlign().value());
897 if (
N->isTargetMemoryOpcode()) {
926 switch (
N->getOpcode()) {
934 for (
unsigned i = 1,
e =
N->getNumValues();
i !=
e; ++
i)
952 if (Node.use_empty())
953 DeadNodes.push_back(&Node);
967 while (!DeadNodes.empty()) {
976 DUL->NodeDeleted(
N,
nullptr);
979 RemoveNodeFromCSEMaps(
N);
990 DeadNodes.push_back(Operand);
1010 RemoveNodeFromCSEMaps(
N);
1014 DeleteNodeNotInCSEMaps(
N);
1017 void SelectionDAG::DeleteNodeNotInCSEMaps(
SDNode *
N) {
1018 assert(
N->getIterator() != AllNodes.begin() &&
1019 "Cannot delete the entry node!");
1020 assert(
N->use_empty() &&
"Cannot delete a node that is not dead!");
1031 ByvalParmDbgValues.push_back(V);
1033 DbgValues.push_back(V);
1036 DbgValMap[Node].push_back(V);
1041 if (
I == DbgValMap.end())
1043 for (
auto &Val:
I->second)
1044 Val->setIsInvalidated();
1048 void SelectionDAG::DeallocateNode(
SDNode *
N) {
1072 switch (
N->getOpcode()) {
1076 EVT VT =
N->getValueType(0);
1077 assert(
N->getNumValues() == 1 &&
"Too many results!");
1079 "Wrong return type!");
1080 assert(
N->getNumOperands() == 2 &&
"Wrong number of operands!");
1081 assert(
N->getOperand(0).getValueType() ==
N->getOperand(1).getValueType() &&
1082 "Mismatched operand types!");
1084 "Wrong operand type!");
1086 "Wrong return type size");
1090 assert(
N->getNumValues() == 1 &&
"Too many results!");
1091 assert(
N->getValueType(0).isVector() &&
"Wrong return type!");
1092 assert(
N->getNumOperands() ==
N->getValueType(0).getVectorNumElements() &&
1093 "Wrong number of operands!");
1094 EVT EltVT =
N->getValueType(0).getVectorElementType();
1096 assert((
Op.getValueType() == EltVT ||
1097 (EltVT.
isInteger() &&
Op.getValueType().isInteger() &&
1098 EltVT.
bitsLE(
Op.getValueType()))) &&
1099 "Wrong operand type!");
1100 assert(
Op.getValueType() ==
N->getOperand(0).getValueType() &&
1101 "Operands must all have the same type");
1113 void SelectionDAG::InsertNode(
SDNode *
N) {
1114 AllNodes.push_back(
N);
1116 N->PersistentId = NextPersistentId++;
1120 DUL->NodeInserted(
N);
1127 bool SelectionDAG::RemoveNodeFromCSEMaps(
SDNode *
N) {
1128 bool Erased =
false;
1129 switch (
N->getOpcode()) {
1132 assert(CondCodeNodes[cast<CondCodeSDNode>(
N)->
get()] &&
1133 "Cond code doesn't exist!");
1134 Erased = CondCodeNodes[cast<CondCodeSDNode>(
N)->get()] !=
nullptr;
1135 CondCodeNodes[cast<CondCodeSDNode>(
N)->get()] =
nullptr;
1138 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(
N)->getSymbol());
1142 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1147 auto *MCSN = cast<MCSymbolSDNode>(
N);
1148 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1152 EVT VT = cast<VTSDNode>(
N)->getVT();
1154 Erased = ExtendedValueTypeNodes.erase(VT);
1165 Erased = CSEMap.RemoveNode(
N);
1172 if (!Erased &&
N->getValueType(
N->getNumValues()-1) !=
MVT::Glue &&
1187 SelectionDAG::AddModifiedNodeToCSEMaps(
SDNode *
N) {
1191 SDNode *Existing = CSEMap.GetOrInsertNode(
N);
1192 if (Existing !=
N) {
1200 DUL->NodeDeleted(
N, Existing);
1201 DeleteNodeNotInCSEMaps(
N);
1208 DUL->NodeUpdated(
N);
1226 Node->intersectFlagsWith(
N->getFlags());
1246 Node->intersectFlagsWith(
N->getFlags());
1264 Node->intersectFlagsWith(
N->getFlags());
1278 : TM(tm), OptLevel(OL),
1281 InsertNode(&EntryNode);
1292 SDAGISelPass = PassPtr;
1296 LibInfo = LibraryInfo;
1301 FnVarLocs = VarLocs;
1305 assert(!UpdateListeners &&
"Dangling registered DAGUpdateListeners");
1307 OperandRecycler.clear(OperandAllocator);
1316 void SelectionDAG::allnodes_clear() {
1317 assert(&*AllNodes.begin() == &EntryNode);
1318 AllNodes.remove(AllNodes.begin());
1319 while (!AllNodes.empty())
1320 DeallocateNode(&AllNodes.front());
1322 NextPersistentId = 0;
1328 SDNode *
N = CSEMap.FindNodeOrInsertPos(
ID, InsertPos);
1330 switch (
N->getOpcode()) {
1335 "debug location. Use another overload.");
1342 const SDLoc &
DL,
void *&InsertPos) {
1343 SDNode *
N = CSEMap.FindNodeOrInsertPos(
ID, InsertPos);
1345 switch (
N->getOpcode()) {
1351 if (
N->getDebugLoc() !=
DL.getDebugLoc())
1358 if (
DL.getIROrder() &&
DL.getIROrder() <
N->getIROrder())
1359 N->setDebugLoc(
DL.getDebugLoc());
1370 if (!Visited.
insert(Node).second)
1373 SDEI[Node].PCSections = MD;
1375 for (
const SDValue &
Op : Node->op_values())
1381 OperandRecycler.clear(OperandAllocator);
1382 OperandAllocator.
Reset();
1385 ExtendedValueTypeNodes.clear();
1386 ExternalSymbols.clear();
1387 TargetExternalSymbols.clear();
1390 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1392 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1393 static_cast<SDNode*
>(
nullptr));
1395 EntryNode.UseList =
nullptr;
1396 InsertNode(&EntryNode);
1402 return VT.
bitsGT(
Op.getValueType())
1408 std::pair<SDValue, SDValue>
1412 "Strict no-op FP extend/round not allowed.");
1419 return std::pair<SDValue, SDValue>(Res,
SDValue(Res.
getNode(), 1));
1423 return VT.
bitsGT(
Op.getValueType()) ?
1429 return VT.
bitsGT(
Op.getValueType()) ?
1435 return VT.
bitsGT(
Op.getValueType()) ?
1450 EVT OpVT =
Op.getValueType();
1452 "Cannot getZeroExtendInReg FP types");
1454 "getZeroExtendInReg type should be vector iff the operand "
1458 "Vector element counts must match in getZeroExtendInReg");
1496 return getNode(ISD::VP_XOR,
DL, VT, Val, TrueValue,
Mask, EVL);
1529 bool isT,
bool isO) {
1533 "getConstant with a uint64_t value that doesn't fit in the type!");
1538 bool isT,
bool isO) {
1543 EVT VT,
bool isT,
bool isO) {
1575 "Can only handle an even split!");
1579 for (
unsigned i = 0;
i != Parts; ++
i)
1582 ViaEltVT, isT, isO));
1587 unsigned ViaVecNumElts = VT.
getSizeInBits() / ViaEltSizeInBits;
1599 ViaEltVT, isT, isO));
1623 "APInt size does not match type size!");
1631 if ((
N = FindNodeOrInsertPos(
ID,
DL, IP)))
1636 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1637 CSEMap.InsertNode(
N, IP);
1654 const SDLoc &
DL,
bool LegalTypes) {
1671 EVT VT,
bool isTarget) {
1685 if ((
N = FindNodeOrInsertPos(
ID,
DL, IP)))
1690 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1691 CSEMap.InsertNode(
N, IP);
1721 EVT VT, int64_t
Offset,
bool isTargetGA,
1722 unsigned TargetFlags) {
1723 assert((TargetFlags == 0 || isTargetGA) &&
1724 "Cannot set target flags on target-independent globals");
1741 ID.AddInteger(TargetFlags);
1746 auto *
N = newSDNode<GlobalAddressSDNode>(
1747 Opc,
DL.getIROrder(),
DL.getDebugLoc(), GV, VT,
Offset, TargetFlags);
1748 CSEMap.InsertNode(
N, IP);
1759 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1762 auto *
N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1763 CSEMap.InsertNode(
N, IP);
1769 unsigned TargetFlags) {
1770 assert((TargetFlags == 0 || isTarget) &&
1771 "Cannot set target flags on target-independent jump tables");
1776 ID.AddInteger(TargetFlags);
1778 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1781 auto *
N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1782 CSEMap.InsertNode(
N, IP);
1789 bool isTarget,
unsigned TargetFlags) {
1790 assert((TargetFlags == 0 || isTarget) &&
1791 "Cannot set target flags on target-independent globals");
1802 ID.AddInteger(TargetFlags);
1804 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1809 CSEMap.InsertNode(
N, IP);
1818 bool isTarget,
unsigned TargetFlags) {
1819 assert((TargetFlags == 0 || isTarget) &&
1820 "Cannot set target flags on target-independent globals");
1828 C->addSelectionDAGCSEId(
ID);
1829 ID.AddInteger(TargetFlags);
1831 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1836 CSEMap.InsertNode(
N, IP);
1842 unsigned TargetFlags) {
1847 ID.AddInteger(TargetFlags);
1849 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1852 auto *
N = newSDNode<TargetIndexSDNode>(
Index, VT,
Offset, TargetFlags);
1853 CSEMap.InsertNode(
N, IP);
1863 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
1866 auto *
N = newSDNode<BasicBlockSDNode>(
MBB);
1867 CSEMap.InsertNode(
N, IP);
1874 ValueTypeNodes.size())
1881 N = newSDNode<VTSDNode>(VT);
1887 SDNode *&
N = ExternalSymbols[Sym];
1889 N = newSDNode<ExternalSymbolSDNode>(
false, Sym, 0, VT);
1898 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1904 unsigned TargetFlags) {
1906 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1908 N = newSDNode<ExternalSymbolSDNode>(
true, Sym, TargetFlags, VT);
1914 if ((
unsigned)
Cond >= CondCodeNodes.size())
1915 CondCodeNodes.resize(
Cond+1);
1917 if (!CondCodeNodes[
Cond]) {
1918 auto *
N = newSDNode<CondCodeSDNode>(
Cond);
1919 CondCodeNodes[
Cond] =
N;
1940 OpsStepConstants.push_back(
1955 "Must have the same number of vector elements as mask elements!");
1957 "Invalid VECTOR_SHUFFLE");
1965 int NElts =
Mask.size();
1967 [&](
int M) {
return M < (NElts * 2) &&
M >= -1; }) &&
1968 "Index out of range");
1976 for (
int i = 0;
i != NElts; ++
i)
1977 if (MaskVec[
i] >= NElts) MaskVec[
i] -= NElts;
1993 for (
int i = 0;
i < NElts; ++
i) {
1998 if (UndefElements[MaskVec[
i] -
Offset]) {
2004 if (!UndefElements[
i])
2008 if (
auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2009 BlendSplat(N1BV, 0);
2010 if (
auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2011 BlendSplat(N2BV, NElts);
2016 bool AllLHS =
true, AllRHS =
true;
2018 for (
int i = 0;
i != NElts; ++
i) {
2019 if (MaskVec[
i] >= NElts) {
2024 }
else if (MaskVec[
i] >= 0) {
2028 if (AllLHS && AllRHS)
2030 if (AllLHS && !N2Undef)
2043 bool Identity =
true, AllSame =
true;
2044 for (
int i = 0;
i != NElts; ++
i) {
2045 if (MaskVec[
i] >= 0 && MaskVec[
i] !=
i) Identity =
false;
2046 if (MaskVec[
i] != MaskVec[0]) AllSame =
false;
2048 if (Identity && NElts)
2061 if (
auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2063 SDValue Splat = BV->getSplatValue(&UndefElements);
2073 if (Splat && UndefElements.
none()) {
2078 if (
auto *
C = dyn_cast<ConstantSDNode>(Splat))
2084 if (AllSame && SameNumElts) {
2085 EVT BuildVT = BV->getValueType(0);
2101 for (
int i = 0;
i != NElts; ++
i)
2102 ID.AddInteger(MaskVec[
i]);
2111 int *MaskAlloc = OperandAllocator.
Allocate<
int>(NElts);
2114 auto *
N = newSDNode<ShuffleVectorSDNode>(VT,
dl.
getIROrder(),
2116 createOperands(
N, Ops);
2118 CSEMap.InsertNode(
N, IP);
2138 ID.AddInteger(RegNo);
2140 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2143 auto *
N = newSDNode<RegisterSDNode>(RegNo, VT);
2145 CSEMap.InsertNode(
N, IP);
2153 ID.AddPointer(RegMask);
2155 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2158 auto *
N = newSDNode<RegisterMaskSDNode>(RegMask);
2159 CSEMap.InsertNode(
N, IP);
2174 ID.AddPointer(Label);
2176 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2181 createOperands(
N, Ops);
2183 CSEMap.InsertNode(
N, IP);
2189 int64_t
Offset,
bool isTarget,
2190 unsigned TargetFlags) {
2197 ID.AddInteger(TargetFlags);
2199 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2202 auto *
N = newSDNode<BlockAddressSDNode>(Opc, VT, BA,
Offset, TargetFlags);
2203 CSEMap.InsertNode(
N, IP);
2214 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2217 auto *
N = newSDNode<SrcValueSDNode>(V);
2218 CSEMap.InsertNode(
N, IP);
2229 if (
SDNode *
E = FindNodeOrInsertPos(
ID, IP))
2232 auto *
N = newSDNode<MDNodeSDNode>(MD);
2233 CSEMap.InsertNode(
N, IP);
2246 unsigned SrcAS,
unsigned DestAS) {
2250 ID.AddInteger(SrcAS);
2251 ID.AddInteger(DestAS);
2259 createOperands(
N, Ops);
2261 CSEMap.InsertNode(
N, IP);
2273 EVT OpTy =
Op.getValueType();
2275 if (OpTy == ShTy || OpTy.
isVector())
return Op;
2283 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2284 EVT VT = Node->getValueType(0);
2285 SDValue Tmp1 = Node->getOperand(0);
2286 SDValue Tmp2 = Node->getOperand(1);
2287 const MaybeAlign MA(Node->getConstantOperandVal(3));
2319 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2320 const Value *
VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2331 Align RedAlign = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2344 unsigned NumIntermediates;
2346 NumIntermediates, RegisterVT);
2348 Align RedAlign2 = UseABI ?
DL.getABITypeAlign(Ty) :
DL.getPrefTypeAlign(Ty);
2349 if (RedAlign2 < RedAlign)
2350 RedAlign = RedAlign2;
2365 false,
nullptr, StackID);
2380 "Don't know how to choose the maximum size when creating a stack "
2440 const APInt &C2 = N2C->getAPIntValue();
2442 const APInt &
C1 = N1C->getAPIntValue();
2449 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2450 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2452 if (N1CFP && N2CFP) {
2513 return getSetCC(
dl, VT, N2, N1, SwappedCond);
2514 }
else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2540 unsigned BitWidth =
Op.getScalarValueSizeInBits();
2548 unsigned Depth)
const {
2556 const APInt &DemandedElts,
2557 unsigned Depth)
const {
2564 unsigned Depth )
const {
2570 unsigned Depth)
const {
2575 const APInt &DemandedElts,
2576 unsigned Depth)
const {
2577 EVT VT =
Op.getValueType();
2584 for (
unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2585 if (!DemandedElts[EltIdx])
2589 KnownZeroElements.
setBit(EltIdx);
2591 return KnownZeroElements;
2605 "scalable demanded bits are ignored");
2626 APInt UndefLHS, UndefRHS;
2631 UndefElts = UndefLHS | UndefRHS;
2661 for (
unsigned i = 0;
i != NumElts; ++
i) {
2667 if (!DemandedElts[
i])
2669 if (Scl && Scl !=
Op)
2680 for (
int i = 0;
i != (
int)NumElts; ++
i) {
2686 if (!DemandedElts[
i])
2688 if (
M < (
int)NumElts)
2691 DemandedRHS.
setBit(
M - NumElts);
2703 auto CheckSplatSrc = [&](
SDValue Src,
const APInt &SrcElts) {
2705 return (SrcElts.countPopulation() == 1) ||
2707 (SrcElts & SrcUndefs).
isZero());
2709 if (!DemandedLHS.
isZero())
2710 return CheckSplatSrc(V.
getOperand(0), DemandedLHS);
2711 return CheckSplatSrc(V.
getOperand(1), DemandedRHS);
2717 if (Src.getValueType().isScalableVector())
2720 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2722 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
2724 UndefElts = UndefSrcElts.
extractBits(NumElts, Idx);
2735 if (Src.getValueType().isScalableVector())
2737 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2739 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts);
2741 UndefElts = UndefSrcElts.
trunc(NumElts);
2748 EVT SrcVT = Src.getValueType();
2758 if ((
BitWidth % SrcBitWidth) == 0) {
2760 unsigned Scale =
BitWidth / SrcBitWidth;
2762 APInt ScaledDemandedElts =
2764 for (
unsigned I = 0;
I != Scale; ++
I) {
2768 SubDemandedElts &= ScaledDemandedElts;
2772 if (!SubUndefElts.
isZero())
2796 (AllowUndefs || !UndefElts);
2838 auto *SVN = cast<ShuffleVectorSDNode>(V);
2839 if (!SVN->isSplat())
2841 int Idx = SVN->getSplatIndex();
2843 SplatIdx = Idx % NumElts;
2860 if (LegalSVT.
bitsLT(SVT))
2871 const APInt &DemandedElts)
const {
2874 "Unknown shift node");
2878 const APInt &ShAmt = SA->getAPIntValue();
2889 "Unknown shift node");
2893 auto *BV = dyn_cast<BuildVectorSDNode>(V.
getOperand(1));
2896 const APInt *MinShAmt =
nullptr;
2897 for (
unsigned i = 0,
e = BV->getNumOperands();
i !=
e; ++
i) {
2898 if (!DemandedElts[
i])
2900 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(
i));
2904 const APInt &ShAmt = SA->getAPIntValue();
2907 if (MinShAmt && MinShAmt->
ule(ShAmt))
2918 "Unknown shift node");
2922 auto *BV = dyn_cast<BuildVectorSDNode>(V.
getOperand(1));
2925 const APInt *MaxShAmt =
nullptr;
2926 for (
unsigned i = 0,
e = BV->getNumOperands();
i !=
e; ++
i) {
2927 if (!DemandedElts[
i])
2929 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(
i));
2933 const APInt &ShAmt = SA->getAPIntValue();
2936 if (MaxShAmt && MaxShAmt->
uge(ShAmt))
2947 EVT VT =
Op.getValueType();
2962 unsigned Depth)
const {
2963 unsigned BitWidth =
Op.getScalarValueSizeInBits();
2967 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
2971 if (
auto *
C = dyn_cast<ConstantFPSDNode>(
Op)) {
2981 assert((!
Op.getValueType().isFixedLengthVector() ||
2982 NumElts ==
Op.getValueType().getVectorNumElements()) &&
2983 "Unexpected vector size");
2988 unsigned Opcode =
Op.getOpcode();
2996 "Expected SPLAT_VECTOR implicit truncation");
3003 assert(!
Op.getValueType().isScalableVector());
3006 for (
unsigned i = 0,
e =
Op.getNumOperands();
i !=
e; ++
i) {
3007 if (!DemandedElts[
i])
3016 "Expected BUILD_VECTOR implicit truncation");
3029 assert(!
Op.getValueType().isScalableVector());
3032 APInt DemandedLHS, DemandedRHS;
3036 DemandedLHS, DemandedRHS))
3041 if (!!DemandedLHS) {
3049 if (!!DemandedRHS) {
3057 if (
Op.getValueType().isScalableVector())
3061 EVT SubVectorVT =
Op.getOperand(0).getValueType();
3063 unsigned NumSubVectors =
Op.getNumOperands();
3064 for (
unsigned i = 0;
i != NumSubVectors; ++
i) {
3066 DemandedElts.
extractBits(NumSubVectorElts,
i * NumSubVectorElts);
3067 if (!!DemandedSub) {
3079 if (
Op.getValueType().isScalableVector())
3088 APInt DemandedSrcElts = DemandedElts;
3093 if (!!DemandedSubElts) {
3098 if (!!DemandedSrcElts) {
3108 if (
Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3111 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3112 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);
3117 if (
Op.getValueType().isScalableVector())
3121 if (DemandedElts != 1)
3132 if (
Op.getValueType().isScalableVector())
3152 if ((
BitWidth % SubBitWidth) == 0) {
3159 unsigned SubScale =
BitWidth / SubBitWidth;
3160 APInt SubDemandedElts(NumElts * SubScale, 0);
3161 for (
unsigned i = 0;
i != NumElts; ++
i)
3162 if (DemandedElts[
i])
3163 SubDemandedElts.
setBit(
i * SubScale);
3165 for (
unsigned i = 0;
i != SubScale; ++
i) {
3168 unsigned Shifts = IsLE ?
i : SubScale - 1 -
i;
3169 Known.
insertBits(Known2, SubBitWidth * Shifts);
3174 if ((SubBitWidth %
BitWidth) == 0) {
3175 assert(
Op.getValueType().isVector() &&
"Expected bitcast to vector");
3180 unsigned SubScale = SubBitWidth /
BitWidth;
3181 APInt SubDemandedElts =
3186 for (
unsigned i = 0;
i != NumElts; ++
i)
3187 if (DemandedElts[
i]) {
3188 unsigned Shifts = IsLE ?
i : NumElts - 1 -
i;
3220 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3224 Op.getOperand(0), DemandedElts,
false,
Depth + 1);
3230 if (
Op->getFlags().hasNoSignedWrap() &&
3231 Op.getOperand(0) ==
Op.getOperand(1) &&
3249 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3252 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3253 if (
Op.getResNo() == 0)
3260 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3263 bool SelfMultiply =
Op.getOperand(0) ==
Op.getOperand(1);
3264 if (
Op.getResNo() == 0)
3303 if (
Op.getResNo() != 1)
3318 unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
3332 if (
const APInt *ShMinAmt =
3342 if (
const APInt *ShMinAmt =
3355 unsigned Amt =
C->getAPIntValue().urem(
BitWidth);
3361 DemandedElts,
Depth + 1);
3387 assert((
Op.getResNo() == 0 ||
Op.getResNo() == 1) &&
"Unknown result");
3390 unsigned LoBits =
Op.getOperand(0).getScalarValueSizeInBits();
3391 unsigned HiBits =
Op.getOperand(1).getScalarValueSizeInBits();
3394 Known = Known2.
concat(Known);
3408 if (
Op.getResNo() == 0)
3416 EVT EVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3457 !
Op.getValueType().isScalableVector()) {
3471 for (
unsigned i = 0;
i != NumElts; ++
i) {
3472 if (!DemandedElts[
i])
3475 if (
auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3481 if (
auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3482 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3493 if (
auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3495 }
else if (
auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3503 EVT VT =
LD->getMemoryVT();
3507 EVT VT =
LD->getValueType(0);
3519 EVT MemVT =
LD->getMemoryVT();
3531 if (
Op.getValueType().isScalableVector())
3533 EVT InVT =
Op.getOperand(0).getValueType();
3545 if (
Op.getValueType().isScalableVector())
3547 EVT InVT =
Op.getOperand(0).getValueType();
3563 if (
Op.getValueType().isScalableVector())
3565 EVT InVT =
Op.getOperand(0).getValueType();
3582 EVT VT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3585 Known.
Zero |= (~InMask);
3590 unsigned LogOfAlign =
Log2(cast<AssertAlignSDNode>(
Op)->
getAlign());
3608 if (
Op.getResNo() == 1) {
3620 "We only compute knownbits for the difference here.");
3636 if (
Op.getResNo() == 1) {
3648 assert(
Op.getResNo() == 0 &&
"We only compute knownbits for the sum here.");
3690 const unsigned Index =
Op.getConstantOperandVal(1);
3691 const unsigned EltBitWidth =
Op.getValueSizeInBits();
3698 Known = Known.
trunc(EltBitWidth);
3714 Known = Known.
trunc(EltBitWidth);
3719 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3720 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3730 if (
Op.getValueType().isScalableVector())
3739 bool DemandedVal =
true;
3740 APInt DemandedVecElts = DemandedElts;
3741 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3742 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3743 unsigned EltIdx = CEltNo->getZExtValue();
3744 DemandedVal = !!DemandedElts[EltIdx];
3753 if (!!DemandedVecElts) {
3771 Known = Known2.
abs();
3802 if (CstLow && CstHigh) {
3807 const APInt &ValueHigh = CstHigh->getAPIntValue();
3808 if (ValueLow.
sle(ValueHigh)) {
3811 unsigned MinSignBits =
std::min(LowSignBits, HighSignBits);
3834 if (IsMax && CstLow) {
3846 EVT VT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3851 if (
Op.getResNo() == 1) {
3878 cast<AtomicSDNode>(
Op)->getMemoryVT().getScalarSizeInBits();
3880 if (
Op.getResNo() == 0) {
3901 if (
Op.getValueType().isScalableVector())
3950 return Const->getAPIntValue().zextOrTrunc(
BitWidth).isPowerOf2();
3956 if (
C &&
C->getAPIntValue() == 1)
3964 if (
C &&
C->getAPIntValue().isSignMask())
3971 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
3972 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
3980 if (
C->getAPIntValue().zextOrTrunc(
BitWidth).isPowerOf2())
3998 EVT VT =
Op.getValueType();
4010 unsigned Depth)
const {
4011 EVT VT =
Op.getValueType();
4016 unsigned FirstAnswer = 1;
4018 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
4019 const APInt &Val =
C->getAPIntValue();
4029 unsigned Opcode =
Op.getOpcode();
4033 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getSizeInBits();
4034 return VTBits-Tmp+1;
4036 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getSizeInBits();
4043 unsigned NumSrcBits =
Op.getOperand(0).getValueSizeInBits();
4045 if (NumSrcSignBits > (NumSrcBits - VTBits))
4046 return NumSrcSignBits - (NumSrcBits - VTBits);
4052 for (
unsigned i = 0,
e =
Op.getNumOperands(); (
i <
e) && (Tmp > 1); ++
i) {
4053 if (!DemandedElts[
i])
4060 if (
SrcOp.getValueSizeInBits() != VTBits) {
4062 "Expected BUILD_VECTOR implicit truncation");
4063 unsigned ExtraBits =
SrcOp.getValueSizeInBits() - VTBits;
4064 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4073 APInt DemandedLHS, DemandedRHS;
4077 DemandedLHS, DemandedRHS))
4083 if (!!DemandedRHS) {
4090 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
4106 if (VTBits == SrcBits)
4112 if ((SrcBits % VTBits) == 0) {
4115 unsigned Scale = SrcBits / VTBits;
4116 APInt SrcDemandedElts =
4126 for (
unsigned i = 0;
i != NumElts; ++
i)
4127 if (DemandedElts[
i]) {
4128 unsigned SubOffset =
i % Scale;
4129 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4130 SubOffset = SubOffset * VTBits;
4131 if (Tmp <= SubOffset)
4133 Tmp2 =
std::min(Tmp2, Tmp - SubOffset);
4142 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getScalarSizeInBits();
4143 return VTBits - Tmp + 1;
4145 Tmp = VTBits -
Op.getOperand(0).getScalarValueSizeInBits();
4149 Tmp = cast<VTSDNode>(
Op.getOperand(1))->getVT().getScalarSizeInBits();
4157 EVT SrcVT = Src.getValueType();
4165 if (
const APInt *ShAmt =
4167 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
4170 if (
const APInt *ShAmt =
4174 if (ShAmt->ult(Tmp))
4175 return Tmp - ShAmt->getZExtValue();
4195 if (Tmp == 1)
return 1;
4200 if (Tmp == 1)
return 1;
4214 if (CstLow && CstHigh) {
4219 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4248 if (
Op.getResNo() != 1)
4262 unsigned OpNo =
Op->isStrictFPOpcode() ? 1 : 0;
4279 unsigned RotAmt =
C->getAPIntValue().urem(VTBits);
4283 RotAmt = (VTBits - RotAmt) % VTBits;
4287 if (Tmp > (RotAmt + 1))
return (Tmp - RotAmt);
4295 if (Tmp == 1)
return 1;
4300 if (CRHS->isAllOnes()) {
4306 if ((Known.
Zero | 1).isAllOnes())
4316 if (Tmp2 == 1)
return 1;
4320 if (Tmp2 == 1)
return 1;
4325 if (CLHS->isZero()) {
4330 if ((Known.
Zero | 1).isAllOnes())
4344 if (Tmp == 1)
return 1;
4349 if (SignBitsOp0 == 1)
4352 if (SignBitsOp1 == 1)
4354 unsigned OutValidBits =
4355 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4356 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4366 unsigned NumSrcBits =
Op.getOperand(0).getScalarValueSizeInBits();
4368 if (NumSrcSignBits > (NumSrcBits - VTBits))
4369 return NumSrcSignBits - (NumSrcBits - VTBits);
4376 const int BitWidth =
Op.getValueSizeInBits();
4377 const int Items =
Op.getOperand(0).getValueSizeInBits() /
BitWidth;
4381 const int rIndex = Items - 1 -
Op.getConstantOperandVal(1);
4396 bool DemandedVal =
true;
4397 APInt DemandedVecElts = DemandedElts;
4398 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4399 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4400 unsigned EltIdx = CEltNo->getZExtValue();
4401 DemandedVal = !!DemandedElts[EltIdx];
4412 if (!!DemandedVecElts) {
4416 assert(Tmp <= VTBits &&
"Failed to determine minimum sign bits");
4427 const unsigned BitWidth =
Op.getValueSizeInBits();
4428 const unsigned EltBitWidth =
Op.getOperand(0).getScalarValueSizeInBits();
4440 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4441 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4451 if (Src.getValueType().isScalableVector())
4454 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4455 APInt DemandedSrcElts = DemandedElts.
zext(NumSrcElts).
shl(Idx);