34#define DEBUG_TYPE "legalize-types"
40void DAGTypeLegalizer::ScalarizeVectorResult(
SDNode *
N,
unsigned ResNo) {
41 LLVM_DEBUG(
dbgs() <<
"Scalarize node result " << ResNo <<
": ";
N->dump(&DAG);
45 switch (
N->getOpcode()) {
48 dbgs() <<
"ScalarizeVectorResult #" << ResNo <<
": ";
60 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(
N);
break;
62 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(
N));
break;
68 case ISD::SETCC: R = ScalarizeVecRes_SETCC(
N);
break;
69 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(
N);
break;
75 R = ScalarizeVecRes_VecInregOp(
N);
114 R = ScalarizeVecRes_UnaryOp(
N);
157 R = ScalarizeVecRes_BinOp(
N);
162 R = ScalarizeVecRes_TernaryOp(
N);
165#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
166 case ISD::STRICT_##DAGN:
167#include "llvm/IR/ConstrainedOps.def"
168 R = ScalarizeVecRes_StrictFPOp(
N);
173 R = ScalarizeVecRes_FP_TO_XINT_SAT(
N);
182 R = ScalarizeVecRes_OverflowOp(
N, ResNo);
192 R = ScalarizeVecRes_FIX(
N);
198 SetScalarizedVector(
SDValue(
N, ResNo), R);
202 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
203 SDValue RHS = GetScalarizedVector(
N->getOperand(1));
205 LHS.getValueType(), LHS, RHS,
N->getFlags());
209 SDValue Op0 = GetScalarizedVector(
N->getOperand(0));
210 SDValue Op1 = GetScalarizedVector(
N->getOperand(1));
211 SDValue Op2 = GetScalarizedVector(
N->getOperand(2));
217 SDValue Op0 = GetScalarizedVector(
N->getOperand(0));
218 SDValue Op1 = GetScalarizedVector(
N->getOperand(1));
225 EVT VT =
N->getValueType(0).getVectorElementType();
226 unsigned NumOpers =
N->getNumOperands();
237 for (
unsigned i = 1; i < NumOpers; ++i) {
243 Oper = GetScalarizedVector(Oper);
254 Opers,
N->getFlags());
265 EVT ResVT =
N->getValueType(0);
266 EVT OvVT =
N->getValueType(1);
270 ScalarLHS = GetScalarizedVector(
N->getOperand(0));
271 ScalarRHS = GetScalarizedVector(
N->getOperand(1));
276 ScalarLHS = ElemsLHS[0];
277 ScalarRHS = ElemsRHS[0];
283 N->getOpcode(),
DL, ScalarVTs, ScalarLHS, ScalarRHS).
getNode();
287 unsigned OtherNo = 1 - ResNo;
288 EVT OtherVT =
N->getValueType(OtherNo);
290 SetScalarizedVector(
SDValue(
N, OtherNo),
SDValue(ScalarNode, OtherNo));
294 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
297 return SDValue(ScalarNode, ResNo);
302 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
303 return GetScalarizedVector(Op);
308 if (
Op.getValueType().isVector()
309 &&
Op.getValueType().getVectorNumElements() == 1
310 && !isSimpleLegalType(
Op.getValueType()))
311 Op = GetScalarizedVector(Op);
312 EVT NewVT =
N->getValueType(0).getVectorElementType();
317SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(
SDNode *
N) {
318 EVT EltVT =
N->getValueType(0).getVectorElementType();
327SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
329 N->getValueType(0).getVectorElementType(),
330 N->getOperand(0),
N->getOperand(1));
336 EVT OpVT =
Op.getValueType();
340 Op = GetScalarizedVector(Op);
347 N->getValueType(0).getVectorElementType(), Op,
352 SDValue Op = GetScalarizedVector(
N->getOperand(0));
354 Op.getValueType(), Op,
N->getOperand(1));
357SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(
SDNode *
N) {
361 EVT EltVT =
N->getValueType(0).getVectorElementType();
362 if (
Op.getValueType() != EltVT)
369 assert(
N->isUnindexed() &&
"Indexed vector load?");
373 N->getValueType(0).getVectorElementType(),
SDLoc(
N),
N->getChain(),
374 N->getBasePtr(), DAG.
getUNDEF(
N->getBasePtr().getValueType()),
375 N->getPointerInfo(),
N->getMemoryVT().getVectorElementType(),
376 N->getOriginalAlign(),
N->getMemOperand()->getFlags(),
N->getAAInfo());
386 EVT DestVT =
N->getValueType(0).getVectorElementType();
388 EVT OpVT =
Op.getValueType();
398 Op = GetScalarizedVector(Op);
404 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), DestVT, Op,
N->getFlags());
408 EVT EltVT =
N->getValueType(0).getVectorElementType();
410 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
419 EVT OpVT =
Op.getValueType();
421 EVT EltVT =
N->getValueType(0).getVectorElementType();
424 Op = GetScalarizedVector(Op);
430 switch (
N->getOpcode()) {
442SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(
SDNode *
N) {
445 EVT EltVT =
N->getValueType(0).getVectorElementType();
454 EVT OpVT =
Cond.getValueType();
467 SDValue LHS = GetScalarizedVector(
N->getOperand(1));
481 EVT OpVT =
Cond->getOperand(0).getValueType();
488 EVT CondVT =
Cond.getValueType();
489 if (ScalarBool != VecBool) {
490 switch (ScalarBool) {
511 auto BoolVT = getSetCCResultType(CondVT);
512 if (BoolVT.bitsLT(CondVT))
517 GetScalarizedVector(
N->getOperand(2)));
521 SDValue LHS = GetScalarizedVector(
N->getOperand(1));
523 LHS.getValueType(),
N->getOperand(0), LHS,
524 GetScalarizedVector(
N->getOperand(2)));
528 SDValue LHS = GetScalarizedVector(
N->getOperand(2));
530 N->getOperand(0),
N->getOperand(1),
531 LHS, GetScalarizedVector(
N->getOperand(3)),
536 return DAG.
getUNDEF(
N->getValueType(0).getVectorElementType());
539SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(
SDNode *
N) {
543 return DAG.
getUNDEF(
N->getValueType(0).getVectorElementType());
544 unsigned Op = !cast<ConstantSDNode>(
Arg)->isZero();
545 return GetScalarizedVector(
N->getOperand(Op));
548SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(
SDNode *
N) {
550 EVT SrcVT = Src.getValueType();
555 Src = GetScalarizedVector(Src);
561 EVT DstVT =
N->getValueType(0).getVectorElementType();
562 return DAG.
getNode(
N->getOpcode(), dl, DstVT, Src,
N->getOperand(1));
566 assert(
N->getValueType(0).isVector() &&
567 N->getOperand(0).getValueType().isVector() &&
568 "Operand types must be vectors");
571 EVT OpVT =
LHS.getValueType();
572 EVT NVT =
N->getValueType(0).getVectorElementType();
577 LHS = GetScalarizedVector(LHS);
578 RHS = GetScalarizedVector(RHS);
594 return DAG.
getNode(ExtendCode,
DL, NVT, Res);
601 EVT ArgVT =
Arg.getValueType();
602 EVT ResultVT =
N->getValueType(0).getVectorElementType();
605 Arg = GetScalarizedVector(
Arg);
618 return DAG.
getNode(ExtendCode,
DL, ResultVT, Res);
625bool DAGTypeLegalizer::ScalarizeVectorOperand(
SDNode *
N,
unsigned OpNo) {
626 LLVM_DEBUG(
dbgs() <<
"Scalarize node operand " << OpNo <<
": ";
N->dump(&DAG);
630 switch (
N->getOpcode()) {
633 dbgs() <<
"ScalarizeVectorOperand Op #" << OpNo <<
": ";
640 Res = ScalarizeVecOp_BITCAST(
N);
650 Res = ScalarizeVecOp_UnaryOp(
N);
656 Res = ScalarizeVecOp_UnaryOp_StrictFP(
N);
659 Res = ScalarizeVecOp_CONCAT_VECTORS(
N);
662 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(
N);
665 Res = ScalarizeVecOp_VSELECT(
N);
668 Res = ScalarizeVecOp_VSETCC(
N);
671 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(
N), OpNo);
674 Res = ScalarizeVecOp_STRICT_FP_ROUND(
N, OpNo);
677 Res = ScalarizeVecOp_FP_ROUND(
N, OpNo);
680 Res = ScalarizeVecOp_STRICT_FP_EXTEND(
N);
683 Res = ScalarizeVecOp_FP_EXTEND(
N);
698 Res = ScalarizeVecOp_VECREDUCE(
N);
702 Res = ScalarizeVecOp_VECREDUCE_SEQ(
N);
707 if (!Res.getNode())
return false;
711 if (Res.getNode() ==
N)
714 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 1 &&
715 "Invalid operand expansion");
717 ReplaceValueWith(
SDValue(
N, 0), Res);
724 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
726 N->getValueType(0), Elt);
732 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
733 "Unexpected vector type!");
734 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
736 N->getValueType(0).getScalarType(), Elt);
744SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(
SDNode *
N) {
745 assert(
N->getValueType(0).getVectorNumElements() == 1 &&
746 "Unexpected vector type!");
747 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
749 {
N->getValueType(0).getScalarType(),
MVT::Other },
750 {
N->getOperand(0), Elt });
760 ReplaceValueWith(
SDValue(
N, 0), Res);
765SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(
SDNode *
N) {
767 for (
unsigned i = 0, e =
N->getNumOperands(); i < e; ++i)
768 Ops[i] = GetScalarizedVector(
N->getOperand(i));
774SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
775 EVT VT =
N->getValueType(0);
776 SDValue Res = GetScalarizedVector(
N->getOperand(0));
788 SDValue ScalarCond = GetScalarizedVector(
N->getOperand(0));
789 EVT VT =
N->getValueType(0);
799 assert(
N->getValueType(0).isVector() &&
800 N->getOperand(0).getValueType().isVector() &&
801 "Operand types must be vectors");
804 EVT VT =
N->getValueType(0);
805 SDValue LHS = GetScalarizedVector(
N->getOperand(0));
806 SDValue RHS = GetScalarizedVector(
N->getOperand(1));
808 EVT OpVT =
N->getOperand(0).getValueType();
820 Res = DAG.
getNode(ExtendCode,
DL, NVT, Res);
828 assert(
N->isUnindexed() &&
"Indexed store of one-element vector?");
829 assert(OpNo == 1 &&
"Do not know how to scalarize this operand!");
832 if (
N->isTruncatingStore())
834 N->getChain(), dl, GetScalarizedVector(
N->getOperand(1)),
835 N->getBasePtr(),
N->getPointerInfo(),
836 N->getMemoryVT().getVectorElementType(),
N->getOriginalAlign(),
837 N->getMemOperand()->getFlags(),
N->getAAInfo());
839 return DAG.
getStore(
N->getChain(), dl, GetScalarizedVector(
N->getOperand(1)),
840 N->getBasePtr(),
N->getPointerInfo(),
841 N->getOriginalAlign(),
N->getMemOperand()->getFlags(),
847SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(
SDNode *
N,
unsigned OpNo) {
848 assert(OpNo == 0 &&
"Wrong operand for scalarization!");
849 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
851 N->getValueType(0).getVectorElementType(), Elt,
856SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(
SDNode *
N,
858 assert(OpNo == 1 &&
"Wrong operand for scalarization!");
859 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
863 {
N->getOperand(0), Elt,
N->getOperand(2) });
872 ReplaceValueWith(
SDValue(
N, 0), Res);
879 SDValue Elt = GetScalarizedVector(
N->getOperand(0));
881 N->getValueType(0).getVectorElementType(), Elt);
887SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(
SDNode *
N) {
888 SDValue Elt = GetScalarizedVector(
N->getOperand(1));
892 {
N->getOperand(0), Elt});
901 ReplaceValueWith(
SDValue(
N, 0), Res);
906 SDValue Res = GetScalarizedVector(
N->getOperand(0));
913SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(
SDNode *
N) {
921 AccOp, Op,
N->getFlags());
932void DAGTypeLegalizer::SplitVectorResult(
SDNode *
N,
unsigned ResNo) {
937 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
940 switch (
N->getOpcode()) {
943 dbgs() <<
"SplitVectorResult #" << ResNo <<
": ";
954 case ISD::VP_SELECT: SplitRes_Select(
N,
Lo,
Hi);
break;
968 SplitVecRes_ScalarOp(
N,
Lo,
Hi);
971 SplitVecRes_STEP_VECTOR(
N,
Lo,
Hi);
975 SplitVecRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
978 SplitVecRes_VP_LOAD(cast<VPLoadSDNode>(
N),
Lo,
Hi);
980 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
981 SplitVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(
N),
Lo,
Hi);
984 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(
N),
Lo,
Hi);
988 SplitVecRes_Gather(cast<MemSDNode>(
N),
Lo,
Hi,
true);
992 SplitVecRes_SETCC(
N,
Lo,
Hi);
995 SplitVecRes_VECTOR_REVERSE(
N,
Lo,
Hi);
998 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(
N),
Lo,
Hi);
1001 SplitVecRes_VECTOR_SPLICE(
N,
Lo,
Hi);
1004 SplitVecRes_VAARG(
N,
Lo,
Hi);
1010 SplitVecRes_ExtVecInRegOp(
N,
Lo,
Hi);
1016 case ISD::VP_BITREVERSE:
1024 case ISD::VP_CTLZ_ZERO_UNDEF:
1026 case ISD::VP_CTTZ_ZERO_UNDEF:
1036 case ISD::VP_FFLOOR:
1041 case ISD::VP_FNEARBYINT:
1046 case ISD::VP_FP_EXTEND:
1048 case ISD::VP_FP_ROUND:
1050 case ISD::VP_FP_TO_SINT:
1052 case ISD::VP_FP_TO_UINT:
1056 case ISD::VP_FROUND:
1058 case ISD::VP_FROUNDEVEN:
1062 case ISD::VP_FROUNDTOZERO:
1064 case ISD::VP_SINT_TO_FP:
1066 case ISD::VP_TRUNCATE:
1068 case ISD::VP_UINT_TO_FP:
1070 SplitVecRes_UnaryOp(
N,
Lo,
Hi);
1076 case ISD::VP_SIGN_EXTEND:
1077 case ISD::VP_ZERO_EXTEND:
1078 SplitVecRes_ExtendOp(
N,
Lo,
Hi);
1098 case ISD::OR:
case ISD::VP_OR:
1118 case ISD::VP_FCOPYSIGN:
1119 SplitVecRes_BinOp(
N,
Lo,
Hi);
1126 SplitVecRes_TernaryOp(
N,
Lo,
Hi);
1129#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1130 case ISD::STRICT_##DAGN:
1131#include "llvm/IR/ConstrainedOps.def"
1132 SplitVecRes_StrictFPOp(
N,
Lo,
Hi);
1137 SplitVecRes_FP_TO_XINT_SAT(
N,
Lo,
Hi);
1146 SplitVecRes_OverflowOp(
N, ResNo,
Lo,
Hi);
1156 SplitVecRes_FIX(
N,
Lo,
Hi);
1165void DAGTypeLegalizer::IncrementPointer(
MemSDNode *
N,
EVT MemVT,
1174 DL,
Ptr.getValueType(),
1175 APInt(
Ptr.getValueSizeInBits().getFixedValue(), IncrementSize));
1177 Flags.setNoUnsignedWrap(
true);
1179 *ScaledOffset += IncrementSize;
1183 MPI =
N->getPointerInfo().getWithOffset(IncrementSize);
1189std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(
SDValue Mask) {
1190 return SplitMask(Mask,
SDLoc(Mask));
1193std::pair<SDValue, SDValue> DAGTypeLegalizer::SplitMask(
SDValue Mask,
1196 EVT MaskVT =
Mask.getValueType();
1198 GetSplitVector(Mask, MaskLo, MaskHi);
1201 return std::make_pair(MaskLo, MaskHi);
1206 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1208 GetSplitVector(
N->getOperand(1), RHSLo, RHSHi);
1212 unsigned Opcode =
N->getOpcode();
1213 if (
N->getNumOperands() == 2) {
1219 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1220 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1223 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(2));
1226 std::tie(EVLLo, EVLHi) =
1227 DAG.
SplitEVL(
N->getOperand(3),
N->getValueType(0), dl);
1230 {LHSLo, RHSLo, MaskLo, EVLLo}, Flags);
1232 {LHSHi, RHSHi, MaskHi, EVLHi}, Flags);
1238 GetSplitVector(
N->getOperand(0), Op0Lo, Op0Hi);
1240 GetSplitVector(
N->getOperand(1), Op1Lo, Op1Hi);
1242 GetSplitVector(
N->getOperand(2), Op2Lo, Op2Hi);
1246 unsigned Opcode =
N->getOpcode();
1247 if (
N->getNumOperands() == 3) {
1253 assert(
N->getNumOperands() == 5 &&
"Unexpected number of operands!");
1254 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1257 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(3));
1260 std::tie(EVLLo, EVLHi) =
1261 DAG.
SplitEVL(
N->getOperand(4),
N->getValueType(0), dl);
1264 {Op0Lo, Op1Lo, Op2Lo, MaskLo, EVLLo}, Flags);
1266 {Op0Hi, Op1Hi, Op2Hi, MaskHi, EVLHi}, Flags);
1271 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1273 GetSplitVector(
N->getOperand(1), RHSLo, RHSHi);
1277 unsigned Opcode =
N->getOpcode();
1296 switch (getTypeAction(InVT)) {
1311 GetExpandedOp(InOp,
Lo,
Hi);
1322 GetSplitVector(InOp,
Lo,
Hi);
1336 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT,
Lo,
Hi);
1359 assert(!(
N->getNumOperands() & 1) &&
"Unsupported CONCAT_VECTORS");
1361 unsigned NumSubvectors =
N->getNumOperands() / 2;
1362 if (NumSubvectors == 1) {
1363 Lo =
N->getOperand(0);
1364 Hi =
N->getOperand(1);
1378void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(
SDNode *
N,
SDValue &
Lo,
1388 uint64_t IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
1400 GetSplitVector(Vec,
Lo,
Hi);
1403 EVT LoVT =
Lo.getValueType();
1412 unsigned IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
1413 if (IdxVal + SubElems <= LoElems) {
1421 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1447 Lo = DAG.
getLoad(
Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1451 auto *
Load = cast<LoadSDNode>(
Lo);
1453 IncrementPointer(Load, LoVT, MPI, StackPtr);
1456 Hi = DAG.
getLoad(
Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1462 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
1470 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1475 EVT RHSVT =
RHS.getValueType();
1477 GetSplitVector(RHS, RHSLo, RHSHi);
1491 SDValue FpValue =
N->getOperand(0);
1493 GetSplitVector(FpValue, ArgLo, ArgHi);
1506 GetSplitVector(
N->getOperand(0), LHSLo, LHSHi);
1510 std::tie(LoVT, HiVT) =
1521 unsigned Opcode =
N->getOpcode();
1528 GetSplitVector(N0, InLo, InHi);
1535 EVT OutLoVT, OutHiVT;
1538 assert((2 * OutNumElements) <= InNumElements &&
1539 "Illegal extend vector in reg split");
1549 for (
unsigned i = 0; i != OutNumElements; ++i)
1550 SplitHi[i] = i + OutNumElements;
1553 Lo = DAG.
getNode(Opcode, dl, OutLoVT, InLo);
1554 Hi = DAG.
getNode(Opcode, dl, OutHiVT, InHi);
1559 unsigned NumOps =
N->getNumOperands();
1573 for (
unsigned i = 1; i < NumOps; ++i) {
1578 EVT InVT =
Op.getValueType();
1583 GetSplitVector(Op, OpLo, OpHi);
1602 Lo.getValue(1),
Hi.getValue(1));
1606 ReplaceValueWith(
SDValue(
N, 1), Chain);
1609SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(
SDNode *
N,
unsigned ResNE) {
1611 EVT VT =
N->getValueType(0);
1622 else if (NE > ResNE)
1630 for (i = 0; i !=
NE; ++i) {
1632 for (
unsigned j = 1, e =
N->getNumOperands(); j != e; ++j) {
1633 SDValue Operand =
N->getOperand(j);
1644 Scalar.getNode()->setFlags(
N->getFlags());
1652 for (; i < ResNE; ++i)
1657 ReplaceValueWith(
SDValue(
N, 1), Chain);
1664void DAGTypeLegalizer::SplitVecRes_OverflowOp(
SDNode *
N,
unsigned ResNo,
1667 EVT ResVT =
N->getValueType(0);
1668 EVT OvVT =
N->getValueType(1);
1669 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1673 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1675 GetSplitVector(
N->getOperand(0), LoLHS, HiLHS);
1676 GetSplitVector(
N->getOperand(1), LoRHS, HiRHS);
1682 unsigned Opcode =
N->getOpcode();
1694 unsigned OtherNo = 1 - ResNo;
1695 EVT OtherVT =
N->getValueType(OtherNo);
1697 SetSplitVector(
SDValue(
N, OtherNo),
1703 ReplaceValueWith(
SDValue(
N, OtherNo), OtherVal);
1707void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(
SDNode *
N,
SDValue &
Lo,
1713 GetSplitVector(Vec,
Lo,
Hi);
1716 unsigned IdxVal = CIdx->getZExtValue();
1717 unsigned LoNumElts =
Lo.getValueType().getVectorMinNumElements();
1718 if (IdxVal < LoNumElts) {
1720 Lo.getValueType(),
Lo, Elt,
Idx);
1730 if (CustomLowerNode(
N,
N->getValueType(0),
true))
1771 Lo = DAG.
getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1774 auto Load = cast<LoadSDNode>(
Lo);
1776 IncrementPointer(Load, LoVT, MPI, StackPtr);
1778 Hi = DAG.
getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1782 if (LoVT !=
Lo.getValueType())
1784 if (HiVT !=
Hi.getValueType())
1792 assert(
N->getValueType(0).isScalableVector() &&
1793 "Only scalable vectors are supported for STEP_VECTOR");
1801 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1816 Lo = DAG.
getNode(
N->getOpcode(), dl, LoVT,
N->getOperand(0));
1836 EVT MemoryVT =
LD->getMemoryVT();
1840 EVT LoMemVT, HiMemVT;
1847 ReplaceValueWith(
SDValue(LD, 1), NewChain);
1852 LD->getPointerInfo(), LoMemVT,
LD->getOriginalAlign(),
1856 IncrementPointer(LD, LoMemVT, MPI,
Ptr);
1859 HiMemVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
1868 ReplaceValueWith(
SDValue(LD, 1), Ch);
1873 assert(
LD->isUnindexed() &&
"Indexed VP load during type legalization!");
1882 assert(
Offset.isUndef() &&
"Unexpected indexed variable-length load offset");
1883 Align Alignment =
LD->getOriginalAlign();
1886 EVT MemoryVT =
LD->getMemoryVT();
1888 EVT LoMemVT, HiMemVT;
1889 bool HiIsEmpty =
false;
1890 std::tie(LoMemVT, HiMemVT) =
1896 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
1899 GetSplitVector(Mask, MaskLo, MaskHi);
1901 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
1906 std::tie(EVLLo, EVLHi) = DAG.
SplitEVL(EVL,
LD->getValueType(0), dl);
1914 MaskLo, EVLLo, LoMemVT, MMO,
LD->isExpandingLoad());
1923 LD->isExpandingLoad());
1929 MPI =
LD->getPointerInfo().getWithOffset(
1934 LD->getAAInfo(),
LD->getRanges());
1937 Offset, MaskHi, EVLHi, HiMemVT, MMO,
1938 LD->isExpandingLoad());
1948 ReplaceValueWith(
SDValue(LD, 1), Ch);
1954 "Indexed VP strided load during type legalization!");
1956 "Unexpected indexed variable-length load offset");
1963 EVT LoMemVT, HiMemVT;
1964 bool HiIsEmpty =
false;
1965 std::tie(LoMemVT, HiMemVT) =
1971 SplitVecRes_SETCC(
Mask.getNode(), LoMask, HiMask);
1974 GetSplitVector(Mask, LoMask, HiMask);
1980 std::tie(LoEVL, HiEVL) =
2018 SLD->
getStride(), HiMask, HiEVL, HiMemVT, MMO,
2029 ReplaceValueWith(
SDValue(SLD, 1), Ch);
2042 assert(
Offset.isUndef() &&
"Unexpected indexed masked load offset");
2051 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
2054 GetSplitVector(Mask, MaskLo, MaskHi);
2056 std::tie(MaskLo, MaskHi) = DAG.
SplitVector(Mask, dl);
2060 EVT LoMemVT, HiMemVT;
2061 bool HiIsEmpty =
false;
2062 std::tie(LoMemVT, HiMemVT) =
2065 SDValue PassThruLo, PassThruHi;
2067 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2069 std::tie(PassThruLo, PassThruHi) = DAG.
SplitVector(PassThru, dl);
2112 ReplaceValueWith(
SDValue(MLD, 1), Ch);
2129 if (
auto *MSC = dyn_cast<MaskedGatherSDNode>(
N)) {
2130 return {MSC->getMask(), MSC->getIndex(), MSC->getScale()};
2132 auto *VPSC = cast<VPGatherSDNode>(
N);
2133 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale()};
2136 EVT MemoryVT =
N->getMemoryVT();
2137 Align Alignment =
N->getOriginalAlign();
2141 if (SplitSETCC && Ops.Mask.getOpcode() ==
ISD::SETCC) {
2142 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
2144 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask, dl);
2147 EVT LoMemVT, HiMemVT;
2152 if (getTypeAction(Ops.Index.getValueType()) ==
2154 GetSplitVector(Ops.Index, IndexLo, IndexHi);
2156 std::tie(IndexLo, IndexHi) = DAG.
SplitVector(Ops.Index, dl);
2162 if (
auto *MGT = dyn_cast<MaskedGatherSDNode>(
N)) {
2163 SDValue PassThru = MGT->getPassThru();
2164 SDValue PassThruLo, PassThruHi;
2167 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2169 std::tie(PassThruLo, PassThruHi) = DAG.
SplitVector(PassThru, dl);
2174 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo,
Ptr, IndexLo, Ops.Scale};
2176 OpsLo, MMO, IndexTy, ExtType);
2178 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi,
Ptr, IndexHi, Ops.Scale};
2180 OpsHi, MMO, IndexTy, ExtType);
2182 auto *VPGT = cast<VPGatherSDNode>(
N);
2184 std::tie(EVLLo, EVLHi) =
2185 DAG.
SplitEVL(VPGT->getVectorLength(), MemoryVT, dl);
2187 SDValue OpsLo[] = {Ch,
Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
2189 MMO, VPGT->getIndexType());
2191 SDValue OpsHi[] = {Ch,
Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
2193 MMO, VPGT->getIndexType());
2203 ReplaceValueWith(
SDValue(
N, 1), Ch);
2207 assert(
N->getValueType(0).isVector() &&
2208 N->getOperand(0).getValueType().isVector() &&
2209 "Operand types must be vectors");
2217 if (getTypeAction(
N->getOperand(0).getValueType()) ==
2219 GetSplitVector(
N->getOperand(0), LL, LH);
2223 if (getTypeAction(
N->getOperand(1).getValueType()) ==
2225 GetSplitVector(
N->getOperand(1), RL, RH);
2230 Lo = DAG.
getNode(
N->getOpcode(),
DL, LoVT, LL, RL,
N->getOperand(2));
2231 Hi = DAG.
getNode(
N->getOpcode(),
DL, HiVT, LH, RH,
N->getOperand(2));
2233 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2234 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
2235 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(3));
2236 std::tie(EVLLo, EVLHi) =
2237 DAG.
SplitEVL(
N->getOperand(4),
N->getValueType(0),
DL);
2238 Lo = DAG.
getNode(
N->getOpcode(),
DL, LoVT, LL, RL,
N->getOperand(2), MaskLo,
2240 Hi = DAG.
getNode(
N->getOpcode(),
DL, HiVT, LH, RH,
N->getOperand(2), MaskHi,
2254 EVT InVT =
N->getOperand(0).getValueType();
2256 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
2261 unsigned Opcode =
N->getOpcode();
2262 if (
N->getNumOperands() <= 2) {
2264 Lo = DAG.
getNode(Opcode, dl, LoVT,
Lo,
N->getOperand(1), Flags);
2265 Hi = DAG.
getNode(Opcode, dl, HiVT,
Hi,
N->getOperand(1), Flags);
2273 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
2274 assert(
N->isVPOpcode() &&
"Expected VP opcode");
2277 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
2280 std::tie(EVLLo, EVLHi) =
2281 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
2290 EVT SrcVT =
N->getOperand(0).getValueType();
2291 EVT DestVT =
N->getValueType(0);
2314 EVT SplitLoVT, SplitHiVT;
2318 LLVM_DEBUG(
dbgs() <<
"Split vector extend via incremental extend:";
2319 N->dump(&DAG);
dbgs() <<
"\n");
2320 if (!
N->isVPOpcode()) {
2323 DAG.
getNode(
N->getOpcode(), dl, NewSrcVT,
N->getOperand(0));
2334 DAG.
getNode(
N->getOpcode(), dl, NewSrcVT,
N->getOperand(0),
2335 N->getOperand(1),
N->getOperand(2));
2340 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
2343 std::tie(EVLLo, EVLHi) =
2344 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
2346 Lo = DAG.
getNode(
N->getOpcode(), dl, LoVT, {Lo, MaskLo, EVLLo});
2347 Hi = DAG.
getNode(
N->getOpcode(), dl, HiVT, {Hi, MaskHi, EVLHi});
2352 SplitVecRes_UnaryOp(
N,
Lo,
Hi);
2360 GetSplitVector(
N->getOperand(0), Inputs[0], Inputs[1]);
2361 GetSplitVector(
N->getOperand(1), Inputs[2], Inputs[3]);
2367 return N.getResNo() == 0 &&
2371 auto &&BuildVector = [NewElts, &DAG = DAG, NewVT, &
DL](
SDValue &Input1,
2376 "Expected build vector node.");
2379 for (
unsigned I = 0;
I < NewElts; ++
I) {
2384 Ops[
I] = Input2.getOperand(
Idx - NewElts);
2386 Ops[
I] = Input1.getOperand(
Idx);
2388 if (Ops[
I].getValueType().bitsGT(EltVT))
2391 return DAG.getBuildVector(NewVT,
DL, Ops);
2399 auto &&TryPeekThroughShufflesInputs = [&Inputs, &NewVT,
this, NewElts,
2403 for (
unsigned Idx = 0;
Idx < std::size(Inputs); ++
Idx) {
2414 for (
auto &
P : ShufflesIdxs) {
2415 if (
P.second.size() < 2)
2419 for (
int &
Idx : Mask) {
2422 unsigned SrcRegIdx =
Idx / NewElts;
2423 if (Inputs[SrcRegIdx].
isUndef()) {
2428 dyn_cast<ShuffleVectorSDNode>(Inputs[SrcRegIdx].getNode());
2431 int MaskElt =
Shuffle->getMaskElt(
Idx % NewElts);
2436 Idx = MaskElt % NewElts +
2437 P.second[
Shuffle->getOperand(MaskElt / NewElts) ==
P.first.first
2443 Inputs[
P.second[0]] =
P.first.first;
2444 Inputs[
P.second[1]] =
P.first.second;
2447 ShufflesIdxs[std::make_pair(
P.first.second,
P.first.first)].clear();
2451 for (
int &
Idx : Mask) {
2454 unsigned SrcRegIdx =
Idx / NewElts;
2455 if (Inputs[SrcRegIdx].
isUndef()) {
2460 getTypeAction(Inputs[SrcRegIdx].getValueType());
2462 Inputs[SrcRegIdx].getNumOperands() == 2 &&
2463 !Inputs[SrcRegIdx].getOperand(1).
isUndef() &&
2466 UsedSubVector.set(2 * SrcRegIdx + (
Idx % NewElts) / (NewElts / 2));
2468 if (UsedSubVector.count() > 1) {
2470 for (
unsigned I = 0;
I < std::size(Inputs); ++
I) {
2471 if (UsedSubVector.test(2 *
I) == UsedSubVector.test(2 *
I + 1))
2473 if (Pairs.
empty() || Pairs.
back().size() == 2)
2475 if (UsedSubVector.test(2 *
I)) {
2476 Pairs.
back().emplace_back(
I, 0);
2478 assert(UsedSubVector.test(2 *
I + 1) &&
2479 "Expected to be used one of the subvectors.");
2480 Pairs.
back().emplace_back(
I, 1);
2483 if (!Pairs.
empty() && Pairs.
front().size() > 1) {
2485 for (
int &
Idx : Mask) {
2488 unsigned SrcRegIdx =
Idx / NewElts;
2490 Pairs, [SrcRegIdx](
ArrayRef<std::pair<unsigned, int>> Idxs) {
2491 return Idxs.front().first == SrcRegIdx ||
2492 Idxs.back().first == SrcRegIdx;
2494 if (It == Pairs.
end())
2496 Idx = It->front().first * NewElts + (
Idx % NewElts) % (NewElts / 2) +
2497 (SrcRegIdx == It->front().first ? 0 : (NewElts / 2));
2500 for (
ArrayRef<std::pair<unsigned, int>> Idxs : Pairs) {
2501 Inputs[Idxs.front().first] = DAG.
getNode(
2503 Inputs[Idxs.front().first].getValueType(),
2504 Inputs[Idxs.front().first].
getOperand(Idxs.front().second),
2505 Inputs[Idxs.back().first].
getOperand(Idxs.back().second));
2514 for (
unsigned I = 0;
I < std::size(Inputs); ++
I) {
2515 auto *
Shuffle = dyn_cast<ShuffleVectorSDNode>(Inputs[
I].getNode());
2518 if (
Shuffle->getOperand(0).getValueType() != NewVT)
2525 !
Shuffle->getOperand(1).isUndef()) {
2527 for (
int &
Idx : Mask) {
2530 unsigned SrcRegIdx =
Idx / NewElts;
2533 int MaskElt =
Shuffle->getMaskElt(
Idx % NewElts);
2538 int OpIdx = MaskElt / NewElts;
2551 for (
int OpIdx = 0; OpIdx < 2; ++OpIdx) {
2552 if (
Shuffle->getOperand(OpIdx).isUndef())
2554 auto *It =
find(Inputs,
Shuffle->getOperand(OpIdx));
2555 if (It == std::end(Inputs))
2557 int FoundOp = std::distance(std::begin(Inputs), It);
2560 for (
int &
Idx : Mask) {
2563 unsigned SrcRegIdx =
Idx / NewElts;
2566 int MaskElt =
Shuffle->getMaskElt(
Idx % NewElts);
2571 int MaskIdx = MaskElt / NewElts;
2572 if (OpIdx == MaskIdx)
2573 Idx = MaskElt % NewElts + FoundOp * NewElts;
2576 Op = (OpIdx + 1) % 2;
2582 Inputs[
I] =
Shuffle->getOperand(Op);
2584 for (
int &
Idx : Mask) {
2587 unsigned SrcRegIdx =
Idx / NewElts;
2590 int MaskElt =
Shuffle->getMaskElt(
Idx % NewElts);
2591 int OpIdx = MaskElt / NewElts;
2594 Idx = MaskElt % NewElts + SrcRegIdx * NewElts;
2600 TryPeekThroughShufflesInputs(OrigMask);
2602 auto &&MakeUniqueInputs = [&Inputs, &
IsConstant,
2606 for (
const auto &
I : Inputs) {
2608 UniqueConstantInputs.
insert(
I);
2609 else if (!
I.isUndef())
2614 if (UniqueInputs.
size() != std::size(Inputs)) {
2615 auto &&UniqueVec = UniqueInputs.
takeVector();
2616 auto &&UniqueConstantVec = UniqueConstantInputs.
takeVector();
2617 unsigned ConstNum = UniqueConstantVec.size();
2618 for (
int &
Idx : Mask) {
2621 unsigned SrcRegIdx =
Idx / NewElts;
2622 if (Inputs[SrcRegIdx].
isUndef()) {
2626 const auto It =
find(UniqueConstantVec, Inputs[SrcRegIdx]);
2627 if (It != UniqueConstantVec.end()) {
2629 NewElts * std::distance(UniqueConstantVec.begin(), It);
2630 assert(
Idx >= 0 &&
"Expected defined mask idx.");
2633 const auto RegIt =
find(UniqueVec, Inputs[SrcRegIdx]);
2634 assert(RegIt != UniqueVec.end() &&
"Cannot find non-const value.");
2636 NewElts * (std::distance(UniqueVec.begin(), RegIt) + ConstNum);
2637 assert(
Idx >= 0 &&
"Expected defined mask idx.");
2639 copy(UniqueConstantVec, std::begin(Inputs));
2640 copy(UniqueVec, std::next(std::begin(Inputs), ConstNum));
2643 MakeUniqueInputs(OrigMask);
2645 copy(Inputs, std::begin(OrigInputs));
2651 unsigned FirstMaskIdx =
High * NewElts;
2654 assert(!Output &&
"Expected default initialized initial value.");
2655 TryPeekThroughShufflesInputs(Mask);
2656 MakeUniqueInputs(Mask);
2658 copy(Inputs, std::begin(TmpInputs));
2661 bool SecondIteration =
false;
2662 auto &&AccumulateResults = [&UsedIdx, &SecondIteration](
unsigned Idx) {
2667 if (UsedIdx >= 0 &&
static_cast<unsigned>(UsedIdx) ==
Idx)
2668 SecondIteration =
true;
2669 return SecondIteration;
2672 Mask, std::size(Inputs), std::size(Inputs),
2674 [&Output, &DAG = DAG, NewVT]() { Output = DAG.getUNDEF(NewVT); },
2675 [&Output, &DAG = DAG, NewVT, &
DL, &Inputs,
2678 Output = BuildVector(Inputs[
Idx], Inputs[
Idx], Mask);
2680 Output = DAG.getVectorShuffle(NewVT,
DL, Inputs[
Idx],
2681 DAG.getUNDEF(NewVT), Mask);
2682 Inputs[
Idx] = Output;
2684 [&AccumulateResults, &Output, &DAG = DAG, NewVT, &
DL, &Inputs,
2687 if (AccumulateResults(Idx1)) {
2690 Output = BuildVector(Inputs[Idx1], Inputs[Idx2], Mask);
2692 Output = DAG.getVectorShuffle(NewVT,
DL, Inputs[Idx1],
2693 Inputs[Idx2], Mask);
2697 Output = BuildVector(TmpInputs[Idx1], TmpInputs[Idx2], Mask);
2699 Output = DAG.getVectorShuffle(NewVT,
DL, TmpInputs[Idx1],
2700 TmpInputs[Idx2], Mask);
2702 Inputs[Idx1] = Output;
2704 copy(OrigInputs, std::begin(Inputs));
2709 EVT OVT =
N->getValueType(0);
2716 const Align Alignment =
2717 DAG.getDataLayout().getABITypeAlign(NVT.
getTypeForEVT(*DAG.getContext()));
2719 Lo = DAG.getVAArg(NVT, dl, Chain,
Ptr, SV, Alignment.
value());
2720 Hi = DAG.getVAArg(NVT, dl,
Lo.getValue(1),
Ptr, SV, Alignment.
value());
2721 Chain =
Hi.getValue(1);
2725 ReplaceValueWith(
SDValue(
N, 1), Chain);
2730 EVT DstVTLo, DstVTHi;
2731 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(
N->getValueType(0));
2735 EVT SrcVT =
N->getOperand(0).getValueType();
2737 GetSplitVector(
N->getOperand(0), SrcLo, SrcHi);
2739 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(
N, 0);
2741 Lo = DAG.getNode(
N->getOpcode(), dl, DstVTLo, SrcLo,
N->getOperand(1));
2742 Hi = DAG.getNode(
N->getOpcode(), dl, DstVTHi, SrcHi,
N->getOperand(1));
2748 GetSplitVector(
N->getOperand(0), InLo, InHi);
2757 EVT VT =
N->getValueType(0);
2761 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
2765 DAG.getVectorIdxConstant(0,
DL));
2779bool DAGTypeLegalizer::SplitVectorOperand(
SDNode *
N,
unsigned OpNo) {
2784 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
2787 switch (
N->getOpcode()) {
2790 dbgs() <<
"SplitVectorOperand Op #" << OpNo <<
": ";
2798 case ISD::SETCC: Res = SplitVecOp_VSETCC(
N);
break;
2804 case ISD::VP_TRUNCATE:
2806 Res = SplitVecOp_TruncateHelper(
N);
2809 case ISD::VP_FP_ROUND:
2813 Res = SplitVecOp_STORE(cast<StoreSDNode>(
N), OpNo);
2816 Res = SplitVecOp_VP_STORE(cast<VPStoreSDNode>(
N), OpNo);
2818 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2819 Res = SplitVecOp_VP_STRIDED_STORE(cast<VPStridedStoreSDNode>(
N), OpNo);
2822 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(
N), OpNo);
2825 case ISD::VP_SCATTER:
2826 Res = SplitVecOp_Scatter(cast<MemSDNode>(
N), OpNo);
2829 case ISD::VP_GATHER:
2830 Res = SplitVecOp_Gather(cast<MemSDNode>(
N), OpNo);
2833 Res = SplitVecOp_VSELECT(
N, OpNo);
2839 case ISD::VP_SINT_TO_FP:
2840 case ISD::VP_UINT_TO_FP:
2841 if (
N->getValueType(0).bitsLT(
2842 N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2843 Res = SplitVecOp_TruncateHelper(
N);
2845 Res = SplitVecOp_UnaryOp(
N);
2849 Res = SplitVecOp_FP_TO_XINT_SAT(
N);
2853 case ISD::VP_FP_TO_SINT:
2854 case ISD::VP_FP_TO_UINT:
2863 Res = SplitVecOp_UnaryOp(
N);
2869 Res = SplitVecOp_ExtVecInRegOp(
N);
2885 Res = SplitVecOp_VECREDUCE(
N, OpNo);
2889 Res = SplitVecOp_VECREDUCE_SEQ(
N);
2891 case ISD::VP_REDUCE_FADD:
2892 case ISD::VP_REDUCE_SEQ_FADD:
2893 case ISD::VP_REDUCE_FMUL:
2894 case ISD::VP_REDUCE_SEQ_FMUL:
2895 case ISD::VP_REDUCE_ADD:
2896 case ISD::VP_REDUCE_MUL:
2897 case ISD::VP_REDUCE_AND:
2898 case ISD::VP_REDUCE_OR:
2899 case ISD::VP_REDUCE_XOR:
2900 case ISD::VP_REDUCE_SMAX:
2901 case ISD::VP_REDUCE_SMIN:
2902 case ISD::VP_REDUCE_UMAX:
2903 case ISD::VP_REDUCE_UMIN:
2904 case ISD::VP_REDUCE_FMAX:
2905 case ISD::VP_REDUCE_FMIN:
2906 Res = SplitVecOp_VP_REDUCE(
N, OpNo);
2911 if (!Res.getNode())
return false;
2915 if (Res.getNode() ==
N)
2918 if (
N->isStrictFPOpcode())
2919 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 2 &&
2920 "Invalid operand expansion");
2922 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 1 &&
2923 "Invalid operand expansion");
2925 ReplaceValueWith(
SDValue(
N, 0), Res);
2929SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(
SDNode *
N,
unsigned OpNo) {
2932 assert(OpNo == 0 &&
"Illegal operand must be mask");
2939 assert(
Mask.getValueType().isVector() &&
"VSELECT without a vector mask?");
2942 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
2943 assert(
Lo.getValueType() ==
Hi.getValueType() &&
2944 "Lo and Hi have differing types");
2947 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2948 assert(LoOpVT == HiOpVT &&
"Asymmetric vector split?");
2950 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2951 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0,
DL);
2952 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1,
DL);
2953 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask,
DL);
2963SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(
SDNode *
N,
unsigned OpNo) {
2964 EVT ResVT =
N->getValueType(0);
2968 SDValue VecOp =
N->getOperand(OpNo);
2970 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
2971 GetSplitVector(VecOp,
Lo,
Hi);
2973 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2979 return DAG.getNode(
N->getOpcode(), dl, ResVT, Partial,
N->getFlags());
2983 EVT ResVT =
N->getValueType(0);
2992 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
2993 GetSplitVector(VecOp,
Lo,
Hi);
2995 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
3001 return DAG.getNode(
N->getOpcode(), dl, ResVT, Partial,
Hi, Flags);
3004SDValue DAGTypeLegalizer::SplitVecOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
3005 assert(
N->isVPOpcode() &&
"Expected VP opcode");
3006 assert(OpNo == 1 &&
"Can only split reduce vector operand");
3008 unsigned Opc =
N->getOpcode();
3009 EVT ResVT =
N->getValueType(0);
3013 SDValue VecOp =
N->getOperand(OpNo);
3015 assert(VecVT.
isVector() &&
"Can only split reduce vector operand");
3016 GetSplitVector(VecOp,
Lo,
Hi);
3019 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(2));
3022 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(
N->getOperand(3), VecVT, dl);
3027 DAG.
getNode(Opc, dl, ResVT, {
N->getOperand(0),
Lo, MaskLo, EVLLo},
Flags);
3028 return DAG.getNode(Opc, dl, ResVT, {ResLo,
Hi, MaskHi, EVLHi},
Flags);
3033 EVT ResVT =
N->getValueType(0);
3036 GetSplitVector(
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0),
Lo,
Hi);
3037 EVT InVT =
Lo.getValueType();
3042 if (
N->isStrictFPOpcode()) {
3043 Lo = DAG.getNode(
N->getOpcode(), dl, { OutVT, MVT::Other },
3044 { N->getOperand(0), Lo });
3045 Hi = DAG.getNode(
N->getOpcode(), dl, { OutVT, MVT::Other },
3046 { N->getOperand(0), Hi });
3055 ReplaceValueWith(
SDValue(
N, 1), Ch);
3056 }
else if (
N->getNumOperands() == 3) {
3057 assert(
N->isVPOpcode() &&
"Expected VP opcode");
3058 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3059 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
3060 std::tie(EVLLo, EVLHi) =
3061 DAG.SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
3062 Lo = DAG.getNode(
N->getOpcode(), dl, OutVT,
Lo, MaskLo, EVLLo);
3063 Hi = DAG.getNode(
N->getOpcode(), dl, OutVT,
Hi, MaskHi, EVLHi);
3065 Lo = DAG.getNode(
N->getOpcode(), dl, OutVT,
Lo);
3066 Hi = DAG.getNode(
N->getOpcode(), dl, OutVT,
Hi);
3077 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3078 Lo = BitConvertToInteger(
Lo);
3079 Hi = BitConvertToInteger(
Hi);
3081 if (DAG.getDataLayout().isBigEndian())
3085 JoinIntegers(
Lo,
Hi));
3090 assert(OpNo == 1 &&
"Invalid OpNo; can only split SubVec.");
3092 EVT ResVT =
N->getValueType(0);
3100 GetSplitVector(SubVec,
Lo,
Hi);
3102 uint64_t IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
3103 uint64_t LoElts =
Lo.getValueType().getVectorMinNumElements();
3109 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
3111 return SecondInsertion;
3114SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
3116 EVT SubVT =
N->getValueType(0);
3121 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3123 uint64_t LoEltsMin =
Lo.getValueType().getVectorMinNumElements();
3124 uint64_t IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
3126 if (IdxVal < LoEltsMin) {
3128 "Extracted subvector crosses vector split!");
3131 N->getOperand(0).getValueType().isScalableVector())
3133 DAG.getVectorIdxConstant(IdxVal - LoEltsMin, dl));
3138 "Extracting scalable subvector from fixed-width unsupported");
3146 "subvector from a scalable predicate vector");
3152 Align SmallestAlign = DAG.getReducedAlign(VecVT,
false);
3154 DAG.CreateStackTemporary(VecVT.
getStoreSize(), SmallestAlign);
3155 auto &MF = DAG.getMachineFunction();
3159 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3166 SubVT, dl, Store, StackPtr,
3170SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
3179 GetSplitVector(Vec,
Lo,
Hi);
3181 uint64_t LoElts =
Lo.getValueType().getVectorMinNumElements();
3183 if (IdxVal < LoElts)
3187 DAG.getConstant(IdxVal - LoElts,
SDLoc(
N),
3188 Idx.getValueType())), 0);
3192 if (CustomLowerNode(
N,
N->getValueType(0),
true))
3208 Align SmallestAlign = DAG.getReducedAlign(VecVT,
false);
3210 DAG.CreateStackTemporary(VecVT.
getStoreSize(), SmallestAlign);
3211 auto &MF = DAG.getMachineFunction();
3214 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
3222 if (
N->getValueType(0).bitsLT(EltVT)) {
3223 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
3225 return DAG.getZExtOrTrunc(Load, dl,
N->getValueType(0));
3228 return DAG.getExtLoad(
3239 SplitVecRes_ExtVecInRegOp(
N,
Lo,
Hi);
3247 SplitVecRes_Gather(
N,
Lo,
Hi);
3250 ReplaceValueWith(
SDValue(
N, 0), Res);
3255 assert(
N->isUnindexed() &&
"Indexed vp_store of vector?");
3259 assert(
Offset.isUndef() &&
"Unexpected VP store offset");
3261 SDValue EVL =
N->getVectorLength();
3263 Align Alignment =
N->getOriginalAlign();
3269 GetSplitVector(
Data, DataLo, DataHi);
3271 std::tie(DataLo, DataHi) = DAG.SplitVector(
Data,
DL);
3276 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
3279 GetSplitVector(Mask, MaskLo, MaskHi);
3281 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask,
DL);
3284 EVT MemoryVT =
N->getMemoryVT();
3285 EVT LoMemVT, HiMemVT;
3286 bool HiIsEmpty =
false;
3287 std::tie(LoMemVT, HiMemVT) =
3288 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.
getValueType(), &HiIsEmpty);
3292 std::tie(EVLLo, EVLHi) = DAG.SplitEVL(EVL,
Data.getValueType(),
DL);
3299 Lo = DAG.getStoreVP(Ch,
DL, DataLo,
Ptr,
Offset, MaskLo, EVLLo, LoMemVT, MMO,
3300 N->getAddressingMode(),
N->isTruncatingStore(),
3301 N->isCompressingStore());
3308 N->isCompressingStore());
3316 MPI =
N->getPointerInfo().getWithOffset(
3319 MMO = DAG.getMachineFunction().getMachineMemOperand(
3321 N->getAAInfo(),
N->getRanges());
3323 Hi = DAG.getStoreVP(Ch,
DL, DataHi,
Ptr,
Offset, MaskHi, EVLHi, HiMemVT, MMO,
3324 N->getAddressingMode(),
N->isTruncatingStore(),
3325 N->isCompressingStore());
3334 assert(
N->isUnindexed() &&
"Indexed vp_strided_store of a vector?");
3335 assert(
N->getOffset().isUndef() &&
"Unexpected VP strided store offset");
3342 GetSplitVector(
Data, LoData, HiData);
3344 std::tie(LoData, HiData) = DAG.SplitVector(
Data,
DL);
3346 EVT LoMemVT, HiMemVT;
3347 bool HiIsEmpty =
false;
3348 std::tie(LoMemVT, HiMemVT) = DAG.GetDependentSplitDestVTs(
3354 SplitVecRes_SETCC(
Mask.getNode(), LoMask, HiMask);
3355 else if (getTypeAction(
Mask.getValueType()) ==
3357 GetSplitVector(Mask, LoMask, HiMask);
3359 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask,
DL);
3362 std::tie(LoEVL, HiEVL) =
3363 DAG.SplitEVL(
N->getVectorLength(),
Data.getValueType(),
DL);
3367 N->getChain(),
DL, LoData,
N->getBasePtr(),
N->getOffset(),
3368 N->getStride(), LoMask, LoEVL, LoMemVT,
N->getMemOperand(),
3369 N->getAddressingMode(),
N->isTruncatingStore(),
N->isCompressingStore());
3380 EVT PtrVT =
N->getBasePtr().getValueType();
3383 DAG.getSExtOrTrunc(
N->getStride(),
DL, PtrVT));
3386 Align Alignment =
N->getOriginalAlign();
3394 N->getAAInfo(),
N->getRanges());
3397 N->getChain(),
DL, HiData,
Ptr,
N->getOffset(),
N->getStride(), HiMask,
3398 HiEVL, HiMemVT, MMO,
N->getAddressingMode(),
N->isTruncatingStore(),
3399 N->isCompressingStore());
3408 assert(
N->isUnindexed() &&
"Indexed masked store of vector?");
3412 assert(
Offset.isUndef() &&
"Unexpected indexed masked store offset");
3415 Align Alignment =
N->getOriginalAlign();
3421 GetSplitVector(
Data, DataLo, DataHi);
3423 std::tie(DataLo, DataHi) = DAG.SplitVector(
Data,
DL);
3428 SplitVecRes_SETCC(
Mask.getNode(), MaskLo, MaskHi);
3431 GetSplitVector(Mask, MaskLo, MaskHi);
3433 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask,
DL);
3436 EVT MemoryVT =
N->getMemoryVT();
3437 EVT LoMemVT, HiMemVT;
3438 bool HiIsEmpty =
false;
3439 std::tie(LoMemVT, HiMemVT) =
3440 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.
getValueType(), &HiIsEmpty);
3447 Lo = DAG.getMaskedStore(Ch,
DL, DataLo,
Ptr,
Offset, MaskLo, LoMemVT, MMO,
3448 N->getAddressingMode(),
N->isTruncatingStore(),
3449 N->isCompressingStore());
3458 N->isCompressingStore());
3466 MPI =
N->getPointerInfo().getWithOffset(
3469 MMO = DAG.getMachineFunction().getMachineMemOperand(
3471 N->getAAInfo(),
N->getRanges());
3473 Hi = DAG.getMaskedStore(Ch,
DL, DataHi,
Ptr,
Offset, MaskHi, HiMemVT, MMO,
3474 N->getAddressingMode(),
N->isTruncatingStore(),
3475 N->isCompressingStore());
3488 EVT MemoryVT =
N->getMemoryVT();
3489 Align Alignment =
N->getOriginalAlign();
3497 if (
auto *MSC = dyn_cast<MaskedScatterSDNode>(
N)) {
3498 return {MSC->getMask(), MSC->getIndex(), MSC->getScale(),
3501 auto *VPSC = cast<VPScatterSDNode>(
N);
3502 return {VPSC->getMask(), VPSC->getIndex(), VPSC->getScale(),
3507 EVT LoMemVT, HiMemVT;
3508 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3513 GetSplitVector(Ops.Data, DataLo, DataHi);
3515 std::tie(DataLo, DataHi) = DAG.SplitVector(Ops.Data,
DL);
3519 if (OpNo == 1 && Ops.Mask.getOpcode() ==
ISD::SETCC) {
3520 SplitVecRes_SETCC(Ops.Mask.getNode(), MaskLo, MaskHi);
3522 std::tie(MaskLo, MaskHi) = SplitMask(Ops.Mask,
DL);
3526 if (getTypeAction(Ops.Index.getValueType()) ==
3528 GetSplitVector(Ops.Index, IndexLo, IndexHi);
3530 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Ops.Index,
DL);
3537 if (
auto *MSC = dyn_cast<MaskedScatterSDNode>(
N)) {
3538 SDValue OpsLo[] = {Ch, DataLo, MaskLo,
Ptr, IndexLo, Ops.Scale};
3540 DAG.getMaskedScatter(DAG.getVTList(
MVT::Other), LoMemVT,
DL, OpsLo, MMO,
3541 MSC->getIndexType(), MSC->isTruncatingStore());
3546 SDValue OpsHi[] = {
Lo, DataHi, MaskHi,
Ptr, IndexHi, Ops.Scale};
3547 return DAG.getMaskedScatter(DAG.getVTList(
MVT::Other), HiMemVT,
DL, OpsHi,
3548 MMO, MSC->getIndexType(),
3549 MSC->isTruncatingStore());
3551 auto *VPSC = cast<VPScatterSDNode>(
N);
3553 std::tie(EVLLo, EVLHi) =
3554 DAG.SplitEVL(VPSC->getVectorLength(), Ops.Data.getValueType(),
DL);
3556 SDValue OpsLo[] = {Ch, DataLo,
Ptr, IndexLo, Ops.Scale, MaskLo, EVLLo};
3557 Lo = DAG.getScatterVP(DAG.getVTList(
MVT::Other), LoMemVT,
DL, OpsLo, MMO,
3558 VPSC->getIndexType());
3563 SDValue OpsHi[] = {
Lo, DataHi,
Ptr, IndexHi, Ops.Scale, MaskHi, EVLHi};
3564 return DAG.getScatterVP(DAG.getVTList(
MVT::Other), HiMemVT,
DL, OpsHi, MMO,
3565 VPSC->getIndexType());
3569 assert(
N->isUnindexed() &&
"Indexed store of vector?");
3570 assert(OpNo == 1 &&
"Can only split the stored value");
3573 bool isTruncating =
N->isTruncatingStore();
3576 EVT MemoryVT =
N->getMemoryVT();
3577 Align Alignment =
N->getOriginalAlign();
3581 GetSplitVector(
N->getOperand(1),
Lo,
Hi);
3583 EVT LoMemVT, HiMemVT;
3584 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
3591 Lo = DAG.getTruncStore(Ch,
DL,
Lo,
Ptr,
N->getPointerInfo(), LoMemVT,
3592 Alignment, MMOFlags, AAInfo);
3594 Lo = DAG.getStore(Ch,
DL,
Lo,
Ptr,
N->getPointerInfo(), Alignment, MMOFlags,
3598 IncrementPointer(
N, LoMemVT, MPI,
Ptr);
3601 Hi = DAG.getTruncStore(Ch,
DL,
Hi,
Ptr, MPI,
3602 HiMemVT, Alignment, MMOFlags, AAInfo);
3604 Hi = DAG.getStore(Ch,
DL,
Hi,
Ptr, MPI, Alignment, MMOFlags, AAInfo);
3618 EVT EltVT =
N->getValueType(0).getVectorElementType();
3619 for (
const SDValue &Op :
N->op_values()) {
3620 for (
unsigned i = 0, e =
Op.getValueType().getVectorNumElements();
3623 DAG.getVectorIdxConstant(i,
DL)));
3627 return DAG.getBuildVector(
N->getValueType(0),
DL, Elts);
3648 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
3649 SDValue InVec =
N->getOperand(OpNo);
3651 EVT OutVT =
N->getValueType(0);
3659 EVT LoOutVT, HiOutVT;
3660 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
3661 assert(LoOutVT == HiOutVT &&
"Unequal split?");
3666 if (isTypeLegal(LoOutVT) ||
3667 InElementSize <= OutElementSize * 2)
3668 return SplitVecOp_UnaryOp(
N);
3677 return SplitVecOp_UnaryOp(
N);
3681 GetSplitVector(InVec, InLoVec, InHiVec);
3687 EVT HalfElementVT = IsFloat ?
3689 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
3696 if (
N->isStrictFPOpcode()) {
3697 HalfLo = DAG.
getNode(
N->getOpcode(),
DL, {HalfVT, MVT::Other},
3698 {N->getOperand(0), InLoVec});
3699 HalfHi = DAG.
getNode(
N->getOpcode(),
DL, {HalfVT, MVT::Other},
3700 {N->getOperand(0), InHiVec});
3706 HalfLo = DAG.
getNode(
N->getOpcode(),
DL, HalfVT, InLoVec);
3707 HalfHi = DAG.
getNode(
N->getOpcode(),
DL, HalfVT, InHiVec);
3719 if (
N->isStrictFPOpcode()) {
3723 DAG.getTargetConstant(0,
DL, TLI.
getPointerTy(DAG.getDataLayout()))});
3731 DAG.getTargetConstant(
3737 assert(
N->getValueType(0).isVector() &&
3738 N->getOperand(0).getValueType().isVector() &&
3739 "Operand types must be vectors");
3741 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
3743 GetSplitVector(
N->getOperand(0), Lo0, Hi0);
3744 GetSplitVector(
N->getOperand(1), Lo1, Hi1);
3755 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
3756 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3757 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(3));
3758 std::tie(EVLLo, EVLHi) =
3759 DAG.SplitEVL(
N->getOperand(4),
N->getValueType(0),
DL);
3760 LoRes = DAG.
getNode(ISD::VP_SETCC,
DL, PartResVT, Lo0, Lo1,
3761 N->getOperand(2), MaskLo, EVLLo);
3762 HiRes = DAG.
getNode(ISD::VP_SETCC,
DL, PartResVT, Hi0, Hi1,
3763 N->getOperand(2), MaskHi, EVLHi);
3767 EVT OpVT =
N->getOperand(0).getValueType();
3770 return DAG.getNode(ExtendCode,
DL,
N->getValueType(0), Con);
3776 EVT ResVT =
N->getValueType(0);
3779 GetSplitVector(
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0),
Lo,
Hi);
3780 EVT InVT =
Lo.getValueType();
3785 if (
N->isStrictFPOpcode()) {
3786 Lo = DAG.getNode(
N->getOpcode(),
DL, { OutVT, MVT::Other },
3787 { N->getOperand(0), Lo, N->getOperand(2) });
3788 Hi = DAG.getNode(
N->getOpcode(),
DL, { OutVT, MVT::Other },
3789 { N->getOperand(0), Hi, N->getOperand(2) });
3793 Lo.getValue(1),
Hi.getValue(1));
3794 ReplaceValueWith(
SDValue(
N, 1), NewChain);
3795 }
else if (
N->getOpcode() == ISD::VP_FP_ROUND) {
3796 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
3797 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
3798 std::tie(EVLLo, EVLHi) =
3799 DAG.SplitEVL(
N->getOperand(2),
N->getValueType(0),
DL);
3800 Lo = DAG.getNode(ISD::VP_FP_ROUND,
DL, OutVT,
Lo, MaskLo, EVLLo);
3801 Hi = DAG.getNode(ISD::VP_FP_ROUND,
DL, OutVT,
Hi, MaskHi, EVLHi);
3816 EVT LHSLoVT, LHSHiVT;
3817 std::tie(LHSLoVT, LHSHiVT) = DAG.GetSplitDestVTs(
N->getValueType(0));
3819 if (!isTypeLegal(LHSLoVT) || !isTypeLegal(LHSHiVT))
3820 return DAG.UnrollVectorOp(
N,
N->getValueType(0).getVectorNumElements());
3823 std::tie(LHSLo, LHSHi) =
3824 DAG.SplitVector(
N->getOperand(0),
DL, LHSLoVT, LHSHiVT);
3827 std::tie(RHSLo, RHSHi) = DAG.SplitVector(
N->getOperand(1),
DL);
3836 EVT ResVT =
N->getValueType(0);
3839 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
3840 EVT InVT =
Lo.getValueType();
3846 Lo = DAG.getNode(
N->getOpcode(), dl, NewResVT,
Lo,
N->getOperand(1));
3847 Hi = DAG.getNode(
N->getOpcode(), dl, NewResVT,
Hi,
N->getOperand(1));
3856void DAGTypeLegalizer::WidenVectorResult(
SDNode *
N,
unsigned ResNo) {
3857 LLVM_DEBUG(
dbgs() <<
"Widen node result " << ResNo <<
": ";
N->dump(&DAG);
3861 if (CustomWidenLowerNode(
N,
N->getValueType(ResNo)))
3866 auto unrollExpandedOp = [&]() {
3871 EVT VT =
N->getValueType(0);
3881 switch (
N->getOpcode()) {
3884 dbgs() <<
"WidenVectorResult #" << ResNo <<
": ";
3895 Res = WidenVecRes_INSERT_SUBVECTOR(
N);
3899 case ISD::LOAD: Res = WidenVecRes_LOAD(
N);
break;
3903 Res = WidenVecRes_ScalarOp(
N);
3908 case ISD::VP_SELECT:
3910 Res = WidenVecRes_Select(
N);
3914 case ISD::SETCC: Res = WidenVecRes_SETCC(
N);
break;
3915 case ISD::UNDEF: Res = WidenVecRes_UNDEF(
N);
break;
3917 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(
N));
3920 Res = WidenVecRes_VP_LOAD(cast<VPLoadSDNode>(
N));
3922 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
3923 Res = WidenVecRes_VP_STRIDED_LOAD(cast<VPStridedLoadSDNode>(
N));
3926 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(
N));
3929 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(
N));
3931 case ISD::VP_GATHER:
3932 Res = WidenVecRes_VP_GATHER(cast<VPGatherSDNode>(
N));
3935 Res = WidenVecRes_VECTOR_REVERSE(
N);
3943 case ISD::OR:
case ISD::VP_OR:
3983 case ISD::VP_FCOPYSIGN:
3984 Res = WidenVecRes_Binary(
N);
3989 if (unrollExpandedOp())
4004 Res = WidenVecRes_BinaryCanTrap(
N);
4013 Res = WidenVecRes_BinaryWithExtraScalarOp(
N);
4016#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
4017 case ISD::STRICT_##DAGN:
4018#include "llvm/IR/ConstrainedOps.def"
4019 Res = WidenVecRes_StrictFP(
N);
4028 Res = WidenVecRes_OverflowOp(
N, ResNo);
4032 Res = WidenVecRes_FCOPYSIGN(
N);
4036 Res = WidenVecRes_IS_FPCLASS(
N);
4040 Res = WidenVecRes_POWI(
N);
4046 Res = WidenVecRes_EXTEND_VECTOR_INREG(
N);
4051 case ISD::VP_FP_EXTEND:
4053 case ISD::VP_FP_ROUND:
4055 case ISD::VP_FP_TO_SINT:
4057 case ISD::VP_FP_TO_UINT:
4059 case ISD::VP_SIGN_EXTEND:
4061 case ISD::VP_SINT_TO_FP:
4062 case ISD::VP_TRUNCATE:
4065 case ISD::VP_UINT_TO_FP:
4067 case ISD::VP_ZERO_EXTEND:
4068 Res = WidenVecRes_Convert(
N);
4073 Res = WidenVecRes_FP_TO_XINT_SAT(
N);
4092 if (unrollExpandedOp())
4102 case ISD::VP_BITREVERSE:
4108 case ISD::VP_CTLZ_ZERO_UNDEF:
4114 case ISD::VP_CTTZ_ZERO_UNDEF:
4119 case ISD::VP_FFLOOR:
4121 case ISD::VP_FNEARBYINT:
4122 case ISD::VP_FROUND:
4123 case ISD::VP_FROUNDEVEN:
4124 case ISD::VP_FROUNDTOZERO: