37#include "llvm/IR/IntrinsicsRISCV.h"
50#define DEBUG_TYPE "riscv-lower"
56 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
57 "instructions that we will consider for VW expansion"),
62 cl::desc(
"Allow the formation of VW_W operations (e.g., "
63 "VWADD_W) with splat constants"),
68 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
69 "transformation to multiplications by the reciprocal"),
74 cl::desc(
"Give the maximum number of instructions that we will "
75 "use for creating a floating-point immediate value"),
80 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
86 if (Subtarget.isRVE())
93 !Subtarget.hasStdExtF()) {
94 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
95 "doesn't support the F instruction set extension (ignoring "
99 !Subtarget.hasStdExtD()) {
100 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
101 "doesn't support the D instruction set extension (ignoring "
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
272 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
288 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
293 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) {
297 }
else if (Subtarget.
is64Bit()) {
305 if (!Subtarget.hasStdExtM()) {
311 }
else if (Subtarget.
is64Bit()) {
314 {MVT::i8, MVT::i16, MVT::i32},
Custom);
331 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
334 }
else if (Subtarget.hasVendorXTHeadBb()) {
347 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
348 Subtarget.hasVendorXTHeadBb())
353 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
354 Subtarget.hasVendorXTHeadBb())
362 if (Subtarget.hasStdExtZbb()) {
381 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb()) {
388 if (!Subtarget.hasStdExtZbb())
400 !Subtarget.hasShortForwardBranchOpt())
404 if (Subtarget.hasShortForwardBranchOpt())
407 if (!Subtarget.hasVendorXTHeadCondMov())
413 static const unsigned FPLegalNodeTypes[] = {
426 static const unsigned FPOpToExpand[] = {
430 static const unsigned FPRndMode[] = {
437 static const unsigned ZfhminZfbfminPromoteOps[] = {
448 if (Subtarget.hasStdExtZfbfmin()) {
507 if (!Subtarget.hasStdExtZfa())
531 if (Subtarget.hasStdExtZfa())
543 if (Subtarget.hasStdExtZfa()) {
619 if (Subtarget.hasStdExtZicbop()) {
623 if (Subtarget.hasStdExtA()) {
626 }
else if (Subtarget.hasForcedAtomics()) {
647 {MVT::i8, MVT::i16},
Custom);
658 static const unsigned IntegerVPOps[] = {
659 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
660 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
661 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
662 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
663 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
664 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
665 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
666 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
667 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
668 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
669 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
670 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE};
672 static const unsigned FloatingPointVPOps[] = {
673 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
674 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
675 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
676 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
677 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
678 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
679 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
680 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
681 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
682 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
683 ISD::EXPERIMENTAL_VP_REVERSE};
685 static const unsigned IntegerVecReduceOps[] = {
690 static const unsigned FloatingPointVecReduceOps[] = {
703 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
704 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
705 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
709 for (
MVT VT : BoolVecVTs) {
736 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
760 ISD::VP_TRUNCATE, ISD::VP_SETCC},
775 for (
MVT VT : IntVecVTs) {
786 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
831 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
832 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
856 if (Subtarget.hasStdExtZvkb()) {
864 if (Subtarget.hasStdExtZvbb()) {
868 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
874 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
883 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
903 static const unsigned ZvfhminPromoteOps[] = {
913 static const unsigned ZvfhminPromoteVPOps[] = {
914 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
915 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
916 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
917 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
918 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
919 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
920 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
921 ISD::VP_FNEARBYINT, ISD::VP_SETCC};
924 const auto SetCommonVFPActions = [&](
MVT VT) {
968 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
969 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1000 const auto SetCommonVFPExtLoadTruncStoreActions =
1002 for (
auto SmallVT : SmallerVTs) {
1009 for (
MVT VT : F16VecVTs) {
1012 SetCommonVFPActions(VT);
1015 for (
MVT VT : F16VecVTs) {
1026 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1036 if (VT == MVT::nxv32f16) {
1049 for (
MVT VT : F32VecVTs) {
1052 SetCommonVFPActions(VT);
1053 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1058 for (
MVT VT : F64VecVTs) {
1061 SetCommonVFPActions(VT);
1062 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1063 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1069 if (!useRVVForFixedLengthVectorVT(VT))
1112 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1139 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1160 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1161 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1196 if (Subtarget.hasStdExtZvkb())
1199 if (Subtarget.hasStdExtZvbb()) {
1221 if (!useRVVForFixedLengthVectorVT(VT))
1242 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1274 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1275 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1326 if (Subtarget.hasStdExtA()) {
1332 if (Subtarget.hasForcedAtomics()) {
1342 if (Subtarget.hasVendorXTHeadMemIdx()) {
1378 if (Subtarget.hasStdExtZbb())
1381 if (Subtarget.hasStdExtZbs() && Subtarget.
is64Bit())
1384 if (Subtarget.hasStdExtZbkb())
1396 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1398 if (Subtarget.hasVendorXTHeadMemPair())
1421MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1426bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1428 bool IsScalable)
const {
1435 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1455 unsigned Intrinsic)
const {
1456 auto &
DL =
I.getModule()->getDataLayout();
1458 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1459 bool IsUnitStrided) {
1461 Info.ptrVal =
I.getArgOperand(PtrOp);
1465 MemTy =
I.getArgOperand(0)->getType();
1468 MemTy =
I.getType();
1483 if (
I.getMetadata(LLVMContext::MD_nontemporal) !=
nullptr)
1487 switch (Intrinsic) {
1490 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1491 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1492 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1493 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1494 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1495 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1496 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1497 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1498 case Intrinsic::riscv_masked_cmpxchg_i32:
1500 Info.memVT = MVT::i32;
1501 Info.ptrVal =
I.getArgOperand(0);
1507 case Intrinsic::riscv_masked_strided_load:
1508 return SetRVVLoadStoreInfo( 1,
false,
1510 case Intrinsic::riscv_masked_strided_store:
1511 return SetRVVLoadStoreInfo( 1,
true,
1513 case Intrinsic::riscv_seg2_load:
1514 case Intrinsic::riscv_seg3_load:
1515 case Intrinsic::riscv_seg4_load:
1516 case Intrinsic::riscv_seg5_load:
1517 case Intrinsic::riscv_seg6_load:
1518 case Intrinsic::riscv_seg7_load:
1519 case Intrinsic::riscv_seg8_load:
1520 return SetRVVLoadStoreInfo( 0,
false,
1522 case Intrinsic::riscv_seg2_store:
1523 case Intrinsic::riscv_seg3_store:
1524 case Intrinsic::riscv_seg4_store:
1525 case Intrinsic::riscv_seg5_store:
1526 case Intrinsic::riscv_seg6_store:
1527 case Intrinsic::riscv_seg7_store:
1528 case Intrinsic::riscv_seg8_store:
1530 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1533 case Intrinsic::riscv_vle:
1534 case Intrinsic::riscv_vle_mask:
1535 case Intrinsic::riscv_vleff:
1536 case Intrinsic::riscv_vleff_mask:
1537 return SetRVVLoadStoreInfo( 1,
1540 case Intrinsic::riscv_vse:
1541 case Intrinsic::riscv_vse_mask:
1542 return SetRVVLoadStoreInfo( 1,
1545 case Intrinsic::riscv_vlse:
1546 case Intrinsic::riscv_vlse_mask:
1547 case Intrinsic::riscv_vloxei:
1548 case Intrinsic::riscv_vloxei_mask:
1549 case Intrinsic::riscv_vluxei:
1550 case Intrinsic::riscv_vluxei_mask:
1551 return SetRVVLoadStoreInfo( 1,
1554 case Intrinsic::riscv_vsse:
1555 case Intrinsic::riscv_vsse_mask:
1556 case Intrinsic::riscv_vsoxei:
1557 case Intrinsic::riscv_vsoxei_mask:
1558 case Intrinsic::riscv_vsuxei:
1559 case Intrinsic::riscv_vsuxei_mask:
1560 return SetRVVLoadStoreInfo( 1,
1563 case Intrinsic::riscv_vlseg2:
1564 case Intrinsic::riscv_vlseg3:
1565 case Intrinsic::riscv_vlseg4:
1566 case Intrinsic::riscv_vlseg5:
1567 case Intrinsic::riscv_vlseg6:
1568 case Intrinsic::riscv_vlseg7:
1569 case Intrinsic::riscv_vlseg8:
1570 case Intrinsic::riscv_vlseg2ff:
1571 case Intrinsic::riscv_vlseg3ff:
1572 case Intrinsic::riscv_vlseg4ff:
1573 case Intrinsic::riscv_vlseg5ff:
1574 case Intrinsic::riscv_vlseg6ff:
1575 case Intrinsic::riscv_vlseg7ff:
1576 case Intrinsic::riscv_vlseg8ff:
1577 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1580 case Intrinsic::riscv_vlseg2_mask:
1581 case Intrinsic::riscv_vlseg3_mask:
1582 case Intrinsic::riscv_vlseg4_mask:
1583 case Intrinsic::riscv_vlseg5_mask:
1584 case Intrinsic::riscv_vlseg6_mask:
1585 case Intrinsic::riscv_vlseg7_mask:
1586 case Intrinsic::riscv_vlseg8_mask:
1587 case Intrinsic::riscv_vlseg2ff_mask:
1588 case Intrinsic::riscv_vlseg3ff_mask:
1589 case Intrinsic::riscv_vlseg4ff_mask:
1590 case Intrinsic::riscv_vlseg5ff_mask:
1591 case Intrinsic::riscv_vlseg6ff_mask:
1592 case Intrinsic::riscv_vlseg7ff_mask:
1593 case Intrinsic::riscv_vlseg8ff_mask:
1594 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1597 case Intrinsic::riscv_vlsseg2:
1598 case Intrinsic::riscv_vlsseg3:
1599 case Intrinsic::riscv_vlsseg4:
1600 case Intrinsic::riscv_vlsseg5:
1601 case Intrinsic::riscv_vlsseg6:
1602 case Intrinsic::riscv_vlsseg7:
1603 case Intrinsic::riscv_vlsseg8:
1604 case Intrinsic::riscv_vloxseg2:
1605 case Intrinsic::riscv_vloxseg3:
1606 case Intrinsic::riscv_vloxseg4:
1607 case Intrinsic::riscv_vloxseg5:
1608 case Intrinsic::riscv_vloxseg6:
1609 case Intrinsic::riscv_vloxseg7:
1610 case Intrinsic::riscv_vloxseg8:
1611 case Intrinsic::riscv_vluxseg2:
1612 case Intrinsic::riscv_vluxseg3:
1613 case Intrinsic::riscv_vluxseg4:
1614 case Intrinsic::riscv_vluxseg5:
1615 case Intrinsic::riscv_vluxseg6:
1616 case Intrinsic::riscv_vluxseg7:
1617 case Intrinsic::riscv_vluxseg8:
1618 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1621 case Intrinsic::riscv_vlsseg2_mask:
1622 case Intrinsic::riscv_vlsseg3_mask:
1623 case Intrinsic::riscv_vlsseg4_mask:
1624 case Intrinsic::riscv_vlsseg5_mask:
1625 case Intrinsic::riscv_vlsseg6_mask:
1626 case Intrinsic::riscv_vlsseg7_mask:
1627 case Intrinsic::riscv_vlsseg8_mask:
1628 case Intrinsic::riscv_vloxseg2_mask:
1629 case Intrinsic::riscv_vloxseg3_mask:
1630 case Intrinsic::riscv_vloxseg4_mask:
1631 case Intrinsic::riscv_vloxseg5_mask:
1632 case Intrinsic::riscv_vloxseg6_mask:
1633 case Intrinsic::riscv_vloxseg7_mask:
1634 case Intrinsic::riscv_vloxseg8_mask:
1635 case Intrinsic::riscv_vluxseg2_mask:
1636 case Intrinsic::riscv_vluxseg3_mask:
1637 case Intrinsic::riscv_vluxseg4_mask:
1638 case Intrinsic::riscv_vluxseg5_mask:
1639 case Intrinsic::riscv_vluxseg6_mask:
1640 case Intrinsic::riscv_vluxseg7_mask:
1641 case Intrinsic::riscv_vluxseg8_mask:
1642 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1645 case Intrinsic::riscv_vsseg2:
1646 case Intrinsic::riscv_vsseg3:
1647 case Intrinsic::riscv_vsseg4:
1648 case Intrinsic::riscv_vsseg5:
1649 case Intrinsic::riscv_vsseg6:
1650 case Intrinsic::riscv_vsseg7:
1651 case Intrinsic::riscv_vsseg8:
1652 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1655 case Intrinsic::riscv_vsseg2_mask:
1656 case Intrinsic::riscv_vsseg3_mask:
1657 case Intrinsic::riscv_vsseg4_mask:
1658 case Intrinsic::riscv_vsseg5_mask:
1659 case Intrinsic::riscv_vsseg6_mask:
1660 case Intrinsic::riscv_vsseg7_mask:
1661 case Intrinsic::riscv_vsseg8_mask:
1662 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1665 case Intrinsic::riscv_vssseg2:
1666 case Intrinsic::riscv_vssseg3:
1667 case Intrinsic::riscv_vssseg4:
1668 case Intrinsic::riscv_vssseg5:
1669 case Intrinsic::riscv_vssseg6:
1670 case Intrinsic::riscv_vssseg7:
1671 case Intrinsic::riscv_vssseg8:
1672 case Intrinsic::riscv_vsoxseg2:
1673 case Intrinsic::riscv_vsoxseg3:
1674 case Intrinsic::riscv_vsoxseg4:
1675 case Intrinsic::riscv_vsoxseg5:
1676 case Intrinsic::riscv_vsoxseg6:
1677 case Intrinsic::riscv_vsoxseg7:
1678 case Intrinsic::riscv_vsoxseg8:
1679 case Intrinsic::riscv_vsuxseg2:
1680 case Intrinsic::riscv_vsuxseg3:
1681 case Intrinsic::riscv_vsuxseg4:
1682 case Intrinsic::riscv_vsuxseg5:
1683 case Intrinsic::riscv_vsuxseg6:
1684 case Intrinsic::riscv_vsuxseg7:
1685 case Intrinsic::riscv_vsuxseg8:
1686 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1689 case Intrinsic::riscv_vssseg2_mask:
1690 case Intrinsic::riscv_vssseg3_mask:
1691 case Intrinsic::riscv_vssseg4_mask:
1692 case Intrinsic::riscv_vssseg5_mask:
1693 case Intrinsic::riscv_vssseg6_mask:
1694 case Intrinsic::riscv_vssseg7_mask:
1695 case Intrinsic::riscv_vssseg8_mask:
1696 case Intrinsic::riscv_vsoxseg2_mask:
1697 case Intrinsic::riscv_vsoxseg3_mask:
1698 case Intrinsic::riscv_vsoxseg4_mask:
1699 case Intrinsic::riscv_vsoxseg5_mask:
1700 case Intrinsic::riscv_vsoxseg6_mask:
1701 case Intrinsic::riscv_vsoxseg7_mask:
1702 case Intrinsic::riscv_vsoxseg8_mask:
1703 case Intrinsic::riscv_vsuxseg2_mask:
1704 case Intrinsic::riscv_vsuxseg3_mask:
1705 case Intrinsic::riscv_vsuxseg4_mask:
1706 case Intrinsic::riscv_vsuxseg5_mask:
1707 case Intrinsic::riscv_vsuxseg6_mask:
1708 case Intrinsic::riscv_vsuxseg7_mask:
1709 case Intrinsic::riscv_vsuxseg8_mask:
1710 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1747 return isInt<12>(Imm);
1751 return isInt<12>(Imm);
1764 return (SrcBits == 64 && DestBits == 32);
1775 return (SrcBits == 64 && DestBits == 32);
1782 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1783 EVT MemVT = LD->getMemoryVT();
1784 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1794 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1802 return Subtarget.hasStdExtZbb();
1806 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb();
1817 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1822 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1826 EVT VT =
Y.getValueType();
1832 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1833 !isa<ConstantSDNode>(
Y);
1838 if (Subtarget.hasStdExtZbs())
1839 return X.getValueType().isScalarInteger();
1840 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1842 if (Subtarget.hasVendorXTHeadBs())
1843 return C !=
nullptr;
1845 return C &&
C->getAPIntValue().ule(10);
1865 if (BitSize > Subtarget.
getXLen())
1869 int64_t Val = Imm.getSExtValue();
1877 if (!Subtarget.enableUnalignedScalarMem())
1893 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1900 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1904 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
1916 case Instruction::Add:
1917 case Instruction::Sub:
1918 case Instruction::Mul:
1919 case Instruction::And:
1920 case Instruction::Or:
1921 case Instruction::Xor:
1922 case Instruction::FAdd:
1923 case Instruction::FSub:
1924 case Instruction::FMul:
1925 case Instruction::FDiv:
1926 case Instruction::ICmp:
1927 case Instruction::FCmp:
1929 case Instruction::Shl:
1930 case Instruction::LShr:
1931 case Instruction::AShr:
1932 case Instruction::UDiv:
1933 case Instruction::SDiv:
1934 case Instruction::URem:
1935 case Instruction::SRem:
1936 return Operand == 1;
1950 auto *II = dyn_cast<IntrinsicInst>(
I);
1954 switch (II->getIntrinsicID()) {
1955 case Intrinsic::fma:
1956 case Intrinsic::vp_fma:
1957 return Operand == 0 || Operand == 1;
1958 case Intrinsic::vp_shl:
1959 case Intrinsic::vp_lshr:
1960 case Intrinsic::vp_ashr:
1961 case Intrinsic::vp_udiv:
1962 case Intrinsic::vp_sdiv:
1963 case Intrinsic::vp_urem:
1964 case Intrinsic::vp_srem:
1965 return Operand == 1;
1967 case Intrinsic::vp_add:
1968 case Intrinsic::vp_mul:
1969 case Intrinsic::vp_and:
1970 case Intrinsic::vp_or:
1971 case Intrinsic::vp_xor:
1972 case Intrinsic::vp_fadd:
1973 case Intrinsic::vp_fmul:
1974 case Intrinsic::vp_icmp:
1975 case Intrinsic::vp_fcmp:
1977 case Intrinsic::vp_sub:
1978 case Intrinsic::vp_fsub:
1979 case Intrinsic::vp_fdiv:
1980 return Operand == 0 || Operand == 1;
1996 for (
auto OpIdx :
enumerate(
I->operands())) {
2000 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2002 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2011 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2016 for (
Use &U :
Op->uses()) {
2068 if (!Subtarget.hasStdExtZfa())
2069 return std::make_pair(-1,
false);
2071 bool IsSupportedVT =
false;
2072 if (VT == MVT::f16) {
2073 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2074 }
else if (VT == MVT::f32) {
2075 IsSupportedVT =
true;
2076 }
else if (VT == MVT::f64) {
2077 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2078 IsSupportedVT =
true;
2082 return std::make_pair(-1,
false);
2085 if (
Index < 0 && Imm.isNegative())
2089 return std::make_pair(
Index,
false);
2093 bool ForCodeSize)
const {
2094 bool IsLegalVT =
false;
2097 else if (VT == MVT::f32)
2099 else if (VT == MVT::f64)
2101 else if (VT == MVT::bf16)
2102 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2114 return Imm.isZero();
2118 if (Imm.isNegZero())
2131 unsigned Index)
const {
2144 if ((ResElts * 2) != SrcElts)
2193 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2195 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2198 IntermediateVT = MVT::i64;
2201 RegisterVT = MVT::i64;
2216 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2222 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2235 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2236 int64_t
C = RHSC->getSExtValue();
2278 switch (KnownSize) {
2306 return RISCV::VRRegClassID;
2308 return RISCV::VRM2RegClassID;
2310 return RISCV::VRM4RegClassID;
2312 return RISCV::VRM8RegClassID;
2322 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2323 "Unexpected subreg numbering");
2324 return RISCV::sub_vrm1_0 +
Index;
2327 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2328 "Unexpected subreg numbering");
2329 return RISCV::sub_vrm2_0 +
Index;
2332 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2333 "Unexpected subreg numbering");
2334 return RISCV::sub_vrm4_0 +
Index;
2341 return RISCV::VRRegClassID;
2350std::pair<unsigned, unsigned>
2352 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2354 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2355 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2356 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2357 "Register classes not ordered");
2366 unsigned SubRegIdx = RISCV::NoSubRegister;
2367 for (
const unsigned RCID :
2368 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2369 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2373 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2378 return {SubRegIdx, InsertExtractIdx};
2383bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2412unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2419 "Unexpected opcode");
2421 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2423 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2426 return Op.getOperand(II->
VLOperand + 1 + HasChain);
2496bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2497 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2506 "Expected legal fixed length vector!");
2509 unsigned MaxELen = Subtarget.
getELen();
2542 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2549 "Expected to convert into a scalable vector!");
2550 assert(V.getValueType().isFixedLengthVector() &&
2551 "Expected a fixed length vector operand!");
2561 "Expected to convert into a fixed length vector!");
2562 assert(V.getValueType().isScalableVector() &&
2563 "Expected a scalable vector operand!");
2591static std::pair<SDValue, SDValue>
2600static std::pair<SDValue, SDValue>
2613static std::pair<SDValue, SDValue>
2639 EVT VT,
unsigned DefinedValues)
const {
2653 std::tie(LMul, Fractional) =
2656 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2658 Cost = (LMul * DLenFactor);
2695 MVT DstVT =
Op.getSimpleValueType();
2696 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2704 Src.getValueType() == MVT::bf16) {
2711 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2719 Opc,
DL, DstVT, Src,
2733 MVT SrcVT = Src.getSimpleValueType();
2739 if (SatVT != DstEltVT)
2743 if (SrcEltSize > (2 * DstEltSize))
2746 MVT DstContainerVT = DstVT;
2747 MVT SrcContainerVT = SrcVT;
2753 "Expected same element count");
2762 {Src, Src, DAG.getCondCode(ISD::SETNE),
2763 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2767 if (DstEltSize > (2 * SrcEltSize)) {
2793 case ISD::VP_FROUNDEVEN:
2797 case ISD::VP_FROUNDTOZERO:
2801 case ISD::VP_FFLOOR:
2809 case ISD::VP_FROUND:
2825 MVT VT =
Op.getSimpleValueType();
2832 MVT ContainerVT = VT;
2839 if (
Op->isVPOpcode()) {
2840 Mask =
Op.getOperand(1);
2844 VL =
Op.getOperand(2);
2866 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
2880 switch (
Op.getOpcode()) {
2886 case ISD::VP_FFLOOR:
2889 case ISD::VP_FROUND:
2890 case ISD::VP_FROUNDEVEN:
2891 case ISD::VP_FROUNDTOZERO: {
2907 case ISD::VP_FNEARBYINT:
2920 Src, Src, Mask, VL);
2935 MVT VT =
Op.getSimpleValueType();
2939 MVT ContainerVT = VT;
2951 MVT MaskVT = Mask.getSimpleValueType();
2954 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
2955 DAG.getUNDEF(MaskVT), Mask, VL});
2959 {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL});
2960 Chain = Src.getValue(1);
2976 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
2988 switch (
Op.getOpcode()) {
2999 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3005 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3009 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3018 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3019 Truncated, Mask, VL);
3025 Src, Src, Mask, VL);
3035 MVT VT =
Op.getSimpleValueType();
3063 MVT VT =
Op.getSimpleValueType();
3068 MVT ContainerVT = VT;
3090 if (
Merge.isUndef())
3102 if (
Merge.isUndef())
3111 "Unexpected vector MVT");
3137 return std::nullopt;
3154 bool IsInteger =
Op.getValueType().isInteger();
3156 std::optional<unsigned> SeqStepDenom;
3157 std::optional<int64_t> SeqStepNum, SeqAddend;
3158 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3159 unsigned EltSizeInBits =
Op.getValueType().getScalarSizeInBits();
3160 for (
unsigned Idx = 0;
Idx < NumElts;
Idx++) {
3163 if (
Op.getOperand(
Idx).isUndef())
3169 if (!isa<ConstantSDNode>(
Op.getOperand(
Idx)))
3170 return std::nullopt;
3171 Val =
Op.getConstantOperandVal(
Idx) &
3172 maskTrailingOnes<uint64_t>(EltSizeInBits);
3175 if (!isa<ConstantFPSDNode>(
Op.getOperand(
Idx)))
3176 return std::nullopt;
3178 cast<ConstantFPSDNode>(
Op.getOperand(
Idx))->getValueAPF(),
3180 Val = *ExactInteger;
3182 return std::nullopt;
3188 unsigned IdxDiff =
Idx - PrevElt->second;
3189 int64_t ValDiff =
SignExtend64(Val - PrevElt->first, EltSizeInBits);
3197 int64_t Remainder = ValDiff % IdxDiff;
3199 if (Remainder != ValDiff) {
3202 return std::nullopt;
3208 SeqStepNum = ValDiff;
3209 else if (ValDiff != SeqStepNum)
3210 return std::nullopt;
3213 SeqStepDenom = IdxDiff;
3214 else if (IdxDiff != *SeqStepDenom)
3215 return std::nullopt;
3219 if (!PrevElt || PrevElt->first != Val)
3220 PrevElt = std::make_pair(Val,
Idx);
3224 if (!SeqStepNum || !SeqStepDenom)
3225 return std::nullopt;
3229 for (
unsigned Idx = 0;
Idx < NumElts;
Idx++) {
3230 if (
Op.getOperand(
Idx).isUndef())
3234 Val =
Op.getConstantOperandVal(
Idx) &
3235 maskTrailingOnes<uint64_t>(EltSizeInBits);
3238 cast<ConstantFPSDNode>(
Op.getOperand(
Idx))->getValueAPF(),
3242 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3243 int64_t Addend =
SignExtend64(Val - ExpectedVal, EltSizeInBits);
3246 else if (Addend != SeqAddend)
3247 return std::nullopt;
3250 assert(SeqAddend &&
"Must have an addend if we have a step");
3252 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3273 MVT ContainerVT = VT;
3301 MVT VT =
Op.getSimpleValueType();
3313 unsigned MostCommonCount = 0;
3315 unsigned NumUndefElts =
3323 unsigned NumScalarLoads = 0;
3329 ValueCounts.
insert(std::make_pair(V, 0));
3330 unsigned &Count = ValueCounts[V];
3332 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3333 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3338 if (++Count >= MostCommonCount) {
3340 MostCommonCount = Count;
3344 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3345 unsigned NumDefElts = NumElts - NumUndefElts;
3346 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3352 ((MostCommonCount > DominantValueCountThreshold) ||
3365 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3366 LastOp != DominantValue) {
3375 Processed.insert(LastOp);
3380 const SDValue &V = OpIdx.value();
3381 if (V.isUndef() || !Processed.insert(V).second)
3383 if (ValueCounts[V] == 1) {
3392 return DAG.getConstant(V == V1, DL, XLenVT);
3408 MVT VT =
Op.getSimpleValueType();
3438 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3439 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3447 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3448 MVT IntegerViaVecVT =
3453 unsigned BitPos = 0, IntegerEltIdx = 0;
3456 for (
unsigned I = 0;
I < NumElts;) {
3458 bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
3459 Bits |= ((
uint64_t)BitValue << BitPos);
3465 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3466 if (NumViaIntegerBits <= 32)
3467 Bits = SignExtend64<32>(Bits);
3469 Elts[IntegerEltIdx] = Elt;
3478 if (NumElts < NumViaIntegerBits) {
3482 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3510 int64_t StepNumerator = SimpleVID->StepNumerator;
3511 unsigned StepDenominator = SimpleVID->StepDenominator;
3512 int64_t Addend = SimpleVID->Addend;
3514 assert(StepNumerator != 0 &&
"Invalid step");
3515 bool Negate =
false;
3516 int64_t SplatStepVal = StepNumerator;
3520 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3522 Negate = StepNumerator < 0;
3524 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3531 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3532 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3534 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3537 MVT VIDContainerVT =
3545 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3546 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3548 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3550 if (StepDenominator != 1) {
3555 if (Addend != 0 || Negate) {
3574 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3575 "Unexpected sequence type");
3579 unsigned ViaVecLen =
3583 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3586 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3587 const auto &SeqV = OpIdx.value();
3588 if (!SeqV.isUndef())
3589 SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
3590 << (OpIdx.index() * EltBitSize));
3595 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3596 SplatValue = SignExtend64<32>(SplatValue);
3618 const auto *BV = cast<BuildVectorSDNode>(
Op);
3619 if (VT.
isInteger() && EltBitSize < 64 &&
3621 BV->getRepeatedSequence(Sequence) &&
3622 (Sequence.size() * EltBitSize) <= 64) {
3623 unsigned SeqLen = Sequence.size();
3625 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3626 ViaIntVT == MVT::i64) &&
3627 "Unexpected sequence type");
3632 const unsigned RequiredVL = NumElts / SeqLen;
3633 const unsigned ViaVecLen =
3635 NumElts : RequiredVL;
3638 unsigned EltIdx = 0;
3639 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3643 for (
const auto &SeqV : Sequence) {
3644 if (!SeqV.isUndef())
3645 SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
3646 << (EltIdx * EltBitSize));
3652 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3653 SplatValue = SignExtend64<32>(SplatValue);
3659 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3660 "Unexpected bitcast sequence");
3661 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3664 MVT ViaContainerVT =
3671 if (ViaVecLen != RequiredVL)
3688 if (EltBitSize - SignBits < 8) {
3692 Source, DAG, Subtarget);
3709 MVT VT =
Op.getSimpleValueType();
3782 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
3786 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
3819 unsigned UndefCount = 0;
3826 LinearBudget -= PerSlideCost;
3829 LinearBudget -= PerSlideCost;
3832 LinearBudget -= PerSlideCost;
3835 if (LinearBudget < 0)
3840 "Illegal type which will result in reserved encoding");
3865 Vec,
Offset, Mask, VL, Policy);
3878 Vec,
Offset, Mask, VL, Policy);
3888 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
3889 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
3890 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
3893 if ((LoC >> 31) == HiC)
3904 (isa<RegisterSDNode>(VL) &&
3905 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
3907 else if (isa<ConstantSDNode>(VL) &&
3908 isUInt<4>(cast<ConstantSDNode>(VL)->getZExtValue()))
3924 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
3925 Hi.getConstantOperandVal(1) == 31)
3944 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
3956 bool HasPassthru = Passthru && !Passthru.
isUndef();
3957 if (!HasPassthru && !Passthru)
3965 if (Scalar.getValueType().bitsLE(XLenVT)) {
3972 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
3976 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
3977 "Unexpected scalar for splat lowering!");
4001 SDValue ExtractedVal = Scalar.getOperand(0);
4003 MVT ExtractedContainerVT = ExtractedVT;
4006 DAG, ExtractedContainerVT, Subtarget);
4010 if (ExtractedContainerVT.
bitsLE(VT))
4024 if (!Scalar.getValueType().bitsLE(XLenVT))
4027 VT,
DL, DAG, Subtarget);
4035 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4061 if (Src != V2.getOperand(0))
4065 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4070 V2.getConstantOperandVal(1) != Mask.size())
4074 if (Mask[0] != 0 && Mask[0] != 1)
4079 for (
unsigned i = 1; i != Mask.size(); ++i)
4080 if (Mask[i] != Mask[i - 1] + 2)
4098 int Size = Mask.size();
4100 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4106 EvenSrc = StartIndexes[0];
4107 OddSrc = StartIndexes[1];
4110 if (EvenSrc != 0 && OddSrc != 0)
4120 int HalfNumElts = NumElts / 2;
4121 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4137 int Size = Mask.size();
4149 for (
int i = 0; i !=
Size; ++i) {
4155 int StartIdx = i - (M %
Size);
4163 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4166 Rotation = CandidateRotation;
4167 else if (Rotation != CandidateRotation)
4172 int MaskSrc = M <
Size ? 0 : 1;
4177 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4182 TargetSrc = MaskSrc;
4183 else if (TargetSrc != MaskSrc)
4190 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4191 assert((LoSrc >= 0 || HiSrc >= 0) &&
4192 "Failed to find a rotated input vector!");
4207 MVT ContainerVT = VT;
4210 assert(Src.getSimpleValueType().isFixedLengthVector());