38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
81 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
91 !Subtarget.hasStdExtF()) {
92 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
93 "doesn't support the F instruction set extension (ignoring "
97 !Subtarget.hasStdExtD()) {
98 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
99 "doesn't support the D instruction set extension (ignoring "
125 if (Subtarget.hasStdExtZfhmin())
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
133 if (Subtarget.hasStdExtZhinxmin())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
274 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
286 if (!Subtarget.hasStdExtZbb())
290 if (Subtarget.hasStdExtZbb()) {
298 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
303 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) {
307 }
else if (Subtarget.
is64Bit()) {
317 if (!Subtarget.hasStdExtM()) {
323 }
else if (Subtarget.
is64Bit()) {
326 {MVT::i8, MVT::i16, MVT::i32},
Custom);
343 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
346 }
else if (Subtarget.hasVendorXTHeadBb()) {
350 }
else if (Subtarget.hasVendorXCVbitmanip()) {
361 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
362 Subtarget.hasVendorXTHeadBb())
367 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
368 Subtarget.hasVendorXTHeadBb())
373 if (Subtarget.hasVendorXCVbitmanip()) {
381 if (Subtarget.hasStdExtZbb()) {
394 }
else if (!Subtarget.hasVendorXCVbitmanip()) {
400 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
401 Subtarget.hasVendorXCVbitmanip()) {
408 if (!Subtarget.hasStdExtZbb())
420 !Subtarget.hasShortForwardBranchOpt())
424 if (Subtarget.hasShortForwardBranchOpt())
427 if (!Subtarget.hasVendorXTHeadCondMov()) {
433 static const unsigned FPLegalNodeTypes[] = {
446 static const unsigned FPOpToExpand[] = {
450 static const unsigned FPRndMode[] = {
457 static const unsigned ZfhminZfbfminPromoteOps[] = {
468 if (Subtarget.hasStdExtZfbfmin()) {
527 if (!Subtarget.hasStdExtZfa())
551 if (Subtarget.hasStdExtZfa())
566 if (Subtarget.hasStdExtZfa()) {
642 if (Subtarget.hasStdExtZicbop()) {
646 if (Subtarget.hasStdExtA()) {
648 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
652 }
else if (Subtarget.hasForcedAtomics()) {
673 {MVT::i8, MVT::i16},
Custom);
684 static const unsigned IntegerVPOps[] = {
685 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
686 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
687 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
688 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
689 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
690 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
691 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
692 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
693 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
694 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
695 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
696 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
697 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
700 static const unsigned FloatingPointVPOps[] = {
701 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
702 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
703 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
704 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
705 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
706 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
707 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
708 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
709 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
710 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
711 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
712 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
713 ISD::EXPERIMENTAL_VP_SPLICE};
715 static const unsigned IntegerVecReduceOps[] = {
720 static const unsigned FloatingPointVecReduceOps[] = {
733 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
734 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
735 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
739 for (
MVT VT : BoolVecVTs) {
766 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
790 ISD::VP_TRUNCATE, ISD::VP_SETCC},
806 for (
MVT VT : IntVecVTs) {
817 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
864 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
865 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
889 if (Subtarget.hasStdExtZvkb()) {
897 if (Subtarget.hasStdExtZvbb()) {
901 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
907 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
916 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
936 static const unsigned ZvfhminPromoteOps[] = {
946 static const unsigned ZvfhminPromoteVPOps[] = {
947 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
948 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
949 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
950 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
951 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
952 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
953 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
954 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
958 const auto SetCommonVFPActions = [&](
MVT VT) {
1003 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1004 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1035 const auto SetCommonVFPExtLoadTruncStoreActions =
1037 for (
auto SmallVT : SmallerVTs) {
1044 for (
MVT VT : F16VecVTs) {
1047 SetCommonVFPActions(VT);
1050 for (
MVT VT : F16VecVTs) {
1061 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1072 if (VT == MVT::nxv32f16) {
1085 for (
MVT VT : F32VecVTs) {
1088 SetCommonVFPActions(VT);
1089 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1094 for (
MVT VT : F64VecVTs) {
1097 SetCommonVFPActions(VT);
1098 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1099 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1105 if (!useRVVForFixedLengthVectorVT(VT))
1148 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1175 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1197 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1198 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1235 if (Subtarget.hasStdExtZvkb())
1238 if (Subtarget.hasStdExtZvbb()) {
1260 if (!useRVVForFixedLengthVectorVT(VT))
1281 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1313 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1314 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1365 if (Subtarget.hasStdExtA()) {
1371 if (Subtarget.hasForcedAtomics()) {
1381 if (Subtarget.hasVendorXTHeadMemIdx()) {
1413 if (Subtarget.hasStdExtZbb())
1416 if (Subtarget.hasStdExtZbs() && Subtarget.
is64Bit())
1419 if (Subtarget.hasStdExtZbkb())
1431 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1434 if (Subtarget.hasVendorXTHeadMemPair())
1457MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1462bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1464 bool IsScalable)
const {
1471 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1488bool RISCVTargetLowering::shouldExpandCttzElements(
EVT VT)
const {
1496 unsigned Intrinsic)
const {
1497 auto &
DL =
I.getModule()->getDataLayout();
1499 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1500 bool IsUnitStrided,
bool UsePtrVal =
false) {
1505 Info.ptrVal =
I.getArgOperand(PtrOp);
1507 Info.fallbackAddressSpace =
1508 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1512 MemTy =
I.getArgOperand(0)->getType();
1515 MemTy =
I.getType();
1530 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1534 switch (Intrinsic) {
1537 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1538 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1539 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1540 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1541 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1542 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1543 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1544 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1545 case Intrinsic::riscv_masked_cmpxchg_i32:
1547 Info.memVT = MVT::i32;
1548 Info.ptrVal =
I.getArgOperand(0);
1554 case Intrinsic::riscv_masked_strided_load:
1555 return SetRVVLoadStoreInfo( 1,
false,
1557 case Intrinsic::riscv_masked_strided_store:
1558 return SetRVVLoadStoreInfo( 1,
true,
1560 case Intrinsic::riscv_seg2_load:
1561 case Intrinsic::riscv_seg3_load:
1562 case Intrinsic::riscv_seg4_load:
1563 case Intrinsic::riscv_seg5_load:
1564 case Intrinsic::riscv_seg6_load:
1565 case Intrinsic::riscv_seg7_load:
1566 case Intrinsic::riscv_seg8_load:
1567 return SetRVVLoadStoreInfo( 0,
false,
1569 case Intrinsic::riscv_seg2_store:
1570 case Intrinsic::riscv_seg3_store:
1571 case Intrinsic::riscv_seg4_store:
1572 case Intrinsic::riscv_seg5_store:
1573 case Intrinsic::riscv_seg6_store:
1574 case Intrinsic::riscv_seg7_store:
1575 case Intrinsic::riscv_seg8_store:
1577 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1580 case Intrinsic::riscv_vle:
1581 case Intrinsic::riscv_vle_mask:
1582 case Intrinsic::riscv_vleff:
1583 case Intrinsic::riscv_vleff_mask:
1584 return SetRVVLoadStoreInfo( 1,
1588 case Intrinsic::riscv_vse:
1589 case Intrinsic::riscv_vse_mask:
1590 return SetRVVLoadStoreInfo( 1,
1594 case Intrinsic::riscv_vlse:
1595 case Intrinsic::riscv_vlse_mask:
1596 case Intrinsic::riscv_vloxei:
1597 case Intrinsic::riscv_vloxei_mask:
1598 case Intrinsic::riscv_vluxei:
1599 case Intrinsic::riscv_vluxei_mask:
1600 return SetRVVLoadStoreInfo( 1,
1603 case Intrinsic::riscv_vsse:
1604 case Intrinsic::riscv_vsse_mask:
1605 case Intrinsic::riscv_vsoxei:
1606 case Intrinsic::riscv_vsoxei_mask:
1607 case Intrinsic::riscv_vsuxei:
1608 case Intrinsic::riscv_vsuxei_mask:
1609 return SetRVVLoadStoreInfo( 1,
1612 case Intrinsic::riscv_vlseg2:
1613 case Intrinsic::riscv_vlseg3:
1614 case Intrinsic::riscv_vlseg4:
1615 case Intrinsic::riscv_vlseg5:
1616 case Intrinsic::riscv_vlseg6:
1617 case Intrinsic::riscv_vlseg7:
1618 case Intrinsic::riscv_vlseg8:
1619 case Intrinsic::riscv_vlseg2ff:
1620 case Intrinsic::riscv_vlseg3ff:
1621 case Intrinsic::riscv_vlseg4ff:
1622 case Intrinsic::riscv_vlseg5ff:
1623 case Intrinsic::riscv_vlseg6ff:
1624 case Intrinsic::riscv_vlseg7ff:
1625 case Intrinsic::riscv_vlseg8ff:
1626 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1629 case Intrinsic::riscv_vlseg2_mask:
1630 case Intrinsic::riscv_vlseg3_mask:
1631 case Intrinsic::riscv_vlseg4_mask:
1632 case Intrinsic::riscv_vlseg5_mask:
1633 case Intrinsic::riscv_vlseg6_mask:
1634 case Intrinsic::riscv_vlseg7_mask:
1635 case Intrinsic::riscv_vlseg8_mask:
1636 case Intrinsic::riscv_vlseg2ff_mask:
1637 case Intrinsic::riscv_vlseg3ff_mask:
1638 case Intrinsic::riscv_vlseg4ff_mask:
1639 case Intrinsic::riscv_vlseg5ff_mask:
1640 case Intrinsic::riscv_vlseg6ff_mask:
1641 case Intrinsic::riscv_vlseg7ff_mask:
1642 case Intrinsic::riscv_vlseg8ff_mask:
1643 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1646 case Intrinsic::riscv_vlsseg2:
1647 case Intrinsic::riscv_vlsseg3:
1648 case Intrinsic::riscv_vlsseg4:
1649 case Intrinsic::riscv_vlsseg5:
1650 case Intrinsic::riscv_vlsseg6:
1651 case Intrinsic::riscv_vlsseg7:
1652 case Intrinsic::riscv_vlsseg8:
1653 case Intrinsic::riscv_vloxseg2:
1654 case Intrinsic::riscv_vloxseg3:
1655 case Intrinsic::riscv_vloxseg4:
1656 case Intrinsic::riscv_vloxseg5:
1657 case Intrinsic::riscv_vloxseg6:
1658 case Intrinsic::riscv_vloxseg7:
1659 case Intrinsic::riscv_vloxseg8:
1660 case Intrinsic::riscv_vluxseg2:
1661 case Intrinsic::riscv_vluxseg3:
1662 case Intrinsic::riscv_vluxseg4:
1663 case Intrinsic::riscv_vluxseg5:
1664 case Intrinsic::riscv_vluxseg6:
1665 case Intrinsic::riscv_vluxseg7:
1666 case Intrinsic::riscv_vluxseg8:
1667 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1670 case Intrinsic::riscv_vlsseg2_mask:
1671 case Intrinsic::riscv_vlsseg3_mask:
1672 case Intrinsic::riscv_vlsseg4_mask:
1673 case Intrinsic::riscv_vlsseg5_mask:
1674 case Intrinsic::riscv_vlsseg6_mask:
1675 case Intrinsic::riscv_vlsseg7_mask:
1676 case Intrinsic::riscv_vlsseg8_mask:
1677 case Intrinsic::riscv_vloxseg2_mask:
1678 case Intrinsic::riscv_vloxseg3_mask:
1679 case Intrinsic::riscv_vloxseg4_mask:
1680 case Intrinsic::riscv_vloxseg5_mask:
1681 case Intrinsic::riscv_vloxseg6_mask:
1682 case Intrinsic::riscv_vloxseg7_mask:
1683 case Intrinsic::riscv_vloxseg8_mask:
1684 case Intrinsic::riscv_vluxseg2_mask:
1685 case Intrinsic::riscv_vluxseg3_mask:
1686 case Intrinsic::riscv_vluxseg4_mask:
1687 case Intrinsic::riscv_vluxseg5_mask:
1688 case Intrinsic::riscv_vluxseg6_mask:
1689 case Intrinsic::riscv_vluxseg7_mask:
1690 case Intrinsic::riscv_vluxseg8_mask:
1691 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1694 case Intrinsic::riscv_vsseg2:
1695 case Intrinsic::riscv_vsseg3:
1696 case Intrinsic::riscv_vsseg4:
1697 case Intrinsic::riscv_vsseg5:
1698 case Intrinsic::riscv_vsseg6:
1699 case Intrinsic::riscv_vsseg7:
1700 case Intrinsic::riscv_vsseg8:
1701 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1704 case Intrinsic::riscv_vsseg2_mask:
1705 case Intrinsic::riscv_vsseg3_mask:
1706 case Intrinsic::riscv_vsseg4_mask:
1707 case Intrinsic::riscv_vsseg5_mask:
1708 case Intrinsic::riscv_vsseg6_mask:
1709 case Intrinsic::riscv_vsseg7_mask:
1710 case Intrinsic::riscv_vsseg8_mask:
1711 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1714 case Intrinsic::riscv_vssseg2:
1715 case Intrinsic::riscv_vssseg3:
1716 case Intrinsic::riscv_vssseg4:
1717 case Intrinsic::riscv_vssseg5:
1718 case Intrinsic::riscv_vssseg6:
1719 case Intrinsic::riscv_vssseg7:
1720 case Intrinsic::riscv_vssseg8:
1721 case Intrinsic::riscv_vsoxseg2:
1722 case Intrinsic::riscv_vsoxseg3:
1723 case Intrinsic::riscv_vsoxseg4:
1724 case Intrinsic::riscv_vsoxseg5:
1725 case Intrinsic::riscv_vsoxseg6:
1726 case Intrinsic::riscv_vsoxseg7:
1727 case Intrinsic::riscv_vsoxseg8:
1728 case Intrinsic::riscv_vsuxseg2:
1729 case Intrinsic::riscv_vsuxseg3:
1730 case Intrinsic::riscv_vsuxseg4:
1731 case Intrinsic::riscv_vsuxseg5:
1732 case Intrinsic::riscv_vsuxseg6:
1733 case Intrinsic::riscv_vsuxseg7:
1734 case Intrinsic::riscv_vsuxseg8:
1735 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1738 case Intrinsic::riscv_vssseg2_mask:
1739 case Intrinsic::riscv_vssseg3_mask:
1740 case Intrinsic::riscv_vssseg4_mask:
1741 case Intrinsic::riscv_vssseg5_mask:
1742 case Intrinsic::riscv_vssseg6_mask:
1743 case Intrinsic::riscv_vssseg7_mask:
1744 case Intrinsic::riscv_vssseg8_mask:
1745 case Intrinsic::riscv_vsoxseg2_mask:
1746 case Intrinsic::riscv_vsoxseg3_mask:
1747 case Intrinsic::riscv_vsoxseg4_mask:
1748 case Intrinsic::riscv_vsoxseg5_mask:
1749 case Intrinsic::riscv_vsoxseg6_mask:
1750 case Intrinsic::riscv_vsoxseg7_mask:
1751 case Intrinsic::riscv_vsoxseg8_mask:
1752 case Intrinsic::riscv_vsuxseg2_mask:
1753 case Intrinsic::riscv_vsuxseg3_mask:
1754 case Intrinsic::riscv_vsuxseg4_mask:
1755 case Intrinsic::riscv_vsuxseg5_mask:
1756 case Intrinsic::riscv_vsuxseg6_mask:
1757 case Intrinsic::riscv_vsuxseg7_mask:
1758 case Intrinsic::riscv_vsuxseg8_mask:
1759 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1796 return isInt<12>(Imm);
1800 return isInt<12>(Imm);
1813 return (SrcBits == 64 && DestBits == 32);
1824 return (SrcBits == 64 && DestBits == 32);
1831 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1832 EVT MemVT = LD->getMemoryVT();
1833 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1843 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1851 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip();
1855 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1856 Subtarget.hasVendorXCVbitmanip();
1867 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1872 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1876 EVT VT =
Y.getValueType();
1882 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1883 !isa<ConstantSDNode>(
Y);
1888 if (Subtarget.hasStdExtZbs())
1889 return X.getValueType().isScalarInteger();
1890 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1892 if (Subtarget.hasVendorXTHeadBs())
1893 return C !=
nullptr;
1895 return C &&
C->getAPIntValue().ule(10);
1915 if (BitSize > Subtarget.
getXLen())
1919 int64_t Val = Imm.getSExtValue();
1927 if (!Subtarget.enableUnalignedScalarMem())
1943 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1950 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1954 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
1966 case Instruction::Add:
1967 case Instruction::Sub:
1968 case Instruction::Mul:
1969 case Instruction::And:
1970 case Instruction::Or:
1971 case Instruction::Xor:
1972 case Instruction::FAdd:
1973 case Instruction::FSub:
1974 case Instruction::FMul:
1975 case Instruction::FDiv:
1976 case Instruction::ICmp:
1977 case Instruction::FCmp:
1979 case Instruction::Shl:
1980 case Instruction::LShr:
1981 case Instruction::AShr:
1982 case Instruction::UDiv:
1983 case Instruction::SDiv:
1984 case Instruction::URem:
1985 case Instruction::SRem:
1986 return Operand == 1;
2000 auto *II = dyn_cast<IntrinsicInst>(
I);
2004 switch (II->getIntrinsicID()) {
2005 case Intrinsic::fma:
2006 case Intrinsic::vp_fma:
2007 return Operand == 0 || Operand == 1;
2008 case Intrinsic::vp_shl:
2009 case Intrinsic::vp_lshr:
2010 case Intrinsic::vp_ashr:
2011 case Intrinsic::vp_udiv:
2012 case Intrinsic::vp_sdiv:
2013 case Intrinsic::vp_urem:
2014 case Intrinsic::vp_srem:
2015 case Intrinsic::ssub_sat:
2016 case Intrinsic::vp_ssub_sat:
2017 case Intrinsic::usub_sat:
2018 case Intrinsic::vp_usub_sat:
2019 return Operand == 1;
2021 case Intrinsic::vp_add:
2022 case Intrinsic::vp_mul:
2023 case Intrinsic::vp_and:
2024 case Intrinsic::vp_or:
2025 case Intrinsic::vp_xor:
2026 case Intrinsic::vp_fadd:
2027 case Intrinsic::vp_fmul:
2028 case Intrinsic::vp_icmp:
2029 case Intrinsic::vp_fcmp:
2030 case Intrinsic::smin:
2031 case Intrinsic::vp_smin:
2032 case Intrinsic::umin:
2033 case Intrinsic::vp_umin:
2034 case Intrinsic::smax:
2035 case Intrinsic::vp_smax:
2036 case Intrinsic::umax:
2037 case Intrinsic::vp_umax:
2038 case Intrinsic::sadd_sat:
2039 case Intrinsic::vp_sadd_sat:
2040 case Intrinsic::uadd_sat:
2041 case Intrinsic::vp_uadd_sat:
2043 case Intrinsic::vp_sub:
2044 case Intrinsic::vp_fsub:
2045 case Intrinsic::vp_fdiv:
2046 return Operand == 0 || Operand == 1;
2067 if (!Subtarget.sinkSplatOperands())
2070 for (
auto OpIdx :
enumerate(
I->operands())) {
2074 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2076 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2085 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2090 for (
Use &U :
Op->uses()) {
2142 if (!Subtarget.hasStdExtZfa())
2143 return std::make_pair(-1,
false);
2145 bool IsSupportedVT =
false;
2146 if (VT == MVT::f16) {
2147 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2148 }
else if (VT == MVT::f32) {
2149 IsSupportedVT =
true;
2150 }
else if (VT == MVT::f64) {
2151 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2152 IsSupportedVT =
true;
2156 return std::make_pair(-1,
false);
2159 if (
Index < 0 && Imm.isNegative())
2163 return std::make_pair(
Index,
false);
2167 bool ForCodeSize)
const {
2168 bool IsLegalVT =
false;
2171 else if (VT == MVT::f32)
2173 else if (VT == MVT::f64)
2175 else if (VT == MVT::bf16)
2176 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2188 return Imm.isZero();
2192 if (Imm.isNegZero())
2205 unsigned Index)
const {
2218 if (EltVT == MVT::i1)
2231 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2238 if ((ResElts * 2) != SrcElts)
2282 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2284 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2287 IntermediateVT = MVT::i64;
2290 RegisterVT = MVT::i64;
2305 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2311 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2324 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2325 int64_t
C = RHSC->getSExtValue();
2367 switch (KnownSize) {
2395 return RISCV::VRRegClassID;
2397 return RISCV::VRM2RegClassID;
2399 return RISCV::VRM4RegClassID;
2401 return RISCV::VRM8RegClassID;
2411 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2412 "Unexpected subreg numbering");
2413 return RISCV::sub_vrm1_0 +
Index;
2416 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2417 "Unexpected subreg numbering");
2418 return RISCV::sub_vrm2_0 +
Index;
2421 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2422 "Unexpected subreg numbering");
2423 return RISCV::sub_vrm4_0 +
Index;
2430 return RISCV::VRRegClassID;
2439std::pair<unsigned, unsigned>
2441 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2443 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2444 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2445 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2446 "Register classes not ordered");
2455 unsigned SubRegIdx = RISCV::NoSubRegister;
2456 for (
const unsigned RCID :
2457 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2458 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2462 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2467 return {SubRegIdx, InsertExtractIdx};
2472bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2501unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2508 "Unexpected opcode");
2510 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2512 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2515 return Op.getOperand(II->
VLOperand + 1 + HasChain);
2585bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2586 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2595 "Expected legal fixed length vector!");
2598 unsigned MaxELen = Subtarget.
getELen();
2631 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2638 "Expected to convert into a scalable vector!");
2639 assert(V.getValueType().isFixedLengthVector() &&
2640 "Expected a fixed length vector operand!");
2650 "Expected to convert into a fixed length vector!");
2651 assert(V.getValueType().isScalableVector() &&
2652 "Expected a scalable vector operand!");
2680 const auto [MinVLMAX, MaxVLMAX] =
2682 if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX)
2688static std::pair<SDValue, SDValue>
2697static std::pair<SDValue, SDValue>
2710static std::pair<SDValue, SDValue>
2727std::pair<unsigned, unsigned>
2743 return std::make_pair(MinVLMAX, MaxVLMAX);
2755 EVT VT,
unsigned DefinedValues)
const {
2769 std::tie(LMul, Fractional) =
2772 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2774 Cost = (LMul * DLenFactor);
2819 MVT DstVT =
Op.getSimpleValueType();
2820 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2828 Src.getValueType() == MVT::bf16) {
2835 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2843 Opc,
DL, DstVT, Src,
2857 MVT SrcVT = Src.getSimpleValueType();
2863 if (SatVT != DstEltVT)
2867 if (SrcEltSize > (2 * DstEltSize))
2870 MVT DstContainerVT = DstVT;
2871 MVT SrcContainerVT = SrcVT;
2877 "Expected same element count");
2886 {Src, Src, DAG.getCondCode(ISD::SETNE),
2887 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2891 if (DstEltSize > (2 * SrcEltSize)) {
2905 Res, DAG.
getUNDEF(DstContainerVT), VL);
2917 case ISD::VP_FROUNDEVEN:
2921 case ISD::VP_FROUNDTOZERO:
2925 case ISD::VP_FFLOOR:
2933 case ISD::VP_FROUND:
2949 MVT VT =
Op.getSimpleValueType();
2956 MVT ContainerVT = VT;
2963 if (
Op->isVPOpcode()) {
2964 Mask =
Op.getOperand(1);
2968 VL =
Op.getOperand(2);
2990 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3004 switch (
Op.getOpcode()) {
3010 case ISD::VP_FFLOOR:
3013 case ISD::VP_FROUND:
3014 case ISD::VP_FROUNDEVEN:
3015 case ISD::VP_FROUNDTOZERO: {
3031 case ISD::VP_FNEARBYINT:
3044 Src, Src, Mask, VL);
3059 MVT VT =
Op.getSimpleValueType();
3063 MVT ContainerVT = VT;
3075 MVT MaskVT = Mask.getSimpleValueType();
3078 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3079 DAG.getUNDEF(MaskVT), Mask, VL});
3083 {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL});
3084 Chain = Src.getValue(1);
3100 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3112 switch (
Op.getOpcode()) {
3123 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3129 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3133 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3142 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3143 Truncated, Mask, VL);
3149 Src, Src, Mask, VL);
3159 MVT VT =
Op.getSimpleValueType();
3187 MVT VT =
Op.getSimpleValueType();
3192 MVT ContainerVT = VT;
3214 if (
Merge.isUndef())
3226 if (
Merge.isUndef())
3235 "Unexpected vector MVT");
3263 return std::nullopt;
3278 unsigned EltSizeInBits) {
3281 return std::nullopt;
3282 bool IsInteger =
Op.getValueType().isInteger();
3284 std::optional<unsigned> SeqStepDenom;
3285 std::optional<int64_t> SeqStepNum, SeqAddend;
3286 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3287 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3292 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3294 if (Elt.isUndef()) {
3295 Elts[
Idx] = std::nullopt;
3299 Elts[
Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3304 return std::nullopt;
3305 Elts[
Idx] = *ExactInteger;
3318 unsigned IdxDiff =
Idx - PrevElt->second;
3319 int64_t ValDiff =
SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3327 int64_t Remainder = ValDiff % IdxDiff;
3329 if (Remainder != ValDiff) {
3332 return std::nullopt;
3338 SeqStepNum = ValDiff;
3339 else if (ValDiff != SeqStepNum)
3340 return std::nullopt;
3343 SeqStepDenom = IdxDiff;
3344 else if (IdxDiff != *SeqStepDenom)
3345 return std::nullopt;
3349 if (!PrevElt || PrevElt->first != *Elt)
3350 PrevElt = std::make_pair(*Elt,
Idx);
3354 if (!SeqStepNum || !SeqStepDenom)
3355 return std::nullopt;
3363 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3364 int64_t Addend =
SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3367 else if (Addend != SeqAddend)
3368 return std::nullopt;
3371 assert(SeqAddend &&
"Must have an addend if we have a step");
3373 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3394 MVT ContainerVT = VT;
3422 MVT VT =
Op.getSimpleValueType();
3434 unsigned MostCommonCount = 0;
3436 unsigned NumUndefElts =
3444 unsigned NumScalarLoads = 0;
3450 ValueCounts.
insert(std::make_pair(V, 0));
3451 unsigned &Count = ValueCounts[V];
3453 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3454 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3459 if (++Count >= MostCommonCount) {
3461 MostCommonCount = Count;
3465 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3466 unsigned NumDefElts = NumElts - NumUndefElts;
3467 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3473 ((MostCommonCount > DominantValueCountThreshold) ||
3486 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3487 LastOp != DominantValue) {
3496 Processed.insert(LastOp);
3501 const SDValue &V = OpIdx.value();
3502 if (V.isUndef() || !Processed.insert(V).second)
3504 if (ValueCounts[V] == 1) {
3513 return DAG.getConstant(V == V1, DL, XLenVT);
3529 MVT VT =
Op.getSimpleValueType();
3559 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3560 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3568 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3569 MVT IntegerViaVecVT =
3574 unsigned BitPos = 0, IntegerEltIdx = 0;
3577 for (
unsigned I = 0;
I < NumElts;) {
3579 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3580 Bits |= ((
uint64_t)BitValue << BitPos);
3586 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3587 if (NumViaIntegerBits <= 32)
3588 Bits = SignExtend64<32>(Bits);
3590 Elts[IntegerEltIdx] = Elt;
3599 if (NumElts < NumViaIntegerBits) {
3603 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3631 int64_t StepNumerator = SimpleVID->StepNumerator;
3632 unsigned StepDenominator = SimpleVID->StepDenominator;
3633 int64_t Addend = SimpleVID->Addend;
3635 assert(StepNumerator != 0 &&
"Invalid step");
3636 bool Negate =
false;
3637 int64_t SplatStepVal = StepNumerator;
3641 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3643 Negate = StepNumerator < 0;
3645 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3652 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3653 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3655 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3658 MVT VIDContainerVT =
3666 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3667 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3669 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3671 if (StepDenominator != 1) {
3676 if (Addend != 0 || Negate) {
3695 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3696 "Unexpected sequence type");
3700 unsigned ViaVecLen =
3704 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3707 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3708 const auto &SeqV = OpIdx.value();
3709 if (!SeqV.isUndef())
3711 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3716 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3717 SplatValue = SignExtend64<32>(SplatValue);
3739 const auto *BV = cast<BuildVectorSDNode>(
Op);
3742 BV->getRepeatedSequence(Sequence) &&
3743 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3744 unsigned SeqLen = Sequence.size();
3746 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3747 ViaIntVT == MVT::i64) &&
3748 "Unexpected sequence type");
3753 const unsigned RequiredVL = NumElts / SeqLen;
3754 const unsigned ViaVecLen =
3756 NumElts : RequiredVL;
3759 unsigned EltIdx = 0;
3760 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3764 for (
const auto &SeqV : Sequence) {
3765 if (!SeqV.isUndef())
3767 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3773 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3774 SplatValue = SignExtend64<32>(SplatValue);
3780 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3781 "Unexpected bitcast sequence");
3782 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3785 MVT ViaContainerVT =
3792 if (ViaVecLen != RequiredVL)
3809 if (EltBitSize - SignBits < 8) {
3813 Source, DAG, Subtarget);
3830 MVT VT =
Op.getSimpleValueType();
3902 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
3906 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
3922 unsigned NumUndefElts =
3924 unsigned NumDefElts = NumElts - NumUndefElts;
3925 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
3932 for (
unsigned i = 0; i < NumElts; i++) {
3934 if (i < NumElts / 2) {
3941 bool SelectMaskVal = (i < NumElts / 2);
3944 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
3945 MaskVals.
size() == NumElts);
3980 unsigned UndefCount = 0;
3987 LinearBudget -= PerSlideCost;
3990 LinearBudget -= PerSlideCost;
3993 LinearBudget -= PerSlideCost;
3996 if (LinearBudget < 0)
4001 "Illegal type which will result in reserved encoding");
4026 Vec,
Offset, Mask, VL, Policy);
4039 Vec,
Offset, Mask, VL, Policy);
4049 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4050 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4051 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4054 if ((LoC >> 31) == HiC)
4065 (isa<RegisterSDNode>(VL) &&
4066 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4068 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4083 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4084 Hi.getConstantOperandVal(1) == 31)
4103 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4115 bool HasPassthru = Passthru && !Passthru.
isUndef();
4116 if (!HasPassthru && !Passthru)
4124 if (Scalar.getValueType().bitsLE(XLenVT)) {
4131 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4135 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4136 "Unexpected scalar for splat lowering!");
4160 SDValue ExtractedVal = Scalar.getOperand(0);
4165 MVT ExtractedContainerVT = ExtractedVT;
4168 DAG, ExtractedContainerVT, Subtarget);
4170 ExtractedVal, DAG, Subtarget);
4172 if (ExtractedContainerVT.
bitsLE(VT))
4187 if (!Scalar.getValueType().bitsLE(XLenVT))
4190 VT,
DL, DAG, Subtarget);
4198 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4224 if (Src != V2.getOperand(0))
4228 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4233 V2.getConstantOperandVal(1) != Mask.size())
4237 if (Mask[0] != 0 && Mask[0] != 1)
4242 for (
unsigned i = 1; i != Mask.size(); ++i)
4243 if (Mask[i] != Mask[i - 1] + 2)
4261 int Size = Mask.size();
4263 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4269 EvenSrc = StartIndexes[0];
4270 OddSrc = StartIndexes[1];
4273 if (EvenSrc != 0 && OddSrc != 0)
4283 int HalfNumElts = NumElts / 2;
4284 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4300 int Size = Mask.size();
4312 for (
int i = 0; i !=
Size; ++i) {
4318 int StartIdx = i - (M %
Size);
4326 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4329 Rotation = CandidateRotation;
4330 else if (Rotation != CandidateRotation)
4335 int MaskSrc = M <
Size ? 0 : 1;
4340 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4345 TargetSrc = MaskSrc;
4346 else if (TargetSrc != MaskSrc)
4353 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4354 assert((LoSrc >= 0 || HiSrc >= 0) &&
4355 "Failed to find a rotated input vector!");
4370 MVT ContainerVT = VT;
4373 assert(Src.getSimpleValueType().isFixedLengthVector());
4377 MVT SrcContainerVT =
4390 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4397 unsigned Shift = EvenElts ? 0 : EltBits;
4403 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4429 auto findNonEXTRACT_SUBVECTORParent =
4430 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4435 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4436 Offset += Parent.getConstantOperandVal(1);
4437 Parent = Parent.getOperand(0);
4439 return std::make_pair(Parent,
Offset);
4442 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4443 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4452 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4453 if (NewMask[i] == -1)
4456 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4457 NewMask[i] = NewMask[i] + V1IndexOffset;
4461 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4467 if (NewMask[0] <= 0)
4471 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4472 if (NewMask[i - 1] + 1 != NewMask[i])
4476 MVT SrcVT = Src.getSimpleValueType();
4507 int NumSubElts,
Index;
4512 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4513 SDValue InPlace = OpsSwapped ? V2 : V1;
4514 SDValue ToInsert = OpsSwapped ? V1 : V2;
4524 if (NumSubElts +
Index >= (
int)NumElts)
4538 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4550 bool OpsSwapped =
false;
4551 if (!isa<BuildVectorSDNode>(V1)) {
4552 if (!isa<BuildVectorSDNode>(V2))
4557 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4565 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4566 for (
unsigned i = S; i != E; ++i)
4567 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4573 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4574 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4577 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4579 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4584 auto OpCode = IsVSlidedown ?
4589 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4592 Splat, TrueMask, VL);
4603 MVT VecContainerVT = VecVT;
4620 MVT WideContainerVT = WideVT;
4626 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4639 }
else if (Subtarget.hasStdExtZvbb()) {
4644 OffsetVec, Passthru, Mask, VL);
4647 Interleaved, EvenV, Passthru, Mask, VL);
4655 Interleaved, OffsetVec, Passthru, Mask, VL);
4663 OddV, Passthru, Mask, VL);
4669 OddV, AllOnesVec, Passthru, Mask, VL);
4677 Interleaved, OddsMul, Passthru, Mask, VL);
4684 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4730 if (ViaEltSize > NumElts)
4739 if (ViaEltSize > NumElts)
4745 if (ViaEltSize > NumElts)
4754 MVT &RotateVT,
unsigned &RotateAmt) {
4760 unsigned NumSubElts;
4762 NumElts, NumSubElts, RotateAmt))
4765 NumElts / NumSubElts);
4827 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
4830 OutMasks(VRegsPerSrc, {-1, {}});
4835 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
4836 int DstVecIdx = DstIdx / ElemsPerVReg;
4837 int DstSubIdx = DstIdx % ElemsPerVReg;
4838 int SrcIdx = Mask[DstIdx];
4839 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
4841 int SrcVecIdx = SrcIdx / ElemsPerVReg;
4842 int SrcSubIdx = SrcIdx % ElemsPerVReg;
4843 if (OutMasks[DstVecIdx].first == -1)
4844 OutMasks[DstVecIdx].first = SrcVecIdx;
4845 if (OutMasks[DstVecIdx].first != SrcVecIdx)
4851 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
4852 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
4866 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
4867 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
4868 if (SrcVecIdx == -1)
4870 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
4877 unsigned InsertIdx = DstVecIdx * NumOpElts;
4890 MVT VT =
Op.getSimpleValueType();
4905 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
4929 V.getOperand(0).getSimpleValueType().getVectorNumElements();
4930 V = V.getOperand(
Offset / OpElements);
4936 auto *Ld = cast<LoadSDNode>(V);
4946 SDValue Ops[] = {Ld->getChain(),
4964 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
4965 Ld->getPointerInfo().getWithOffset(
Offset),
4966 Ld->getOriginalAlign(),
4970 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
4971 Ld->getOriginalAlign(),
4972 Ld->getMemOperand()->getFlags());
4983 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
4986 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5008 if (Subtarget.hasStdExtZvkb())
5019 LoV = LoSrc == 0 ? V1 : V2;
5023 HiV = HiSrc == 0 ? V1 : V2;
5029 unsigned InvRotate = NumElts - Rotation;
5039 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5059 int EvenSrc, OddSrc;
5064 int Size = Mask.size();
5066 assert(EvenSrc >= 0 &&
"Undef source?");
5067 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5071 assert(OddSrc >= 0 &&
"Undef source?");
5072 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5081 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5090 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5119 MVT IndexContainerVT =
5124 for (
int MaskIndex : Mask) {
5125 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5134 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5148 int MaskIndex = MaskIdx.value();
5149 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
5154 for (
int MaskIndex : Mask) {
5155 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ SwapOps;
5162 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5175 for (
int MaskIndex : Mask) {
5176 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5178 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5179 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5181 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5186 std::swap(ShuffleMaskLHS, ShuffleMaskRHS);
5189 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5225RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5227 MVT VT =
Op.getSimpleValueType();
5231 MVT ContainerVT = VT;
5234 if (
Op->isVPOpcode()) {
5235 Mask =
Op.getOperand(1);
5239 VL =
Op.getOperand(2);
5245 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5247 FloatEltVT = MVT::f32;
5254 "Expected legal float type!");
5261 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5264 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5269 if (FloatVT.
bitsGT(VT)) {
5270 if (
Op->isVPOpcode())
5271 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5280 if (!
Op->isVPOpcode())
5284 MVT ContainerFloatVT =
5287 Src, Mask, RTZRM, VL);
5294 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5298 if (
Op->isVPOpcode()) {
5307 else if (IntVT.
bitsGT(VT))
5312 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5317 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5318 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5323 unsigned Adjust = ExponentBias + (EltSize - 1);
5325 if (
Op->isVPOpcode())
5335 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5336 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5347 auto *
Load = cast<LoadSDNode>(
Op);
5348 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5351 Load->getMemoryVT(),
5352 *
Load->getMemOperand()))
5356 MVT VT =
Op.getSimpleValueType();
5358 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5359 "Unexpected unaligned RVV load type");
5363 "Expecting equally-sized RVV vector types to be legal");
5365 Load->getPointerInfo(),
Load->getOriginalAlign(),
5366 Load->getMemOperand()->getFlags());
5376 auto *
Store = cast<StoreSDNode>(
Op);
5377 assert(Store &&
Store->getValue().getValueType().isVector() &&
5378 "Expected vector store");
5381 Store->getMemoryVT(),
5382 *
Store->getMemOperand()))
5389 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5390 "Unexpected unaligned RVV store type");
5394 "Expecting equally-sized RVV vector types to be legal");
5395 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5397 Store->getPointerInfo(),
Store->getOriginalAlign(),
5398 Store->getMemOperand()->getFlags());
5403 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5405 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5432 unsigned ShiftAmt, AddOpc;
5449 if (Subtarget.hasStdExtZtso()) {
5472 "Unexpected custom legalisation");
5493 "Unexpected custom legalisation");
5508 "Unexpected custom legalisation");
5509 if (isa<ConstantSDNode>(
Op.getOperand(1)))
5529 "Unexpected custom legalisation");
5545 MVT VT =
Op.getSimpleValueType();
5547 unsigned Check =
Op.getConstantOperandVal(1);
5548 unsigned TDCMask = 0;
5576 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5581 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5583 VL =
Op.getOperand(3);
5586 VL,
Op->getFlags());
5601 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5603 MVT MaskContainerVT =
5606 VL =
Op.getOperand(3);
5611 Mask, VL,
Op->getFlags());
5614 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5619 DAG.
getUNDEF(ContainerVT), Mask, VL});
5623 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5627 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5631 DAG.
getUNDEF(ContainerVT), Mask, VL});
5647 MVT VT =
Op.getSimpleValueType();
5674 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5681 MVT ContainerVT = VT;
5689 if (
Op->isVPOpcode()) {
5690 Mask =
Op.getOperand(2);
5694 VL =
Op.getOperand(3);
5702 {X, X, DAG.getCondCode(ISD::SETOEQ),
5703 DAG.getUNDEF(ContainerVT), Mask, VL});
5711 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5712 DAG.getUNDEF(ContainerVT), Mask, VL});
5722 DAG.
getUNDEF(ContainerVT), Mask, VL);
5730#define OP_CASE(NODE) \
5732 return RISCVISD::NODE##_VL;
5733#define VP_CASE(NODE) \
5734 case ISD::VP_##NODE: \
5735 return RISCVISD::NODE##_VL;
5737 switch (
Op.getOpcode()) {
5813 case ISD::VP_CTLZ_ZERO_UNDEF:
5816 case ISD::VP_CTTZ_ZERO_UNDEF:
5825 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5830 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5835 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5838 case ISD::VP_SELECT:
5847 case ISD::VP_SIGN_EXTEND:
5849 case ISD::VP_ZERO_EXTEND:
5851 case ISD::VP_FP_TO_SINT:
5853 case ISD::VP_FP_TO_UINT:
5856 case ISD::VP_FMINNUM:
5859 case ISD::VP_FMAXNUM:
5864 case ISD::VP_LLRINT:
5876 "not a RISC-V target specific op");
5882 "adding target specific op should update this function");
5902 "not a RISC-V target specific op");
5908 "adding target specific op should update this function");
5927 if (!
Op.getOperand(j).getValueType().isVector()) {
5928 LoOperands[j] =
Op.getOperand(j);
5929 HiOperands[j] =
Op.getOperand(j);
5932 std::tie(LoOperands[j], HiOperands[j]) =
5937 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
5939 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
5954 std::tie(LoOperands[j], HiOperands[j]) =
5958 if (!
Op.getOperand(j).getValueType().isVector()) {
5959 LoOperands[j] =
Op.getOperand(j);
5960 HiOperands[j] =
Op.getOperand(j);
5963 std::tie(LoOperands[j], HiOperands[j]) =
5968 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
5970 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
5980 auto [EVLLo, EVLHi] =
5981 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
5985 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
5987 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6005 if (!
Op.getOperand(j).getValueType().isVector()) {
6006 LoOperands[j] =
Op.getOperand(j);
6007 HiOperands[j] =
Op.getOperand(j);
6010 std::tie(LoOperands[j], HiOperands[j]) =
6015 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6018 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6027 switch (
Op.getOpcode()) {
6033 return lowerGlobalAddress(
Op, DAG);
6035 return lowerBlockAddress(
Op, DAG);
6037 return lowerConstantPool(
Op, DAG);
6039 return lowerJumpTable(
Op, DAG);
6041 return lowerGlobalTLSAddress(
Op, DAG);
6045 return lowerSELECT(
Op, DAG);
6047 return lowerBRCOND(
Op, DAG);
6049 return lowerVASTART(
Op, DAG);
6051 return lowerFRAMEADDR(
Op, DAG);
6053 return lowerRETURNADDR(
Op, DAG);
6060 return lowerShiftLeftParts(
Op, DAG);
6062 return lowerShiftRightParts(
Op, DAG,
true);
6064 return lowerShiftRightParts(
Op, DAG,
false);
6067 if (
Op.getValueType().isFixedLengthVector()) {
6068 assert(Subtarget.hasStdExtZvkb());
6069 return lowerToScalableOp(
Op, DAG);
6071 assert(Subtarget.hasVendorXTHeadBb() &&
6072 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6073 "Unexpected custom legalization");
6075 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6080 EVT VT =
Op.getValueType();
6084 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6090 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6091 Subtarget.hasStdExtZfbfmin()) {
6096 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6103 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6120 "Unexpected types");
6154 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6156 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6158 return LowerINTRINSIC_VOID(
Op, DAG);
6160 return LowerIS_FPCLASS(
Op, DAG);
6162 MVT VT =
Op.getSimpleValueType();
6164 assert(Subtarget.hasStdExtZvbb());
6165 return lowerToScalableOp(
Op, DAG);
6168 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6176 if (!
Op.getSimpleValueType().isVector())
6178 return lowerVectorTruncLike(
Op, DAG);
6181 if (
Op.getOperand(0).getValueType().isVector() &&
6182 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6183 return lowerVectorMaskExt(
Op, DAG, 1);
6186 if (
Op.getOperand(0).getValueType().isVector() &&
6187 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6188 return lowerVectorMaskExt(
Op, DAG, -1);
6191 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6193 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6195 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6197 MVT VT =
Op.getSimpleValueType();
6205 MVT ContainerVT = VT;
6211 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6218 MVT VT =
Op.getSimpleValueType();
6238 }
else if ((Val % 8) == 0) {
6254 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6255 Op.getOperand(1).getValueType() == MVT::i32) {
6267 if (
Op.getValueType() == MVT::nxv32f16 &&
6274 EVT VT =
Op.getValueType();
6277 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6279 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6285 if (!
Op.getValueType().isVector())
6287 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6291 EVT VT =
Op.getValueType();
6294 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6296 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6304 if (!
Op.getValueType().isVector())
6306 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6310 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6313 if (
Op.getValueType().isVector() &&
6314 Op.getValueType().getScalarType() == MVT::f16 &&
6317 if (
Op.getValueType() == MVT::nxv32f16)
6332 Op1.getValueType().isVector() &&
6333 Op1.getValueType().getScalarType() == MVT::f16 &&
6336 if (Op1.getValueType() == MVT::nxv32f16)
6341 Op1.getValueType().getVectorElementCount());
6344 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6354 MVT VT =
Op.getSimpleValueType();
6358 bool IsStrict =
Op->isStrictFPOpcode();
6359 SDValue Src =
Op.getOperand(0 + IsStrict);
6361 MVT SrcVT = Src.getSimpleValueType();
6366 "Unexpected vector element types");
6370 if (EltSize > (2 * SrcEltSize)) {
6382 Op.getOperand(0), Ext);
6386 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6391 auto [FExt, Chain] =
6393 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6400 if (SrcEltSize > (2 * EltSize)) {
6403 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6408 Op.getOperand(0), Src);
6423 Op.getOperand(0), Src);
6437 unsigned RVVOpc = 0;
6438 switch (
Op.getOpcode()) {
6470 "Expected same element count");
6477 Op.getOperand(0), Src, Mask, VL);
6481 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6496 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6503 MVT VT =
Op.getSimpleValueType();
6525 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6541 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6561 return lowerVECREDUCE(
Op, DAG);
6565 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6566 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6567 return lowerVECREDUCE(
Op, DAG);
6574 return lowerFPVECREDUCE(
Op, DAG);
6575 case ISD::VP_REDUCE_ADD:
6576 case ISD::VP_REDUCE_UMAX:
6577 case ISD::VP_REDUCE_SMAX:
6578 case ISD::VP_REDUCE_UMIN:
6579 case ISD::VP_REDUCE_SMIN:
6580 case ISD::VP_REDUCE_FADD:
6581 case ISD::VP_REDUCE_SEQ_FADD:
6582 case ISD::VP_REDUCE_FMIN:
6583 case ISD::VP_REDUCE_FMAX:
6584 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6588 return lowerVPREDUCE(
Op, DAG);
6589 case ISD::VP_REDUCE_AND:
6590 case ISD::VP_REDUCE_OR:
6591 case ISD::VP_REDUCE_XOR:
6592 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6593 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6594 return lowerVPREDUCE(
Op, DAG);
6598 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6601 return lowerINSERT_SUBVECTOR(
Op, DAG);
6603 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6605 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6607 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6609 return lowerSTEP_VECTOR(
Op, DAG);
6611 return lowerVECTOR_REVERSE(
Op, DAG);
6613 return lowerVECTOR_SPLICE(
Op, DAG);
6617 if (
Op.getValueType().getScalarType() == MVT::f16 &&
6620 if (
Op.getValueType() == MVT::nxv32f16)
6632 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6633 return lowerVectorMaskSplat(
Op, DAG);
6641 MVT VT =
Op.getSimpleValueType();
6642 MVT ContainerVT = VT;
6660 Op->ops().take_front(HalfNumOps));
6662 Op->ops().drop_front(HalfNumOps));
6666 unsigned NumOpElts =
6667 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6670 SDValue SubVec = OpIdx.value();
6681 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6683 if (
Op.getValueType().isFixedLengthVector())
6684 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6687 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6689 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6690 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6694 return lowerMaskedLoad(
Op, DAG);
6697 return lowerMaskedStore(
Op, DAG);
6706 EVT VT =
Op.getValueType();
6717 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6719 MVT VT =
Op.getSimpleValueType();
6724 "Unexpected CondCode");
6732 if (isa<ConstantSDNode>(
RHS)) {
6733 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6734 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6753 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6758 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
6774 return lowerToScalableOp(
Op, DAG);
6778 if (
Op.getSimpleValueType().isFixedLengthVector())
6779 return lowerToScalableOp(
Op, DAG);
6781 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
6782 "Unexpected custom legalisation");
6794 if (
Op.getValueType() == MVT::nxv32f16 &&
6805 return lowerToScalableOp(
Op, DAG);
6808 if (!
Op.getValueType().isVector())
6810 return lowerToScalableOp(
Op, DAG);
6813 if (!
Op.getValueType().isVector())
6815 return lowerToScalableOp(
Op, DAG);
6819 EVT VT =
Op->getValueType(0);
6834 return lowerABS(
Op, DAG);
6839 if (Subtarget.hasStdExtZvbb())
6840 return lowerToScalableOp(
Op, DAG);
6842 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6844 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
6846 if (
Op.getValueType() == MVT::nxv32f16 &&
6850 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
6857 if (
Op.getValueType() == MVT::nxv32f16 &&
6861 return lowerToScalableOp(
Op, DAG);
6864 return lowerVectorStrictFSetcc(
Op, DAG);
6874 case ISD::VP_GATHER:
6875 return lowerMaskedGather(
Op, DAG);
6877 case ISD::VP_SCATTER:
6878 return lowerMaskedScatter(
Op, DAG);
6880 return lowerGET_ROUNDING(
Op, DAG);
6882 return lowerSET_ROUNDING(
Op, DAG);
6884 return lowerEH_DWARF_CFA(
Op, DAG);
6885 case ISD::VP_SELECT:
6894 case ISD::VP_UADDSAT:
6895 case ISD::VP_USUBSAT:
6896 case ISD::VP_SADDSAT:
6897 case ISD::VP_SSUBSAT:
6899 case ISD::VP_LLRINT:
6900 return lowerVPOp(
Op, DAG);
6904 return lowerLogicVPOp(
Op, DAG);
6913 case ISD::VP_FMINNUM:
6914 case ISD::VP_FMAXNUM:
6915 case ISD::VP_FCOPYSIGN:
6916 if (
Op.getValueType() == MVT::nxv32f16 &&
6924 return lowerVPOp(
Op, DAG);
6925 case ISD::VP_IS_FPCLASS:
6926 return LowerIS_FPCLASS(
Op, DAG);
6927 case ISD::VP_SIGN_EXTEND:
6928 case ISD::VP_ZERO_EXTEND:
6929 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
6930 return lowerVPExtMaskOp(
Op, DAG);
6931 return lowerVPOp(
Op, DAG);
6932 case ISD::VP_TRUNCATE:
6933 return lowerVectorTruncLike(
Op, DAG);
6934 case ISD::VP_FP_EXTEND:
6935 case ISD::VP_FP_ROUND:
6936 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6937 case ISD::VP_SINT_TO_FP:
6938 case ISD::VP_UINT_TO_FP:
6939 if (
Op.getValueType().isVector() &&
6940 Op.getValueType().getScalarType() == MVT::f16 &&
6943 if (
Op.getValueType() == MVT::nxv32f16)
6955 case ISD::VP_FP_TO_SINT:
6956 case ISD::VP_FP_TO_UINT:
6958 Op1.getValueType().isVector() &&
6959 Op1.getValueType().getScalarType() == MVT::f16 &&
6962 if (Op1.getValueType() == MVT::nxv32f16)
6967 Op1.getValueType().getVectorElementCount());
6971 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
6973 return lowerVPFPIntConvOp(
Op, DAG);
6975 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6979 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
6980 return lowerVPSetCCMaskOp(
Op, DAG);
6986 case ISD::VP_BITREVERSE:
6988 return lowerVPOp(
Op, DAG);
6990 case ISD::VP_CTLZ_ZERO_UNDEF:
6991 if (Subtarget.hasStdExtZvbb())
6992 return lowerVPOp(
Op, DAG);
6993 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6995 case ISD::VP_CTTZ_ZERO_UNDEF:
6996 if (Subtarget.hasStdExtZvbb())
6997 return lowerVPOp(
Op, DAG);
6998 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7000 return lowerVPOp(
Op, DAG);
7001 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7002 return lowerVPStridedLoad(
Op, DAG);
7003 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7004 return lowerVPStridedStore(
Op, DAG);
7006 case ISD::VP_FFLOOR:
7008 case ISD::VP_FNEARBYINT:
7009 case ISD::VP_FROUND:
7010 case ISD::VP_FROUNDEVEN:
7011 case ISD::VP_FROUNDTOZERO:
7012 if (
Op.getValueType() == MVT::nxv32f16 &&
7017 case ISD::VP_FMAXIMUM:
7018 case ISD::VP_FMINIMUM:
7019 if (
Op.getValueType() == MVT::nxv32f16 &&
7024 case ISD::EXPERIMENTAL_VP_SPLICE:
7025 return lowerVPSpliceExperimental(
Op, DAG);
7026 case ISD::EXPERIMENTAL_VP_REVERSE:
7027 return lowerVPReverseExperimental(
Op, DAG);
7045 N->getOffset(), Flags);
7053template <
class NodeTy>
7055 bool IsLocal,
bool IsExternWeak)
const {
7065 if (IsLocal && !Subtarget.allowTaggedGlobals())
7127 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7136 return getAddr(
N, DAG);
7143 return getAddr(
N, DAG);
7150 return getAddr(
N, DAG);
7155 bool UseGOT)
const {
7219 Args.push_back(Entry);
7252 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7266 Addr = getStaticTLSAddr(
N, DAG,
false);
7269 Addr = getStaticTLSAddr(
N, DAG,
true);
7274 : getDynamicTLSAddr(
N, DAG);
7291 if (
LHS == LHS2 &&
RHS == RHS2) {
7296 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7304 return std::nullopt;
7312 MVT VT =
N->getSimpleValueType(0);
7342 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7345 if (~TrueVal == FalseVal) {
7385 if (Subtarget.hasShortForwardBranchOpt())
7388 unsigned SelOpNo = 0;
7398 unsigned ConstSelOpNo = 1;
7399 unsigned OtherSelOpNo = 2;
7400 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7405 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7406 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7410 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7411 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7417 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7419 std::swap(NewConstOps[0], NewConstOps[1]);
7431 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7433 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7436 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7437 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7446 MVT VT =
Op.getSimpleValueType();
7460 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7488 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7492 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7494 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7495 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7497 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7502 DL, VT, LHSVal, CondV);
7518 if (
Op.hasOneUse()) {
7519 unsigned UseOpc =
Op->use_begin()->getOpcode();
7525 return lowerSELECT(NewSel, DAG);
7553 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7574 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7578 if (TrueVal - 1 == FalseVal)
7580 if (TrueVal + 1 == FalseVal)
7587 RHS == TrueV && LHS == FalseV) {
7604 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7630 LHS, RHS, TargetCC,
Op.getOperand(2));
7648 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7660 int XLenInBytes = Subtarget.
getXLen() / 8;
7662 EVT VT =
Op.getValueType();
7665 unsigned Depth =
Op.getConstantOperandVal(0);
7667 int Offset = -(XLenInBytes * 2);
7683 int XLenInBytes = Subtarget.
getXLen() / 8;
7688 EVT VT =
Op.getValueType();
7690 unsigned Depth =
Op.getConstantOperandVal(0);
7692 int Off = -XLenInBytes;
7693 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7712 EVT VT =
Lo.getValueType();
7751 EVT VT =
Lo.getValueType();
7802 MVT VT =
Op.getSimpleValueType();
7828 MVT VecVT =
Op.getSimpleValueType();
7830 "Unexpected SPLAT_VECTOR_PARTS lowering");
7836 MVT ContainerVT = VecVT;
7856 int64_t ExtTrueVal)
const {
7858 MVT VecVT =
Op.getSimpleValueType();
7861 assert(Src.getValueType().isVector() &&
7862 Src.getValueType().getVectorElementType() == MVT::i1);
7883 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7885 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
7888 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
7893SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
7895 MVT ExtVT =
Op.getSimpleValueType();
7899 MVT VT =
Op.getOperand(0).getSimpleValueType();
7925 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
7927 EVT MaskVT =
Op.getValueType();
7930 "Unexpected type for vector mask lowering");
7932 MVT VecVT = Src.getSimpleValueType();
7936 VL =
Op.getOperand(2);
7939 MVT ContainerVT = VecVT;
7945 MVT MaskContainerVT =
7952 std::tie(Mask, VL) =
7960 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
7962 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7966 DAG.
getUNDEF(ContainerVT), Mask, VL);
7969 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
7977 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
7980 MVT VT =
Op.getSimpleValueType();
7982 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
7986 return lowerVectorMaskTruncLike(
Op, DAG);
7994 MVT SrcVT = Src.getSimpleValueType();
7999 "Unexpected vector truncate lowering");
8001 MVT ContainerVT = SrcVT;
8005 VL =
Op.getOperand(2);
8018 std::tie(Mask, VL) =
8029 }
while (SrcEltVT != DstEltVT);
8038RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8043 MVT VT =
Op.getSimpleValueType();
8044 MVT SrcVT = Src.getSimpleValueType();
8045 MVT ContainerVT = VT;
8066 Chain, Src, Mask, VL);
8067 Chain = Src.getValue(1);
8074 Chain, Src, Mask, VL);
8085RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8088 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8095 MVT VT =
Op.getSimpleValueType();
8097 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8100 MVT SrcVT = Src.getSimpleValueType();
8107 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8110 MVT ContainerVT = VT;
8114 VL =
Op.getOperand(2);
8128 std::tie(Mask, VL) =
8134 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8140 unsigned InterConvOpc =
8145 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8147 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8158static std::optional<MVT>
8164 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8166 if (MaxIdx < MinVLMAX)
8168 else if (MaxIdx < MinVLMAX * 2)
8170 else if (MaxIdx < MinVLMAX * 4)
8175 return std::nullopt;
8188 MVT VecVT =
Op.getSimpleValueType();
8202 MVT ContainerVT = VecVT;
8211 MVT OrigContainerVT = ContainerVT;
8214 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8215 const unsigned OrigIdx = IdxC->getZExtValue();
8218 DL, DAG, Subtarget)) {
8219 ContainerVT = *ShrunkVT;
8228 VLEN && ContainerVT.
bitsGT(M1VT)) {
8231 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8232 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8233 unsigned ExtractIdx =
8252 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8253 const auto *CVal = cast<ConstantSDNode>(Val);
8254 if (isInt<32>(CVal->getSExtValue())) {
8255 IsLegalInsert =
true;
8264 if (IsLegalInsert) {
8270 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8286 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8287 MVT I32ContainerVT =
8298 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8303 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8305 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8310 ValInVec, AlignedIdx);
8320 DAG.
getUNDEF(I32ContainerVT), ValLo,
8321 I32Mask, InsertI64VL);
8323 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8324 I32Mask, InsertI64VL);
8326 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8339 Idx, Mask, InsertVL, Policy);
8343 Slideup, AlignedIdx);
8358 EVT EltVT =
Op.getValueType();
8365 MVT ContainerVT = VecVT;
8381 unsigned WidenVecLen;
8384 unsigned MaxEEW = Subtarget.
getELen();
8389 "the number of elements should be power of 2");
8393 ExtractBitIdx =
Idx;
8395 WideEltVT = LargestEltVT;
8398 ExtractElementIdx = DAG.
getNode(
8409 Vec, ExtractElementIdx);
8425 MVT ContainerVT = VecVT;
8436 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8437 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8439 unsigned OrigIdx = IdxC->getZExtValue();
8442 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8443 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8444 unsigned ExtractIdx =
8454 std::optional<uint64_t> MaxIdx;
8457 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8458 MaxIdx = IdxC->getZExtValue();
8460 if (
auto SmallerVT =
8462 ContainerVT = *SmallerVT;
8509 "Unexpected opcode");
8516 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8521 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8538 if (OpVT.
bitsLT(XLenVT)) {
8545 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8556 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8559 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8570 case Intrinsic::riscv_vslide1up:
8571 case Intrinsic::riscv_vslide1down:
8572 case Intrinsic::riscv_vslide1up_mask:
8573 case Intrinsic::riscv_vslide1down_mask: {
8576 bool IsMasked = NumOps == 7;
8582 std::tie(ScalarLo, ScalarHi) =
8590 if (isa<ConstantSDNode>(AVL)) {
8591 const auto [MinVLMAX, MaxVLMAX] =
8595 if (AVLInt <= MinVLMAX) {
8597 }
else if (AVLInt >= 2 * MaxVLMAX) {
8604 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
8638 if (IntNo == Intrinsic::riscv_vslide1up ||
8639 IntNo == Intrinsic::riscv_vslide1up_mask) {
8641 ScalarHi, I32Mask, I32VL);
8643 ScalarLo, I32Mask, I32VL);
8646 ScalarLo, I32Mask, I32VL);
8648 ScalarHi, I32Mask, I32VL);
8697 const unsigned ElementWidth = 8;
8702 [[maybe_unused]]
unsigned MinVF =
8705 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8709 bool Fractional = VF < LMul1VF;
8710 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8731 MVT ContainerVT = OpVT;
8758 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8762 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8779 if (OpVT.
bitsLT(XLenVT)) {
8782 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8795 EVT ValType = V.getValueType();
8796 if (ValType.isVector() && ValType.isFloatingPoint()) {
8799 ValType.getVectorElementCount());
8802 if (ValType.isFixedLengthVector()) {
8804 DAG, V.getSimpleValueType(), Subtarget);
8820 unsigned IntNo =
Op.getConstantOperandVal(0);
8827 case Intrinsic::thread_pointer: {
8831 case Intrinsic::riscv_orc_b:
8832 case Intrinsic::riscv_brev8:
8833 case Intrinsic::riscv_sha256sig0:
8834 case Intrinsic::riscv_sha256sig1:
8835 case Intrinsic::riscv_sha256sum0:
8836 case Intrinsic::riscv_sha256sum1:
8837 case Intrinsic::riscv_sm3p0:
8838 case Intrinsic::riscv_sm3p1: {
8858 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8860 case Intrinsic::riscv_sm4ks:
8861 case Intrinsic::riscv_sm4ed: {
8871 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
Op.getOperand(3));
8875 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
8878 case Intrinsic::riscv_zip:
8879 case Intrinsic::riscv_unzip: {
8882 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8884 case Intrinsic::riscv_mopr: {
8897 case Intrinsic::riscv_moprr: {
8909 Op.getOperand(2),
Op.getOperand(3));
8911 case Intrinsic::riscv_clmul:
8922 case Intrinsic::riscv_clmulh:
8923 case Intrinsic::riscv_clmulr: {
8941 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
8943 case Intrinsic::experimental_get_vector_length:
8945 case Intrinsic::experimental_cttz_elts:
8947 case Intrinsic::riscv_vmv_x_s: {
8951 case Intrinsic::riscv_vfmv_f_s:
8954 case Intrinsic::riscv_vmv_v_x:
8956 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
8958 case Intrinsic::riscv_vfmv_v_f:
8960 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
8961 case Intrinsic::riscv_vmv_s_x: {
8964 if (
Scalar.getValueType().bitsLE(XLenVT)) {
8967 Op.getOperand(1), Scalar,
Op.getOperand(3));
8970 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
8987 MVT VT =
Op.getSimpleValueType();
8992 if (
Op.getOperand(1).isUndef())
9008 case Intrinsic::riscv_vfmv_s_f:
9010 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9012 case Intrinsic::riscv_vaesdf_vv:
9013 case Intrinsic::riscv_vaesdf_vs:
9014 case Intrinsic::riscv_vaesdm_vv:
9015 case Intrinsic::riscv_vaesdm_vs:
9016 case Intrinsic::riscv_vaesef_vv:
9017 case Intrinsic::riscv_vaesef_vs:
9018 case Intrinsic::riscv_vaesem_vv:
9019 case Intrinsic::riscv_vaesem_vs:
9020 case Intrinsic::riscv_vaeskf1:
9021 case Intrinsic::riscv_vaeskf2:
9022 case Intrinsic::riscv_vaesz_vs:
9023 case Intrinsic::riscv_vsm4k:
9024 case Intrinsic::riscv_vsm4r_vv:
9025 case Intrinsic::riscv_vsm4r_vs: {
9026 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9027 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9028 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9033 case Intrinsic::riscv_vsm3c:
9034 case Intrinsic::riscv_vsm3me: {
9035 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9036 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9041 case Intrinsic::riscv_vsha2ch:
9042 case Intrinsic::riscv_vsha2cl:
9043 case Intrinsic::riscv_vsha2ms: {
9044 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9045 !Subtarget.hasStdExtZvknhb())
9047 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9048 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9049 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9053 case Intrinsic::riscv_sf_vc_v_x:
9054 case Intrinsic::riscv_sf_vc_v_i:
9055 case Intrinsic::riscv_sf_vc_v_xv:
9056 case Intrinsic::riscv_sf_vc_v_iv:
9057 case Intrinsic::riscv_sf_vc_v_vv:
9058 case Intrinsic::riscv_sf_vc_v_fv:
9059 case Intrinsic::riscv_sf_vc_v_xvv:
9060 case Intrinsic::riscv_sf_vc_v_ivv:
9061 case Intrinsic::riscv_sf_vc_v_vvv:
9062 case Intrinsic::riscv_sf_vc_v_fvv:
9063 case Intrinsic::riscv_sf_vc_v_xvw:
9064 case Intrinsic::riscv_sf_vc_v_ivw:
9065 case Intrinsic::riscv_sf_vc_v_vvw:
9066 case Intrinsic::riscv_sf_vc_v_fvw: {
9067 MVT VT =
Op.getSimpleValueType();
9104 MVT VT =
Op.getSimpleValueType();
9108 if (VT.isFloatingPoint()) {
9113 if (VT.isFixedLengthVector())
9123 if (VT.isFixedLengthVector())
9125 if (VT.isFloatingPoint())
9144 unsigned IntNo =
Op.getConstantOperandVal(1);
9148 case Intrinsic::riscv_masked_strided_load: {
9157 MVT VT =
Op->getSimpleValueType(0);
9158 MVT ContainerVT = VT;
9171 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9183 ScalarVT,
Load->getMemOperand());
9189 Load->getMemOperand());
9194 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask,
DL,
9199 Ops.push_back(DAG.
getUNDEF(ContainerVT));
9201 Ops.push_back(PassThru);
9203 Ops.push_back(Stride);
9205 Ops.push_back(Mask);
9210 Ops.push_back(Policy);
9216 Load->getMemoryVT(),
Load->getMemOperand());
9217 Chain =
Result.getValue(1);
9223 case Intrinsic::riscv_seg2_load:
9224 case Intrinsic::riscv_seg3_load:
9225 case Intrinsic::riscv_seg4_load:
9226 case Intrinsic::riscv_seg5_load:
9227 case Intrinsic::riscv_seg6_load:
9228 case Intrinsic::riscv_seg7_load:
9229 case Intrinsic::riscv_seg8_load: {
9232 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9233 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9234 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9235 Intrinsic::riscv_vlseg8};
9236 unsigned NF =
Op->getNumValues() - 1;
9237 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9239 MVT VT =
Op->getSimpleValueType(0);
9245 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9247 ContainerVTs.push_back(MVT::Other);
9255 Load->getMemoryVT(),
Load->getMemOperand());
9257 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9263 case Intrinsic::riscv_sf_vc_v_x_se:
9265 case Intrinsic::riscv_sf_vc_v_i_se:
9267 case Intrinsic::riscv_sf_vc_v_xv_se:
9269 case Intrinsic::riscv_sf_vc_v_iv_se:
9271 case Intrinsic::riscv_sf_vc_v_vv_se:
9273 case Intrinsic::riscv_sf_vc_v_fv_se:
9275 case Intrinsic::riscv_sf_vc_v_xvv_se:
9277 case Intrinsic::riscv_sf_vc_v_ivv_se:
9279 case Intrinsic::riscv_sf_vc_v_vvv_se:
9281 case Intrinsic::riscv_sf_vc_v_fvv_se:
9283 case Intrinsic::riscv_sf_vc_v_xvw_se:
9285 case Intrinsic::riscv_sf_vc_v_ivw_se:
9287 case Intrinsic::riscv_sf_vc_v_vvw_se:
9289 case Intrinsic::riscv_sf_vc_v_fvw_se:
9298 unsigned IntNo =
Op.getConstantOperandVal(1);
9302 case Intrinsic::riscv_masked_strided_store: {
9313 MVT ContainerVT = VT;
9327 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask,
DL,
9330 auto *
Store = cast<MemIntrinsicSDNode>(
Op);
9340 Ops,
Store->getMemoryVT(),
9341 Store->getMemOperand());
9343 case Intrinsic::riscv_seg2_store:
9344 case Intrinsic::riscv_seg3_store:
9345 case Intrinsic::riscv_seg4_store:
9346 case Intrinsic::riscv_seg5_store:
9347 case Intrinsic::riscv_seg6_store:
9348 case Intrinsic::riscv_seg7_store:
9349 case Intrinsic::riscv_seg8_store: {
9352 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9353 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9354 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9355 Intrinsic::riscv_vsseg8};
9358 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9360 MVT VT =
Op->getOperand(2).getSimpleValueType();
9368 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9370 for (
unsigned i = 0; i < NF; i++)
9372 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9377 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9379 case Intrinsic::riscv_sf_vc_xv_se:
9381 case Intrinsic::riscv_sf_vc_iv_se:
9383 case Intrinsic::riscv_sf_vc_vv_se:
9385 case Intrinsic::riscv_sf_vc_fv_se:
9387 case Intrinsic::riscv_sf_vc_xvv_se:
9389 case Intrinsic::riscv_sf_vc_ivv_se:
9391 case Intrinsic::riscv_sf_vc_vvv_se:
9393 case Intrinsic::riscv_sf_vc_fvv_se:
9395 case Intrinsic::riscv_sf_vc_xvw_se:
9397 case Intrinsic::riscv_sf_vc_ivw_se:
9399 case Intrinsic::riscv_sf_vc_vvw_se:
9401 case Intrinsic::riscv_sf_vc_fvw_se:
9409 switch (ISDOpcode) {
9412 case ISD::VP_REDUCE_ADD:
9415 case ISD::VP_REDUCE_UMAX:
9418 case ISD::VP_REDUCE_SMAX:
9421 case ISD::VP_REDUCE_UMIN:
9424 case ISD::VP_REDUCE_SMIN:
9427 case ISD::VP_REDUCE_AND:
9430 case ISD::VP_REDUCE_OR:
9433 case ISD::VP_REDUCE_XOR:
9436 case ISD::VP_REDUCE_FADD:
9438 case ISD::VP_REDUCE_SEQ_FADD:
9440 case ISD::VP_REDUCE_FMAX:
9442 case ISD::VP_REDUCE_FMIN:
9452 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9457 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9458 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9459 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9460 "Unexpected reduction lowering");
9464 MVT ContainerVT = VecVT;
9473 VL =
Op.getOperand(3);
9475 std::tie(Mask, VL) =
9483 switch (
Op.getOpcode()) {
9487 case ISD::VP_REDUCE_AND: {
9497 case ISD::VP_REDUCE_OR:
9504 case ISD::VP_REDUCE_XOR: {
9527 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9531 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9532 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9533 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9534 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9550 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9554 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9557 if (M1VT != InnerVT)
9563 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9582 VecEVT =
Lo.getValueType();
9595 MVT ContainerVT = VecVT;
9615 Mask, VL,
DL, DAG, Subtarget);
9621static std::tuple<unsigned, SDValue, SDValue>
9625 auto Flags =
Op->getFlags();
9626 unsigned Opcode =
Op.getOpcode();
9650 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9658 MVT VecEltVT =
Op.getSimpleValueType();
9662 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9666 MVT ContainerVT = VecVT;
9672 MVT ResVT =
Op.getSimpleValueType();
9675 VL,
DL, DAG, Subtarget);
9680 if (
Op->getFlags().hasNoNaNs())
9686 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9687 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9693 DL, ResVT, NoNaNs, Res,
9720 Vec, Mask, VL,
DL, DAG, Subtarget);
9732 unsigned OrigIdx =
Op.getConstantOperandVal(2);
9741 (OrigIdx != 0 || !Vec.
isUndef())) {
9744 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9747 "Unexpected mask vector lowering");
9780 MVT ContainerVT = VecVT;
9816 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
9817 SlideupAmt, Mask, VL, Policy);
9825 unsigned SubRegIdx, RemIdx;
9826 std::tie(SubRegIdx, RemIdx) =
9828 VecVT, SubVecVT, OrigIdx,
TRI);
9847 if (RemIdx == 0 && (!IsSubVecPartReg || Vec.
isUndef()))
9855 MVT InterSubVT = VecVT;
9857 unsigned AlignedIdx = OrigIdx - RemIdx;
9893 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
9894 SlideupAmt, Mask, VL, Policy);
9899 if (VecVT.
bitsGT(InterSubVT))
9905 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
9911 MVT SubVecVT =
Op.getSimpleValueType();
9916 unsigned OrigIdx =
Op.getConstantOperandVal(1);
9927 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9930 "Unexpected mask vector lowering");
9969 MVT ContainerVT = VecVT;
9979 ContainerVT = *ShrunkVT;
9993 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10005 MVT ContainerSubVecVT = SubVecVT;
10009 unsigned SubRegIdx;
10019 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10020 SubRegIdx = Decompose.first;
10022 (OrigIdx % Vscale));
10026 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10027 SubRegIdx = Decompose.first;
10050 MVT InterSubVT = VecVT;
10054 assert(SubRegIdx != RISCV::NoSubRegister);
10068 Vec, SlidedownAmt, Mask, VL);
10077 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10084 MVT VT =
N.getSimpleValueType();
10088 assert(
Op.getSimpleValueType() == VT &&
10089 "Operands and result must be same type");
10093 unsigned NumVals =
N->getNumValues();
10096 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10099 for (
unsigned I = 0;
I < NumVals;
I++) {
10105 if (TruncVals.
size() > 1)
10107 return TruncVals.
front();
10113 MVT VecVT =
Op.getSimpleValueType();
10116 "vector_interleave on non-scalable vector!");
10127 EVT SplitVT = Op0Lo.getValueType();
10130 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10132 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10146 Op.getOperand(0),
Op.getOperand(1));
10173 Concat, EvenIdx, Passthru, Mask, VL);
10175 Concat, OddIdx, Passthru, Mask, VL);
10189 MVT VecVT =
Op.getSimpleValueType();
10192 "vector_interleave on non-scalable vector!");
10205 EVT SplitVT = Op0Lo.getValueType();
10208 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10210 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10232 Op.getOperand(0),
Op.getOperand(1));
10280 MVT VT =
Op.getSimpleValueType();
10285 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10286 if (StepValImm != 1) {
10295 VL, VT,
DL, DAG, Subtarget);
10310 MVT VecVT =
Op.getSimpleValueType();
10320 unsigned MaxVLMAX =
10330 if (MaxVLMAX > 256 && EltSize == 8) {
10377 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10387 MVT VecVT =
Op.getSimpleValueType();
10391 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10392 SDValue DownOffset, UpOffset;
10393 if (ImmValue >= 0) {
10409 DownOffset, TrueMask, UpOffset);
10410 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10416RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10419 auto *
Load = cast<LoadSDNode>(
Op);
10422 Load->getMemoryVT(),
10423 *
Load->getMemOperand()) &&
10424 "Expecting a correctly-aligned load");
10426 MVT VT =
Op.getSimpleValueType();
10432 const auto [MinVLMAX, MaxVLMAX] =
10435 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10449 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10458 Load->getMemoryVT(),
Load->getMemOperand());
10465RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10468 auto *
Store = cast<StoreSDNode>(
Op);
10471 Store->getMemoryVT(),
10472 *
Store->getMemOperand()) &&
10473 "Expecting a correctly-aligned store");
10495 const auto [MinVLMAX, MaxVLMAX] =
10498 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10510 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10513 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10514 Store->getMemoryVT(),
Store->getMemOperand());
10520 MVT VT =
Op.getSimpleValueType();
10522 const auto *MemSD = cast<MemSDNode>(
Op);
10523 EVT MemVT = MemSD->getMemoryVT();
10525 SDValue Chain = MemSD->getChain();
10529 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10530 Mask = VPLoad->getMask();
10532 VL = VPLoad->getVectorLength();
10534 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10535 Mask = MLoad->getMask();
10536 PassThru = MLoad->getPassThru();
10543 MVT ContainerVT = VT;
10557 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10574 Chain =
Result.getValue(1);
10586 const auto *MemSD = cast<MemSDNode>(
Op);
10587 EVT MemVT = MemSD->getMemoryVT();
10589 SDValue Chain = MemSD->getChain();
10593 bool IsCompressingStore =
false;
10594 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10595 Val = VPStore->getValue();
10596 Mask = VPStore->getMask();
10597 VL = VPStore->getVectorLength();
10599 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10600 Val = MStore->getValue();
10601 Mask = MStore->getMask();
10602 IsCompressingStore = MStore->isCompressingStore();
10611 MVT ContainerVT = VT;
10616 if (!IsUnmasked || IsCompressingStore) {
10625 if (IsCompressingStore) {
10628 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10635 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10644 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10648RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10650 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10653 MVT VT =
Op.getSimpleValueType();
10667 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
10674 unsigned Opc =
Op.getOpcode();
10681 MVT VT =
Op.getSimpleValueType();
10714 MVT ContainerInVT = InVT;
10733 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10737 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10745 {Chain, Op1, Op2, CC, Mask, Mask, VL});
10750 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
10763 MVT VT =
Op.getSimpleValueType();
10767 "Unexpected type for ISD::ABS");
10769 MVT ContainerVT = VT;
10776 if (
Op->getOpcode() == ISD::VP_ABS) {
10777 Mask =
Op->getOperand(1);
10781 VL =
Op->getOperand(2);
10789 DAG.
getUNDEF(ContainerVT), Mask, VL);
10791 DAG.
getUNDEF(ContainerVT), Mask, VL);
10798SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
10801 MVT VT =
Op.getSimpleValueType();
10805 "Can only handle COPYSIGN with matching types.");
10814 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
10819SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
10821 MVT VT =
Op.getSimpleValueType();
10824 MVT I1ContainerVT =
10838 Op2, DAG.
getUNDEF(ContainerVT), VL);
10849 MVT VT =
Op.getSimpleValueType();
10854 for (
const SDValue &V :
Op->op_values()) {
10855 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10858 if (!
V.getValueType().isVector()) {
10864 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
10865 "Only fixed length vectors are supported!");
10879 if (
Op->isStrictFPOpcode()) {
10888 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
10902 MVT VT =
Op.getSimpleValueType();
10905 MVT ContainerVT = VT;
10911 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10917 if (*MaskIdx == OpIdx.index())
10921 if (
Op.getOpcode() == ISD::VP_MERGE) {
10925 assert(
Op.getOpcode() == ISD::VP_SELECT);
10932 if (!
V.getValueType().isFixedLengthVector()) {
10937 MVT OpVT =
V.getSimpleValueType();
10939 assert(useRVVForFixedLengthVectorVT(OpVT) &&
10940 "Only fixed length vectors are supported!");
10945 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
10955 MVT VT =
Op.getSimpleValueType();
10961 MVT ContainerVT = VT;
10971 DAG.
getUNDEF(ContainerVT), Zero, VL);
10974 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
10976 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
10979 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
10988 MVT VT =
Op.getSimpleValueType();
10992 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
10996 MVT ContainerVT = VT;
11006 switch (Condition) {
11074 MVT DstVT =
Op.getSimpleValueType();
11075 MVT SrcVT = Src.getSimpleValueType();
11088 if (DstEltSize >= SrcEltSize) {
11097 if (SrcEltSize == 1) {
11108 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11109 }
else if (DstEltSize > (2 * SrcEltSize)) {
11113 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11119 "Wrong input/output vector types");
11122 if (DstEltSize > (2 * SrcEltSize)) {
11138 MVT InterimFVT = DstVT;
11139 if (SrcEltSize > (2 * DstEltSize)) {
11140 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11147 if (InterimFVT != DstVT) {
11153 "Wrong input/output vector types");
11157 if (DstEltSize == 1) {
11160 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11170 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11180 while (InterimIVT != DstVT) {
11192 MVT VT =
Op.getSimpleValueType();
11199RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11211 MVT VT =
Op.getSimpleValueType();
11212 MVT ContainerVT = VT;
11225 if (IsMaskVector) {
11236 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11245 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11248 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11249 SDValue DownOffset, UpOffset;
11250 if (ImmValue >= 0) {
11264 Op1, DownOffset, Mask, UpOffset);
11268 if (IsMaskVector) {
11272 {Result, DAG.getConstant(0, DL, ContainerVT),
11273 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11283RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11286 MVT VT =
Op.getSimpleValueType();
11293 MVT ContainerVT = VT;
11301 MVT GatherVT = ContainerVT;
11305 if (IsMaskVector) {
11316 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11322 unsigned MaxVLMAX =
11331 if (MaxVLMAX > 256 && EltSize == 8) {
11359 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11361 if (IsMaskVector) {
11384 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11386 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11388 DAG.
getUNDEF(GatherVT), Mask, EVL);
11390 if (IsMaskVector) {
11405 MVT VT =
Op.getSimpleValueType();
11407 return lowerVPOp(
Op, DAG);
11414 MVT ContainerVT = VT;
11433 MVT VT =
Op.getSimpleValueType();
11434 MVT ContainerVT = VT;
11440 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11446 : Intrinsic::riscv_vlse_mask,
11449 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11450 VPNode->getStride()};
11458 Ops.
push_back(VPNode->getVectorLength());
11466 VPNode->getMemoryVT(), VPNode->getMemOperand());
11480 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11481 SDValue StoreVal = VPNode->getValue();
11483 MVT ContainerVT = VT;
11494 : Intrinsic::riscv_vsse_mask,
11497 VPNode->getBasePtr(), VPNode->getStride()};
11505 Ops.
push_back(VPNode->getVectorLength());
11508 Ops, VPNode->getMemoryVT(),
11509 VPNode->getMemOperand());
11521 MVT VT =
Op.getSimpleValueType();
11523 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11524 EVT MemVT = MemSD->getMemoryVT();
11526 SDValue Chain = MemSD->getChain();
11532 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11533 Index = VPGN->getIndex();
11534 Mask = VPGN->getMask();
11536 VL = VPGN->getVectorLength();
11541 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11542 Index = MGN->getIndex();
11543 Mask = MGN->getMask();
11544 PassThru = MGN->getPassThru();
11548 MVT IndexVT =
Index.getSimpleValueType();
11552 "Unexpected VTs!");
11553 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11556 "Unexpected extending MGATHER/VP_GATHER");
11562 MVT ContainerVT = VT;
11586 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11603 Chain =
Result.getValue(1);
11620 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11621 EVT MemVT = MemSD->getMemoryVT();
11623 SDValue Chain = MemSD->getChain();
11626 [[maybe_unused]]
bool IsTruncatingStore =
false;
11629 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11630 Index = VPSN->getIndex();
11631 Mask = VPSN->getMask();
11632 Val = VPSN->getValue();
11633 VL = VPSN->getVectorLength();
11635 IsTruncatingStore =
false;
11638 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
11639 Index = MSN->getIndex();
11640 Mask = MSN->getMask();
11641 Val = MSN->getValue();
11642 IsTruncatingStore = MSN->isTruncatingStore();
11646 MVT IndexVT =
Index.getSimpleValueType();
11650 "Unexpected VTs!");
11651 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11654 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
11660 MVT ContainerVT = VT;
11684 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
11694 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11703 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11711 static const int Table =
11735 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11741 static const unsigned Table =
11764 bool isRISCV64 = Subtarget.
is64Bit();
11828 switch (
N->getOpcode()) {
11830 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
11836 "Unexpected custom legalisation");
11837 bool IsStrict =
N->isStrictFPOpcode();
11840 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
11858 Opc,
DL, VTs, Chain, Op0,
11892 std::tie(Result, Chain) =
11893 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
11921 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
11932 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
11933 "has custom type legalization on riscv32");
11935 SDValue LoCounter, HiCounter;
11939 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
11941 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
11944 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
11946 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
11950 N->getOperand(0), LoCounter, HiCounter);
11974 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
11975 unsigned XLen = Subtarget.
getXLen();
11978 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
11986 if (LHSIsU == RHSIsU)
12003 if (RHSIsU && LHSIsS && !RHSIsS)
12005 else if (LHSIsU && RHSIsS && !LHSIsS)
12015 "Unexpected custom legalisation");
12022 "Unexpected custom legalisation");
12025 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12051 "Unexpected custom legalisation");
12052 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12053 Subtarget.hasVendorXTHeadBb()) &&
12054 "Unexpected custom legalization");
12055 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12056 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12065 "Unexpected custom legalisation");
12079 MVT VT =
N->getSimpleValueType(0);
12080 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12081 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12082 "Unexpected custom legalisation");
12094 if (VT != MVT::i32)
12103 "Unexpected custom legalisation");
12107 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12124 EVT OType =
N->getValueType(1);
12137 "Unexpected custom legalisation");
12154 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12158 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12176 "Unexpected custom legalisation");
12177 if (Subtarget.hasStdExtZbb()) {
12198 "Unexpected custom legalisation");
12204 "Unexpected custom legalisation");
12206 if (Subtarget.hasStdExtZbb()) {
12240 EVT VT =
N->getValueType(0);
12245 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12249 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12250 Subtarget.hasStdExtZfbfmin()) {
12253 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12258 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12260 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12280 MVT VT =
N->getSimpleValueType(0);
12282 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12283 "Unexpected custom legalisation");
12284 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
12310 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12312 "Unexpected EXTRACT_VECTOR_ELT legalization");
12315 MVT ContainerVT = VecVT;
12343 DAG.
getUNDEF(ContainerVT), Mask, VL);
12351 unsigned IntNo =
N->getConstantOperandVal(0);
12355 "Don't know how to custom type legalize this intrinsic!");
12356 case Intrinsic::experimental_get_vector_length: {
12361 case Intrinsic::experimental_cttz_elts: {
12367 case Intrinsic::riscv_orc_b:
12368 case Intrinsic::riscv_brev8:
12369 case Intrinsic::riscv_sha256sig0:
12370 case Intrinsic::riscv_sha256sig1:
12371 case Intrinsic::riscv_sha256sum0:
12372 case Intrinsic::riscv_sha256sum1:
12373 case Intrinsic::riscv_sm3p0:
12374 case Intrinsic::riscv_sm3p1: {
12375 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12395 case Intrinsic::riscv_sm4ks:
12396 case Intrinsic::riscv_sm4ed: {
12404 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12408 case Intrinsic::riscv_mopr: {
12409 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12419 case Intrinsic::riscv_moprr: {
12420 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12432 case Intrinsic::riscv_clmul: {
12433 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12444 case Intrinsic::riscv_clmulh:
12445 case Intrinsic::riscv_clmulr: {
12446 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12474 case Intrinsic::riscv_vmv_x_s: {
12475 EVT VT =
N->getValueType(0);
12477 if (VT.
bitsLT(XLenVT)) {
12486 "Unexpected custom legalization");
12524 case ISD::VP_REDUCE_ADD:
12525 case ISD::VP_REDUCE_AND:
12526 case ISD::VP_REDUCE_OR:
12527 case ISD::VP_REDUCE_XOR:
12528 case ISD::VP_REDUCE_SMAX:
12529 case ISD::VP_REDUCE_UMAX:
12530 case ISD::VP_REDUCE_SMIN:
12531 case ISD::VP_REDUCE_UMIN:
12595 const EVT VT =
N->getValueType(0);
12596 const unsigned Opc =
N->getOpcode();
12603 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12608 "Inconsistent mappings");
12619 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12622 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
12637 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
12639 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
12640 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
12644 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
12651 if (
LHS.getOpcode() != ReduceOpc)
12666 auto Flags = ReduceVec->
getFlags();
12667 Flags.intersectWith(
N->getFlags());
12668 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
12678 auto BinOpToRVVReduce = [](
unsigned Opc) {
12707 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
12710 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
12713 unsigned Opc =
N->getOpcode();
12714 unsigned ReduceIdx;
12715 if (IsReduction(
N->getOperand(0), Opc))
12717 else if (IsReduction(
N->getOperand(1), Opc))
12723 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
12726 SDValue Extract =
N->getOperand(ReduceIdx);
12758 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
12785 if (!Subtarget.hasStdExtZba())
12789 EVT VT =
N->getValueType(0);
12801 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12802 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
12805 int64_t C0 = N0C->getSExtValue();
12806 int64_t C1 = N1C->getSExtValue();
12807 if (C0 <= 0 || C1 <= 0)
12811 int64_t Bits = std::min(C0, C1);
12812 int64_t Diff = std::abs(C0 - C1);
12813 if (Diff != 1 && Diff != 2 && Diff != 3)
12841 EVT VT =
N->getValueType(0);
12849 if ((!Subtarget.hasStdExtZicond() &&
12850 !Subtarget.hasVendorXVentanaCondOps()) ||
12872 bool SwapSelectOps;
12878 SwapSelectOps =
false;
12879 NonConstantVal = FalseVal;
12881 SwapSelectOps =
true;
12882 NonConstantVal = TrueVal;
12888 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
12936 EVT VT =
N->getValueType(0);
12944 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12945 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
12951 if (!N0C->hasOneUse())
12953 int64_t C0 = N0C->getSExtValue();
12954 int64_t C1 = N1C->getSExtValue();
12956 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
12959 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
12960 !isInt<12>(C0 * (C1 / C0))) {
12963 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
12964 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
12967 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
12968 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
12995 EVT VT =
N->getValueType(0);
13026 unsigned OuterExtend =
13030 OuterExtend,
SDLoc(
N), VT,
13038 EVT VT =
N->getValueType(0);
13083 EVT VT =
N->getValueType(0);
13087 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13093 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13103 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13125 EVT VT =
N->getValueType(0);
13154 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13178 EVT VT =
N->getValueType(0);
13198 EVT VT =
N->getValueType(0);
13284 EVT VT =
N->getValueType(0);
13355 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13360 const APInt &Imm = ConstN00->getAPIntValue();
13361 if ((Imm + 1).isSignedIntN(12))
13378 LHS.getValueType());
13399 EVT VT =
N->getValueType(0);
13411 if (!Subtarget.hasStdExtZba() && !Subtarget.hasVendorXTHeadBa())
13425 for (
uint64_t Divisor : {3, 5, 9}) {
13426 if (MulAmt % Divisor != 0)
13428 uint64_t MulAmt2 = MulAmt / Divisor;
13435 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13451 if (ScaleShift >= 1 && ScaleShift < 4) {
13452 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13466 for (
uint64_t Divisor : {3, 5, 9}) {
13471 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13482 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13484 if (ScaleShift >= 1 && ScaleShift < 4) {
13485 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13516 EVT VT =
N->getValueType(0);
13524 unsigned AddSubOpc;
13530 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
13531 AddSubOpc = V->getOpcode();
13533 SDValue Opnd = V->getOperand(1);
13534 MulOper = V->getOperand(0);
13543 if (IsAddSubWith1(N0)) {
13545 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
13548 if (IsAddSubWith1(N1)) {
13550 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
13562 if (isIndexTypeSigned(IndexType))
13565 if (!
N->hasOneUse())
13568 EVT VT =
N.getValueType();
13607 EVT SrcVT = Src.getValueType();
13611 NewElen = std::max(NewElen, 8U);
13634 EVT VT =
N->getValueType(0);
13637 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
13641 auto *N1C = dyn_cast<ConstantSDNode>(N1);
13653 if (!isIntEqualitySetCC(
Cond))
13662 const APInt &C1 = N1C->getAPIntValue();
13680 EVT VT =
N->getValueType(0);
13684 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
13686 Src.getOperand(0));
13694struct CombineResult;
13696enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
13723struct NodeExtensionHelper {
13732 bool SupportsFPExt;
13735 bool EnforceOneUse;
13750 return OrigOperand;
13761 unsigned getExtOpc(ExtKind SupportsExt)
const {
13762 switch (SupportsExt) {
13763 case ExtKind::SExt:
13765 case ExtKind::ZExt:
13767 case ExtKind::FPExt:
13778 std::optional<ExtKind> SupportsExt)
const {
13779 if (!SupportsExt.has_value())
13780 return OrigOperand;
13782 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
13786 if (
Source.getValueType() == NarrowVT)
13789 unsigned ExtOpc = getExtOpc(*SupportsExt);
13793 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
13800 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
13818 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
13824 MVT EltVT = SupportsExt == ExtKind::FPExt
13826 :
MVT::getIntegerVT(NarrowSize);
13828 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
13829 "Trying to extend something we can't represent");
13836 static unsigned getSExtOpcode(
unsigned Opcode) {
13859 static unsigned getZExtOpcode(
unsigned Opcode) {
13885 static unsigned getFPExtOpcode(
unsigned Opcode) {
13902 static unsigned getSUOpcode(
unsigned Opcode) {
13904 "SU is only supported for MUL");
13910 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
13930 using CombineToTry = std::function<std::optional<CombineResult>(
13931 SDNode * ,
const NodeExtensionHelper & ,
13936 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
13940 unsigned Opc = OrigOperand.
getOpcode();
13944 "Unexpected Opcode");
13957 unsigned ScalarBits =
Op.getValueSizeInBits();
13960 if (ScalarBits < EltBits)
13966 if (NarrowSize < 8)
13970 SupportsSExt =
true;
13974 SupportsZExt =
true;
13976 EnforceOneUse =
false;
13983 SupportsZExt =
false;
13984 SupportsSExt =
false;
13985 SupportsFPExt =
false;
13986 EnforceOneUse =
true;
13987 unsigned Opc = OrigOperand.
getOpcode();
14009 SupportsZExt =
true;
14012 SupportsSExt =
true;
14015 SupportsFPExt =
true;
14019 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14027 static bool isSupportedRoot(
const SDNode *Root,
14056 Subtarget.hasStdExtZvbb();
14058 return Subtarget.hasStdExtZvbb();
14067 assert(isSupportedRoot(Root, Subtarget) &&
14068 "Trying to build an helper with an "
14069 "unsupported root");
14070 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14086 if (OperandIdx == 1) {
14095 EnforceOneUse =
false;
14100 fillUpExtensionSupport(Root, DAG, Subtarget);
14106 static std::pair<SDValue, SDValue>
14109 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14128 switch (
N->getOpcode()) {
14166struct CombineResult {
14168 unsigned TargetOpcode;
14170 std::optional<ExtKind> LHSExt;
14171 std::optional<ExtKind> RHSExt;
14175 NodeExtensionHelper
LHS;
14177 NodeExtensionHelper
RHS;
14179 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14180 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14181 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14182 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14191 std::tie(Mask, VL) =
14192 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14206 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14207 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14222static std::optional<CombineResult>
14223canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14224 const NodeExtensionHelper &RHS,
14227 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14228 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14229 Root, LHS, {ExtKind::ZExt}, RHS,
14231 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14232 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14233 Root, LHS, {ExtKind::SExt}, RHS,
14235 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14236 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14237 Root, LHS, {ExtKind::FPExt}, RHS,
14239 return std::nullopt;
14248static std::optional<CombineResult>
14249canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14252 return canFoldToVWWithSameExtensionImpl(
14253 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14261static std::optional<CombineResult>
14262canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14265 if (
RHS.SupportsFPExt)
14266 return CombineResult(
14267 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14268 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14275 return CombineResult(
14276 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14277 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14279 return CombineResult(
14280 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14281 LHS, std::nullopt, RHS, {ExtKind::SExt});
14282 return std::nullopt;
14289static std::optional<CombineResult>
14290canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14293 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14301static std::optional<CombineResult>
14302canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14305 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14313static std::optional<CombineResult>
14314canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14317 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14325static std::optional<CombineResult>
14326canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14330 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14331 return std::nullopt;
14332 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14333 Root, LHS, {ExtKind::SExt}, RHS,
14338NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14349 Strategies.
push_back(canFoldToVWWithSameExtension);
14354 Strategies.
push_back(canFoldToVWWithSameExtension);
14359 Strategies.
push_back(canFoldToVWWithSameExtension);
14366 Strategies.
push_back(canFoldToVWWithZEXT);
14371 Strategies.
push_back(canFoldToVWWithSEXT);
14376 Strategies.
push_back(canFoldToVWWithZEXT);
14381 Strategies.
push_back(canFoldToVWWithFPEXT);
14410 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14416 Inserted.insert(
N);
14419 while (!Worklist.
empty()) {
14421 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
14424 NodeExtensionHelper
LHS(
N, 0, DAG, Subtarget);
14425 NodeExtensionHelper
RHS(
N, 1, DAG, Subtarget);
14426 auto AppendUsersIfNeeded = [&Worklist,
14427 &Inserted](
const NodeExtensionHelper &
Op) {
14428 if (
Op.needToPromoteOtherUsers()) {
14429 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
14430 if (Inserted.insert(TheUse).second)
14442 NodeExtensionHelper::getSupportedFoldings(
N);
14444 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
14445 bool Matched =
false;
14446 for (
int Attempt = 0;
14447 (Attempt != 1 + NodeExtensionHelper::isCommutative(
N)) && !Matched;
14450 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
14451 FoldingStrategies) {
14452 std::optional<CombineResult> Res =
14453 FoldingStrategy(
N,
LHS,
RHS, DAG, Subtarget);
14460 if (Res->LHSExt.has_value())
14461 AppendUsersIfNeeded(
LHS);
14462 if (Res->RHSExt.has_value())
14463 AppendUsersIfNeeded(
RHS);
14474 SDValue InputRootReplacement;
14481 for (CombineResult Res : CombinesToApply) {
14482 SDValue NewValue = Res.materialize(DAG, Subtarget);
14483 if (!InputRootReplacement) {
14485 "First element is expected to be the current node");
14486 InputRootReplacement = NewValue;
14491 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
14495 return InputRootReplacement;
14502 unsigned Opc =
N->getOpcode();
14507 SDValue MergeOp =
N->getOperand(1);
14508 unsigned MergeOpc = MergeOp.
getOpcode();
14519 SDValue Passthru =
N->getOperand(2);
14533 Z = Z.getOperand(1);
14539 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
14546 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
14575 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
14581 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
14583 if (MemVT == MVT::i32)
14589 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
14624 if (!Subtarget.hasVendorXTHeadMemPair())
14636 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
14638 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
14639 return {
Ptr->getOperand(0), C1->getZExtValue()};
14643 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
14666 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
14669 if (Base1 != Base2)
14673 bool Valid =
false;
14674 if (MemVT == MVT::i32) {
14676 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
14678 }
else if (MemVT == MVT::i64) {
14680 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
14714 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14722 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14732 EVT VT =
N->getValueType(0);
14735 MVT SrcVT = Src.getSimpleValueType();
14736 MVT SrcContainerVT = SrcVT;
14738 SDValue XVal = Src.getOperand(0);
14765 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14769 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14773 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
14786 if (VT != MVT::i32 && VT != XLenVT)
14816 EVT DstVT =
N->getValueType(0);
14817 if (DstVT != XLenVT)
14823 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14831 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14834 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
14843 if (SatVT == DstVT)
14845 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
14851 Src = Src.getOperand(0);
14872 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
14878 EVT VT =
N->getValueType(0);
14931 unsigned Offset =
N->isTargetStrictFPOpcode();
14938 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
14940 V.getOperand(2) == VL) {
14942 V = V.getOperand(0);
14949 bool NegA = invertIfNegative(
A);
14950 bool NegB = invertIfNegative(
B);
14951 bool NegC = invertIfNegative(
C);
14954 if (!NegA && !NegB && !NegC)
14958 if (
N->isTargetStrictFPOpcode())
14960 {N->getOperand(0), A, B, C, Mask, VL});
14970 if (
N->getValueType(0).isScalableVector() &&
14971 N->getValueType(0).getVectorElementType() == MVT::f32 &&
14978 if (
N->isTargetStrictFPOpcode())
15003 switch (
N->getOpcode()) {
15023 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
15024 N->getOperand(2), Mask, VL);
15031 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
15034 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15036 uint64_t ShAmt =
N->getConstantOperandVal(1);
15047 cast<VTSDNode>(N0.
getOperand(1))->getVT() == MVT::i32 &&
15075 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15088 !isa<ConstantSDNode>(U->getOperand(1)) ||
15089 U->getConstantOperandVal(1) > 32)
15144 if (!
Cond.hasOneUse())
15163 EVT VT =
Cond.getValueType();
15208 LHS =
LHS.getOperand(0);
15218 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15222 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15226 RHS =
LHS.getOperand(1);
15227 LHS =
LHS.getOperand(0);
15236 RHS =
LHS.getOperand(1);
15237 LHS =
LHS.getOperand(0);
15253 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15294 bool Commutative =
true;
15295 unsigned Opc = TrueVal.getOpcode();
15303 Commutative =
false;
15311 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15315 if (FalseVal == TrueVal.getOperand(0))
15317 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15322 EVT VT =
N->getValueType(0);
15324 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15330 assert(IdentityOperand &&
"No identity operand!");
15335 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15336 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15357 CountZeroes =
N->getOperand(2);
15358 ValOnZero =
N->getOperand(1);
15360 CountZeroes =
N->getOperand(1);
15361 ValOnZero =
N->getOperand(2);
15380 if (
Cond->getOperand(0) != CountZeroesArgument)
15396 CountZeroes, BitWidthMinusOne);
15406 EVT VT =
N->getValueType(0);
15407 EVT CondVT =
Cond.getValueType();
15415 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15421 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
15442 SDValue TrueVal =
N->getOperand(1);
15443 SDValue FalseVal =
N->getOperand(2);
15458 EVT VT =
N->getValueType(0);
15465 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
15480 if (
Op.isUndef()) {
15493 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
15497 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
15498 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
15502 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
15530 const unsigned InVecOpcode = InVec->
getOpcode();
15540 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
15547 InVecLHS, InValLHS, EltNo);
15549 InVecRHS, InValRHS, EltNo);
15558 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
15561 unsigned Elt = IndexC->getZExtValue();
15569 unsigned ConcatOpIdx = Elt / ConcatNumElts;
15572 ConcatOp, InVal, NewIdx);
15576 ConcatOps[ConcatOpIdx] = ConcatOp;
15588 EVT VT =
N->getValueType(0);
15598 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
15600 !
SDValue(BaseLd, 0).hasOneUse())
15603 EVT BaseLdVT = BaseLd->getValueType(0);
15610 auto *Ld = dyn_cast<LoadSDNode>(
Op);
15611 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
15613 Ld->getValueType(0) != BaseLdVT)
15622 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
15624 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
15629 if (BIO1.equalBaseIndex(BIO2, DAG))
15630 return {{BIO2.getOffset() - BIO1.getOffset(),
false}};
15634 SDValue P2 = Ld2->getBasePtr();
15640 return std::nullopt;
15644 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
15649 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
15650 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
15658 unsigned WideScalarBitWidth =
15671 auto [StrideVariant, MustNegateStride] = *BaseDiff;
15672 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
15673 ? std::get<SDValue>(StrideVariant)
15676 if (MustNegateStride)
15689 BaseLd->getBasePtr(), Stride, AllOneMask};
15692 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
15693 ConstStride && ConstStride->getSExtValue() >= 0)
15697 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
15703 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
15707 Ops, WideVecVT, MMO);
15719 if (
N->getValueType(0).isFixedLengthVector())
15722 SDValue Addend =
N->getOperand(0);
15726 SDValue AddMergeOp =
N->getOperand(2);
15731 auto IsVWMulOpc = [](
unsigned Opc) {
15760 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
15761 }(
N, DAG, Subtarget);
15766 if (AddMask != MulMask || AddVL != MulVL)
15771 "Unexpected opcode after VWMACC_VL");
15773 "Unexpected opcode after VWMACC_VL!");
15775 "Unexpected opcode after VWMUL_VL!");
15777 "Unexpected opcode after VWMUL_VL!");
15780 EVT VT =
N->getValueType(0);
15796 const EVT IndexVT =
Index.getValueType();
15800 if (!isIndexTypeSigned(IndexType))
15832 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
15835 if (
Index->getOperand(i)->isUndef())
15838 if (
C % ElementSize != 0)
15840 C =
C / ElementSize;
15844 ActiveLanes.
set(
C);
15846 return ActiveLanes.
all();
15864 if (NumElems % 2 != 0)
15868 const unsigned WiderElementSize = ElementSize * 2;
15869 if (WiderElementSize > ST.getELen()/8)
15872 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
15875 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
15878 if (
Index->getOperand(i)->isUndef())
15884 if (
C % WiderElementSize != 0)
15889 if (
C !=
Last + ElementSize)
15905 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
15916 switch (
N->getOpcode()) {
15936 APInt V =
C->getValueAPF().bitcastToAPInt();
15971 if (SimplifyDemandedLowBitsHelper(0, 32) ||
15972 SimplifyDemandedLowBitsHelper(1, 5))
15980 if (SimplifyDemandedLowBitsHelper(0, 32))
15997 MVT VT =
N->getSimpleValueType(0);
16006 "Unexpected value type!");
16029 EVT VT =
N->getValueType(0);
16094 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16099 Src.getOperand(0));
16104 Src.getOperand(0), Src.getOperand(1));
16119 auto IsTruncNode = [](
SDValue V) {
16122 SDValue VL = V.getOperand(2);
16123 auto *
C = dyn_cast<ConstantSDNode>(VL);
16125 bool IsVLMAXForVMSET = (
C &&
C->isAllOnes()) ||
16126 (isa<RegisterSDNode>(VL) &&
16127 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16136 while (IsTruncNode(
Op)) {
16137 if (!
Op.hasOneUse())
16139 Op =
Op.getOperand(0);
16170 if (
N->getOperand(1).getOpcode() ==
ISD::XOR &&
16179 N->getOperand(0),
Cond);
16191 SDValue FalseV =
N->getOperand(4);
16193 EVT VT =
N->getValueType(0);
16196 if (TrueV == FalseV)
16201 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16207 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16208 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16211 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16212 isInt<12>(TrueSImm - FalseSImm)) {
16228 {LHS, RHS, CC, TrueV, FalseV});
16295 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
16308 EVT VT =
N->getValueType(0);
16332 const auto *MGN = dyn_cast<MaskedGatherSDNode>(
N);
16333 const EVT VT =
N->getValueType(0);
16335 SDValue ScaleOp = MGN->getScale();
16337 assert(!MGN->isIndexScaled() &&
16338 "Scaled gather/scatter should not be formed");
16343 N->getVTList(), MGN->getMemoryVT(),
DL,
16344 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16345 MGN->getBasePtr(), Index, ScaleOp},
16346 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16350 N->getVTList(), MGN->getMemoryVT(),
DL,
16351 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16352 MGN->getBasePtr(), Index, ScaleOp},
16353 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16359 if (std::optional<VIDSequence> SimpleVID =
16361 SimpleVID && SimpleVID->StepDenominator == 1) {
16362 const int64_t StepNumerator = SimpleVID->StepNumerator;
16363 const int64_t Addend = SimpleVID->Addend;
16370 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
16380 {MGN->getChain(), IntID, MGN->getPassThru(), BasePtr,
16381 DAG.
getConstant(StepNumerator,
DL, XLenVT), MGN->getMask()};
16383 Ops, VT, MGN->getMemOperand());
16391 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
16393 MGN->getMemoryVT(), MGN->getMemOperand(),
16402 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
16404 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
16406 EVT IndexVT =
Index.getValueType()
16413 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
16415 EltCnt.divideCoefficientBy(2));
16418 EltCnt.divideCoefficientBy(2));
16423 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
16432 const auto *MSN = dyn_cast<MaskedScatterSDNode>(
N);
16434 SDValue ScaleOp = MSN->getScale();
16436 assert(!MSN->isIndexScaled() &&
16437 "Scaled gather/scatter should not be formed");
16442 N->getVTList(), MSN->getMemoryVT(),
DL,
16443 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16445 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16449 N->getVTList(), MSN->getMemoryVT(),
DL,
16450 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16452 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16454 EVT VT = MSN->getValue()->getValueType(0);
16456 if (!MSN->isTruncatingStore() &&
16460 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
16461 DAG.
getUNDEF(XLenVT), MSN->getMask(),
16462 MSN->getMemoryVT(), MSN->getMemOperand(),
16467 case ISD::VP_GATHER: {
16468 const auto *VPGN = dyn_cast<VPGatherSDNode>(
N);
16470 SDValue ScaleOp = VPGN->getScale();
16472 assert(!VPGN->isIndexScaled() &&
16473 "Scaled gather/scatter should not be formed");
16478 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16479 ScaleOp, VPGN->getMask(),
16480 VPGN->getVectorLength()},
16481 VPGN->getMemOperand(), IndexType);
16485 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16486 ScaleOp, VPGN->getMask(),
16487 VPGN->getVectorLength()},
16488 VPGN->getMemOperand(), IndexType);
16492 case ISD::VP_SCATTER: {
16493 const auto *VPSN = dyn_cast<VPScatterSDNode>(
N);
16495 SDValue ScaleOp = VPSN->getScale();
16497 assert(!VPSN->isIndexScaled() &&
16498 "Scaled gather/scatter should not be formed");
16503 {VPSN->getChain(), VPSN->getValue(),
16504 VPSN->getBasePtr(), Index, ScaleOp,
16505 VPSN->getMask(), VPSN->getVectorLength()},
16506 VPSN->getMemOperand(), IndexType);
16510 {VPSN->getChain(), VPSN->getValue(),
16511 VPSN->getBasePtr(), Index, ScaleOp,
16512 VPSN->getMask(), VPSN->getVectorLength()},
16513 VPSN->getMemOperand(), IndexType);
16527 EVT VT =
N->getValueType(0);
16530 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
16531 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
16549 EVT VT =
N->getValueType(0);
16553 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
16583 if (
N->getValueType(0).isScalableVector() &&
16584 N->getValueType(0).getVectorElementType() == MVT::f32 &&
16599 auto *Store = cast<StoreSDNode>(
N);
16600 SDValue Chain = Store->getChain();
16601 EVT MemVT = Store->getMemoryVT();
16602 SDValue Val = Store->getValue();
16605 bool IsScalarizable =
16607 Store->isSimple() &&
16637 NewVT, *Store->getMemOperand())) {
16639 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
16640 Store->getPointerInfo(), Store->getOriginalAlign(),
16641 Store->getMemOperand()->getFlags());
16649 if (
auto *L = dyn_cast<LoadSDNode>(Val);
16651 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
16653 L->getMemoryVT() == MemVT) {
16656 NewVT, *Store->getMemOperand()) &&
16658 NewVT, *L->getMemOperand())) {
16660 L->getPointerInfo(), L->getOriginalAlign(),
16661 L->getMemOperand()->getFlags());
16662 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
16663 Store->getPointerInfo(), Store->getOriginalAlign(),
16664 Store->getMemOperand()->getFlags());
16676 MVT VecVT = Src.getSimpleValueType();
16683 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
16686 Store->getMemOperand(), Store->getAddressingMode(),
16687 Store->isTruncatingStore(),
false);
16694 EVT VT =
N->getValueType(0);
16716 const MVT VT =
N->getSimpleValueType(0);
16717 SDValue Passthru =
N->getOperand(0);
16718 SDValue Scalar =
N->getOperand(1);
16727 const MVT VT =
N->getSimpleValueType(0);
16728 SDValue Passthru =
N->getOperand(0);
16729 SDValue Scalar =
N->getOperand(1);
16734 unsigned ScalarSize = Scalar.getValueSizeInBits();
16736 if (ScalarSize > EltWidth && Passthru.
isUndef())
16737 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
16744 (!Const || Const->isZero() ||
16745 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
16755 if (
N->getOperand(0).isUndef() &&
16758 Src.getOperand(0).getValueType().isScalableVector()) {
16759 EVT VT =
N->getValueType(0);
16760 EVT SrcVT = Src.getOperand(0).getValueType();
16764 return Src.getOperand(0);
16770 const MVT VT =
N->getSimpleValueType(0);
16771 SDValue Passthru =
N->getOperand(0);
16772 SDValue Scalar =
N->getOperand(1);
16782 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
16792 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
16800 MVT VecVT =
N->getOperand(0).getSimpleValueType();
16802 if (M1VT.
bitsLT(VecVT)) {
16813 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
16818 case Intrinsic::riscv_masked_strided_load: {
16819 MVT VT =
N->getSimpleValueType(0);
16820 auto *Load = cast<MemIntrinsicSDNode>(
N);
16821 SDValue PassThru =
N->getOperand(2);
16823 SDValue Stride =
N->getOperand(4);
16829 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
16830 StrideC && StrideC->getZExtValue() == ElementSize)
16832 DAG.
getUNDEF(XLenVT), Mask, PassThru,
16833 Load->getMemoryVT(), Load->getMemOperand(),
16837 case Intrinsic::riscv_masked_strided_store: {
16838 auto *Store = cast<MemIntrinsicSDNode>(
N);
16841 SDValue Stride =
N->getOperand(4);
16846 const unsigned ElementSize =
Value.getValueType().getScalarStoreSize();
16847 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
16848 StrideC && StrideC->getZExtValue() == ElementSize)
16851 Value.getValueType(), Store->getMemOperand(),
16855 case Intrinsic::riscv_vcpop:
16856 case Intrinsic::riscv_vcpop_mask:
16857 case Intrinsic::riscv_vfirst:
16858 case Intrinsic::riscv_vfirst_mask: {
16860 if (IntNo == Intrinsic::riscv_vcpop_mask ||
16861 IntNo == Intrinsic::riscv_vfirst_mask)
16862 VL =
N->getOperand(3);
16867 EVT VT =
N->getValueType(0);
16868 if (IntNo == Intrinsic::riscv_vfirst ||
16869 IntNo == Intrinsic::riscv_vfirst_mask)
16878 EVT VT =
N->getValueType(0);
16882 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
16901 EVT XVT,
unsigned KeptBits)
const {
16906 if (XVT != MVT::i32 && XVT != MVT::i64)
16910 if (KeptBits == 32 || KeptBits == 64)
16914 return Subtarget.hasStdExtZbb() &&
16915 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
16923 "Expected shift op");
16934 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
16935 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
16937 const APInt &C1Int = C1->getAPIntValue();
16938 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
16964 if (C1Cost < ShiftedC1Cost)
16978 EVT VT =
Op.getValueType();
16982 unsigned Opcode =
Op.getOpcode();
16990 const APInt &Mask =
C->getAPIntValue();
16999 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17000 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17002 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17003 if (NewMask == Mask)
17008 Op.getOperand(0), NewC);
17021 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17022 if (IsLegalMask(NewMask))
17023 return UseMask(NewMask);
17026 if (VT == MVT::i64) {
17028 if (IsLegalMask(NewMask))
17029 return UseMask(NewMask);
17044 APInt NewMask = ShrunkMask;
17045 if (MinSignedBits <= 12)
17047 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17053 assert(IsLegalMask(NewMask));
17054 return UseMask(NewMask);
17058 static const uint64_t GREVMasks[] = {
17059 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17060 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17062 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17063 unsigned Shift = 1 << Stage;
17064 if (ShAmt & Shift) {
17066 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17078 const APInt &DemandedElts,
17080 unsigned Depth)
const {
17082 unsigned Opc =
Op.getOpcode();
17087 "Should use MaskedValueIsZero if you don't know whether Op"
17088 " is a target node!");
17171 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17174 if (MaxVLenB == MinVLenB)
17191 case Intrinsic::riscv_vsetvli:
17192 case Intrinsic::riscv_vsetvlimax: {
17193 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17194 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17200 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17203 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17204 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17206 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17219 unsigned Depth)
const {
17220 switch (
Op.getOpcode()) {
17226 if (Tmp == 1)
return 1;
17229 return std::min(Tmp, Tmp2);
17241 if (Tmp < 33)
return 1;
17266 unsigned XLen = Subtarget.
getXLen();
17267 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17268 if (EltBits <= XLen)
17269 return XLen - EltBits + 1;
17273 unsigned IntNo =
Op.getConstantOperandVal(1);
17277 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
17278 case Intrinsic::riscv_masked_atomicrmw_add_i64:
17279 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
17280 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
17281 case Intrinsic::riscv_masked_atomicrmw_max_i64:
17282 case Intrinsic::riscv_masked_atomicrmw_min_i64:
17283 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
17284 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
17285 case Intrinsic::riscv_masked_cmpxchg_i64:
17293 assert(Subtarget.hasStdExtA());
17308 switch (
Op.getOpcode()) {
17314 return !
Op.getValueType().isInteger();
17322 assert(Ld &&
"Unexpected null LoadSDNode");
17330 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
17331 if (!CNode || CNode->isMachineConstantPoolEntry() ||
17332 CNode->getOffset() != 0)
17340 auto *CNode = GetSupportedConstantPool(
Ptr);
17341 if (!CNode || CNode->getTargetFlags() != 0)
17344 return CNode->getConstVal();
17352 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
17353 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
17359 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
17362 return CNodeLo->getConstVal();
17367 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
17399 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
17402 int64_t LoCounter =
MI.getOperand(2).getImm();
17403 int64_t HiCounter =
MI.getOperand(3).getImm();
17413 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
17425 MI.eraseFromParent();
17433 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
17441 Register SrcReg =
MI.getOperand(2).getReg();
17461 MI.eraseFromParent();
17468 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
17469 "Unexpected instruction");
17475 Register DstReg =
MI.getOperand(0).getReg();
17498 MI.eraseFromParent();
17503 switch (
MI.getOpcode()) {
17506 case RISCV::Select_GPR_Using_CC_GPR:
17507 case RISCV::Select_FPR16_Using_CC_GPR:
17508 case RISCV::Select_FPR16INX_Using_CC_GPR:
17509 case RISCV::Select_FPR32_Using_CC_GPR:
17510 case RISCV::Select_FPR32INX_Using_CC_GPR:
17511 case RISCV::Select_FPR64_Using_CC_GPR:
17512 case RISCV::Select_FPR64INX_Using_CC_GPR:
17513 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
17519 unsigned RelOpcode,
unsigned EqOpcode,
17522 Register DstReg =
MI.getOperand(0).getReg();
17523 Register Src1Reg =
MI.getOperand(1).getReg();
17524 Register Src2Reg =
MI.getOperand(2).getReg();
17526 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17550 MI.eraseFromParent();
17601 F->insert(It, FirstMBB);
17602 F->insert(It, SecondMBB);
17603 F->insert(It, SinkMBB);
17652 First.eraseFromParent();
17695 SelectDests.
insert(
MI.getOperand(0).getReg());
17699 if (
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR && Next != BB->
end() &&
17700 Next->getOpcode() ==
MI.getOpcode() &&
17701 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
17702 Next->getOperand(5).isKill()) {
17707 SequenceMBBI != E; ++SequenceMBBI) {
17708 if (SequenceMBBI->isDebugInstr())
17711 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
17712 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
17713 SequenceMBBI->getOperand(3).getImm() !=
CC ||
17714 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
17715 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
17717 LastSelectPseudo = &*SequenceMBBI;
17719 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
17722 if (SequenceMBBI->hasUnmodeledSideEffects() ||
17723 SequenceMBBI->mayLoadOrStore() ||
17724 SequenceMBBI->usesCustomInsertionHook())
17727 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
17742 F->insert(
I, IfFalseMBB);
17743 F->insert(
I, TailMBB);
17747 TailMBB->
push_back(DebugInstr->removeFromParent());
17751 TailMBB->
splice(TailMBB->
end(), HeadMBB,
17770 auto SelectMBBI =
MI.getIterator();
17771 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
17772 auto InsertionPoint = TailMBB->
begin();
17773 while (SelectMBBI != SelectEnd) {
17774 auto Next = std::next(SelectMBBI);
17777 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
17778 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
17779 .
addReg(SelectMBBI->getOperand(4).getReg())
17781 .
addReg(SelectMBBI->getOperand(5).getReg())
17796 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
17797 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
17799 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
17800 assert(Masked &&
"Could not find masked instruction for LMUL and SEW pair");
17806 unsigned CVTXOpc) {
17812 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17825 .
add(
MI.getOperand(1))
17826 .
add(
MI.getOperand(2))
17827 .
add(
MI.getOperand(3))
17829 .
add(
MI.getOperand(4))
17830 .
add(
MI.getOperand(5))
17831 .
add(
MI.getOperand(6))
17846 .
add(
MI.getOperand(0))
17847 .
add(
MI.getOperand(1))
17849 .
add(
MI.getOperand(3))
17851 .
add(
MI.getOperand(4))
17852 .
add(
MI.getOperand(5))
17853 .
add(
MI.getOperand(6))
17863 MI.eraseFromParent();
17869 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
17871 switch (
MI.getOpcode()) {
17874 case RISCV::PseudoFROUND_H:
17875 CmpOpc = RISCV::FLT_H;
17876 F2IOpc = RISCV::FCVT_W_H;
17877 I2FOpc = RISCV::FCVT_H_W;
17878 FSGNJOpc = RISCV::FSGNJ_H;
17879 FSGNJXOpc = RISCV::FSGNJX_H;
17880 RC = &RISCV::FPR16RegClass;
17882 case RISCV::PseudoFROUND_H_INX:
17883 CmpOpc = RISCV::FLT_H_INX;
17884 F2IOpc = RISCV::FCVT_W_H_INX;
17885 I2FOpc = RISCV::FCVT_H_W_INX;
17886 FSGNJOpc = RISCV::FSGNJ_H_INX;
17887 FSGNJXOpc = RISCV::FSGNJX_H_INX;
17888 RC = &RISCV::GPRF16RegClass;
17890 case RISCV::PseudoFROUND_S:
17891 CmpOpc = RISCV::FLT_S;
17892 F2IOpc = RISCV::FCVT_W_S;
17893 I2FOpc = RISCV::FCVT_S_W;
17894 FSGNJOpc = RISCV::FSGNJ_S;
17895 FSGNJXOpc = RISCV::FSGNJX_S;
17896 RC = &RISCV::FPR32RegClass;
17898 case RISCV::PseudoFROUND_S_INX:
17899 CmpOpc = RISCV::FLT_S_INX;
17900 F2IOpc = RISCV::FCVT_W_S_INX;
17901 I2FOpc = RISCV::FCVT_S_W_INX;
17902 FSGNJOpc = RISCV::FSGNJ_S_INX;
17903 FSGNJXOpc = RISCV::FSGNJX_S_INX;
17904 RC = &RISCV::GPRF32RegClass;
17906 case RISCV::PseudoFROUND_D:
17908 CmpOpc = RISCV::FLT_D;
17909 F2IOpc = RISCV::FCVT_L_D;
17910 I2FOpc = RISCV::FCVT_D_L;
17911 FSGNJOpc = RISCV::FSGNJ_D;
17912 FSGNJXOpc = RISCV::FSGNJX_D;
17913 RC = &RISCV::FPR64RegClass;
17915 case RISCV::PseudoFROUND_D_INX:
17917 CmpOpc = RISCV::FLT_D_INX;
17918 F2IOpc = RISCV::FCVT_L_D_INX;
17919 I2FOpc = RISCV::FCVT_D_L_INX;
17920 FSGNJOpc = RISCV::FSGNJ_D_INX;
17921 FSGNJXOpc = RISCV::FSGNJX_D_INX;
17922 RC = &RISCV::GPRRegClass;
17934 F->insert(
I, CvtMBB);
17935 F->insert(
I, DoneMBB);
17946 Register DstReg =
MI.getOperand(0).getReg();
17947 Register SrcReg =
MI.getOperand(1).getReg();
17948 Register MaxReg =
MI.getOperand(2).getReg();
17949 int64_t FRM =
MI.getOperand(3).getImm();
17954 Register FabsReg =
MRI.createVirtualRegister(RC);
17958 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17973 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17995 MI.eraseFromParent();
18002 switch (
MI.getOpcode()) {
18005 case RISCV::ReadCounterWide:
18007 "ReadCounterWide is only to be used on riscv32");
18009 case RISCV::Select_GPR_Using_CC_GPR:
18010 case RISCV::Select_FPR16_Using_CC_GPR:
18011 case RISCV::Select_FPR16INX_Using_CC_GPR:
18012 case RISCV::Select_FPR32_Using_CC_GPR:
18013 case RISCV::Select_FPR32INX_Using_CC_GPR:
18014 case RISCV::Select_FPR64_Using_CC_GPR:
18015 case RISCV::Select_FPR64INX_Using_CC_GPR:
18016 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18018 case RISCV::BuildPairF64Pseudo:
18020 case RISCV::SplitF64Pseudo:
18022 case RISCV::PseudoQuietFLE_H:
18024 case RISCV::PseudoQuietFLE_H_INX:
18025 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18026 case RISCV::PseudoQuietFLT_H:
18028 case RISCV::PseudoQuietFLT_H_INX:
18029 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18030 case RISCV::PseudoQuietFLE_S:
18032 case RISCV::PseudoQuietFLE_S_INX:
18033 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18034 case RISCV::PseudoQuietFLT_S:
18036 case RISCV::PseudoQuietFLT_S_INX:
18037 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18038 case RISCV::PseudoQuietFLE_D:
18040 case RISCV::PseudoQuietFLE_D_INX:
18041 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18042 case RISCV::PseudoQuietFLE_D_IN32X:
18045 case RISCV::PseudoQuietFLT_D:
18047 case RISCV::PseudoQuietFLT_D_INX:
18048 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18049 case RISCV::PseudoQuietFLT_D_IN32X:
18053 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18055 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18057 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18059 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18061 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18063 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18065 case RISCV::PseudoFROUND_H:
18066 case RISCV::PseudoFROUND_H_INX:
18067 case RISCV::PseudoFROUND_S:
18068 case RISCV::PseudoFROUND_S_INX:
18069 case RISCV::PseudoFROUND_D:
18070 case RISCV::PseudoFROUND_D_INX:
18071 case RISCV::PseudoFROUND_D_IN32X:
18073 case TargetOpcode::STATEPOINT:
18079 MI.addOperand(*
MI.getMF(),
18085 case TargetOpcode::STACKMAP:
18086 case TargetOpcode::PATCHPOINT:
18089 "supported on 64-bit targets");
18107 if (
MI.readsRegister(RISCV::FRM,
nullptr))
18139 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18140 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18143 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18144 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18147 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18148 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18152 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18153 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18154 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18156 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18157 RISCV::V20M2, RISCV::V22M2};
18165 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18166 RISCV::X13, RISCV::X14, RISCV::X15,
18167 RISCV::X16, RISCV::X17};
18169 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18170 RISCV::X13, RISCV::X14, RISCV::X15};
18181 static const MCPhysReg FastCCIGPRs[] = {
18182 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
18183 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
18184 RISCV::X29, RISCV::X30, RISCV::X31};
18187 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18188 RISCV::X13, RISCV::X14, RISCV::X15,
18203 unsigned XLenInBytes = XLen / 8;
18216 Align StackAlign(XLenInBytes);
18217 if (!
EABI || XLen != 32)
18249 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18250 assert(XLen == 32 || XLen == 64);
18251 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18255 if (ArgFlags.
isNest()) {
18256 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
18264 if (!LocVT.
isVector() && IsRet && ValNo > 1)
18269 bool UseGPRForF16_F32 =
true;
18272 bool UseGPRForF64 =
true;
18284 UseGPRForF16_F32 = !IsFixed;
18288 UseGPRForF16_F32 = !IsFixed;
18289 UseGPRForF64 = !IsFixed;
18295 UseGPRForF16_F32 =
true;
18296 UseGPRForF64 =
true;
18303 if (UseGPRForF16_F32 &&
18304 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
18307 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
18324 unsigned TwoXLenInBytes = (2 * XLen) / 8;
18326 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
18330 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
18339 "PendingLocs and PendingArgFlags out of sync");
18343 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
18344 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
18392 PendingLocs.
size() <= 2) {
18393 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
18398 PendingLocs.
clear();
18399 PendingArgFlags.
clear();
18401 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
18407 unsigned StoreSizeBytes = XLen / 8;
18410 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
18412 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
18414 else if (ValVT == MVT::f64 && !UseGPRForF64)
18450 if (!PendingLocs.
empty()) {
18452 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
18454 for (
auto &It : PendingLocs) {
18456 It.convertToReg(Reg);
18461 PendingLocs.clear();
18462 PendingArgFlags.
clear();
18466 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
18468 "Expected an XLenVT or vector types at this stage");
18486template <
typename ArgTy>
18488 for (
const auto &ArgIdx :
enumerate(Args)) {
18489 MVT ArgVT = ArgIdx.value().VT;
18491 return ArgIdx.index();
18493 return std::nullopt;
18496void RISCVTargetLowering::analyzeInputArgs(
18499 RISCVCCAssignFn Fn)
const {
18500 unsigned NumArgs =
Ins.size();
18513 for (
unsigned i = 0; i != NumArgs; ++i) {
18517 Type *ArgTy =
nullptr;
18519 ArgTy = FType->getReturnType();
18520 else if (Ins[i].isOrigArg())
18521 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
18525 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
18527 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
18534void RISCVTargetLowering::analyzeOutputArgs(
18537 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
18538 unsigned NumArgs = Outs.
size();
18548 for (
unsigned i = 0; i != NumArgs; i++) {
18549 MVT ArgVT = Outs[i].VT;
18551 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
18555 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
18557 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
18611 if (In.isOrigArg()) {
18616 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
18617 (
BitWidth < 32 && In.Flags.isZExt())) {
18646 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
18692 ExtType,
DL, LocVT, Chain, FIN,
18709 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18722 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18732 unsigned ValNo,
MVT ValVT,
MVT LocVT,
18735 bool IsFixed,
bool IsRet,
Type *OrigTy,
18738 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18747 if (LocVT == MVT::f16 &&
18748 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
18750 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
18751 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
18752 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
18753 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
18754 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
18760 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18762 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
18763 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
18764 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
18765 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
18766 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
18772 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
18774 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
18775 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
18776 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
18777 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
18778 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
18785 if ((LocVT == MVT::f16 &&
18786 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
18787 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
18788 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
18789 Subtarget.hasStdExtZdinx())) {
18796 if (LocVT == MVT::f16) {
18802 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
18808 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
18816 if (AllocatedVReg) {
18851 if (ArgFlags.
isNest()) {
18853 "Attribute 'nest' is not supported in GHC calling convention");
18857 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
18858 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
18860 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18872 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18875 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
18876 RISCV::F18_F, RISCV::F19_F,
18877 RISCV::F20_F, RISCV::F21_F};
18878 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
18884 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
18887 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
18888 RISCV::F24_D, RISCV::F25_D,
18889 RISCV::F26_D, RISCV::F27_D};
18890 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
18896 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
18897 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
18917 switch (CallConv) {
18927 if (Subtarget.hasStdExtE())
18931 "(Zdinx/D) instruction set extensions");
18935 if (Func.hasFnAttribute(
"interrupt")) {
18936 if (!Func.arg_empty())
18938 "Functions with the interrupt attribute cannot have arguments!");
18943 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
18945 "Function interrupt attribute argument not supported!");
18950 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
18952 std::vector<SDValue> OutChains;
18961 analyzeInputArgs(MF, CCInfo, Ins,
false,
18965 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
18986 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
18987 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
18989 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
18991 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19020 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19025 if (VarArgsSaveSize == 0) {
19029 int VaArgOffset = -VarArgsSaveSize;
19037 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19038 VarArgsSaveSize += XLenInBytes;
19045 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19050 Chain,
DL, ArgValue, FIN,
19052 OutChains.push_back(Store);
19066 if (!OutChains.empty()) {
19067 OutChains.push_back(Chain);
19077bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19081 auto CalleeCC = CLI.CallConv;
19082 auto &Outs = CLI.Outs;
19084 auto CallerCC = Caller.getCallingConv();
19091 if (Caller.hasFnAttribute(
"interrupt"))
19106 for (
auto &VA : ArgLocs)
19112 auto IsCallerStructRet = Caller.hasStructRetAttr();
19113 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19114 if (IsCallerStructRet || IsCalleeStructRet)
19119 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19120 if (CalleeCC != CallerCC) {
19121 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19122 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19129 for (
auto &Arg : Outs)
19130 if (Arg.Flags.isByVal())
19165 if (Subtarget.hasStdExtE())
19169 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19175 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19181 "site marked musttail");
19188 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19190 if (!Flags.isByVal())
19194 unsigned Size = Flags.getByValSize();
19195 Align Alignment = Flags.getNonZeroByValAlign();
19202 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19216 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19219 SDValue ArgValue = OutVals[OutIdx];
19239 if (!StackPtr.getNode())
19250 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19268 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19269 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19275 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19276 SDValue PartValue = OutVals[OutIdx + 1];
19277 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19289 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
19291 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19293 for (
const auto &Part : Parts) {
19294 SDValue PartValue = Part.first;
19295 SDValue PartOffset = Part.second;
19302 ArgValue = SpillSlot;
19308 if (Flags.isByVal())
19309 ArgValue = ByValArgs[j++];
19316 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19317 "for passing parameters");
19320 if (!StackPtr.getNode())
19333 if (!MemOpChains.
empty())
19339 for (
auto &Reg : RegsToPass) {
19340 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19347 validateCCReservedRegs(RegsToPass, MF);
19352 "Return address register required, but has been reserved."});
19371 for (
auto &Reg : RegsToPass)
19377 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19378 assert(Mask &&
"Missing call preserved mask for calling convention");
19387 "Unexpected CFI type for a direct call");
19417 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
19418 auto &VA = RVLocs[i];
19426 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
19427 assert(VA.needsCustom());
19452 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19453 MVT VT = Outs[i].VT;
19457 ArgFlags, CCInfo,
true,
true,
19458 nullptr, *
this, Dispatcher))
19490 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
19491 SDValue Val = OutVals[OutIdx];
19500 DAG.
getVTList(MVT::i32, MVT::i32), Val);
19504 Register RegHi = RVLocs[++i].getLocReg();
19510 "Return value register required, but has been reserved."});
19526 "Return value register required, but has been reserved."});
19548 if (Func.hasFnAttribute(
"interrupt")) {
19549 if (!Func.getReturnType()->isVoidTy())
19551 "Functions with the interrupt attribute must have void return type!");
19557 if (Kind ==
"supervisor")
19563 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
19566void RISCVTargetLowering::validateCCReservedRegs(
19567 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
19576 F,
"Argument register required, but has been reserved."});
19582 if (
N->getNumValues() != 1)
19584 if (!
N->hasNUsesOfValue(1, 0))
19587 SDNode *Copy = *
N->use_begin();
19601 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
19605 bool HasRet =
false;
19606 for (
SDNode *Node : Copy->uses()) {
19614 Chain = Copy->getOperand(0);
19623#define NODE_NAME_CASE(NODE) \
19624 case RISCVISD::NODE: \
19625 return "RISCVISD::" #NODE;
19878#undef NODE_NAME_CASE
19885 if (Constraint.
size() == 1) {
19886 switch (Constraint[0]) {
19902 if (Constraint ==
"vr" || Constraint ==
"vm")
19908std::pair<unsigned, const TargetRegisterClass *>
19914 if (Constraint.
size() == 1) {
19915 switch (Constraint[0]) {
19920 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
19921 return std::make_pair(0U, &RISCV::GPRF16RegClass);
19922 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
19923 return std::make_pair(0U, &RISCV::GPRF32RegClass);
19924 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
19925 return std::make_pair(0U, &RISCV::GPRPairRegClass);
19926 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
19928 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
19929 return std::make_pair(0U, &RISCV::FPR16RegClass);
19930 if (Subtarget.hasStdExtF() && VT == MVT::f32)
19931 return std::make_pair(0U, &RISCV::FPR32RegClass);
19932 if (Subtarget.hasStdExtD() && VT == MVT::f64)
19933 return std::make_pair(0U, &RISCV::FPR64RegClass);
19938 }
else if (Constraint ==
"vr") {
19939 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
19940 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
19942 return std::make_pair(0U, RC);
19944 }
else if (Constraint ==
"vm") {
19945 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
19946 return std::make_pair(0U, &RISCV::VMV0RegClass);
19954 .
Case(
"{zero}", RISCV::X0)
19955 .
Case(
"{ra}", RISCV::X1)
19956 .
Case(
"{sp}", RISCV::X2)
19957 .
Case(
"{gp}", RISCV::X3)
19958 .
Case(
"{tp}", RISCV::X4)
19959 .
Case(
"{t0}", RISCV::X5)
19960 .
Case(
"{t1}", RISCV::X6)
19961 .
Case(
"{t2}", RISCV::X7)
19962 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
19963 .
Case(
"{s1}", RISCV::X9)
19964 .
Case(
"{a0}", RISCV::X10)
19965 .
Case(
"{a1}", RISCV::X11)
19966 .
Case(
"{a2}", RISCV::X12)
19967 .
Case(
"{a3}", RISCV::X13)
19968 .
Case(
"{a4}", RISCV::X14)
19969 .
Case(
"{a5}", RISCV::X15)
19970 .
Case(
"{a6}", RISCV::X16)
19971 .
Case(
"{a7}", RISCV::X17)
19972 .
Case(
"{s2}", RISCV::X18)
19973 .
Case(
"{s3}", RISCV::X19)
19974 .
Case(
"{s4}", RISCV::X20)
19975 .
Case(
"{s5}", RISCV::X21)
19976 .
Case(
"{s6}", RISCV::X22)
19977 .
Case(
"{s7}", RISCV::X23)
19978 .
Case(
"{s8}", RISCV::X24)
19979 .
Case(
"{s9}", RISCV::X25)
19980 .
Case(
"{s10}", RISCV::X26)
19981 .
Case(
"{s11}", RISCV::X27)
19982 .
Case(
"{t3}", RISCV::X28)
19983 .
Case(
"{t4}", RISCV::X29)
19984 .
Case(
"{t5}", RISCV::X30)
19985 .
Case(
"{t6}", RISCV::X31)
19987 if (XRegFromAlias != RISCV::NoRegister)
19988 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
19997 if (Subtarget.hasStdExtF()) {
19999 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20000 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20001 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20002 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20003 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20004 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20005 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20006 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20007 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20008 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20009 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20010 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20011 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20012 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20013 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20014 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20015 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20016 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20017 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20018 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20019 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20020 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20021 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20022 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20023 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20024 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20025 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20026 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20027 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20028 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20029 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20030 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20032 if (FReg != RISCV::NoRegister) {
20033 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20034 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20035 unsigned RegNo = FReg - RISCV::F0_F;
20036 unsigned DReg = RISCV::F0_D + RegNo;
20037 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20039 if (VT == MVT::f32 || VT == MVT::Other)
20040 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20041 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20042 unsigned RegNo = FReg - RISCV::F0_F;
20043 unsigned HReg = RISCV::F0_H + RegNo;
20044 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20051 .
Case(
"{v0}", RISCV::V0)
20052 .
Case(
"{v1}", RISCV::V1)
20053 .
Case(
"{v2}", RISCV::V2)
20054 .
Case(
"{v3}", RISCV::V3)
20055 .
Case(
"{v4}", RISCV::V4)
20056 .
Case(
"{v5}", RISCV::V5)
20057 .
Case(
"{v6}", RISCV::V6)
20058 .
Case(
"{v7}", RISCV::V7)
20059 .
Case(
"{v8}", RISCV::V8)
20060 .
Case(
"{v9}", RISCV::V9)
20061 .
Case(
"{v10}", RISCV::V10)
20062 .
Case(
"{v11}", RISCV::V11)
20063 .
Case(
"{v12}", RISCV::V12)
20064 .
Case(
"{v13}", RISCV::V13)
20065 .
Case(
"{v14}", RISCV::V14)
20066 .
Case(
"{v15}", RISCV::V15)
20067 .
Case(
"{v16}", RISCV::V16)
20068 .
Case(
"{v17}", RISCV::V17)
20069 .
Case(
"{v18}", RISCV::V18)
20070 .
Case(
"{v19}", RISCV::V19)
20071 .
Case(
"{v20}", RISCV::V20)
20072 .
Case(
"{v21}", RISCV::V21)
20073 .
Case(
"{v22}", RISCV::V22)
20074 .
Case(
"{v23}", RISCV::V23)
20075 .
Case(
"{v24}", RISCV::V24)
20076 .
Case(
"{v25}", RISCV::V25)
20077 .
Case(
"{v26}", RISCV::V26)
20078 .
Case(
"{v27}", RISCV::V27)
20079 .
Case(
"{v28}", RISCV::V28)
20080 .
Case(
"{v29}", RISCV::V29)
20081 .
Case(
"{v30}", RISCV::V30)
20082 .
Case(
"{v31}", RISCV::V31)
20084 if (VReg != RISCV::NoRegister) {
20085 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20086 return std::make_pair(VReg, &RISCV::VMRegClass);
20087 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20088 return std::make_pair(VReg, &RISCV::VRRegClass);
20089 for (
const auto *RC :
20090 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20091 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20092 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20093 return std::make_pair(VReg, RC);
20099 std::pair<Register, const TargetRegisterClass *> Res =
20105 if (Res.second == &RISCV::GPRF16RegClass ||
20106 Res.second == &RISCV::GPRF32RegClass ||
20107 Res.second == &RISCV::GPRPairRegClass)
20108 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20116 if (ConstraintCode.
size() == 1) {
20117 switch (ConstraintCode[0]) {
20132 if (Constraint.
size() == 1) {
20133 switch (Constraint[0]) {
20136 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20138 if (isInt<12>(CVal))
20151 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20153 if (isUInt<5>(CVal))
20171 if (Subtarget.hasStdExtZtso()) {
20187 if (Subtarget.hasStdExtZtso()) {
20195 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Inst) &&
20212 if (Subtarget.hasForcedAtomics())
20217 if (Subtarget.hasStdExtZacas() &&
20218 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20224 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20237 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20239 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20241 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20243 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20245 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20247 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20249 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20251 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20260 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20262 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20264 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20266 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20268 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20270 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20272 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20274 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20300 unsigned XLen = Subtarget.
getXLen();
20324 unsigned ValWidth =
20329 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20332 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20344 if (Subtarget.hasForcedAtomics())
20348 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20357 unsigned XLen = Subtarget.
getXLen();
20359 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20364 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20370 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20377 EVT DataVT)
const {
20393 return Subtarget.hasStdExtZfhmin();
20395 return Subtarget.hasStdExtF();
20397 return Subtarget.hasStdExtD();
20429 "RVVBitsPerBlock changed, audit needed");
20438 if (!Subtarget.hasVendorXTHeadMemIdx())
20444 Base =
Op->getOperand(0);
20446 int64_t RHSC =
RHS->getSExtValue();
20452 bool isLegalIndexedOffset =
false;
20453 for (
unsigned i = 0; i < 4; i++)
20454 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
20455 isLegalIndexedOffset =
true;
20459 if (!isLegalIndexedOffset)
20476 VT = LD->getMemoryVT();
20477 Ptr = LD->getBasePtr();
20478 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20479 VT = ST->getMemoryVT();
20480 Ptr = ST->getBasePtr();
20499 VT = LD->getMemoryVT();
20500 Ptr = LD->getBasePtr();
20501 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20502 VT = ST->getMemoryVT();
20503 Ptr = ST->getBasePtr();
20546 const Constant *PersonalityFn)
const {
20551 const Constant *PersonalityFn)
const {
20575 const bool HasExtMOrZmmul =
20576 Subtarget.hasStdExtM() || Subtarget.hasStdExtZmmul();
20585 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(
C.getNode())) {
20587 const APInt &Imm = ConstNode->getAPIntValue();
20588 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
20589 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
20593 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
20594 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
20595 (Imm - 8).isPowerOf2()))
20600 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
20601 ConstNode->hasOneUse()) {
20602 APInt ImmS = Imm.ashr(Imm.countr_zero());
20603 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
20604 (1 - ImmS).isPowerOf2())
20628 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
20637 unsigned *
Fast)
const {
20640 *
Fast = Subtarget.enableUnalignedScalarMem();
20641 return Subtarget.enableUnalignedScalarMem();
20657 *
Fast = Subtarget.enableUnalignedVectorMem();
20658 return Subtarget.enableUnalignedVectorMem();
20667 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
20679 if (
Op.size() < MinVLenInBytes)
20689 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
20693 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
20695 if (
Op.isFixedDstAlign())
20696 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
20698 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
20706 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
20707 bool IsABIRegCopy =
CC.has_value();
20709 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20710 PartVT == MVT::f32) {
20728 if (PartVTBitSize % ValueVTBitSize == 0) {
20729 assert(PartVTBitSize >= ValueVTBitSize);
20736 if (ValueEltVT != PartEltVT) {
20737 if (PartVTBitSize > ValueVTBitSize) {
20739 assert(Count != 0 &&
"The number of element should not be zero.");
20740 EVT SameEltTypeVT =
20761 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
20762 bool IsABIRegCopy =
CC.has_value();
20763 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20764 PartVT == MVT::f32) {
20781 if (PartVTBitSize % ValueVTBitSize == 0) {
20782 assert(PartVTBitSize >= ValueVTBitSize);
20783 EVT SameEltTypeVT = ValueVT;
20790 if (ValueEltVT != PartEltVT) {
20792 assert(Count != 0 &&
"The number of element should not be zero.");
20809 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
20816 unsigned Opc =
N->getOpcode();
20840 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
20854 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
20859 if (FVTy->getNumElements() < 2)
20869 return Factor * LMUL <= 8;
20873 Align Alignment)
const {
20885 if (!Subtarget.enableUnalignedVectorMem() &&
20893 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
20894 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
20895 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
20896 Intrinsic::riscv_seg8_load};
20915 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
20925 {VTy, LI->getPointerOperandType(), XLenTy});
20927 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
20932 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
20934 Shuffles[i]->replaceAllUsesWith(SubVec);
20941 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
20942 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
20943 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
20944 Intrinsic::riscv_seg8_store};
20964 unsigned Factor)
const {
20966 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
20969 ShuffleVTy->getNumElements() / Factor);
20971 SI->getPointerAddressSpace(),
20972 SI->getModule()->getDataLayout()))
20979 {VTy, SI->getPointerOperandType(), XLenTy});
20984 for (
unsigned i = 0; i < Factor; i++) {
20993 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
20994 Ops.
append({SI->getPointerOperand(), VL});
21007 if (DI->
getIntrinsicID() != Intrinsic::experimental_vector_deinterleave2)
21010 unsigned Factor = 2;
21025 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21028 {ResVTy, LI->getPointerOperandType(), XLenTy});
21029 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21032 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21033 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21034 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21035 Intrinsic::riscv_vlseg8};
21057 if (II->
getIntrinsicID() != Intrinsic::experimental_vector_interleave2)
21060 unsigned Factor = 2;
21066 SI->getPointerAddressSpace(),
21067 SI->getModule()->getDataLayout()))
21074 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21077 {InVTy, SI->getPointerOperandType(), XLenTy});
21078 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21081 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21082 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21083 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21084 Intrinsic::riscv_vsseg8};
21092 SI->getPointerOperand(), VL});
21102 "Invalid call instruction for a KCFI check");
21104 MBBI->getOpcode()));
21107 Target.setIsRenamable(
false);
21115#define GET_REGISTER_MATCHER
21116#include "RISCVGenAsmMatcher.inc"
21122 if (Reg == RISCV::NoRegister)
21124 if (Reg == RISCV::NoRegister)
21136 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21138 if (NontemporalInfo ==
nullptr)
21146 int NontemporalLevel = 5;
21147 const MDNode *RISCVNontemporalInfo =
21148 I.getMetadata(
"riscv-nontemporal-domain");
21149 if (RISCVNontemporalInfo !=
nullptr)
21152 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21156 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21157 "RISC-V target doesn't support this non-temporal domain.");
21159 NontemporalLevel -= 2;
21161 if (NontemporalLevel & 0b1)
21163 if (NontemporalLevel & 0b10)
21176 return TargetFlags;
21186 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21189 return Subtarget.hasStdExtZbb() &&
21202 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21203 Op == Instruction::And ||
Op == Instruction::Or ||
21204 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21205 Op == Instruction::ShuffleVector ||
Op == Instruction::Load)
21213 !isa<ReturnInst>(&Inst))
21216 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21217 if (AI->getAllocatedType()->isScalableTy())
21225RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21233 if (!Subtarget.hasShortForwardBranchOpt())
21235 EVT VT =
N->getValueType(0);
21236 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21240 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21245bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21246 EVT VT,
const APInt &AndMask)
const {
21247 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21248 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21252unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21257template <
typename Arg>
21258void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
21261 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
21263 auto It = ArgList.
begin();
21264 MVT FirstArgRegType = It->VT;
21267 if (It == ArgList.
end() || It->Flags.isSplit())
21279 for (; It != ArgList.
end(); ++It)
21280 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
21286 if (isHomogeneousScalableVectorType(ArgList)) {
21288 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
21291 bool FirstVMaskAssigned =
false;
21292 for (
const auto &OutArg : ArgList) {
21293 MVT RegisterVT = OutArg.VT;
21303 RVVArgInfos.push_back({1, RegisterVT,
true});
21304 FirstVMaskAssigned =
true;
21308 RVVArgInfos.push_back({1, RegisterVT,
false});
21315void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
21320 bool FirstVMaskAssigned =
false;
21321 for (
Type *Ty : TypeList) {
21331 RVVArgInfos.push_back(
21337 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
21352 if (!FirstVMaskAssigned &&
21354 RVVArgInfos.push_back({1, RegisterVT,
true});
21355 FirstVMaskAssigned =
true;
21359 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
21365void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
21366 unsigned StartReg) {
21367 assert((StartReg % LMul) == 0 &&
21368 "Start register number should be multiple of lmul");
21387 for (
unsigned i = 0; i < NF; ++i)
21389 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
21391 AllocatedPhysRegs.push_back(
MCPhysReg());
21397void RVVArgDispatcher::compute() {
21399 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
21402 AllocatedPhysRegs.push_back(RISCV::V0);
21408 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
21409 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
21410 StartReg += RegsNeeded) {
21411 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
21412 if ((AssignedMap & Map) == 0) {
21413 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
21414 AssignedMap |=
Map;
21419 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
21422 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
21423 allocate(RVVArgInfos[i]);
21427 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
21428 return AllocatedPhysRegs[CurIdx++];
21433#define GET_RISCVVIntrinsicsTable_IMPL
21434#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static bool IsSelect(MachineInstr &MI)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary operation to its equivalent VW or VW_W form.
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG)
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getScalarStoreSize() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
bool hasVInstructionsBF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Helper struct to store a base, index and offset that forms an address.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
bool hasScalarOperand() const
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)