38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
81 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
91 !Subtarget.hasStdExtF()) {
92 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
93 "doesn't support the F instruction set extension (ignoring "
97 !Subtarget.hasStdExtD()) {
98 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
99 "doesn't support the D instruction set extension (ignoring "
125 if (Subtarget.hasStdExtZfhmin())
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
133 if (Subtarget.hasStdExtZhinxmin())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
253 if (!Subtarget.hasVendorXCValu())
257 if (!Subtarget.hasVendorXCValu())
276 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
288 if (!Subtarget.hasStdExtZbb())
292 if (Subtarget.hasStdExtZbb()) {
299 if (!Subtarget.hasStdExtZmmul()) {
303 }
else if (Subtarget.
is64Bit()) {
313 if (!Subtarget.hasStdExtM()) {
319 }
else if (Subtarget.
is64Bit()) {
322 {MVT::i8, MVT::i16, MVT::i32},
Custom);
339 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
342 }
else if (Subtarget.hasVendorXTHeadBb()) {
346 }
else if (Subtarget.hasVendorXCVbitmanip()) {
357 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
358 Subtarget.hasVendorXTHeadBb())
363 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
364 Subtarget.hasVendorXTHeadBb())
369 if (Subtarget.hasVendorXCVbitmanip()) {
377 if (Subtarget.hasStdExtZbb()) {
390 }
else if (!Subtarget.hasVendorXCVbitmanip()) {
396 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
397 Subtarget.hasVendorXCVbitmanip()) {
404 if (!Subtarget.hasStdExtZbb())
416 !Subtarget.hasShortForwardBranchOpt())
420 if (Subtarget.hasShortForwardBranchOpt())
423 if (!Subtarget.hasVendorXTHeadCondMov()) {
429 static const unsigned FPLegalNodeTypes[] = {
442 static const unsigned FPOpToExpand[] = {
446 static const unsigned FPRndMode[] = {
453 static const unsigned ZfhminZfbfminPromoteOps[] = {
464 if (Subtarget.hasStdExtZfbfmin()) {
547 if (Subtarget.hasStdExtZfa()) {
564 if (Subtarget.hasStdExtZfa()) {
641 if (Subtarget.hasStdExtZicbop()) {
645 if (Subtarget.hasStdExtA()) {
647 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
651 }
else if (Subtarget.hasForcedAtomics()) {
677 {MVT::i8, MVT::i16},
Custom);
688 static const unsigned IntegerVPOps[] = {
689 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
690 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
691 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
692 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
693 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
694 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
695 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
696 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
697 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
698 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
699 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
700 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
701 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
702 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
703 ISD::EXPERIMENTAL_VP_SPLAT};
705 static const unsigned FloatingPointVPOps[] = {
706 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
707 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
708 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
709 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
710 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
711 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
712 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
713 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
714 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
715 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
716 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
717 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
718 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
719 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
721 static const unsigned IntegerVecReduceOps[] = {
726 static const unsigned FloatingPointVecReduceOps[] = {
739 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
740 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
741 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
745 for (
MVT VT : BoolVecVTs) {
775 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
799 ISD::VP_TRUNCATE, ISD::VP_SETCC},
815 for (
MVT VT : IntVecVTs) {
826 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
874 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
875 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
899 if (Subtarget.hasStdExtZvkb()) {
907 if (Subtarget.hasStdExtZvbb()) {
911 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
917 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
926 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
946 static const unsigned ZvfhminPromoteOps[] = {
956 static const unsigned ZvfhminPromoteVPOps[] = {
957 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
958 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
959 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
960 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
961 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
962 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
963 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
964 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
965 ISD::VP_FMAXIMUM, ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
968 const auto SetCommonVFPActions = [&](
MVT VT) {
1013 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1014 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1045 const auto SetCommonVFPExtLoadTruncStoreActions =
1047 for (
auto SmallVT : SmallerVTs) {
1054 for (
MVT VT : F16VecVTs) {
1057 SetCommonVFPActions(VT);
1060 for (
MVT VT : F16VecVTs) {
1071 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1076 if (Subtarget.hasStdExtZfhmin())
1082 if (VT == MVT::nxv32f16) {
1096 for (
MVT VT : BF16VecVTs) {
1107 if (Subtarget.hasStdExtZfbfmin())
1117 for (
MVT VT : F32VecVTs) {
1120 SetCommonVFPActions(VT);
1121 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1126 for (
MVT VT : F64VecVTs) {
1129 SetCommonVFPActions(VT);
1130 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1131 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1137 if (!useRVVForFixedLengthVectorVT(VT))
1180 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1207 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1229 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1230 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1267 if (Subtarget.hasStdExtZvkb())
1270 if (Subtarget.hasStdExtZvbb()) {
1292 if (!useRVVForFixedLengthVectorVT(VT))
1322 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1325 if (Subtarget.hasStdExtZfhmin()) {
1363 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1364 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1410 if (Subtarget.hasStdExtA()) {
1416 if (Subtarget.hasForcedAtomics()) {
1426 if (Subtarget.hasVendorXTHeadMemIdx()) {
1442 if (Subtarget.hasVendorXCVmem()) {
1452 if (Subtarget.hasVendorXCValu()) {
1478 if (Subtarget.hasStdExtZbb())
1481 if ((Subtarget.hasStdExtZbs() && Subtarget.
is64Bit()) ||
1482 Subtarget.hasStdExtV())
1485 if (Subtarget.hasStdExtZbkb())
1497 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1500 if (Subtarget.hasVendorXTHeadMemPair())
1528MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1533bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1535 bool IsScalable)
const {
1542 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1567 unsigned Intrinsic)
const {
1568 auto &
DL =
I.getDataLayout();
1570 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1571 bool IsUnitStrided,
bool UsePtrVal =
false) {
1576 Info.ptrVal =
I.getArgOperand(PtrOp);
1578 Info.fallbackAddressSpace =
1579 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1583 MemTy =
I.getArgOperand(0)->getType();
1586 MemTy =
I.getType();
1601 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1605 switch (Intrinsic) {
1608 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1609 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1610 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1611 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1612 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1613 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1614 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1615 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1616 case Intrinsic::riscv_masked_cmpxchg_i32:
1618 Info.memVT = MVT::i32;
1619 Info.ptrVal =
I.getArgOperand(0);
1625 case Intrinsic::riscv_seg2_load:
1626 case Intrinsic::riscv_seg3_load:
1627 case Intrinsic::riscv_seg4_load:
1628 case Intrinsic::riscv_seg5_load:
1629 case Intrinsic::riscv_seg6_load:
1630 case Intrinsic::riscv_seg7_load:
1631 case Intrinsic::riscv_seg8_load:
1632 return SetRVVLoadStoreInfo( 0,
false,
1634 case Intrinsic::riscv_seg2_store:
1635 case Intrinsic::riscv_seg3_store:
1636 case Intrinsic::riscv_seg4_store:
1637 case Intrinsic::riscv_seg5_store:
1638 case Intrinsic::riscv_seg6_store:
1639 case Intrinsic::riscv_seg7_store:
1640 case Intrinsic::riscv_seg8_store:
1642 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1645 case Intrinsic::riscv_vle:
1646 case Intrinsic::riscv_vle_mask:
1647 case Intrinsic::riscv_vleff:
1648 case Intrinsic::riscv_vleff_mask:
1649 return SetRVVLoadStoreInfo( 1,
1653 case Intrinsic::riscv_vse:
1654 case Intrinsic::riscv_vse_mask:
1655 return SetRVVLoadStoreInfo( 1,
1659 case Intrinsic::riscv_vlse:
1660 case Intrinsic::riscv_vlse_mask:
1661 case Intrinsic::riscv_vloxei:
1662 case Intrinsic::riscv_vloxei_mask:
1663 case Intrinsic::riscv_vluxei:
1664 case Intrinsic::riscv_vluxei_mask:
1665 return SetRVVLoadStoreInfo( 1,
1668 case Intrinsic::riscv_vsse:
1669 case Intrinsic::riscv_vsse_mask:
1670 case Intrinsic::riscv_vsoxei:
1671 case Intrinsic::riscv_vsoxei_mask:
1672 case Intrinsic::riscv_vsuxei:
1673 case Intrinsic::riscv_vsuxei_mask:
1674 return SetRVVLoadStoreInfo( 1,
1677 case Intrinsic::riscv_vlseg2:
1678 case Intrinsic::riscv_vlseg3:
1679 case Intrinsic::riscv_vlseg4:
1680 case Intrinsic::riscv_vlseg5:
1681 case Intrinsic::riscv_vlseg6:
1682 case Intrinsic::riscv_vlseg7:
1683 case Intrinsic::riscv_vlseg8:
1684 case Intrinsic::riscv_vlseg2ff:
1685 case Intrinsic::riscv_vlseg3ff:
1686 case Intrinsic::riscv_vlseg4ff:
1687 case Intrinsic::riscv_vlseg5ff:
1688 case Intrinsic::riscv_vlseg6ff:
1689 case Intrinsic::riscv_vlseg7ff:
1690 case Intrinsic::riscv_vlseg8ff:
1691 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1694 case Intrinsic::riscv_vlseg2_mask:
1695 case Intrinsic::riscv_vlseg3_mask:
1696 case Intrinsic::riscv_vlseg4_mask:
1697 case Intrinsic::riscv_vlseg5_mask:
1698 case Intrinsic::riscv_vlseg6_mask:
1699 case Intrinsic::riscv_vlseg7_mask:
1700 case Intrinsic::riscv_vlseg8_mask:
1701 case Intrinsic::riscv_vlseg2ff_mask:
1702 case Intrinsic::riscv_vlseg3ff_mask:
1703 case Intrinsic::riscv_vlseg4ff_mask:
1704 case Intrinsic::riscv_vlseg5ff_mask:
1705 case Intrinsic::riscv_vlseg6ff_mask:
1706 case Intrinsic::riscv_vlseg7ff_mask:
1707 case Intrinsic::riscv_vlseg8ff_mask:
1708 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1711 case Intrinsic::riscv_vlsseg2:
1712 case Intrinsic::riscv_vlsseg3:
1713 case Intrinsic::riscv_vlsseg4:
1714 case Intrinsic::riscv_vlsseg5:
1715 case Intrinsic::riscv_vlsseg6:
1716 case Intrinsic::riscv_vlsseg7:
1717 case Intrinsic::riscv_vlsseg8:
1718 case Intrinsic::riscv_vloxseg2:
1719 case Intrinsic::riscv_vloxseg3:
1720 case Intrinsic::riscv_vloxseg4:
1721 case Intrinsic::riscv_vloxseg5:
1722 case Intrinsic::riscv_vloxseg6:
1723 case Intrinsic::riscv_vloxseg7:
1724 case Intrinsic::riscv_vloxseg8:
1725 case Intrinsic::riscv_vluxseg2:
1726 case Intrinsic::riscv_vluxseg3:
1727 case Intrinsic::riscv_vluxseg4:
1728 case Intrinsic::riscv_vluxseg5:
1729 case Intrinsic::riscv_vluxseg6:
1730 case Intrinsic::riscv_vluxseg7:
1731 case Intrinsic::riscv_vluxseg8:
1732 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1735 case Intrinsic::riscv_vlsseg2_mask:
1736 case Intrinsic::riscv_vlsseg3_mask:
1737 case Intrinsic::riscv_vlsseg4_mask:
1738 case Intrinsic::riscv_vlsseg5_mask:
1739 case Intrinsic::riscv_vlsseg6_mask:
1740 case Intrinsic::riscv_vlsseg7_mask:
1741 case Intrinsic::riscv_vlsseg8_mask:
1742 case Intrinsic::riscv_vloxseg2_mask:
1743 case Intrinsic::riscv_vloxseg3_mask:
1744 case Intrinsic::riscv_vloxseg4_mask:
1745 case Intrinsic::riscv_vloxseg5_mask:
1746 case Intrinsic::riscv_vloxseg6_mask:
1747 case Intrinsic::riscv_vloxseg7_mask:
1748 case Intrinsic::riscv_vloxseg8_mask:
1749 case Intrinsic::riscv_vluxseg2_mask:
1750 case Intrinsic::riscv_vluxseg3_mask:
1751 case Intrinsic::riscv_vluxseg4_mask:
1752 case Intrinsic::riscv_vluxseg5_mask:
1753 case Intrinsic::riscv_vluxseg6_mask:
1754 case Intrinsic::riscv_vluxseg7_mask:
1755 case Intrinsic::riscv_vluxseg8_mask:
1756 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1759 case Intrinsic::riscv_vsseg2:
1760 case Intrinsic::riscv_vsseg3:
1761 case Intrinsic::riscv_vsseg4:
1762 case Intrinsic::riscv_vsseg5:
1763 case Intrinsic::riscv_vsseg6:
1764 case Intrinsic::riscv_vsseg7:
1765 case Intrinsic::riscv_vsseg8:
1766 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1769 case Intrinsic::riscv_vsseg2_mask:
1770 case Intrinsic::riscv_vsseg3_mask:
1771 case Intrinsic::riscv_vsseg4_mask:
1772 case Intrinsic::riscv_vsseg5_mask:
1773 case Intrinsic::riscv_vsseg6_mask:
1774 case Intrinsic::riscv_vsseg7_mask:
1775 case Intrinsic::riscv_vsseg8_mask:
1776 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1779 case Intrinsic::riscv_vssseg2:
1780 case Intrinsic::riscv_vssseg3:
1781 case Intrinsic::riscv_vssseg4:
1782 case Intrinsic::riscv_vssseg5:
1783 case Intrinsic::riscv_vssseg6:
1784 case Intrinsic::riscv_vssseg7:
1785 case Intrinsic::riscv_vssseg8:
1786 case Intrinsic::riscv_vsoxseg2:
1787 case Intrinsic::riscv_vsoxseg3:
1788 case Intrinsic::riscv_vsoxseg4:
1789 case Intrinsic::riscv_vsoxseg5:
1790 case Intrinsic::riscv_vsoxseg6:
1791 case Intrinsic::riscv_vsoxseg7:
1792 case Intrinsic::riscv_vsoxseg8:
1793 case Intrinsic::riscv_vsuxseg2:
1794 case Intrinsic::riscv_vsuxseg3:
1795 case Intrinsic::riscv_vsuxseg4:
1796 case Intrinsic::riscv_vsuxseg5:
1797 case Intrinsic::riscv_vsuxseg6:
1798 case Intrinsic::riscv_vsuxseg7:
1799 case Intrinsic::riscv_vsuxseg8:
1800 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1803 case Intrinsic::riscv_vssseg2_mask:
1804 case Intrinsic::riscv_vssseg3_mask:
1805 case Intrinsic::riscv_vssseg4_mask:
1806 case Intrinsic::riscv_vssseg5_mask:
1807 case Intrinsic::riscv_vssseg6_mask:
1808 case Intrinsic::riscv_vssseg7_mask:
1809 case Intrinsic::riscv_vssseg8_mask:
1810 case Intrinsic::riscv_vsoxseg2_mask:
1811 case Intrinsic::riscv_vsoxseg3_mask:
1812 case Intrinsic::riscv_vsoxseg4_mask:
1813 case Intrinsic::riscv_vsoxseg5_mask:
1814 case Intrinsic::riscv_vsoxseg6_mask:
1815 case Intrinsic::riscv_vsoxseg7_mask:
1816 case Intrinsic::riscv_vsoxseg8_mask:
1817 case Intrinsic::riscv_vsuxseg2_mask:
1818 case Intrinsic::riscv_vsuxseg3_mask:
1819 case Intrinsic::riscv_vsuxseg4_mask:
1820 case Intrinsic::riscv_vsuxseg5_mask:
1821 case Intrinsic::riscv_vsuxseg6_mask:
1822 case Intrinsic::riscv_vsuxseg7_mask:
1823 case Intrinsic::riscv_vsuxseg8_mask:
1824 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1861 return isInt<12>(Imm);
1865 return isInt<12>(Imm);
1878 return (SrcBits == 64 && DestBits == 32);
1889 return (SrcBits == 64 && DestBits == 32);
1895 if (Subtarget.hasStdExtV() &&
1900 if (SrcBits == DestBits * 2) {
1911 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1912 EVT MemVT = LD->getMemoryVT();
1913 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1923 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1931 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip();
1935 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1936 Subtarget.hasVendorXCVbitmanip();
1947 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1952 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1956 EVT VT =
Y.getValueType();
1962 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1963 (!isa<ConstantSDNode>(
Y) || cast<ConstantSDNode>(
Y)->isOpaque());
1968 if (Subtarget.hasStdExtZbs())
1969 return X.getValueType().isScalarInteger();
1970 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1972 if (Subtarget.hasVendorXTHeadBs())
1973 return C !=
nullptr;
1975 return C &&
C->getAPIntValue().ule(10);
1995 if (BitSize > Subtarget.
getXLen())
1999 int64_t Val = Imm.getSExtValue();
2007 if (!Subtarget.enableUnalignedScalarMem())
2023 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
2030 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
2034 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
2046 case Instruction::Add:
2047 case Instruction::Sub:
2048 case Instruction::Mul:
2049 case Instruction::And:
2050 case Instruction::Or:
2051 case Instruction::Xor:
2052 case Instruction::FAdd:
2053 case Instruction::FSub:
2054 case Instruction::FMul:
2055 case Instruction::FDiv:
2056 case Instruction::ICmp:
2057 case Instruction::FCmp:
2059 case Instruction::Shl:
2060 case Instruction::LShr:
2061 case Instruction::AShr:
2062 case Instruction::UDiv:
2063 case Instruction::SDiv:
2064 case Instruction::URem:
2065 case Instruction::SRem:
2066 case Instruction::Select:
2067 return Operand == 1;
2081 auto *
II = dyn_cast<IntrinsicInst>(
I);
2085 switch (
II->getIntrinsicID()) {
2086 case Intrinsic::fma:
2087 case Intrinsic::vp_fma:
2088 return Operand == 0 || Operand == 1;
2089 case Intrinsic::vp_shl:
2090 case Intrinsic::vp_lshr:
2091 case Intrinsic::vp_ashr:
2092 case Intrinsic::vp_udiv:
2093 case Intrinsic::vp_sdiv:
2094 case Intrinsic::vp_urem:
2095 case Intrinsic::vp_srem:
2096 case Intrinsic::ssub_sat:
2097 case Intrinsic::vp_ssub_sat:
2098 case Intrinsic::usub_sat:
2099 case Intrinsic::vp_usub_sat:
2100 return Operand == 1;
2102 case Intrinsic::vp_add:
2103 case Intrinsic::vp_mul:
2104 case Intrinsic::vp_and:
2105 case Intrinsic::vp_or:
2106 case Intrinsic::vp_xor:
2107 case Intrinsic::vp_fadd:
2108 case Intrinsic::vp_fmul:
2109 case Intrinsic::vp_icmp:
2110 case Intrinsic::vp_fcmp:
2111 case Intrinsic::smin:
2112 case Intrinsic::vp_smin:
2113 case Intrinsic::umin:
2114 case Intrinsic::vp_umin:
2115 case Intrinsic::smax:
2116 case Intrinsic::vp_smax:
2117 case Intrinsic::umax:
2118 case Intrinsic::vp_umax:
2119 case Intrinsic::sadd_sat:
2120 case Intrinsic::vp_sadd_sat:
2121 case Intrinsic::uadd_sat:
2122 case Intrinsic::vp_uadd_sat:
2124 case Intrinsic::vp_sub:
2125 case Intrinsic::vp_fsub:
2126 case Intrinsic::vp_fdiv:
2127 return Operand == 0 || Operand == 1;
2148 if (!Subtarget.sinkSplatOperands())
2151 for (
auto OpIdx :
enumerate(
I->operands())) {
2155 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2157 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2166 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2171 for (
Use &U :
Op->uses()) {
2223 if (!Subtarget.hasStdExtZfa())
2224 return std::make_pair(-1,
false);
2226 bool IsSupportedVT =
false;
2227 if (VT == MVT::f16) {
2228 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2229 }
else if (VT == MVT::f32) {
2230 IsSupportedVT =
true;
2231 }
else if (VT == MVT::f64) {
2232 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2233 IsSupportedVT =
true;
2237 return std::make_pair(-1,
false);
2240 if (
Index < 0 && Imm.isNegative())
2244 return std::make_pair(
Index,
false);
2248 bool ForCodeSize)
const {
2249 bool IsLegalVT =
false;
2252 else if (VT == MVT::f32)
2254 else if (VT == MVT::f64)
2256 else if (VT == MVT::bf16)
2257 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2269 return Imm.isZero();
2273 if (Imm.isNegZero())
2286 unsigned Index)
const {
2299 if (EltVT == MVT::i1)
2312 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2319 if ((ResElts * 2) != SrcElts)
2363 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2365 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2368 IntermediateVT = MVT::i64;
2371 RegisterVT = MVT::i64;
2386 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2392 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2405 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2406 int64_t
C = RHSC->getSExtValue();
2448 switch (KnownSize) {
2476 return RISCV::VRRegClassID;
2478 return RISCV::VRM2RegClassID;
2480 return RISCV::VRM4RegClassID;
2482 return RISCV::VRM8RegClassID;
2492 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2493 "Unexpected subreg numbering");
2494 return RISCV::sub_vrm1_0 +
Index;
2497 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2498 "Unexpected subreg numbering");
2499 return RISCV::sub_vrm2_0 +
Index;
2502 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2503 "Unexpected subreg numbering");
2504 return RISCV::sub_vrm4_0 +
Index;
2511 return RISCV::VRRegClassID;
2520std::pair<unsigned, unsigned>
2522 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2524 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2525 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2526 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2527 "Register classes not ordered");
2536 unsigned SubRegIdx = RISCV::NoSubRegister;
2537 for (
const unsigned RCID :
2538 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2539 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2543 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2548 return {SubRegIdx, InsertExtractIdx};
2553bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2582unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2589 "Unexpected opcode");
2591 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2593 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2596 return Op.getOperand(
II->VLOperand + 1 + HasChain);
2670bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2671 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2680 "Expected legal fixed length vector!");
2683 unsigned MaxELen = Subtarget.
getELen();
2717 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2724 "Expected to convert into a scalable vector!");
2725 assert(V.getValueType().isFixedLengthVector() &&
2726 "Expected a fixed length vector operand!");
2736 "Expected to convert into a fixed length vector!");
2737 assert(V.getValueType().isScalableVector() &&
2738 "Expected a scalable vector operand!");
2766 const auto [MinVLMAX, MaxVLMAX] =
2768 if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX)
2774static std::pair<SDValue, SDValue>
2783static std::pair<SDValue, SDValue>
2796static std::pair<SDValue, SDValue>
2813std::pair<unsigned, unsigned>
2829 return std::make_pair(MinVLMAX, MaxVLMAX);
2841 EVT VT,
unsigned DefinedValues)
const {
2855 std::tie(LMul, Fractional) =
2858 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2860 Cost = (LMul * DLenFactor);
2905 MVT DstVT =
Op.getSimpleValueType();
2906 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2914 Src.getValueType() == MVT::bf16) {
2921 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2929 Opc,
DL, DstVT, Src,
2943 MVT SrcVT = Src.getSimpleValueType();
2949 if (SatVT != DstEltVT)
2952 MVT DstContainerVT = DstVT;
2953 MVT SrcContainerVT = SrcVT;
2959 "Expected same element count");
2968 {Src, Src, DAG.getCondCode(ISD::SETNE),
2969 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2973 if (DstEltSize > (2 * SrcEltSize)) {
2979 MVT CvtContainerVT = DstContainerVT;
2980 MVT CvtEltVT = DstEltVT;
2981 if (SrcEltSize > (2 * DstEltSize)) {
2990 while (CvtContainerVT != DstContainerVT) {
2996 Res = DAG.
getNode(ClipOpc,
DL, CvtContainerVT, Res, Mask, VL);
3003 Res, DAG.
getUNDEF(DstContainerVT), VL);
3015 case ISD::VP_FROUNDEVEN:
3019 case ISD::VP_FROUNDTOZERO:
3023 case ISD::VP_FFLOOR:
3031 case ISD::VP_FROUND:
3047 MVT VT =
Op.getSimpleValueType();
3054 MVT ContainerVT = VT;
3061 if (
Op->isVPOpcode()) {
3062 Mask =
Op.getOperand(1);
3066 VL =
Op.getOperand(2);
3088 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3102 switch (
Op.getOpcode()) {
3108 case ISD::VP_FFLOOR:
3111 case ISD::VP_FROUND:
3112 case ISD::VP_FROUNDEVEN:
3113 case ISD::VP_FROUNDTOZERO: {
3129 case ISD::VP_FNEARBYINT:
3142 Src, Src, Mask, VL);
3157 MVT VT =
Op.getSimpleValueType();
3161 MVT ContainerVT = VT;
3173 MVT MaskVT = Mask.getSimpleValueType();
3176 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3177 DAG.getUNDEF(MaskVT), Mask, VL});
3181 {Chain, Src, Src, Src, Unorder, VL});
3182 Chain = Src.getValue(1);
3198 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3210 switch (
Op.getOpcode()) {
3221 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3227 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3231 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3240 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3241 Truncated, Mask, VL);
3247 Src, Src, Mask, VL);
3257 MVT VT =
Op.getSimpleValueType();
3285 MVT VT =
Op.getSimpleValueType();
3290 MVT ContainerVT = VT;
3312 if (
Merge.isUndef())
3324 if (
Merge.isUndef())
3333 "Unexpected vector MVT");
3361 return std::nullopt;
3376 unsigned EltSizeInBits) {
3379 return std::nullopt;
3380 bool IsInteger =
Op.getValueType().isInteger();
3382 std::optional<unsigned> SeqStepDenom;
3383 std::optional<int64_t> SeqStepNum, SeqAddend;
3384 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3385 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3390 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3392 if (Elt.isUndef()) {
3393 Elts[
Idx] = std::nullopt;
3397 Elts[
Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3402 return std::nullopt;
3403 Elts[
Idx] = *ExactInteger;
3416 unsigned IdxDiff =
Idx - PrevElt->second;
3417 int64_t ValDiff =
SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3425 int64_t Remainder = ValDiff % IdxDiff;
3427 if (Remainder != ValDiff) {
3430 return std::nullopt;
3436 SeqStepNum = ValDiff;
3437 else if (ValDiff != SeqStepNum)
3438 return std::nullopt;
3441 SeqStepDenom = IdxDiff;
3442 else if (IdxDiff != *SeqStepDenom)
3443 return std::nullopt;
3447 if (!PrevElt || PrevElt->first != *Elt)
3448 PrevElt = std::make_pair(*Elt,
Idx);
3452 if (!SeqStepNum || !SeqStepDenom)
3453 return std::nullopt;
3461 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3462 int64_t Addend =
SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3465 else if (Addend != SeqAddend)
3466 return std::nullopt;
3469 assert(SeqAddend &&
"Must have an addend if we have a step");
3471 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3492 MVT ContainerVT = VT;
3520 MVT VT =
Op.getSimpleValueType();
3532 unsigned MostCommonCount = 0;
3534 unsigned NumUndefElts =
3542 unsigned NumScalarLoads = 0;
3548 ValueCounts.
insert(std::make_pair(V, 0));
3549 unsigned &Count = ValueCounts[V];
3551 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3552 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3557 if (++Count >= MostCommonCount) {
3559 MostCommonCount = Count;
3563 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3564 unsigned NumDefElts = NumElts - NumUndefElts;
3565 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3571 ((MostCommonCount > DominantValueCountThreshold) ||
3584 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3585 LastOp != DominantValue) {
3594 Processed.insert(LastOp);
3599 const SDValue &V = OpIdx.value();
3600 if (V.isUndef() || !Processed.insert(V).second)
3602 if (ValueCounts[V] == 1) {
3611 return DAG.getConstant(V == V1, DL, XLenVT);
3627 MVT VT =
Op.getSimpleValueType();
3657 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3658 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3666 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3667 MVT IntegerViaVecVT =
3672 unsigned BitPos = 0, IntegerEltIdx = 0;
3675 for (
unsigned I = 0;
I < NumElts;) {
3677 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3678 Bits |= ((
uint64_t)BitValue << BitPos);
3684 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3685 if (NumViaIntegerBits <= 32)
3686 Bits = SignExtend64<32>(Bits);
3688 Elts[IntegerEltIdx] = Elt;
3697 if (NumElts < NumViaIntegerBits) {
3701 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3729 int64_t StepNumerator = SimpleVID->StepNumerator;
3730 unsigned StepDenominator = SimpleVID->StepDenominator;
3731 int64_t Addend = SimpleVID->Addend;
3733 assert(StepNumerator != 0 &&
"Invalid step");
3734 bool Negate =
false;
3735 int64_t SplatStepVal = StepNumerator;
3739 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3741 Negate = StepNumerator < 0;
3743 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3750 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3751 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3753 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3756 MVT VIDContainerVT =
3764 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3765 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3767 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3769 if (StepDenominator != 1) {
3774 if (Addend != 0 || Negate) {
3793 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3794 "Unexpected sequence type");
3798 unsigned ViaVecLen =
3802 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3805 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3806 const auto &SeqV = OpIdx.value();
3807 if (!SeqV.isUndef())
3809 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3814 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3815 SplatValue = SignExtend64<32>(SplatValue);
3837 const auto *BV = cast<BuildVectorSDNode>(
Op);
3840 BV->getRepeatedSequence(Sequence) &&
3841 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3842 unsigned SeqLen = Sequence.size();
3844 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3845 ViaIntVT == MVT::i64) &&
3846 "Unexpected sequence type");
3851 const unsigned RequiredVL = NumElts / SeqLen;
3852 const unsigned ViaVecLen =
3854 NumElts : RequiredVL;
3857 unsigned EltIdx = 0;
3858 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3862 for (
const auto &SeqV : Sequence) {
3863 if (!SeqV.isUndef())
3865 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3871 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3872 SplatValue = SignExtend64<32>(SplatValue);
3878 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3879 "Unexpected bitcast sequence");
3880 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3883 MVT ViaContainerVT =
3890 if (ViaVecLen != RequiredVL)
3907 if (EltBitSize - SignBits < 8) {
3911 Source, DAG, Subtarget);
3932 return RISCV::PACKH;
3934 return Subtarget.
is64Bit() ? RISCV::PACKW : RISCV::PACK;
3949 MVT VT =
Op.getSimpleValueType();
3957 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
3962 if (ElemSizeInBits >= std::min(Subtarget.
getELen(), Subtarget.
getXLen()) ||
3976 if (Subtarget.hasStdExtZbkb())
3981 ElemDL, XLenVT,
A,
B),
3988 Flags.setDisjoint(
true);
3994 NewOperands.
reserve(NumElts / 2);
3996 NewOperands.
push_back(pack(
Op.getOperand(i),
Op.getOperand(i + 1)));
4006 MVT VT =
Op.getSimpleValueType();
4017 MVT VT =
Op.getSimpleValueType();
4022 !Subtarget.hasStdExtZfhmin())
4094 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
4098 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4121 unsigned NumUndefElts =
4123 unsigned NumDefElts = NumElts - NumUndefElts;
4124 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4131 for (
unsigned i = 0; i < NumElts; i++) {
4133 if (i < NumElts / 2) {
4140 bool SelectMaskVal = (i < NumElts / 2);
4143 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
4144 MaskVals.
size() == NumElts);
4179 unsigned UndefCount = 0;
4186 LinearBudget -= PerSlideCost;
4189 LinearBudget -= PerSlideCost;
4192 LinearBudget -= PerSlideCost;
4195 if (LinearBudget < 0)
4200 "Illegal type which will result in reserved encoding");
4225 Vec,
Offset, Mask, VL, Policy);
4238 Vec,
Offset, Mask, VL, Policy);
4248 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4249 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4250 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4253 if ((LoC >> 31) == HiC)
4264 (isa<RegisterSDNode>(VL) &&
4265 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4267 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4282 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4283 Hi.getConstantOperandVal(1) == 31)
4302 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4314 bool HasPassthru = Passthru && !Passthru.
isUndef();
4315 if (!HasPassthru && !Passthru)
4323 if (Scalar.getValueType().bitsLE(XLenVT)) {
4330 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4334 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4335 "Unexpected scalar for splat lowering!");
4359 SDValue ExtractedVal = Scalar.getOperand(0);
4364 MVT ExtractedContainerVT = ExtractedVT;
4367 DAG, ExtractedContainerVT, Subtarget);
4369 ExtractedVal, DAG, Subtarget);
4371 if (ExtractedContainerVT.
bitsLE(VT))
4386 if (!Scalar.getValueType().bitsLE(XLenVT))
4389 VT,
DL, DAG, Subtarget);
4397 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4423 if (Src != V2.getOperand(0))
4427 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4432 V2.getConstantOperandVal(1) != Mask.size())
4436 if (Mask[0] != 0 && Mask[0] != 1)
4441 for (
unsigned i = 1; i != Mask.size(); ++i)
4442 if (Mask[i] != Mask[i - 1] + 2)
4460 int Size = Mask.size();
4462 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4468 EvenSrc = StartIndexes[0];
4469 OddSrc = StartIndexes[1];
4472 if (EvenSrc != 0 && OddSrc != 0)
4482 int HalfNumElts = NumElts / 2;
4483 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4499 int Size = Mask.size();
4511 for (
int i = 0; i !=
Size; ++i) {
4517 int StartIdx = i - (M %
Size);
4525 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4528 Rotation = CandidateRotation;
4529 else if (Rotation != CandidateRotation)
4534 int MaskSrc = M <
Size ? 0 : 1;
4539 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4544 TargetSrc = MaskSrc;
4545 else if (TargetSrc != MaskSrc)
4552 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4553 assert((LoSrc >= 0 || HiSrc >= 0) &&
4554 "Failed to find a rotated input vector!");
4569 MVT ContainerVT = VT;
4572 assert(Src.getSimpleValueType().isFixedLengthVector());
4576 MVT SrcContainerVT =
4589 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4596 unsigned Shift = EvenElts ? 0 : EltBits;
4602 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4628 auto findNonEXTRACT_SUBVECTORParent =
4629 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4634 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4635 Offset += Parent.getConstantOperandVal(1);
4636 Parent = Parent.getOperand(0);
4638 return std::make_pair(Parent,
Offset);
4641 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4642 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4651 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4652 if (NewMask[i] == -1)
4655 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4656 NewMask[i] = NewMask[i] + V1IndexOffset;
4660 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4666 if (NewMask[0] <= 0)
4670 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4671 if (NewMask[i - 1] + 1 != NewMask[i])
4675 MVT SrcVT = Src.getSimpleValueType();
4706 int NumSubElts,
Index;
4711 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4712 SDValue InPlace = OpsSwapped ? V2 : V1;
4713 SDValue ToInsert = OpsSwapped ? V1 : V2;
4723 if (NumSubElts +
Index >= (
int)NumElts)
4737 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4749 bool OpsSwapped =
false;
4750 if (!isa<BuildVectorSDNode>(V1)) {
4751 if (!isa<BuildVectorSDNode>(V2))
4756 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4764 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4765 for (
unsigned i = S; i != E; ++i)
4766 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4772 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4773 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4776 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4778 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4783 auto OpCode = IsVSlidedown ?
4788 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4791 Splat, TrueMask, VL);
4802 MVT VecContainerVT = VecVT;
4819 MVT WideContainerVT = WideVT;
4825 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4838 }
else if (Subtarget.hasStdExtZvbb()) {
4843 OffsetVec, Passthru, Mask, VL);
4846 Interleaved, EvenV, Passthru, Mask, VL);
4854 Interleaved, OffsetVec, Passthru, Mask, VL);
4862 OddV, Passthru, Mask, VL);
4868 OddV, AllOnesVec, Passthru, Mask, VL);
4876 Interleaved, OddsMul, Passthru, Mask, VL);
4883 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4929 if (ViaEltSize > NumElts)
4938 if (ViaEltSize > NumElts)
4944 if (ViaEltSize > NumElts)
4953 MVT &RotateVT,
unsigned &RotateAmt) {
4959 unsigned NumSubElts;
4961 NumElts, NumSubElts, RotateAmt))
4964 NumElts / NumSubElts);
5026 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
5029 OutMasks(VRegsPerSrc, {-1, {}});
5034 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
5035 int DstVecIdx = DstIdx / ElemsPerVReg;
5036 int DstSubIdx = DstIdx % ElemsPerVReg;
5037 int SrcIdx = Mask[DstIdx];
5038 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
5040 int SrcVecIdx = SrcIdx / ElemsPerVReg;
5041 int SrcSubIdx = SrcIdx % ElemsPerVReg;
5042 if (OutMasks[DstVecIdx].first == -1)
5043 OutMasks[DstVecIdx].first = SrcVecIdx;
5044 if (OutMasks[DstVecIdx].first != SrcVecIdx)
5050 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
5051 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
5065 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
5066 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
5067 if (SrcVecIdx == -1)
5069 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
5076 unsigned InsertIdx = DstVecIdx * NumOpElts;
5089 MVT VT =
Op.getSimpleValueType();
5104 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
5128 V.getOperand(0).getSimpleValueType().getVectorNumElements();
5129 V = V.getOperand(
Offset / OpElements);
5135 auto *Ld = cast<LoadSDNode>(V);
5145 SDValue Ops[] = {Ld->getChain(),
5159 MVT SplatVT = ContainerVT;
5162 if (SVT == MVT::f16 && !Subtarget.hasStdExtZfh()) {
5171 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
5172 Ld->getPointerInfo().getWithOffset(
Offset),
5173 Ld->getOriginalAlign(),
5177 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
5178 Ld->getOriginalAlign(),
5179 Ld->getMemOperand()->getFlags());
5191 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
5194 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5216 if (Subtarget.hasStdExtZvkb())
5227 LoV = LoSrc == 0 ? V1 : V2;
5231 HiV = HiSrc == 0 ? V1 : V2;
5237 unsigned InvRotate = NumElts - Rotation;
5247 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5267 int EvenSrc, OddSrc;
5272 int Size = Mask.size();
5274 assert(EvenSrc >= 0 &&
"Undef source?");
5275 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5279 assert(OddSrc >= 0 &&
"Undef source?");
5280 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5289 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5298 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5327 MVT IndexContainerVT =
5332 for (
int MaskIndex : Mask) {
5333 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5342 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5352 for (
int MaskIndex : Mask) {
5353 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5354 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5356 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5371 for (
int MaskIndex : Mask) {
5372 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5376 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5408RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5410 MVT VT =
Op.getSimpleValueType();
5414 MVT ContainerVT = VT;
5417 if (
Op->isVPOpcode()) {
5418 Mask =
Op.getOperand(1);
5422 VL =
Op.getOperand(2);
5428 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5430 FloatEltVT = MVT::f32;
5437 "Expected legal float type!");
5444 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5447 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5452 if (FloatVT.
bitsGT(VT)) {
5453 if (
Op->isVPOpcode())
5454 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5463 if (!
Op->isVPOpcode())
5467 MVT ContainerFloatVT =
5470 Src, Mask, RTZRM, VL);
5477 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5481 if (
Op->isVPOpcode()) {
5490 else if (IntVT.
bitsGT(VT))
5495 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5500 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5501 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5506 unsigned Adjust = ExponentBias + (EltSize - 1);
5508 if (
Op->isVPOpcode())
5518 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5519 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5529 MVT SrcVT =
Source.getSimpleValueType();
5538 SrcVT = ContainerVT;
5551 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5568 auto *
Load = cast<LoadSDNode>(
Op);
5569 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5572 Load->getMemoryVT(),
5573 *
Load->getMemOperand()))
5577 MVT VT =
Op.getSimpleValueType();
5579 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5580 "Unexpected unaligned RVV load type");
5584 "Expecting equally-sized RVV vector types to be legal");
5586 Load->getPointerInfo(),
Load->getOriginalAlign(),
5587 Load->getMemOperand()->getFlags());
5597 auto *
Store = cast<StoreSDNode>(
Op);
5598 assert(Store &&
Store->getValue().getValueType().isVector() &&
5599 "Expected vector store");
5602 Store->getMemoryVT(),
5603 *
Store->getMemOperand()))
5610 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5611 "Unexpected unaligned RVV store type");
5615 "Expecting equally-sized RVV vector types to be legal");
5616 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5618 Store->getPointerInfo(),
Store->getOriginalAlign(),
5619 Store->getMemOperand()->getFlags());
5624 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5626 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5653 unsigned ShiftAmt, AddOpc;
5670 if (Subtarget.hasStdExtZtso()) {
5693 "Unexpected custom legalisation");
5714 "Unexpected custom legalisation");
5729 "Unexpected custom legalisation");
5730 if (isa<ConstantSDNode>(
Op.getOperand(1)))
5750 "Unexpected custom legalisation");
5766 MVT VT =
Op.getSimpleValueType();
5768 unsigned Check =
Op.getConstantOperandVal(1);
5769 unsigned TDCMask = 0;
5797 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5802 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5804 VL =
Op.getOperand(3);
5807 VL,
Op->getFlags());
5822 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5824 MVT MaskContainerVT =
5827 VL =
Op.getOperand(3);
5832 Mask, VL,
Op->getFlags());
5835 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5840 DAG.
getUNDEF(ContainerVT), Mask, VL});
5844 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5848 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5852 DAG.
getUNDEF(ContainerVT), Mask, VL});
5868 MVT VT =
Op.getSimpleValueType();
5895 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5902 MVT ContainerVT = VT;
5910 if (
Op->isVPOpcode()) {
5911 Mask =
Op.getOperand(2);
5915 VL =
Op.getOperand(3);
5923 {X, X, DAG.getCondCode(ISD::SETOEQ),
5924 DAG.getUNDEF(ContainerVT), Mask, VL});
5932 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5933 DAG.getUNDEF(ContainerVT), Mask, VL});
5943 DAG.
getUNDEF(ContainerVT), Mask, VL);
5951#define OP_CASE(NODE) \
5953 return RISCVISD::NODE##_VL;
5954#define VP_CASE(NODE) \
5955 case ISD::VP_##NODE: \
5956 return RISCVISD::NODE##_VL;
5958 switch (
Op.getOpcode()) {
6036 case ISD::VP_CTLZ_ZERO_UNDEF:
6039 case ISD::VP_CTTZ_ZERO_UNDEF:
6048 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6053 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6058 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
6061 case ISD::VP_SELECT:
6070 case ISD::VP_SIGN_EXTEND:
6072 case ISD::VP_ZERO_EXTEND:
6074 case ISD::VP_FP_TO_SINT:
6076 case ISD::VP_FP_TO_UINT:
6079 case ISD::VP_FMINNUM:
6082 case ISD::VP_FMAXNUM:
6087 case ISD::VP_LLRINT:
6099 "not a RISC-V target specific op");
6105 "adding target specific op should update this function");
6125 "not a RISC-V target specific op");
6131 "adding target specific op should update this function");
6150 if (!
Op.getOperand(j).getValueType().isVector()) {
6151 LoOperands[j] =
Op.getOperand(j);
6152 HiOperands[j] =
Op.getOperand(j);
6155 std::tie(LoOperands[j], HiOperands[j]) =
6160 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6162 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6177 std::tie(LoOperands[j], HiOperands[j]) =
6181 if (!
Op.getOperand(j).getValueType().isVector()) {
6182 LoOperands[j] =
Op.getOperand(j);
6183 HiOperands[j] =
Op.getOperand(j);
6186 std::tie(LoOperands[j], HiOperands[j]) =
6191 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6193 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6203 auto [EVLLo, EVLHi] =
6204 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6208 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6210 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6228 if (!
Op.getOperand(j).getValueType().isVector()) {
6229 LoOperands[j] =
Op.getOperand(j);
6230 HiOperands[j] =
Op.getOperand(j);
6233 std::tie(LoOperands[j], HiOperands[j]) =
6238 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6241 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6250 switch (
Op.getOpcode()) {
6256 return lowerGlobalAddress(
Op, DAG);
6258 return lowerBlockAddress(
Op, DAG);
6260 return lowerConstantPool(
Op, DAG);
6262 return lowerJumpTable(
Op, DAG);
6264 return lowerGlobalTLSAddress(
Op, DAG);
6268 return lowerSELECT(
Op, DAG);
6270 return lowerBRCOND(
Op, DAG);
6272 return lowerVASTART(
Op, DAG);
6274 return lowerFRAMEADDR(
Op, DAG);
6276 return lowerRETURNADDR(
Op, DAG);
6283 return lowerShiftLeftParts(
Op, DAG);
6285 return lowerShiftRightParts(
Op, DAG,
true);
6287 return lowerShiftRightParts(
Op, DAG,
false);
6290 if (
Op.getValueType().isFixedLengthVector()) {
6291 assert(Subtarget.hasStdExtZvkb());
6292 return lowerToScalableOp(
Op, DAG);
6294 assert(Subtarget.hasVendorXTHeadBb() &&
6295 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6296 "Unexpected custom legalization");
6298 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6303 EVT VT =
Op.getValueType();
6307 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6313 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6314 Subtarget.hasStdExtZfbfmin()) {
6319 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6326 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6343 "Unexpected types");
6377 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6379 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6381 return LowerINTRINSIC_VOID(
Op, DAG);
6383 return LowerIS_FPCLASS(
Op, DAG);
6385 MVT VT =
Op.getSimpleValueType();
6387 assert(Subtarget.hasStdExtZvbb());
6388 return lowerToScalableOp(
Op, DAG);
6391 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6399 if (!
Op.getSimpleValueType().isVector())
6401 return lowerVectorTruncLike(
Op, DAG);
6404 if (
Op.getOperand(0).getValueType().isVector() &&
6405 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6406 return lowerVectorMaskExt(
Op, DAG, 1);
6409 if (
Op.getOperand(0).getValueType().isVector() &&
6410 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6411 return lowerVectorMaskExt(
Op, DAG, -1);
6414 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6416 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6418 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6420 MVT VT =
Op.getSimpleValueType();
6428 MVT ContainerVT = VT;
6434 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6441 MVT VT =
Op.getSimpleValueType();
6461 }
else if ((Val % 8) == 0) {
6477 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6478 Op.getOperand(1).getValueType() == MVT::i32) {
6490 if (
Op.getValueType() == MVT::nxv32f16 &&
6497 EVT VT =
Op.getValueType();
6500 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6502 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6508 if (!
Op.getValueType().isVector())
6510 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6514 EVT VT =
Op.getValueType();
6517 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6519 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6527 if (!
Op.getValueType().isVector())
6529 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6533 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6536 if (
Op.getValueType().isVector() &&
6537 Op.getValueType().getScalarType() == MVT::f16 &&
6540 if (
Op.getValueType() == MVT::nxv32f16)
6555 Op1.getValueType().isVector() &&
6556 Op1.getValueType().getScalarType() == MVT::f16 &&
6559 if (Op1.getValueType() == MVT::nxv32f16)
6564 Op1.getValueType().getVectorElementCount());
6567 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6577 MVT VT =
Op.getSimpleValueType();
6581 bool IsStrict =
Op->isStrictFPOpcode();
6582 SDValue Src =
Op.getOperand(0 + IsStrict);
6584 MVT SrcVT = Src.getSimpleValueType();
6589 "Unexpected vector element types");
6593 if (EltSize > (2 * SrcEltSize)) {
6605 Op.getOperand(0), Ext);
6609 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6614 auto [FExt, Chain] =
6616 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6623 if (SrcEltSize > (2 * EltSize)) {
6626 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6631 Op.getOperand(0), Src);
6646 Op.getOperand(0), Src);
6660 unsigned RVVOpc = 0;
6661 switch (
Op.getOpcode()) {
6693 "Expected same element count");
6700 Op.getOperand(0), Src, Mask, VL);
6704 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6719 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6726 MVT VT =
Op.getSimpleValueType();
6748 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6764 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6784 return lowerVECREDUCE(
Op, DAG);
6788 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6789 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6790 return lowerVECREDUCE(
Op, DAG);
6797 return lowerFPVECREDUCE(
Op, DAG);
6798 case ISD::VP_REDUCE_ADD:
6799 case ISD::VP_REDUCE_UMAX:
6800 case ISD::VP_REDUCE_SMAX:
6801 case ISD::VP_REDUCE_UMIN:
6802 case ISD::VP_REDUCE_SMIN:
6803 case ISD::VP_REDUCE_FADD:
6804 case ISD::VP_REDUCE_SEQ_FADD:
6805 case ISD::VP_REDUCE_FMIN:
6806 case ISD::VP_REDUCE_FMAX:
6807 case ISD::VP_REDUCE_FMINIMUM:
6808 case ISD::VP_REDUCE_FMAXIMUM:
6809 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6813 return lowerVPREDUCE(
Op, DAG);
6814 case ISD::VP_REDUCE_AND:
6815 case ISD::VP_REDUCE_OR:
6816 case ISD::VP_REDUCE_XOR:
6817 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6818 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6819 return lowerVPREDUCE(
Op, DAG);
6820 case ISD::VP_CTTZ_ELTS:
6821 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
6822 return lowerVPCttzElements(
Op, DAG);
6826 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6829 return lowerINSERT_SUBVECTOR(
Op, DAG);
6831 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6833 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6835 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6837 return lowerSTEP_VECTOR(
Op, DAG);
6839 return lowerVECTOR_REVERSE(
Op, DAG);
6841 return lowerVECTOR_SPLICE(
Op, DAG);
6845 if ((
Op.getValueType().getScalarType() == MVT::f16 &&
6849 (
Op.getValueType().getScalarType() == MVT::bf16 &&
6851 if (
Op.getValueType() == MVT::nxv32f16 ||
6852 Op.getValueType() == MVT::nxv32bf16)
6864 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6865 return lowerVectorMaskSplat(
Op, DAG);
6873 MVT VT =
Op.getSimpleValueType();
6874 MVT ContainerVT = VT;
6892 Op->ops().take_front(HalfNumOps));
6894 Op->ops().drop_front(HalfNumOps));
6898 unsigned NumOpElts =
6899 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6902 SDValue SubVec = OpIdx.value();
6913 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6915 if (
Op.getValueType().isFixedLengthVector())
6916 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6919 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6921 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6922 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6926 return lowerMaskedLoad(
Op, DAG);
6929 return lowerMaskedStore(
Op, DAG);
6938 EVT VT =
Op.getValueType();
6949 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6951 MVT VT =
Op.getSimpleValueType();
6956 "Unexpected CondCode");
6964 if (isa<ConstantSDNode>(
RHS)) {
6965 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6966 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6985 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6990 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
7006 return lowerToScalableOp(
Op, DAG);
7010 if (
Op.getSimpleValueType().isFixedLengthVector())
7011 return lowerToScalableOp(
Op, DAG);
7013 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
7014 "Unexpected custom legalisation");
7026 if (
Op.getValueType() == MVT::nxv32f16 &&
7039 return lowerToScalableOp(
Op, DAG);
7042 if (!
Op.getValueType().isVector())
7044 return lowerToScalableOp(
Op, DAG);
7047 if (!
Op.getValueType().isVector())
7049 return lowerToScalableOp(
Op, DAG);
7053 EVT VT =
Op->getValueType(0);
7068 return lowerABS(
Op, DAG);
7073 if (Subtarget.hasStdExtZvbb())
7074 return lowerToScalableOp(
Op, DAG);
7076 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7078 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
7080 if (
Op.getValueType() == MVT::nxv32f16 &&
7084 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
7091 if (
Op.getValueType() == MVT::nxv32f16 &&
7095 return lowerToScalableOp(
Op, DAG);
7098 return lowerVectorStrictFSetcc(
Op, DAG);
7108 case ISD::VP_GATHER:
7109 return lowerMaskedGather(
Op, DAG);
7111 case ISD::VP_SCATTER:
7112 return lowerMaskedScatter(
Op, DAG);
7114 return lowerGET_ROUNDING(
Op, DAG);
7116 return lowerSET_ROUNDING(
Op, DAG);
7118 return lowerEH_DWARF_CFA(
Op, DAG);
7119 case ISD::VP_SELECT:
7128 case ISD::VP_UADDSAT:
7129 case ISD::VP_USUBSAT:
7130 case ISD::VP_SADDSAT:
7131 case ISD::VP_SSUBSAT:
7133 case ISD::VP_LLRINT:
7134 return lowerVPOp(
Op, DAG);
7138 return lowerLogicVPOp(
Op, DAG);
7147 case ISD::VP_FMINNUM:
7148 case ISD::VP_FMAXNUM:
7149 case ISD::VP_FCOPYSIGN:
7150 if (
Op.getValueType() == MVT::nxv32f16 &&
7158 return lowerVPOp(
Op, DAG);
7159 case ISD::VP_IS_FPCLASS:
7160 return LowerIS_FPCLASS(
Op, DAG);
7161 case ISD::VP_SIGN_EXTEND:
7162 case ISD::VP_ZERO_EXTEND:
7163 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7164 return lowerVPExtMaskOp(
Op, DAG);
7165 return lowerVPOp(
Op, DAG);
7166 case ISD::VP_TRUNCATE:
7167 return lowerVectorTruncLike(
Op, DAG);
7168 case ISD::VP_FP_EXTEND:
7169 case ISD::VP_FP_ROUND:
7170 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
7171 case ISD::VP_SINT_TO_FP:
7172 case ISD::VP_UINT_TO_FP:
7173 if (
Op.getValueType().isVector() &&
7174 Op.getValueType().getScalarType() == MVT::f16 &&
7177 if (
Op.getValueType() == MVT::nxv32f16)
7189 case ISD::VP_FP_TO_SINT:
7190 case ISD::VP_FP_TO_UINT:
7192 Op1.getValueType().isVector() &&
7193 Op1.getValueType().getScalarType() == MVT::f16 &&
7196 if (Op1.getValueType() == MVT::nxv32f16)
7201 Op1.getValueType().getVectorElementCount());
7205 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7207 return lowerVPFPIntConvOp(
Op, DAG);
7209 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
7213 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7214 return lowerVPSetCCMaskOp(
Op, DAG);
7220 case ISD::VP_BITREVERSE:
7222 return lowerVPOp(
Op, DAG);
7224 case ISD::VP_CTLZ_ZERO_UNDEF:
7225 if (Subtarget.hasStdExtZvbb())
7226 return lowerVPOp(
Op, DAG);
7227 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7229 case ISD::VP_CTTZ_ZERO_UNDEF:
7230 if (Subtarget.hasStdExtZvbb())
7231 return lowerVPOp(
Op, DAG);
7232 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7234 return lowerVPOp(
Op, DAG);
7235 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7236 return lowerVPStridedLoad(
Op, DAG);
7237 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7238 return lowerVPStridedStore(
Op, DAG);
7240 case ISD::VP_FFLOOR:
7242 case ISD::VP_FNEARBYINT:
7243 case ISD::VP_FROUND:
7244 case ISD::VP_FROUNDEVEN:
7245 case ISD::VP_FROUNDTOZERO:
7246 if (
Op.getValueType() == MVT::nxv32f16 &&
7251 case ISD::VP_FMAXIMUM:
7252 case ISD::VP_FMINIMUM:
7253 if (
Op.getValueType() == MVT::nxv32f16 &&
7258 case ISD::EXPERIMENTAL_VP_SPLICE:
7259 return lowerVPSpliceExperimental(
Op, DAG);
7260 case ISD::EXPERIMENTAL_VP_REVERSE:
7261 return lowerVPReverseExperimental(
Op, DAG);
7262 case ISD::EXPERIMENTAL_VP_SPLAT:
7263 return lowerVPSplatExperimental(
Op, DAG);
7266 "llvm.clear_cache only needs custom lower on Linux targets");
7269 return emitFlushICache(DAG,
Op.getOperand(0),
Op.getOperand(1),
7270 Op.getOperand(2), Flags,
DL);
7278 MakeLibCallOptions CallOptions;
7279 std::pair<SDValue, SDValue> CallResult =
7280 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
7281 {Start,
End, Flags}, CallOptions,
DL, InChain);
7284 return CallResult.second;
7301 N->getOffset(), Flags);
7309template <
class NodeTy>
7311 bool IsLocal,
bool IsExternWeak)
const {
7321 if (IsLocal && !Subtarget.allowTaggedGlobals())
7383 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7392 return getAddr(
N, DAG);
7399 return getAddr(
N, DAG);
7406 return getAddr(
N, DAG);
7411 bool UseGOT)
const {
7475 Args.push_back(Entry);
7508 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7522 Addr = getStaticTLSAddr(
N, DAG,
false);
7525 Addr = getStaticTLSAddr(
N, DAG,
true);
7530 : getDynamicTLSAddr(
N, DAG);
7547 if (
LHS == LHS2 &&
RHS == RHS2) {
7552 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7560 return std::nullopt;
7568 MVT VT =
N->getSimpleValueType(0);
7598 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7601 if (~TrueVal == FalseVal) {
7641 if (Subtarget.hasShortForwardBranchOpt())
7644 unsigned SelOpNo = 0;
7654 unsigned ConstSelOpNo = 1;
7655 unsigned OtherSelOpNo = 2;
7656 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7661 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7662 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7666 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7667 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7673 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7675 std::swap(NewConstOps[0], NewConstOps[1]);
7687 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7689 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7692 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7693 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7702 MVT VT =
Op.getSimpleValueType();
7716 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7744 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7748 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7750 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7751 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7753 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7758 DL, VT, LHSVal, CondV);
7774 if (
Op.hasOneUse()) {
7775 unsigned UseOpc =
Op->use_begin()->getOpcode();
7784 return lowerSELECT(NewSel, DAG);
7813 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7834 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7838 if (TrueVal - 1 == FalseVal)
7840 if (TrueVal + 1 == FalseVal)
7847 RHS == TrueV && LHS == FalseV) {
7864 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7890 LHS, RHS, TargetCC,
Op.getOperand(2));
7908 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7920 int XLenInBytes = Subtarget.
getXLen() / 8;
7922 EVT VT =
Op.getValueType();
7925 unsigned Depth =
Op.getConstantOperandVal(0);
7927 int Offset = -(XLenInBytes * 2);
7943 int XLenInBytes = Subtarget.
getXLen() / 8;
7948 EVT VT =
Op.getValueType();
7950 unsigned Depth =
Op.getConstantOperandVal(0);
7952 int Off = -XLenInBytes;
7953 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7972 EVT VT =
Lo.getValueType();
8011 EVT VT =
Lo.getValueType();
8062 MVT VT =
Op.getSimpleValueType();
8088 MVT VecVT =
Op.getSimpleValueType();
8090 "Unexpected SPLAT_VECTOR_PARTS lowering");
8096 MVT ContainerVT = VecVT;
8116 int64_t ExtTrueVal)
const {
8118 MVT VecVT =
Op.getSimpleValueType();
8121 assert(Src.getValueType().isVector() &&
8122 Src.getValueType().getVectorElementType() == MVT::i1);
8143 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8145 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
8148 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
8153SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
8155 MVT ExtVT =
Op.getSimpleValueType();
8159 MVT VT =
Op.getOperand(0).getSimpleValueType();
8185 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8187 EVT MaskVT =
Op.getValueType();
8190 "Unexpected type for vector mask lowering");
8192 MVT VecVT = Src.getSimpleValueType();
8196 VL =
Op.getOperand(2);
8199 MVT ContainerVT = VecVT;
8205 MVT MaskContainerVT =
8212 std::tie(Mask, VL) =
8220 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8222 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8226 DAG.
getUNDEF(ContainerVT), Mask, VL);
8229 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
8237 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8240 MVT VT =
Op.getSimpleValueType();
8242 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8246 return lowerVectorMaskTruncLike(
Op, DAG);
8254 MVT SrcVT = Src.getSimpleValueType();
8259 "Unexpected vector truncate lowering");
8261 MVT ContainerVT = SrcVT;
8265 VL =
Op.getOperand(2);
8278 std::tie(Mask, VL) =
8287 }
while (SrcEltVT != DstEltVT);
8296RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8301 MVT VT =
Op.getSimpleValueType();
8302 MVT SrcVT = Src.getSimpleValueType();
8303 MVT ContainerVT = VT;
8326 Chain, Src, Mask, VL);
8327 Chain = Src.getValue(1);
8334 Chain, Src, Mask, VL);
8345RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8348 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8355 MVT VT =
Op.getSimpleValueType();
8357 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8360 MVT SrcVT = Src.getSimpleValueType();
8362 bool IsDirectExtend =
8370 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8373 MVT ContainerVT = VT;
8377 VL =
Op.getOperand(2);
8391 std::tie(Mask, VL) =
8397 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8403 unsigned InterConvOpc =
8408 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8410 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8421static std::optional<MVT>
8427 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8429 if (MaxIdx < MinVLMAX)
8431 else if (MaxIdx < MinVLMAX * 2)
8433 else if (MaxIdx < MinVLMAX * 4)
8438 return std::nullopt;
8451 MVT VecVT =
Op.getSimpleValueType();
8465 MVT ContainerVT = VecVT;
8474 MVT OrigContainerVT = ContainerVT;
8477 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8478 const unsigned OrigIdx = IdxC->getZExtValue();
8481 DL, DAG, Subtarget)) {
8482 ContainerVT = *ShrunkVT;
8491 VLEN && ContainerVT.
bitsGT(M1VT)) {
8494 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8495 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8496 unsigned ExtractIdx =
8515 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8516 const auto *CVal = cast<ConstantSDNode>(Val);
8517 if (isInt<32>(CVal->getSExtValue())) {
8518 IsLegalInsert =
true;
8527 if (IsLegalInsert) {
8533 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8549 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8550 MVT I32ContainerVT =
8561 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8566 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8568 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8573 ValInVec, AlignedIdx);
8583 DAG.
getUNDEF(I32ContainerVT), ValLo,
8584 I32Mask, InsertI64VL);
8586 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8587 I32Mask, InsertI64VL);
8589 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8602 Idx, Mask, InsertVL, Policy);
8606 Slideup, AlignedIdx);
8621 EVT EltVT =
Op.getValueType();
8628 MVT ContainerVT = VecVT;
8644 unsigned WidenVecLen;
8647 unsigned MaxEEW = Subtarget.
getELen();
8652 "the number of elements should be power of 2");
8656 ExtractBitIdx =
Idx;
8658 WideEltVT = LargestEltVT;
8661 ExtractElementIdx = DAG.
getNode(
8672 Vec, ExtractElementIdx);
8688 MVT ContainerVT = VecVT;
8699 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8700 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8702 unsigned OrigIdx = IdxC->getZExtValue();
8705 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8706 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8707 unsigned ExtractIdx =
8717 std::optional<uint64_t> MaxIdx;
8720 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8721 MaxIdx = IdxC->getZExtValue();
8723 if (
auto SmallerVT =
8725 ContainerVT = *SmallerVT;
8772 "Unexpected opcode");
8779 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8784 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8785 if (!
II || !
II->hasScalarOperand())
8788 unsigned SplatOp =
II->ScalarOperand + 1 + HasChain;
8801 if (OpVT.
bitsLT(XLenVT)) {
8808 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8818 assert(
II->ScalarOperand > 0 &&
"Unexpected splat operand!");
8819 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8822 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8833 case Intrinsic::riscv_vslide1up:
8834 case Intrinsic::riscv_vslide1down:
8835 case Intrinsic::riscv_vslide1up_mask:
8836 case Intrinsic::riscv_vslide1down_mask: {
8839 bool IsMasked = NumOps == 7;
8845 std::tie(ScalarLo, ScalarHi) =
8853 if (isa<ConstantSDNode>(AVL)) {
8854 const auto [MinVLMAX, MaxVLMAX] =
8858 if (AVLInt <= MinVLMAX) {
8860 }
else if (AVLInt >= 2 * MaxVLMAX) {
8867 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
8901 if (IntNo == Intrinsic::riscv_vslide1up ||
8902 IntNo == Intrinsic::riscv_vslide1up_mask) {
8904 ScalarHi, I32Mask, I32VL);
8906 ScalarLo, I32Mask, I32VL);
8909 ScalarLo, I32Mask, I32VL);
8911 ScalarHi, I32Mask, I32VL);
8960 const unsigned ElementWidth = 8;
8965 [[maybe_unused]]
unsigned MinVF =
8968 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8972 bool Fractional = VF < LMul1VF;
8973 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8994 MVT ContainerVT = OpVT;
9021 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
9025 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
9026 if (!
II || !
II->hasScalarOperand())
9029 unsigned SplatOp =
II->ScalarOperand + 1;
9042 if (OpVT.
bitsLT(XLenVT)) {
9045 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
9058 EVT ValType = V.getValueType();
9059 if (ValType.isVector() && ValType.isFloatingPoint()) {
9062 ValType.getVectorElementCount());
9065 if (ValType.isFixedLengthVector()) {
9067 DAG, V.getSimpleValueType(), Subtarget);
9083 unsigned IntNo =
Op.getConstantOperandVal(0);
9090 case Intrinsic::thread_pointer: {
9094 case Intrinsic::riscv_orc_b:
9095 case Intrinsic::riscv_brev8:
9096 case Intrinsic::riscv_sha256sig0:
9097 case Intrinsic::riscv_sha256sig1:
9098 case Intrinsic::riscv_sha256sum0:
9099 case Intrinsic::riscv_sha256sum1:
9100 case Intrinsic::riscv_sm3p0:
9101 case Intrinsic::riscv_sm3p1: {
9121 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9123 case Intrinsic::riscv_sm4ks:
9124 case Intrinsic::riscv_sm4ed: {
9134 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
Op.getOperand(3));
9138 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
9141 case Intrinsic::riscv_zip:
9142 case Intrinsic::riscv_unzip: {
9145 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
9147 case Intrinsic::riscv_mopr: {
9160 case Intrinsic::riscv_moprr: {
9172 Op.getOperand(2),
Op.getOperand(3));
9174 case Intrinsic::riscv_clmul:
9185 case Intrinsic::riscv_clmulh:
9186 case Intrinsic::riscv_clmulr: {
9204 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
9206 case Intrinsic::experimental_get_vector_length:
9208 case Intrinsic::experimental_cttz_elts:
9210 case Intrinsic::riscv_vmv_x_s: {
9214 case Intrinsic::riscv_vfmv_f_s:
9217 case Intrinsic::riscv_vmv_v_x:
9219 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
9221 case Intrinsic::riscv_vfmv_v_f:
9223 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9224 case Intrinsic::riscv_vmv_s_x: {
9227 if (
Scalar.getValueType().bitsLE(XLenVT)) {
9230 Op.getOperand(1), Scalar,
Op.getOperand(3));
9233 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9250 MVT VT =
Op.getSimpleValueType();
9255 if (
Op.getOperand(1).isUndef())
9271 case Intrinsic::riscv_vfmv_s_f:
9273 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9275 case Intrinsic::riscv_vaesdf_vv:
9276 case Intrinsic::riscv_vaesdf_vs:
9277 case Intrinsic::riscv_vaesdm_vv:
9278 case Intrinsic::riscv_vaesdm_vs:
9279 case Intrinsic::riscv_vaesef_vv:
9280 case Intrinsic::riscv_vaesef_vs:
9281 case Intrinsic::riscv_vaesem_vv:
9282 case Intrinsic::riscv_vaesem_vs:
9283 case Intrinsic::riscv_vaeskf1:
9284 case Intrinsic::riscv_vaeskf2:
9285 case Intrinsic::riscv_vaesz_vs:
9286 case Intrinsic::riscv_vsm4k:
9287 case Intrinsic::riscv_vsm4r_vv:
9288 case Intrinsic::riscv_vsm4r_vs: {
9289 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9290 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9291 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9296 case Intrinsic::riscv_vsm3c:
9297 case Intrinsic::riscv_vsm3me: {
9298 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9299 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9304 case Intrinsic::riscv_vsha2ch:
9305 case Intrinsic::riscv_vsha2cl:
9306 case Intrinsic::riscv_vsha2ms: {
9307 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9308 !Subtarget.hasStdExtZvknhb())
9310 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9311 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9312 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9316 case Intrinsic::riscv_sf_vc_v_x:
9317 case Intrinsic::riscv_sf_vc_v_i:
9318 case Intrinsic::riscv_sf_vc_v_xv:
9319 case Intrinsic::riscv_sf_vc_v_iv:
9320 case Intrinsic::riscv_sf_vc_v_vv:
9321 case Intrinsic::riscv_sf_vc_v_fv:
9322 case Intrinsic::riscv_sf_vc_v_xvv:
9323 case Intrinsic::riscv_sf_vc_v_ivv:
9324 case Intrinsic::riscv_sf_vc_v_vvv:
9325 case Intrinsic::riscv_sf_vc_v_fvv:
9326 case Intrinsic::riscv_sf_vc_v_xvw:
9327 case Intrinsic::riscv_sf_vc_v_ivw:
9328 case Intrinsic::riscv_sf_vc_v_vvw:
9329 case Intrinsic::riscv_sf_vc_v_fvw: {
9330 MVT VT =
Op.getSimpleValueType();
9367 MVT VT =
Op.getSimpleValueType();
9371 if (VT.isFloatingPoint()) {
9376 if (VT.isFixedLengthVector())
9386 if (VT.isFixedLengthVector())
9388 if (VT.isFloatingPoint())
9407 unsigned IntNo =
Op.getConstantOperandVal(1);
9411 case Intrinsic::riscv_seg2_load:
9412 case Intrinsic::riscv_seg3_load:
9413 case Intrinsic::riscv_seg4_load:
9414 case Intrinsic::riscv_seg5_load:
9415 case Intrinsic::riscv_seg6_load:
9416 case Intrinsic::riscv_seg7_load:
9417 case Intrinsic::riscv_seg8_load: {
9420 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9421 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9422 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9423 Intrinsic::riscv_vlseg8};
9424 unsigned NF =
Op->getNumValues() - 1;
9425 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9427 MVT VT =
Op->getSimpleValueType(0);
9433 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9435 ContainerVTs.push_back(MVT::Other);
9443 Load->getMemoryVT(),
Load->getMemOperand());
9445 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9451 case Intrinsic::riscv_sf_vc_v_x_se:
9453 case Intrinsic::riscv_sf_vc_v_i_se:
9455 case Intrinsic::riscv_sf_vc_v_xv_se:
9457 case Intrinsic::riscv_sf_vc_v_iv_se:
9459 case Intrinsic::riscv_sf_vc_v_vv_se:
9461 case Intrinsic::riscv_sf_vc_v_fv_se:
9463 case Intrinsic::riscv_sf_vc_v_xvv_se:
9465 case Intrinsic::riscv_sf_vc_v_ivv_se:
9467 case Intrinsic::riscv_sf_vc_v_vvv_se:
9469 case Intrinsic::riscv_sf_vc_v_fvv_se:
9471 case Intrinsic::riscv_sf_vc_v_xvw_se:
9473 case Intrinsic::riscv_sf_vc_v_ivw_se:
9475 case Intrinsic::riscv_sf_vc_v_vvw_se:
9477 case Intrinsic::riscv_sf_vc_v_fvw_se:
9486 unsigned IntNo =
Op.getConstantOperandVal(1);
9490 case Intrinsic::riscv_seg2_store:
9491 case Intrinsic::riscv_seg3_store:
9492 case Intrinsic::riscv_seg4_store:
9493 case Intrinsic::riscv_seg5_store:
9494 case Intrinsic::riscv_seg6_store:
9495 case Intrinsic::riscv_seg7_store:
9496 case Intrinsic::riscv_seg8_store: {
9499 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9500 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9501 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9502 Intrinsic::riscv_vsseg8};
9505 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9507 MVT VT =
Op->getOperand(2).getSimpleValueType();
9515 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9517 for (
unsigned i = 0; i < NF; i++)
9519 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9524 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9526 case Intrinsic::riscv_sf_vc_xv_se:
9528 case Intrinsic::riscv_sf_vc_iv_se:
9530 case Intrinsic::riscv_sf_vc_vv_se:
9532 case Intrinsic::riscv_sf_vc_fv_se:
9534 case Intrinsic::riscv_sf_vc_xvv_se:
9536 case Intrinsic::riscv_sf_vc_ivv_se:
9538 case Intrinsic::riscv_sf_vc_vvv_se:
9540 case Intrinsic::riscv_sf_vc_fvv_se:
9542 case Intrinsic::riscv_sf_vc_xvw_se:
9544 case Intrinsic::riscv_sf_vc_ivw_se:
9546 case Intrinsic::riscv_sf_vc_vvw_se:
9548 case Intrinsic::riscv_sf_vc_fvw_se:
9556 switch (ISDOpcode) {
9559 case ISD::VP_REDUCE_ADD:
9562 case ISD::VP_REDUCE_UMAX:
9565 case ISD::VP_REDUCE_SMAX:
9568 case ISD::VP_REDUCE_UMIN:
9571 case ISD::VP_REDUCE_SMIN:
9574 case ISD::VP_REDUCE_AND:
9577 case ISD::VP_REDUCE_OR:
9580 case ISD::VP_REDUCE_XOR:
9583 case ISD::VP_REDUCE_FADD:
9585 case ISD::VP_REDUCE_SEQ_FADD:
9587 case ISD::VP_REDUCE_FMAX:
9588 case ISD::VP_REDUCE_FMAXIMUM:
9590 case ISD::VP_REDUCE_FMIN:
9591 case ISD::VP_REDUCE_FMINIMUM:
9601 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9606 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9607 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9608 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9609 "Unexpected reduction lowering");
9613 MVT ContainerVT = VecVT;
9622 VL =
Op.getOperand(3);
9624 std::tie(Mask, VL) =
9632 switch (
Op.getOpcode()) {
9636 case ISD::VP_REDUCE_AND: {
9646 case ISD::VP_REDUCE_OR:
9653 case ISD::VP_REDUCE_XOR: {
9676 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9680 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9681 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9682 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9683 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9699 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9703 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9706 if (M1VT != InnerVT)
9712 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9731 VecEVT =
Lo.getValueType();
9744 MVT ContainerVT = VecVT;
9764 Mask, VL,
DL, DAG, Subtarget);
9770static std::tuple<unsigned, SDValue, SDValue>
9774 auto Flags =
Op->getFlags();
9775 unsigned Opcode =
Op.getOpcode();
9799 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9807 MVT VecEltVT =
Op.getSimpleValueType();
9811 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9815 MVT ContainerVT = VecVT;
9821 MVT ResVT =
Op.getSimpleValueType();
9824 VL,
DL, DAG, Subtarget);
9829 if (
Op->getFlags().hasNoNaNs())
9835 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9836 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9842 DL, ResVT, NoNaNs, Res,
9850 unsigned Opc =
Op.getOpcode();
9873 Vec, Mask, VL,
DL, DAG, Subtarget);
9874 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
9875 Op->getFlags().hasNoNaNs())
9892 DL, ResVT, NoNaNs, Res,
9906 unsigned OrigIdx =
Op.getConstantOperandVal(2);
9915 (OrigIdx != 0 || !Vec.
isUndef())) {
9918 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9921 "Unexpected mask vector lowering");
9955 MVT ContainerVT = VecVT;
9991 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
9992 SlideupAmt, Mask, VL, Policy);
10000 MVT ContainerVecVT = VecVT;
10006 MVT ContainerSubVecVT = SubVecVT;
10012 unsigned SubRegIdx;
10022 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10023 SubRegIdx = Decompose.first;
10025 (OrigIdx % Vscale));
10029 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
10030 SubRegIdx = Decompose.first;
10037 bool ExactlyVecRegSized =
10039 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
10054 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
10058 if (SubRegIdx == RISCV::NoSubRegister) {
10077 MVT InterSubVT = ContainerVecVT;
10078 SDValue AlignedExtract = Vec;
10116 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
10117 SlideupAmt, Mask, VL, Policy);
10122 if (ContainerVecVT.
bitsGT(InterSubVT))
10131 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
10137 MVT SubVecVT =
Op.getSimpleValueType();
10142 unsigned OrigIdx =
Op.getConstantOperandVal(1);
10153 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10156 "Unexpected mask vector lowering");
10195 MVT ContainerVT = VecVT;
10203 if (
auto ShrunkVT =
10205 ContainerVT = *ShrunkVT;
10219 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10231 MVT ContainerSubVecVT = SubVecVT;
10235 unsigned SubRegIdx;
10245 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10246 SubRegIdx = Decompose.first;
10248 (OrigIdx % Vscale));
10252 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10253 SubRegIdx = Decompose.first;
10276 MVT InterSubVT = VecVT;
10280 assert(SubRegIdx != RISCV::NoSubRegister);
10294 Vec, SlidedownAmt, Mask, VL);
10303 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10310 MVT VT =
N.getSimpleValueType();
10314 assert(
Op.getSimpleValueType() == VT &&
10315 "Operands and result must be same type");
10319 unsigned NumVals =
N->getNumValues();
10322 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10325 for (
unsigned I = 0;
I < NumVals;
I++) {
10331 if (TruncVals.
size() > 1)
10333 return TruncVals.
front();
10339 MVT VecVT =
Op.getSimpleValueType();
10342 "vector_interleave on non-scalable vector!");
10353 EVT SplitVT = Op0Lo.getValueType();
10356 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10358 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10372 Op.getOperand(0),
Op.getOperand(1));
10399 Concat, EvenIdx, Passthru, Mask, VL);
10401 Concat, OddIdx, Passthru, Mask, VL);
10415 MVT VecVT =
Op.getSimpleValueType();
10418 "vector_interleave on non-scalable vector!");
10431 EVT SplitVT = Op0Lo.getValueType();
10434 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10436 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10458 Op.getOperand(0),
Op.getOperand(1));
10506 MVT VT =
Op.getSimpleValueType();
10511 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10512 if (StepValImm != 1) {
10521 VL, VT,
DL, DAG, Subtarget);
10536 MVT VecVT =
Op.getSimpleValueType();
10546 unsigned MaxVLMAX =
10556 if (MaxVLMAX > 256 && EltSize == 8) {
10603 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10613 MVT VecVT =
Op.getSimpleValueType();
10617 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10618 SDValue DownOffset, UpOffset;
10619 if (ImmValue >= 0) {
10635 DownOffset, TrueMask, UpOffset);
10636 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10642RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10645 auto *
Load = cast<LoadSDNode>(
Op);
10648 Load->getMemoryVT(),
10649 *
Load->getMemOperand()) &&
10650 "Expecting a correctly-aligned load");
10652 MVT VT =
Op.getSimpleValueType();
10658 const auto [MinVLMAX, MaxVLMAX] =
10661 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10675 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10684 Load->getMemoryVT(),
Load->getMemOperand());
10691RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10694 auto *
Store = cast<StoreSDNode>(
Op);
10697 Store->getMemoryVT(),
10698 *
Store->getMemOperand()) &&
10699 "Expecting a correctly-aligned store");
10721 const auto [MinVLMAX, MaxVLMAX] =
10724 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10736 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10739 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10740 Store->getMemoryVT(),
Store->getMemOperand());
10746 MVT VT =
Op.getSimpleValueType();
10748 const auto *MemSD = cast<MemSDNode>(
Op);
10749 EVT MemVT = MemSD->getMemoryVT();
10751 SDValue Chain = MemSD->getChain();
10755 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10756 Mask = VPLoad->getMask();
10758 VL = VPLoad->getVectorLength();
10760 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10761 Mask = MLoad->getMask();
10762 PassThru = MLoad->getPassThru();
10769 MVT ContainerVT = VT;
10783 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10800 Chain =
Result.getValue(1);
10812 const auto *MemSD = cast<MemSDNode>(
Op);
10813 EVT MemVT = MemSD->getMemoryVT();
10815 SDValue Chain = MemSD->getChain();
10819 bool IsCompressingStore =
false;
10820 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10821 Val = VPStore->getValue();
10822 Mask = VPStore->getMask();
10823 VL = VPStore->getVectorLength();
10825 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10826 Val = MStore->getValue();
10827 Mask = MStore->getMask();
10828 IsCompressingStore = MStore->isCompressingStore();
10837 MVT ContainerVT = VT;
10842 if (!IsUnmasked || IsCompressingStore) {
10851 if (IsCompressingStore) {
10854 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10861 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10870 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10874RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10876 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10879 MVT VT =
Op.getSimpleValueType();
10893 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
10900 unsigned Opc =
Op.getOpcode();
10907 MVT VT =
Op.getSimpleValueType();
10940 MVT ContainerInVT = InVT;
10959 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10963 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10971 {Chain, Op1, Op2, CC, Mask, Mask, VL});
10976 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
10989 MVT VT =
Op.getSimpleValueType();
10993 "Unexpected type for ISD::ABS");
10995 MVT ContainerVT = VT;
11002 if (
Op->getOpcode() == ISD::VP_ABS) {
11003 Mask =
Op->getOperand(1);
11007 VL =
Op->getOperand(2);
11015 DAG.
getUNDEF(ContainerVT), Mask, VL);
11017 DAG.
getUNDEF(ContainerVT), Mask, VL);
11024SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
11027 MVT VT =
Op.getSimpleValueType();
11031 "Can only handle COPYSIGN with matching types.");
11040 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
11045SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
11047 MVT VT =
Op.getSimpleValueType();
11050 MVT I1ContainerVT =
11064 Op2, DAG.
getUNDEF(ContainerVT), VL);
11075 MVT VT =
Op.getSimpleValueType();
11080 for (
const SDValue &V :
Op->op_values()) {
11081 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11084 if (!
V.getValueType().isVector()) {
11090 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
11091 "Only fixed length vectors are supported!");
11105 if (
Op->isStrictFPOpcode()) {
11114 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
11128 MVT VT =
Op.getSimpleValueType();
11131 MVT ContainerVT = VT;
11137 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11143 if (*MaskIdx == OpIdx.index())
11147 if (
Op.getOpcode() == ISD::VP_MERGE) {
11151 assert(
Op.getOpcode() == ISD::VP_SELECT);
11158 if (!
V.getValueType().isFixedLengthVector()) {
11163 MVT OpVT =
V.getSimpleValueType();
11165 assert(useRVVForFixedLengthVectorVT(OpVT) &&
11166 "Only fixed length vectors are supported!");
11171 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
11181 MVT VT =
Op.getSimpleValueType();
11187 MVT ContainerVT = VT;
11197 DAG.
getUNDEF(ContainerVT), Zero, VL);
11200 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
11202 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
11205 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
11214 MVT VT =
Op.getSimpleValueType();
11218 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
11222 MVT ContainerVT = VT;
11232 switch (Condition) {
11300 MVT DstVT =
Op.getSimpleValueType();
11301 MVT SrcVT = Src.getSimpleValueType();
11314 if (DstEltSize >= SrcEltSize) {
11323 if (SrcEltSize == 1) {
11334 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11335 }
else if (DstEltSize > (2 * SrcEltSize)) {
11339 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11345 "Wrong input/output vector types");
11348 if (DstEltSize > (2 * SrcEltSize)) {
11364 MVT InterimFVT = DstVT;
11365 if (SrcEltSize > (2 * DstEltSize)) {
11366 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11373 if (InterimFVT != DstVT) {
11379 "Wrong input/output vector types");
11383 if (DstEltSize == 1) {
11386 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11396 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11406 while (InterimIVT != DstVT) {
11418 MVT VT =
Op.getSimpleValueType();
11425RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11437 MVT VT =
Op.getSimpleValueType();
11438 MVT ContainerVT = VT;
11451 if (IsMaskVector) {
11462 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11471 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11474 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11475 SDValue DownOffset, UpOffset;
11476 if (ImmValue >= 0) {
11490 Op1, DownOffset, Mask, UpOffset);
11494 if (IsMaskVector) {
11498 {Result, DAG.getConstant(0, DL, ContainerVT),
11499 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11514 MVT VT =
Op.getSimpleValueType();
11516 MVT ContainerVT = VT;
11532RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11535 MVT VT =
Op.getSimpleValueType();
11542 MVT ContainerVT = VT;
11550 MVT GatherVT = ContainerVT;
11554 if (IsMaskVector) {
11565 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11571 unsigned MaxVLMAX =
11580 if (MaxVLMAX > 256 && EltSize == 8) {
11608 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11610 if (IsMaskVector) {
11633 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11635 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11637 DAG.
getUNDEF(GatherVT), Mask, EVL);
11639 if (IsMaskVector) {
11654 MVT VT =
Op.getSimpleValueType();
11656 return lowerVPOp(
Op, DAG);
11663 MVT ContainerVT = VT;
11682 MVT VT =
Op.getSimpleValueType();
11683 MVT ContainerVT = VT;
11689 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11695 : Intrinsic::riscv_vlse_mask,
11698 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11699 VPNode->getStride()};
11707 Ops.
push_back(VPNode->getVectorLength());
11715 VPNode->getMemoryVT(), VPNode->getMemOperand());
11729 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11730 SDValue StoreVal = VPNode->getValue();
11732 MVT ContainerVT = VT;
11743 : Intrinsic::riscv_vsse_mask,
11746 VPNode->getBasePtr(), VPNode->getStride()};
11754 Ops.
push_back(VPNode->getVectorLength());
11757 Ops, VPNode->getMemoryVT(),
11758 VPNode->getMemOperand());
11770 MVT VT =
Op.getSimpleValueType();
11772 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11773 EVT MemVT = MemSD->getMemoryVT();
11775 SDValue Chain = MemSD->getChain();
11781 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11782 Index = VPGN->getIndex();
11783 Mask = VPGN->getMask();
11785 VL = VPGN->getVectorLength();
11790 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11791 Index = MGN->getIndex();
11792 Mask = MGN->getMask();
11793 PassThru = MGN->getPassThru();
11797 MVT IndexVT =
Index.getSimpleValueType();
11801 "Unexpected VTs!");
11802 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11805 "Unexpected extending MGATHER/VP_GATHER");
11811 MVT ContainerVT = VT;
11835 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11852 Chain =
Result.getValue(1);
11869 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11870 EVT MemVT = MemSD->getMemoryVT();
11872 SDValue Chain = MemSD->getChain();
11875 [[maybe_unused]]
bool IsTruncatingStore =
false;
11878 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11879 Index = VPSN->getIndex();
11880 Mask = VPSN->getMask();
11881 Val = VPSN->getValue();
11882 VL = VPSN->getVectorLength();
11884 IsTruncatingStore =
false;
11887 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
11888 Index = MSN->getIndex();
11889 Mask = MSN->getMask();
11890 Val = MSN->getValue();
11891 IsTruncatingStore = MSN->isTruncatingStore();
11895 MVT IndexVT =
Index.getSimpleValueType();
11899 "Unexpected VTs!");
11900 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11903 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
11909 MVT ContainerVT = VT;
11933 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
11943 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11952 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11960 static const int Table =
11984 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11990 static const unsigned Table =
12013 bool isRISCV64 = Subtarget.
is64Bit();
12077 switch (
N->getOpcode()) {
12079 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
12085 "Unexpected custom legalisation");
12086 bool IsStrict =
N->isStrictFPOpcode();
12089 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
12107 Opc,
DL, VTs, Chain, Op0,
12141 std::tie(Result, Chain) =
12142 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
12170 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
12181 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
12182 "has custom type legalization on riscv32");
12184 SDValue LoCounter, HiCounter;
12188 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
12190 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
12193 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
12195 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
12199 N->getOperand(0), LoCounter, HiCounter);
12223 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
12224 unsigned XLen = Subtarget.
getXLen();
12227 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
12235 if (LHSIsU == RHSIsU)
12252 if (RHSIsU && LHSIsS && !RHSIsS)
12254 else if (LHSIsU && RHSIsS && !LHSIsS)
12264 "Unexpected custom legalisation");
12271 "Unexpected custom legalisation");
12274 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12300 "Unexpected custom legalisation");
12301 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12302 Subtarget.hasVendorXTHeadBb()) &&
12303 "Unexpected custom legalization");
12304 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12305 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12314 "Unexpected custom legalisation");
12328 MVT VT =
N->getSimpleValueType(0);
12329 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12330 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12331 "Unexpected custom legalisation");
12343 if (VT != MVT::i32)
12352 "Unexpected custom legalisation");
12356 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12373 EVT OType =
N->getValueType(1);
12386 "Unexpected custom legalisation");
12403 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12407 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12425 "Unexpected custom legalisation");
12426 if (Subtarget.hasStdExtZbb()) {
12447 "Unexpected custom legalisation");
12453 "Unexpected custom legalisation");
12455 if (Subtarget.hasStdExtZbb()) {
12489 EVT VT =
N->getValueType(0);
12494 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12498 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12499 Subtarget.hasStdExtZfbfmin()) {
12502 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12507 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12509 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12530 MVT VT =
N->getSimpleValueType(0);
12532 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12533 "Unexpected custom legalisation");
12536 "Unexpected extension");
12562 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12564 "Unexpected EXTRACT_VECTOR_ELT legalization");
12567 MVT ContainerVT = VecVT;
12595 DAG.
getUNDEF(ContainerVT), Mask, VL);
12603 unsigned IntNo =
N->getConstantOperandVal(0);
12607 "Don't know how to custom type legalize this intrinsic!");
12608 case Intrinsic::experimental_get_vector_length: {
12613 case Intrinsic::experimental_cttz_elts: {
12619 case Intrinsic::riscv_orc_b:
12620 case Intrinsic::riscv_brev8:
12621 case Intrinsic::riscv_sha256sig0:
12622 case Intrinsic::riscv_sha256sig1:
12623 case Intrinsic::riscv_sha256sum0:
12624 case Intrinsic::riscv_sha256sum1:
12625 case Intrinsic::riscv_sm3p0:
12626 case Intrinsic::riscv_sm3p1: {
12627 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12647 case Intrinsic::riscv_sm4ks:
12648 case Intrinsic::riscv_sm4ed: {
12656 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12660 case Intrinsic::riscv_mopr: {
12661 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12671 case Intrinsic::riscv_moprr: {
12672 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12684 case Intrinsic::riscv_clmul: {
12685 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12696 case Intrinsic::riscv_clmulh:
12697 case Intrinsic::riscv_clmulr: {
12698 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12726 case Intrinsic::riscv_vmv_x_s: {
12727 EVT VT =
N->getValueType(0);
12729 if (VT.
bitsLT(XLenVT)) {
12738 "Unexpected custom legalization");
12776 case ISD::VP_REDUCE_ADD:
12777 case ISD::VP_REDUCE_AND:
12778 case ISD::VP_REDUCE_OR:
12779 case ISD::VP_REDUCE_XOR:
12780 case ISD::VP_REDUCE_SMAX:
12781 case ISD::VP_REDUCE_UMAX:
12782 case ISD::VP_REDUCE_SMIN:
12783 case ISD::VP_REDUCE_UMIN:
12847 const EVT VT =
N->getValueType(0);
12848 const unsigned Opc =
N->getOpcode();
12855 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12860 "Inconsistent mappings");
12871 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12874 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
12889 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
12891 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
12892 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
12896 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
12903 if (
LHS.getOpcode() != ReduceOpc)
12918 auto Flags = ReduceVec->
getFlags();
12919 Flags.intersectWith(
N->getFlags());
12920 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
12930 auto BinOpToRVVReduce = [](
unsigned Opc) {
12959 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
12962 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
12965 unsigned Opc =
N->getOpcode();
12966 unsigned ReduceIdx;
12967 if (IsReduction(
N->getOperand(0), Opc))
12969 else if (IsReduction(
N->getOperand(1), Opc))
12975 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
12978 SDValue Extract =
N->getOperand(ReduceIdx);
13010 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
13037 if (!Subtarget.hasStdExtZba())
13041 EVT VT =
N->getValueType(0);
13053 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13054 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
13057 int64_t C0 = N0C->getSExtValue();
13058 int64_t C1 = N1C->getSExtValue();
13059 if (C0 <= 0 || C1 <= 0)
13063 int64_t Bits = std::min(C0, C1);
13064 int64_t Diff = std::abs(C0 - C1);
13065 if (Diff != 1 && Diff != 2 && Diff != 3)
13092 EVT VT =
N->getValueType(0);
13100 if ((!Subtarget.hasStdExtZicond() &&
13101 !Subtarget.hasVendorXVentanaCondOps()) ||
13123 bool SwapSelectOps;
13129 SwapSelectOps =
false;
13130 NonConstantVal = FalseVal;
13132 SwapSelectOps =
true;
13133 NonConstantVal = TrueVal;
13139 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
13187 EVT VT =
N->getValueType(0);
13195 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13196 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13202 if (!N0C->hasOneUse())
13204 int64_t C0 = N0C->getSExtValue();
13205 int64_t C1 = N1C->getSExtValue();
13207 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
13210 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
13211 !isInt<12>(C0 * (C1 / C0))) {
13214 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
13215 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
13218 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
13219 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
13246 EVT VT =
N->getValueType(0);
13277 unsigned OuterExtend =
13281 OuterExtend,
SDLoc(
N), VT,
13289 EVT VT =
N->getValueType(0);
13337 EVT VT =
N->getValueType(0);
13341 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13347 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13357 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13378 if (!Subtarget.hasStdExtZbb())
13381 EVT VT =
N->getValueType(0);
13383 if (VT != Subtarget.
getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
13392 auto *ShAmtC = dyn_cast<ConstantSDNode>(N0.
getOperand(1));
13393 if (!ShAmtC || ShAmtC->getZExtValue() != 8)
13408 EVT VT =
N->getValueType(0);
13439 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13463 EVT VT =
N->getValueType(0);
13487 EVT VT =
N->getValueType(0);
13514 if (CondLHS != True)
13521 if (!CondRHSC || CondRHSC->
getAPIntValue() != (1ULL << ScalarBits))
13533 if (!FalseRHSC || !FalseRHSC->
isZero())
13553 EVT VT =
N->getValueType(0);
13639 EVT VT =
N->getValueType(0);
13710 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13715 const APInt &Imm = ConstN00->getAPIntValue();
13716 if ((Imm + 1).isSignedIntN(12))
13733 LHS.getValueType());
13754 EVT VT =
N->getValueType(0);
13766 const bool HasShlAdd =
13767 Subtarget.hasStdExtZba() || Subtarget.hasVendorXTHeadBa();
13781 for (
uint64_t Divisor : {3, 5, 9}) {
13782 if (MulAmt % Divisor != 0)
13784 uint64_t MulAmt2 = MulAmt / Divisor;
13791 if (
X.getOpcode() ==
ISD::AND && isa<ConstantSDNode>(
X.getOperand(1)) &&
13792 X.getConstantOperandVal(1) == UINT64_C(0xffffffff)) {
13809 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13825 if (ScaleShift >= 1 && ScaleShift < 4) {
13826 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13840 for (
uint64_t Divisor : {3, 5, 9}) {
13845 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13856 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13858 if (ScaleShift >= 1 && ScaleShift < 4) {
13859 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13885 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
13887 uint64_t ShiftAmt1 = MulAmt + MulAmtLowBit;
13904 EVT VT =
N->getValueType(0);
13911 if (
N->getOperand(0).getOpcode() !=
ISD::AND ||
13912 N->getOperand(0).getOperand(0).getOpcode() !=
ISD::SRL)
13925 if (!V1.
isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
13926 V3 != (HalfSize - 1))
13942 EVT VT =
N->getValueType(0);
13950 unsigned AddSubOpc;
13956 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
13957 AddSubOpc = V->getOpcode();
13959 SDValue Opnd = V->getOperand(1);
13960 MulOper = V->getOperand(0);
13969 if (IsAddSubWith1(N0)) {
13971 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
13974 if (IsAddSubWith1(N1)) {
13976 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
13991 if (isIndexTypeSigned(IndexType))
13994 if (!
N->hasOneUse())
13997 EVT VT =
N.getValueType();
14036 EVT SrcVT = Src.getValueType();
14040 NewElen = std::max(NewElen, 8U);
14063 EVT VT =
N->getValueType(0);
14066 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
14070 auto *N1C = dyn_cast<ConstantSDNode>(N1);
14082 if (!isIntEqualitySetCC(
Cond))
14091 const APInt &C1 = N1C->getAPIntValue();
14109 EVT VT =
N->getValueType(0);
14113 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
14115 Src.getOperand(0));
14123struct CombineResult;
14125enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
14152struct NodeExtensionHelper {
14161 bool SupportsFPExt;
14164 bool EnforceOneUse;
14179 return OrigOperand;
14190 unsigned getExtOpc(ExtKind SupportsExt)
const {
14191 switch (SupportsExt) {
14192 case ExtKind::SExt:
14194 case ExtKind::ZExt:
14196 case ExtKind::FPExt:
14207 std::optional<ExtKind> SupportsExt)
const {
14208 if (!SupportsExt.has_value())
14209 return OrigOperand;
14211 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
14215 if (
Source.getValueType() == NarrowVT)
14218 unsigned ExtOpc = getExtOpc(*SupportsExt);
14222 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
14229 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
14241 DAG.
getUNDEF(NarrowVT), Source, VL);
14254 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
14260 MVT EltVT = SupportsExt == ExtKind::FPExt
14262 :
MVT::getIntegerVT(NarrowSize);
14264 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
14265 "Trying to extend something we can't represent");
14272 static unsigned getSExtOpcode(
unsigned Opcode) {
14295 static unsigned getZExtOpcode(
unsigned Opcode) {
14321 static unsigned getFPExtOpcode(
unsigned Opcode) {
14338 static unsigned getSUOpcode(
unsigned Opcode) {
14340 "SU is only supported for MUL");
14346 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
14366 using CombineToTry = std::function<std::optional<CombineResult>(
14367 SDNode * ,
const NodeExtensionHelper & ,
14372 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
14376 unsigned Opc = OrigOperand.
getOpcode();
14380 "Unexpected Opcode");
14393 unsigned ScalarBits =
Op.getValueSizeInBits();
14395 if (ScalarBits < EltBits) {
14398 !Subtarget.
is64Bit() &&
"Unexpected splat");
14400 SupportsSExt =
true;
14404 SupportsZExt =
true;
14406 EnforceOneUse =
false;
14410 unsigned NarrowSize = EltBits / 2;
14413 if (NarrowSize < 8)
14417 SupportsSExt =
true;
14421 SupportsZExt =
true;
14423 EnforceOneUse =
false;
14430 SupportsZExt =
false;
14431 SupportsSExt =
false;
14432 SupportsFPExt =
false;
14433 EnforceOneUse =
true;
14434 unsigned Opc = OrigOperand.
getOpcode();
14456 SupportsZExt =
true;
14459 SupportsSExt =
true;
14462 SupportsFPExt =
true;
14466 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14479 unsigned ScalarBits =
Op.getOperand(0).getValueSizeInBits();
14480 if (NarrowSize != ScalarBits)
14483 SupportsFPExt =
true;
14492 static bool isSupportedRoot(
const SDNode *Root,
14521 Subtarget.hasStdExtZvbb();
14523 return Subtarget.hasStdExtZvbb();
14532 assert(isSupportedRoot(Root, Subtarget) &&
14533 "Trying to build an helper with an "
14534 "unsupported root");
14535 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14551 if (OperandIdx == 1) {
14560 EnforceOneUse =
false;
14565 fillUpExtensionSupport(Root, DAG, Subtarget);
14571 static std::pair<SDValue, SDValue>
14574 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14593 switch (
N->getOpcode()) {
14631struct CombineResult {
14633 unsigned TargetOpcode;
14635 std::optional<ExtKind> LHSExt;
14636 std::optional<ExtKind> RHSExt;
14640 NodeExtensionHelper
LHS;
14642 NodeExtensionHelper
RHS;
14644 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14645 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14646 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14647 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14656 std::tie(Mask, VL) =
14657 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14671 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14672 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14687static std::optional<CombineResult>
14688canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14689 const NodeExtensionHelper &RHS,
14692 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14693 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14694 Root, LHS, {ExtKind::ZExt}, RHS,
14696 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14697 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14698 Root, LHS, {ExtKind::SExt}, RHS,
14700 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14701 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14702 Root, LHS, {ExtKind::FPExt}, RHS,
14704 return std::nullopt;
14713static std::optional<CombineResult>
14714canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14717 return canFoldToVWWithSameExtensionImpl(
14718 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14726static std::optional<CombineResult>
14727canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14730 if (
RHS.SupportsFPExt)
14731 return CombineResult(
14732 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14733 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14740 return CombineResult(
14741 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14742 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14744 return CombineResult(
14745 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14746 LHS, std::nullopt, RHS, {ExtKind::SExt});
14747 return std::nullopt;
14754static std::optional<CombineResult>
14755canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14758 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14766static std::optional<CombineResult>
14767canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14770 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14778static std::optional<CombineResult>
14779canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14782 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14790static std::optional<CombineResult>
14791canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14795 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14796 return std::nullopt;
14797 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14798 Root, LHS, {ExtKind::SExt}, RHS,
14803NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14814 Strategies.
push_back(canFoldToVWWithSameExtension);
14819 Strategies.
push_back(canFoldToVWWithSameExtension);
14824 Strategies.
push_back(canFoldToVWWithSameExtension);
14831 Strategies.
push_back(canFoldToVWWithZEXT);
14836 Strategies.
push_back(canFoldToVWWithSEXT);
14841 Strategies.
push_back(canFoldToVWWithZEXT);
14846 Strategies.
push_back(canFoldToVWWithFPEXT);
14875 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14881 Inserted.insert(
N);
14884 while (!Worklist.
empty()) {
14886 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
14889 NodeExtensionHelper
LHS(Root, 0, DAG, Subtarget);
14890 NodeExtensionHelper
RHS(Root, 1, DAG, Subtarget);
14891 auto AppendUsersIfNeeded = [&Worklist,
14892 &Inserted](
const NodeExtensionHelper &
Op) {
14893 if (
Op.needToPromoteOtherUsers()) {
14894 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
14895 if (Inserted.insert(TheUse).second)
14907 NodeExtensionHelper::getSupportedFoldings(Root);
14909 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
14910 bool Matched =
false;
14911 for (
int Attempt = 0;
14912 (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
14915 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
14916 FoldingStrategies) {
14917 std::optional<CombineResult> Res =
14918 FoldingStrategy(Root,
LHS,
RHS, DAG, Subtarget);
14925 if (Res->LHSExt.has_value())
14926 AppendUsersIfNeeded(
LHS);
14927 if (Res->RHSExt.has_value())
14928 AppendUsersIfNeeded(
RHS);
14939 SDValue InputRootReplacement;
14946 for (CombineResult Res : CombinesToApply) {
14947 SDValue NewValue = Res.materialize(DAG, Subtarget);
14948 if (!InputRootReplacement) {
14950 "First element is expected to be the current node");
14951 InputRootReplacement = NewValue;
14956 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
14960 return InputRootReplacement;
14967 unsigned Opc =
N->getOpcode();
14972 SDValue MergeOp =
N->getOperand(1);
14973 unsigned MergeOpc = MergeOp.
getOpcode();
14984 SDValue Passthru =
N->getOperand(2);
14998 Z = Z.getOperand(1);
15004 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
15011 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
15040 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
15046 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
15048 if (MemVT == MVT::i32)
15054 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
15089 if (!Subtarget.hasVendorXTHeadMemPair())
15101 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
15103 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
15104 return {
Ptr->getOperand(0), C1->getZExtValue()};
15108 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
15131 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
15134 if (Base1 != Base2)
15138 bool Valid =
false;
15139 if (MemVT == MVT::i32) {
15141 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
15143 }
else if (MemVT == MVT::i64) {
15145 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
15179 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15187 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15197 EVT VT =
N->getValueType(0);
15200 MVT SrcVT = Src.getSimpleValueType();
15201 MVT SrcContainerVT = SrcVT;
15203 SDValue XVal = Src.getOperand(0);
15230 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
15234 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
15238 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
15251 if (VT != MVT::i32 && VT != XLenVT)
15281 EVT DstVT =
N->getValueType(0);
15282 if (DstVT != XLenVT)
15288 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
15296 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
15299 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
15308 if (SatVT == DstVT)
15310 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
15316 Src = Src.getOperand(0);
15337 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
15343 EVT VT =
N->getValueType(0);
15396 unsigned Offset =
N->isTargetStrictFPOpcode();
15403 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
15405 V.getOperand(2) == VL) {
15407 V = V.getOperand(0);
15414 bool NegA = invertIfNegative(
A);
15415 bool NegB = invertIfNegative(
B);
15416 bool NegC = invertIfNegative(
C);
15419 if (!NegA && !NegB && !NegC)
15423 if (
N->isTargetStrictFPOpcode())
15425 {N->getOperand(0), A, B, C, Mask, VL});
15435 if (
N->getValueType(0).getVectorElementType() == MVT::f32 &&
15440 if (
N->isTargetStrictFPOpcode())
15465 switch (
N->getOpcode()) {
15485 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
15486 N->getOperand(2), Mask, VL);
15493 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
15496 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15498 uint64_t ShAmt =
N->getConstantOperandVal(1);
15509 cast<VTSDNode>(N0.
getOperand(1))->getVT() == MVT::i32 &&
15537 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15550 !isa<ConstantSDNode>(U->getOperand(1)) ||
15551 U->getConstantOperandVal(1) > 32)
15606 if (!
Cond.hasOneUse())
15625 EVT VT =
Cond.getValueType();
15670 LHS =
LHS.getOperand(0);
15680 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15684 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15688 RHS =
LHS.getOperand(1);
15689 LHS =
LHS.getOperand(0);
15698 RHS =
LHS.getOperand(1);
15699 LHS =
LHS.getOperand(0);
15715 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15756 bool Commutative =
true;
15757 unsigned Opc = TrueVal.getOpcode();
15765 Commutative =
false;
15773 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15777 if (FalseVal == TrueVal.getOperand(0))
15779 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15784 EVT VT =
N->getValueType(0);
15786 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15792 assert(IdentityOperand &&
"No identity operand!");
15797 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15798 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15819 CountZeroes =
N->getOperand(2);
15820 ValOnZero =
N->getOperand(1);
15822 CountZeroes =
N->getOperand(1);
15823 ValOnZero =
N->getOperand(2);
15842 if (
Cond->getOperand(0) != CountZeroesArgument)
15858 CountZeroes, BitWidthMinusOne);
15868 EVT VT =
N->getValueType(0);
15869 EVT CondVT =
Cond.getValueType();
15877 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15883 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
15904 SDValue TrueVal =
N->getOperand(1);
15905 SDValue FalseVal =
N->getOperand(2);
15920 EVT VT =
N->getValueType(0);
15927 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
15942 if (
Op.isUndef()) {
15955 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
15959 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
15960 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
15964 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
15992 const unsigned InVecOpcode = InVec->
getOpcode();
16002 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
16009 InVecLHS, InValLHS, EltNo);
16011 InVecRHS, InValRHS, EltNo);
16020 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
16023 unsigned Elt = IndexC->getZExtValue();
16031 unsigned ConcatOpIdx = Elt / ConcatNumElts;
16034 ConcatOp, InVal, NewIdx);
16038 ConcatOps[ConcatOpIdx] = ConcatOp;
16050 EVT VT =
N->getValueType(0);
16060 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
16062 !
SDValue(BaseLd, 0).hasOneUse())
16065 EVT BaseLdVT = BaseLd->getValueType(0);
16072 auto *Ld = dyn_cast<LoadSDNode>(
Op);
16073 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
16075 Ld->getValueType(0) != BaseLdVT)
16084 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
16086 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
16091 if (BIO1.equalBaseIndex(BIO2, DAG))
16096 SDValue P2 = Ld2->getBasePtr();
16102 return std::nullopt;
16106 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
16111 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
16112 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
16120 unsigned WideScalarBitWidth =
16133 auto [StrideVariant, MustNegateStride] = *BaseDiff;
16134 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
16135 ? std::get<SDValue>(StrideVariant)
16138 if (MustNegateStride)
16146 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
16147 ConstStride && ConstStride->getSExtValue() >= 0)
16151 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
16157 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
16161 WideVecVT,
DL, BaseLd->getChain(), BaseLd->getBasePtr(), Stride,
16176 if (
N->getValueType(0).isFixedLengthVector())
16179 SDValue Addend =
N->getOperand(0);
16183 SDValue AddMergeOp =
N->getOperand(2);
16188 auto IsVWMulOpc = [](
unsigned Opc) {
16217 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
16218 }(
N, DAG, Subtarget);
16223 if (AddMask != MulMask || AddVL != MulVL)
16228 "Unexpected opcode after VWMACC_VL");
16230 "Unexpected opcode after VWMACC_VL!");
16232 "Unexpected opcode after VWMUL_VL!");
16234 "Unexpected opcode after VWMUL_VL!");
16237 EVT VT =
N->getValueType(0);
16253 const EVT IndexVT =
Index.getValueType();
16257 if (!isIndexTypeSigned(IndexType))
16289 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16292 if (
Index->getOperand(i)->isUndef())
16295 if (
C % ElementSize != 0)
16297 C =
C / ElementSize;
16301 ActiveLanes.
set(
C);
16303 return ActiveLanes.
all();
16321 if (NumElems % 2 != 0)
16325 const unsigned WiderElementSize = ElementSize * 2;
16326 if (WiderElementSize > ST.getELen()/8)
16329 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
16332 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
16335 if (
Index->getOperand(i)->isUndef())
16341 if (
C % WiderElementSize != 0)
16346 if (
C !=
Last + ElementSize)
16363 (isa<RegisterSDNode>(VL) &&
16364 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16366 Mask.getOperand(0) != VL)
16369 auto IsTruncNode = [&](
SDValue V) {
16371 V.getOperand(1) == Mask && V.getOperand(2) == VL;
16378 while (IsTruncNode(
Op)) {
16379 if (!
Op.hasOneUse())
16381 Op =
Op.getOperand(0);
16416 MVT VT =
N->getSimpleValueType(0);
16421 auto MatchMinMax = [&VL, &Mask](
SDValue V,
unsigned Opc,
unsigned OpcVL,
16423 if (V.getOpcode() != Opc &&
16424 !(V.getOpcode() == OpcVL && V.getOperand(2).isUndef() &&
16425 V.getOperand(3) == Mask && V.getOperand(4) == VL))
16433 Op.getOperand(1).getValueType().isFixedLengthVector() &&
16435 Op.getOperand(1).getOperand(0).getValueType() ==
Op.getValueType() &&
16437 Op =
Op.getOperand(1).getOperand(0);
16440 return V.getOperand(0);
16443 Op.getOperand(2) == VL) {
16444 if (
auto *Op1 = dyn_cast<ConstantSDNode>(
Op.getOperand(1))) {
16446 Op1->getAPIntValue().sextOrTrunc(
Op.getScalarValueSizeInBits());
16447 return V.getOperand(0);
16456 auto DetectUSatPattern = [&](
SDValue V) {
16481 V.getOperand(1), DAG.
getUNDEF(V.getValueType()),
16487 auto DetectSSatPattern = [&](
SDValue V) {
16489 unsigned NumSrcBits = V.getScalarValueSizeInBits();
16497 if (HiC == SignedMax && LoC == SignedMin)
16503 if (HiC == SignedMax && LoC == SignedMin)
16513 Src.getOperand(1) == Mask && Src.getOperand(2) == VL &&
16515 Src = Src.getOperand(0);
16519 if ((Val = DetectUSatPattern(Src)))
16521 else if ((Val = DetectSSatPattern(Src)))
16531 Val = DAG.
getNode(ClipOpc,
DL, ValVT, Val, Mask, VL);
16532 }
while (ValVT != VT);
16546 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
16557 switch (
N->getOpcode()) {
16577 APInt V =
C->getValueAPF().bitcastToAPInt();
16612 if (SimplifyDemandedLowBitsHelper(0, 32) ||
16613 SimplifyDemandedLowBitsHelper(1, 5))
16621 if (SimplifyDemandedLowBitsHelper(0, 32))
16638 MVT VT =
N->getSimpleValueType(0);
16647 "Unexpected value type!");
16670 EVT VT =
N->getValueType(0);
16735 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16740 Src.getOperand(0));
16745 Src.getOperand(0), Src.getOperand(1));
16766 unsigned Opc =
N->getOpcode();
16781 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
16791 N->getValueType(0), Val,
Cond.getOperand(0));
16802 SDValue FalseV =
N->getOperand(4);
16804 EVT VT =
N->getValueType(0);
16807 if (TrueV == FalseV)
16812 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16818 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16819 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16822 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16823 isInt<12>(TrueSImm - FalseSImm)) {
16839 {LHS, RHS, CC, TrueV, FalseV});
16906 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
16919 EVT VT =
N->getValueType(0);
16943 const auto *MGN = cast<MaskedGatherSDNode>(
N);
16944 const EVT VT =
N->getValueType(0);
16946 SDValue ScaleOp = MGN->getScale();
16948 assert(!MGN->isIndexScaled() &&
16949 "Scaled gather/scatter should not be formed");
16954 N->getVTList(), MGN->getMemoryVT(),
DL,
16955 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16956 MGN->getBasePtr(), Index, ScaleOp},
16957 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16961 N->getVTList(), MGN->getMemoryVT(),
DL,
16962 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16963 MGN->getBasePtr(), Index, ScaleOp},
16964 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16970 if (std::optional<VIDSequence> SimpleVID =
16972 SimpleVID && SimpleVID->StepDenominator == 1) {
16973 const int64_t StepNumerator = SimpleVID->StepNumerator;
16974 const int64_t Addend = SimpleVID->Addend;
16981 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
16991 MGN->getMask(), EVL, MGN->getMemOperand());
16993 StridedLoad, MGN->getPassThru(), EVL);
17003 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
17005 MGN->getMemoryVT(), MGN->getMemOperand(),
17014 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
17016 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
17018 EVT IndexVT =
Index.getValueType()
17025 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
17027 EltCnt.divideCoefficientBy(2));
17030 EltCnt.divideCoefficientBy(2));
17035 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
17044 const auto *MSN = cast<MaskedScatterSDNode>(
N);
17046 SDValue ScaleOp = MSN->getScale();
17048 assert(!MSN->isIndexScaled() &&
17049 "Scaled gather/scatter should not be formed");
17054 N->getVTList(), MSN->getMemoryVT(),
DL,
17055 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17057 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17061 N->getVTList(), MSN->getMemoryVT(),
DL,
17062 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
17064 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
17066 EVT VT = MSN->getValue()->getValueType(0);
17068 if (!MSN->isTruncatingStore() &&
17072 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
17073 DAG.
getUNDEF(XLenVT), MSN->getMask(),
17074 MSN->getMemoryVT(), MSN->getMemOperand(),
17079 case ISD::VP_GATHER: {
17080 const auto *VPGN = cast<VPGatherSDNode>(
N);
17082 SDValue ScaleOp = VPGN->getScale();
17084 assert(!VPGN->isIndexScaled() &&
17085 "Scaled gather/scatter should not be formed");
17090 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17091 ScaleOp, VPGN->getMask(),
17092 VPGN->getVectorLength()},
17093 VPGN->getMemOperand(), IndexType);
17097 {VPGN->getChain(), VPGN->getBasePtr(), Index,
17098 ScaleOp, VPGN->getMask(),
17099 VPGN->getVectorLength()},
17100 VPGN->getMemOperand(), IndexType);
17104 case ISD::VP_SCATTER: {
17105 const auto *VPSN = cast<VPScatterSDNode>(
N);
17107 SDValue ScaleOp = VPSN->getScale();
17109 assert(!VPSN->isIndexScaled() &&
17110 "Scaled gather/scatter should not be formed");
17115 {VPSN->getChain(), VPSN->getValue(),
17116 VPSN->getBasePtr(), Index, ScaleOp,
17117 VPSN->getMask(), VPSN->getVectorLength()},
17118 VPSN->getMemOperand(), IndexType);
17122 {VPSN->getChain(), VPSN->getValue(),
17123 VPSN->getBasePtr(), Index, ScaleOp,
17124 VPSN->getMask(), VPSN->getVectorLength()},
17125 VPSN->getMemOperand(), IndexType);
17139 EVT VT =
N->getValueType(0);
17142 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
17143 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
17161 EVT VT =
N->getValueType(0);
17165 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
17195 if (
N->getValueType(0).getVectorElementType() == MVT::f32 &&
17209 auto *Store = cast<StoreSDNode>(
N);
17210 SDValue Chain = Store->getChain();
17211 EVT MemVT = Store->getMemoryVT();
17212 SDValue Val = Store->getValue();
17215 bool IsScalarizable =
17217 Store->isSimple() &&
17247 NewVT, *Store->getMemOperand())) {
17249 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
17250 Store->getPointerInfo(), Store->getOriginalAlign(),
17251 Store->getMemOperand()->getFlags());
17259 if (
auto *L = dyn_cast<LoadSDNode>(Val);
17261 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
17263 L->getMemoryVT() == MemVT) {
17266 NewVT, *Store->getMemOperand()) &&
17268 NewVT, *L->getMemOperand())) {
17270 L->getPointerInfo(), L->getOriginalAlign(),
17271 L->getMemOperand()->getFlags());
17272 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
17273 Store->getPointerInfo(), Store->getOriginalAlign(),
17274 Store->getMemOperand()->getFlags());
17286 MVT VecVT = Src.getSimpleValueType();
17293 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
17296 Store->getMemOperand(), Store->getAddressingMode(),
17297 Store->isTruncatingStore(),
false);
17304 EVT VT =
N->getValueType(0);
17326 const MVT VT =
N->getSimpleValueType(0);
17327 SDValue Passthru =
N->getOperand(0);
17328 SDValue Scalar =
N->getOperand(1);
17337 const MVT VT =
N->getSimpleValueType(0);
17338 SDValue Passthru =
N->getOperand(0);
17339 SDValue Scalar =
N->getOperand(1);
17344 unsigned ScalarSize = Scalar.getValueSizeInBits();
17346 if (ScalarSize > EltWidth && Passthru.
isUndef())
17347 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
17354 (!Const || Const->isZero() ||
17355 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
17365 if (
N->getOperand(0).isUndef() &&
17368 Src.getOperand(0).getValueType().isScalableVector()) {
17369 EVT VT =
N->getValueType(0);
17370 EVT SrcVT = Src.getOperand(0).getValueType();
17374 return Src.getOperand(0);
17380 const MVT VT =
N->getSimpleValueType(0);
17381 SDValue Passthru =
N->getOperand(0);
17382 SDValue Scalar =
N->getOperand(1);
17386 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
17387 return Scalar.getOperand(0);
17396 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
17406 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
17414 MVT VecVT =
N->getOperand(0).getSimpleValueType();
17416 if (M1VT.
bitsLT(VecVT)) {
17427 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
17432 case Intrinsic::riscv_vcpop:
17433 case Intrinsic::riscv_vcpop_mask:
17434 case Intrinsic::riscv_vfirst:
17435 case Intrinsic::riscv_vfirst_mask: {
17437 if (IntNo == Intrinsic::riscv_vcpop_mask ||
17438 IntNo == Intrinsic::riscv_vfirst_mask)
17439 VL =
N->getOperand(3);
17444 EVT VT =
N->getValueType(0);
17445 if (IntNo == Intrinsic::riscv_vfirst ||
17446 IntNo == Intrinsic::riscv_vfirst_mask)
17455 EVT VT =
N->getValueType(0);
17459 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
17478 EVT XVT,
unsigned KeptBits)
const {
17483 if (XVT != MVT::i32 && XVT != MVT::i64)
17487 if (KeptBits == 32 || KeptBits == 64)
17491 return Subtarget.hasStdExtZbb() &&
17492 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
17500 "Expected shift op");
17511 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
17512 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17514 const APInt &C1Int = C1->getAPIntValue();
17515 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
17541 if (C1Cost < ShiftedC1Cost)
17555 EVT VT =
Op.getValueType();
17559 unsigned Opcode =
Op.getOpcode();
17567 const APInt &Mask =
C->getAPIntValue();
17576 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17577 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17579 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17580 if (NewMask == Mask)
17585 Op.getOperand(0), NewC);
17598 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17599 if (IsLegalMask(NewMask))
17600 return UseMask(NewMask);
17603 if (VT == MVT::i64) {
17605 if (IsLegalMask(NewMask))
17606 return UseMask(NewMask);
17621 APInt NewMask = ShrunkMask;
17622 if (MinSignedBits <= 12)
17624 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17630 assert(IsLegalMask(NewMask));
17631 return UseMask(NewMask);
17635 static const uint64_t GREVMasks[] = {
17636 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17637 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17639 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17640 unsigned Shift = 1 << Stage;
17641 if (ShAmt & Shift) {
17643 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17655 const APInt &DemandedElts,
17657 unsigned Depth)
const {
17659 unsigned Opc =
Op.getOpcode();
17664 "Should use MaskedValueIsZero if you don't know whether Op"
17665 " is a target node!");
17748 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17751 if (MaxVLenB == MinVLenB)
17768 case Intrinsic::riscv_vsetvli:
17769 case Intrinsic::riscv_vsetvlimax: {
17770 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17771 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17777 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17780 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17781 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17783 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17796 unsigned Depth)
const {
17797 switch (
Op.getOpcode()) {
17803 if (Tmp == 1)
return 1;
17806 return std::min(Tmp, Tmp2);
17818 if (Tmp < 33)
return 1;
17843 unsigned XLen = Subtarget.
getXLen();
17844 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17845 if (EltBits <= XLen)
17846 return XLen - EltBits + 1;
17850 unsigned IntNo =
Op.getConstantOperandVal(1);
17854 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
17855 case Intrinsic::riscv_masked_atomicrmw_add_i64:
17856 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
17857 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
17858 case Intrinsic::riscv_masked_atomicrmw_max_i64:
17859 case Intrinsic::riscv_masked_atomicrmw_min_i64:
17860 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
17861 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
17862 case Intrinsic::riscv_masked_cmpxchg_i64:
17870 assert(Subtarget.hasStdExtA());
17885 switch (
Op.getOpcode()) {
17891 return !
Op.getValueType().isInteger();
17899 assert(Ld &&
"Unexpected null LoadSDNode");
17907 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
17908 if (!CNode || CNode->isMachineConstantPoolEntry() ||
17909 CNode->getOffset() != 0)
17917 auto *CNode = GetSupportedConstantPool(
Ptr);
17918 if (!CNode || CNode->getTargetFlags() != 0)
17921 return CNode->getConstVal();
17929 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
17930 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
17936 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
17939 return CNodeLo->getConstVal();
17944 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
17976 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
17979 int64_t LoCounter =
MI.getOperand(2).getImm();
17980 int64_t HiCounter =
MI.getOperand(3).getImm();
17990 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
18002 MI.eraseFromParent();
18010 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
18018 Register SrcReg =
MI.getOperand(2).getReg();
18038 MI.eraseFromParent();
18045 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
18046 "Unexpected instruction");
18052 Register DstReg =
MI.getOperand(0).getReg();
18075 MI.eraseFromParent();
18080 switch (
MI.getOpcode()) {
18083 case RISCV::Select_GPR_Using_CC_GPR:
18084 case RISCV::Select_GPR_Using_CC_Imm:
18085 case RISCV::Select_FPR16_Using_CC_GPR:
18086 case RISCV::Select_FPR16INX_Using_CC_GPR:
18087 case RISCV::Select_FPR32_Using_CC_GPR:
18088 case RISCV::Select_FPR32INX_Using_CC_GPR:
18089 case RISCV::Select_FPR64_Using_CC_GPR:
18090 case RISCV::Select_FPR64INX_Using_CC_GPR:
18091 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18097 unsigned RelOpcode,
unsigned EqOpcode,
18100 Register DstReg =
MI.getOperand(0).getReg();
18101 Register Src1Reg =
MI.getOperand(1).getReg();
18102 Register Src2Reg =
MI.getOperand(2).getReg();
18104 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18128 MI.eraseFromParent();
18179 F->insert(It, FirstMBB);
18180 F->insert(It, SecondMBB);
18181 F->insert(It, SinkMBB);
18230 First.eraseFromParent();
18269 if ((
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
18270 MI.getOpcode() != RISCV::Select_GPR_Using_CC_Imm) &&
18271 Next != BB->
end() && Next->getOpcode() ==
MI.getOpcode() &&
18272 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
18273 Next->getOperand(5).isKill())
18278 if (
MI.getOperand(2).isReg())
18279 RHS =
MI.getOperand(2).getReg();
18284 SelectDests.
insert(
MI.getOperand(0).getReg());
18288 SequenceMBBI != E; ++SequenceMBBI) {
18289 if (SequenceMBBI->isDebugInstr())
18292 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
18293 !SequenceMBBI->getOperand(2).isReg() ||
18294 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
18295 SequenceMBBI->getOperand(3).getImm() !=
CC ||
18296 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
18297 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
18299 LastSelectPseudo = &*SequenceMBBI;
18301 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
18304 if (SequenceMBBI->hasUnmodeledSideEffects() ||
18305 SequenceMBBI->mayLoadOrStore() ||
18306 SequenceMBBI->usesCustomInsertionHook())
18309 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
18324 F->insert(
I, IfFalseMBB);
18325 F->insert(
I, TailMBB);
18328 unsigned CallFrameSize =
TII.getCallFrameSizeAt(*LastSelectPseudo);
18334 TailMBB->
push_back(DebugInstr->removeFromParent());
18338 TailMBB->
splice(TailMBB->
end(), HeadMBB,
18348 if (
MI.getOperand(2).isImm())
18351 .
addImm(
MI.getOperand(2).getImm())
18363 auto SelectMBBI =
MI.getIterator();
18364 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
18365 auto InsertionPoint = TailMBB->
begin();
18366 while (SelectMBBI != SelectEnd) {
18367 auto Next = std::next(SelectMBBI);
18370 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
18371 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
18372 .
addReg(SelectMBBI->getOperand(4).getReg())
18374 .
addReg(SelectMBBI->getOperand(5).getReg())
18389 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
18390 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
18392 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
18393 assert(
Masked &&
"Could not find masked instruction for LMUL and SEW pair");
18399 unsigned CVTXOpc) {
18405 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18418 .
add(
MI.getOperand(1))
18419 .
add(
MI.getOperand(2))
18420 .
add(
MI.getOperand(3))
18422 .
add(
MI.getOperand(4))
18423 .
add(
MI.getOperand(5))
18424 .
add(
MI.getOperand(6))
18439 .
add(
MI.getOperand(0))
18440 .
add(
MI.getOperand(1))
18442 .
add(
MI.getOperand(3))
18444 .
add(
MI.getOperand(4))
18445 .
add(
MI.getOperand(5))
18446 .
add(
MI.getOperand(6))
18456 MI.eraseFromParent();
18462 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
18464 switch (
MI.getOpcode()) {
18467 case RISCV::PseudoFROUND_H:
18468 CmpOpc = RISCV::FLT_H;
18469 F2IOpc = RISCV::FCVT_W_H;
18470 I2FOpc = RISCV::FCVT_H_W;
18471 FSGNJOpc = RISCV::FSGNJ_H;
18472 FSGNJXOpc = RISCV::FSGNJX_H;
18473 RC = &RISCV::FPR16RegClass;
18475 case RISCV::PseudoFROUND_H_INX:
18476 CmpOpc = RISCV::FLT_H_INX;
18477 F2IOpc = RISCV::FCVT_W_H_INX;
18478 I2FOpc = RISCV::FCVT_H_W_INX;
18479 FSGNJOpc = RISCV::FSGNJ_H_INX;
18480 FSGNJXOpc = RISCV::FSGNJX_H_INX;
18481 RC = &RISCV::GPRF16RegClass;
18483 case RISCV::PseudoFROUND_S:
18484 CmpOpc = RISCV::FLT_S;
18485 F2IOpc = RISCV::FCVT_W_S;
18486 I2FOpc = RISCV::FCVT_S_W;
18487 FSGNJOpc = RISCV::FSGNJ_S;
18488 FSGNJXOpc = RISCV::FSGNJX_S;
18489 RC = &RISCV::FPR32RegClass;
18491 case RISCV::PseudoFROUND_S_INX:
18492 CmpOpc = RISCV::FLT_S_INX;
18493 F2IOpc = RISCV::FCVT_W_S_INX;
18494 I2FOpc = RISCV::FCVT_S_W_INX;
18495 FSGNJOpc = RISCV::FSGNJ_S_INX;
18496 FSGNJXOpc = RISCV::FSGNJX_S_INX;
18497 RC = &RISCV::GPRF32RegClass;
18499 case RISCV::PseudoFROUND_D:
18501 CmpOpc = RISCV::FLT_D;
18502 F2IOpc = RISCV::FCVT_L_D;
18503 I2FOpc = RISCV::FCVT_D_L;
18504 FSGNJOpc = RISCV::FSGNJ_D;
18505 FSGNJXOpc = RISCV::FSGNJX_D;
18506 RC = &RISCV::FPR64RegClass;
18508 case RISCV::PseudoFROUND_D_INX:
18510 CmpOpc = RISCV::FLT_D_INX;
18511 F2IOpc = RISCV::FCVT_L_D_INX;
18512 I2FOpc = RISCV::FCVT_D_L_INX;
18513 FSGNJOpc = RISCV::FSGNJ_D_INX;
18514 FSGNJXOpc = RISCV::FSGNJX_D_INX;
18515 RC = &RISCV::GPRRegClass;
18527 F->insert(
I, CvtMBB);
18528 F->insert(
I, DoneMBB);
18539 Register DstReg =
MI.getOperand(0).getReg();
18540 Register SrcReg =
MI.getOperand(1).getReg();
18541 Register MaxReg =
MI.getOperand(2).getReg();
18542 int64_t FRM =
MI.getOperand(3).getImm();
18547 Register FabsReg =
MRI.createVirtualRegister(RC);
18551 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18566 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18588 MI.eraseFromParent();
18595 switch (
MI.getOpcode()) {
18598 case RISCV::ReadCounterWide:
18600 "ReadCounterWide is only to be used on riscv32");
18602 case RISCV::Select_GPR_Using_CC_GPR:
18603 case RISCV::Select_GPR_Using_CC_Imm:
18604 case RISCV::Select_FPR16_Using_CC_GPR:
18605 case RISCV::Select_FPR16INX_Using_CC_GPR:
18606 case RISCV::Select_FPR32_Using_CC_GPR:
18607 case RISCV::Select_FPR32INX_Using_CC_GPR:
18608 case RISCV::Select_FPR64_Using_CC_GPR:
18609 case RISCV::Select_FPR64INX_Using_CC_GPR:
18610 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18612 case RISCV::BuildPairF64Pseudo:
18614 case RISCV::SplitF64Pseudo:
18616 case RISCV::PseudoQuietFLE_H:
18618 case RISCV::PseudoQuietFLE_H_INX:
18619 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18620 case RISCV::PseudoQuietFLT_H:
18622 case RISCV::PseudoQuietFLT_H_INX:
18623 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18624 case RISCV::PseudoQuietFLE_S:
18626 case RISCV::PseudoQuietFLE_S_INX:
18627 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18628 case RISCV::PseudoQuietFLT_S:
18630 case RISCV::PseudoQuietFLT_S_INX:
18631 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18632 case RISCV::PseudoQuietFLE_D:
18634 case RISCV::PseudoQuietFLE_D_INX:
18635 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18636 case RISCV::PseudoQuietFLE_D_IN32X:
18639 case RISCV::PseudoQuietFLT_D:
18641 case RISCV::PseudoQuietFLT_D_INX:
18642 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18643 case RISCV::PseudoQuietFLT_D_IN32X:
18647 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18649 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18651 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18653 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18655 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18657 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18659 case RISCV::PseudoFROUND_H:
18660 case RISCV::PseudoFROUND_H_INX:
18661 case RISCV::PseudoFROUND_S:
18662 case RISCV::PseudoFROUND_S_INX:
18663 case RISCV::PseudoFROUND_D:
18664 case RISCV::PseudoFROUND_D_INX:
18665 case RISCV::PseudoFROUND_D_IN32X:
18667 case TargetOpcode::STATEPOINT:
18673 MI.addOperand(*
MI.getMF(),
18679 case TargetOpcode::STACKMAP:
18680 case TargetOpcode::PATCHPOINT:
18683 "supported on 64-bit targets");
18701 if (
MI.readsRegister(RISCV::FRM,
nullptr))
18733 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18734 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18737 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18738 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18741 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18742 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18746 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18747 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18748 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18750 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18751 RISCV::V20M2, RISCV::V22M2};
18759 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18760 RISCV::X13, RISCV::X14, RISCV::X15,
18761 RISCV::X16, RISCV::X17};
18763 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18764 RISCV::X13, RISCV::X14, RISCV::X15};
18776 static const MCPhysReg FastCCIGPRs[] = {
18777 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14, RISCV::X15,
18778 RISCV::X16, RISCV::X17, RISCV::X28, RISCV::X29, RISCV::X30, RISCV::X31};
18781 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18782 RISCV::X13, RISCV::X14, RISCV::X15};
18796 unsigned XLenInBytes = XLen / 8;
18809 Align StackAlign(XLenInBytes);
18810 if (!
EABI || XLen != 32)
18842 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18843 assert(XLen == 32 || XLen == 64);
18844 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18848 if (ArgFlags.
isNest()) {
18849 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
18857 if (!LocVT.
isVector() && IsRet && ValNo > 1)
18862 bool UseGPRForF16_F32 =
true;
18865 bool UseGPRForF64 =
true;
18877 UseGPRForF16_F32 = !IsFixed;
18881 UseGPRForF16_F32 = !IsFixed;
18882 UseGPRForF64 = !IsFixed;
18888 UseGPRForF16_F32 =
true;
18889 UseGPRForF64 =
true;
18896 if (UseGPRForF16_F32 &&
18897 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
18900 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
18917 unsigned TwoXLenInBytes = (2 * XLen) / 8;
18919 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
18923 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
18932 "PendingLocs and PendingArgFlags out of sync");
18936 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
18937 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
18985 PendingLocs.
size() <= 2) {
18986 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
18991 PendingLocs.
clear();
18992 PendingArgFlags.
clear();
18994 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
19000 unsigned StoreSizeBytes = XLen / 8;
19003 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
19005 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
19007 else if (ValVT == MVT::f64 && !UseGPRForF64)
19043 if (!PendingLocs.
empty()) {
19045 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
19047 for (
auto &It : PendingLocs) {
19049 It.convertToReg(Reg);
19054 PendingLocs.clear();
19055 PendingArgFlags.
clear();
19059 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
19061 "Expected an XLenVT or vector types at this stage");
19079template <
typename ArgTy>
19081 for (
const auto &ArgIdx :
enumerate(Args)) {
19082 MVT ArgVT = ArgIdx.value().VT;
19084 return ArgIdx.index();
19086 return std::nullopt;
19089void RISCVTargetLowering::analyzeInputArgs(
19092 RISCVCCAssignFn Fn)
const {
19093 unsigned NumArgs =
Ins.size();
19106 for (
unsigned i = 0; i != NumArgs; ++i) {
19110 Type *ArgTy =
nullptr;
19112 ArgTy = FType->getReturnType();
19113 else if (Ins[i].isOrigArg())
19114 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
19118 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
19120 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
19127void RISCVTargetLowering::analyzeOutputArgs(
19130 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
19131 unsigned NumArgs = Outs.
size();
19141 for (
unsigned i = 0; i != NumArgs; i++) {
19142 MVT ArgVT = Outs[i].VT;
19144 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
19148 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
19150 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
19204 if (In.isOrigArg()) {
19209 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
19210 (
BitWidth < 32 && In.Flags.isZExt())) {
19239 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
19285 ExtType,
DL, LocVT, Chain, FIN,
19302 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19315 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
19325 unsigned ValNo,
MVT ValVT,
MVT LocVT,
19328 bool IsFixed,
bool IsRet,
Type *OrigTy,
19331 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19340 if (LocVT == MVT::f16 &&
19341 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
19343 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
19344 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
19345 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
19346 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
19347 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
19353 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19355 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
19356 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
19357 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
19358 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
19359 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19365 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19367 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
19368 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
19369 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
19370 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
19371 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19378 if ((LocVT == MVT::f16 &&
19379 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
19380 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19381 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
19382 Subtarget.hasStdExtZdinx())) {
19389 if (LocVT == MVT::f16) {
19395 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
19401 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
19409 if (AllocatedVReg) {
19444 if (ArgFlags.
isNest()) {
19446 "Attribute 'nest' is not supported in GHC calling convention");
19450 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
19451 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
19453 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
19465 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
19468 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
19469 RISCV::F18_F, RISCV::F19_F,
19470 RISCV::F20_F, RISCV::F21_F};
19471 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19477 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19480 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
19481 RISCV::F24_D, RISCV::F25_D,
19482 RISCV::F26_D, RISCV::F27_D};
19483 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19489 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19490 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
19510 switch (CallConv) {
19520 if (Subtarget.hasStdExtE())
19524 "(Zdinx/D) instruction set extensions");
19528 if (Func.hasFnAttribute(
"interrupt")) {
19529 if (!Func.arg_empty())
19531 "Functions with the interrupt attribute cannot have arguments!");
19536 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
19538 "Function interrupt attribute argument not supported!");
19543 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
19545 std::vector<SDValue> OutChains;
19554 analyzeInputArgs(MF, CCInfo, Ins,
false,
19558 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19579 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19580 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19582 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19584 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19613 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19618 if (VarArgsSaveSize == 0) {
19622 int VaArgOffset = -VarArgsSaveSize;
19630 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19631 VarArgsSaveSize += XLenInBytes;
19638 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19643 Chain,
DL, ArgValue, FIN,
19645 OutChains.push_back(Store);
19659 if (!OutChains.empty()) {
19660 OutChains.push_back(Chain);
19670bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19674 auto CalleeCC = CLI.CallConv;
19675 auto &Outs = CLI.Outs;
19677 auto CallerCC = Caller.getCallingConv();
19684 if (Caller.hasFnAttribute(
"interrupt"))
19699 for (
auto &VA : ArgLocs)
19705 auto IsCallerStructRet = Caller.hasStructRetAttr();
19706 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19707 if (IsCallerStructRet || IsCalleeStructRet)
19712 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19713 if (CalleeCC != CallerCC) {
19714 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19715 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19722 for (
auto &Arg : Outs)
19723 if (Arg.Flags.isByVal())
19758 if (Subtarget.hasStdExtE())
19762 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19768 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19774 "site marked musttail");
19781 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19783 if (!Flags.isByVal())
19787 unsigned Size = Flags.getByValSize();
19788 Align Alignment = Flags.getNonZeroByValAlign();
19795 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19797 false,
nullptr, IsTailCall,
19809 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19812 SDValue ArgValue = OutVals[OutIdx];
19832 if (!StackPtr.getNode())
19843 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19861 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19862 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19868 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19869 SDValue PartValue = OutVals[OutIdx + 1];
19870 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19882 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
19884 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19886 for (
const auto &Part : Parts) {
19887 SDValue PartValue = Part.first;
19888 SDValue PartOffset = Part.second;
19895 ArgValue = SpillSlot;
19901 if (Flags.isByVal())
19902 ArgValue = ByValArgs[j++];
19909 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19910 "for passing parameters");
19913 if (!StackPtr.getNode())
19926 if (!MemOpChains.
empty())
19932 for (
auto &Reg : RegsToPass) {
19933 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19940 validateCCReservedRegs(RegsToPass, MF);
19945 "Return address register required, but has been reserved."});
19964 for (
auto &Reg : RegsToPass)
19970 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19971 assert(Mask &&
"Missing call preserved mask for calling convention");
19980 "Unexpected CFI type for a direct call");
20010 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
20011 auto &VA = RVLocs[i];
20019 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
20020 assert(VA.needsCustom());
20041 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
20045 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
20046 MVT VT = Outs[i].VT;
20050 ArgFlags, CCInfo,
true,
true,
20051 nullptr, *
this, Dispatcher))
20083 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
20084 SDValue Val = OutVals[OutIdx];
20093 DAG.
getVTList(MVT::i32, MVT::i32), Val);
20097 Register RegHi = RVLocs[++i].getLocReg();
20103 "Return value register required, but has been reserved."});
20119 "Return value register required, but has been reserved."});
20141 if (Func.hasFnAttribute(
"interrupt")) {
20142 if (!Func.getReturnType()->isVoidTy())
20144 "Functions with the interrupt attribute must have void return type!");
20150 if (Kind ==
"supervisor")
20156 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
20159void RISCVTargetLowering::validateCCReservedRegs(
20160 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
20169 F,
"Argument register required, but has been reserved."});
20175 if (
N->getNumValues() != 1)
20177 if (!
N->hasNUsesOfValue(1, 0))
20180 SDNode *Copy = *
N->use_begin();
20194 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
20198 bool HasRet =
false;
20199 for (
SDNode *Node : Copy->uses()) {
20207 Chain = Copy->getOperand(0);
20216#define NODE_NAME_CASE(NODE) \
20217 case RISCVISD::NODE: \
20218 return "RISCVISD::" #NODE;
20476#undef NODE_NAME_CASE
20483 if (Constraint.
size() == 1) {
20484 switch (Constraint[0]) {
20500 if (Constraint ==
"vr" || Constraint ==
"vm")
20506std::pair<unsigned, const TargetRegisterClass *>
20512 if (Constraint.
size() == 1) {
20513 switch (Constraint[0]) {
20518 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20519 return std::make_pair(0U, &RISCV::GPRF16RegClass);
20520 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20521 return std::make_pair(0U, &RISCV::GPRF32RegClass);
20522 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20523 return std::make_pair(0U, &RISCV::GPRPairRegClass);
20524 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20526 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
20527 return std::make_pair(0U, &RISCV::FPR16RegClass);
20528 if (Subtarget.hasStdExtF() && VT == MVT::f32)
20529 return std::make_pair(0U, &RISCV::FPR32RegClass);
20530 if (Subtarget.hasStdExtD() && VT == MVT::f64)
20531 return std::make_pair(0U, &RISCV::FPR64RegClass);
20536 }
else if (Constraint ==
"vr") {
20537 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
20538 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20540 return std::make_pair(0U, RC);
20542 }
else if (Constraint ==
"vm") {
20543 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
20544 return std::make_pair(0U, &RISCV::VMV0RegClass);
20552 .
Case(
"{zero}", RISCV::X0)
20553 .
Case(
"{ra}", RISCV::X1)
20554 .
Case(
"{sp}", RISCV::X2)
20555 .
Case(
"{gp}", RISCV::X3)
20556 .
Case(
"{tp}", RISCV::X4)
20557 .
Case(
"{t0}", RISCV::X5)
20558 .
Case(
"{t1}", RISCV::X6)
20559 .
Case(
"{t2}", RISCV::X7)
20560 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20561 .
Case(
"{s1}", RISCV::X9)
20562 .
Case(
"{a0}", RISCV::X10)
20563 .
Case(
"{a1}", RISCV::X11)
20564 .
Case(
"{a2}", RISCV::X12)
20565 .
Case(
"{a3}", RISCV::X13)
20566 .
Case(
"{a4}", RISCV::X14)
20567 .
Case(
"{a5}", RISCV::X15)
20568 .
Case(
"{a6}", RISCV::X16)
20569 .
Case(
"{a7}", RISCV::X17)
20570 .
Case(
"{s2}", RISCV::X18)
20571 .
Case(
"{s3}", RISCV::X19)
20572 .
Case(
"{s4}", RISCV::X20)
20573 .
Case(
"{s5}", RISCV::X21)
20574 .
Case(
"{s6}", RISCV::X22)
20575 .
Case(
"{s7}", RISCV::X23)
20576 .
Case(
"{s8}", RISCV::X24)
20577 .
Case(
"{s9}", RISCV::X25)
20578 .
Case(
"{s10}", RISCV::X26)
20579 .
Case(
"{s11}", RISCV::X27)
20580 .
Case(
"{t3}", RISCV::X28)
20581 .
Case(
"{t4}", RISCV::X29)
20582 .
Case(
"{t5}", RISCV::X30)
20583 .
Case(
"{t6}", RISCV::X31)
20585 if (XRegFromAlias != RISCV::NoRegister)
20586 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20595 if (Subtarget.hasStdExtF()) {
20597 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20598 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20599 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20600 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20601 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20602 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20603 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20604 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20605 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20606 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20607 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20608 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20609 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20610 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20611 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20612 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20613 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20614 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20615 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20616 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20617 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20618 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20619 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20620 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20621 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20622 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20623 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20624 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20625 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20626 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20627 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20628 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20630 if (FReg != RISCV::NoRegister) {
20631 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20632 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20633 unsigned RegNo = FReg - RISCV::F0_F;
20634 unsigned DReg = RISCV::F0_D + RegNo;
20635 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20637 if (VT == MVT::f32 || VT == MVT::Other)
20638 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20639 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20640 unsigned RegNo = FReg - RISCV::F0_F;
20641 unsigned HReg = RISCV::F0_H + RegNo;
20642 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20649 .
Case(
"{v0}", RISCV::V0)
20650 .
Case(
"{v1}", RISCV::V1)
20651 .
Case(
"{v2}", RISCV::V2)
20652 .
Case(
"{v3}", RISCV::V3)
20653 .
Case(
"{v4}", RISCV::V4)
20654 .
Case(
"{v5}", RISCV::V5)
20655 .
Case(
"{v6}", RISCV::V6)
20656 .
Case(
"{v7}", RISCV::V7)
20657 .
Case(
"{v8}", RISCV::V8)
20658 .
Case(
"{v9}", RISCV::V9)
20659 .
Case(
"{v10}", RISCV::V10)
20660 .
Case(
"{v11}", RISCV::V11)
20661 .
Case(
"{v12}", RISCV::V12)
20662 .
Case(
"{v13}", RISCV::V13)
20663 .
Case(
"{v14}", RISCV::V14)
20664 .
Case(
"{v15}", RISCV::V15)
20665 .
Case(
"{v16}", RISCV::V16)
20666 .
Case(
"{v17}", RISCV::V17)
20667 .
Case(
"{v18}", RISCV::V18)
20668 .
Case(
"{v19}", RISCV::V19)
20669 .
Case(
"{v20}", RISCV::V20)
20670 .
Case(
"{v21}", RISCV::V21)
20671 .
Case(
"{v22}", RISCV::V22)
20672 .
Case(
"{v23}", RISCV::V23)
20673 .
Case(
"{v24}", RISCV::V24)
20674 .
Case(
"{v25}", RISCV::V25)
20675 .
Case(
"{v26}", RISCV::V26)
20676 .
Case(
"{v27}", RISCV::V27)
20677 .
Case(
"{v28}", RISCV::V28)
20678 .
Case(
"{v29}", RISCV::V29)
20679 .
Case(
"{v30}", RISCV::V30)
20680 .
Case(
"{v31}", RISCV::V31)
20682 if (VReg != RISCV::NoRegister) {
20683 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20684 return std::make_pair(VReg, &RISCV::VMRegClass);
20685 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20686 return std::make_pair(VReg, &RISCV::VRRegClass);
20687 for (
const auto *RC :
20688 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20689 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20690 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20691 return std::make_pair(VReg, RC);
20697 std::pair<Register, const TargetRegisterClass *> Res =
20703 if (Res.second == &RISCV::GPRF16RegClass ||
20704 Res.second == &RISCV::GPRF32RegClass ||
20705 Res.second == &RISCV::GPRPairRegClass)
20706 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20714 if (ConstraintCode.
size() == 1) {
20715 switch (ConstraintCode[0]) {
20730 if (Constraint.
size() == 1) {
20731 switch (Constraint[0]) {
20734 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20736 if (isInt<12>(CVal))
20749 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20751 if (isUInt<5>(CVal))
20769 if (Subtarget.hasStdExtZtso()) {
20785 if (Subtarget.hasStdExtZtso()) {
20793 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Inst) &&
20810 if (Subtarget.hasForcedAtomics())
20815 if (Subtarget.hasStdExtZacas() &&
20816 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20822 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20835 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20837 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20839 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20841 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20843 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20845 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20847 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20849 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20858 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20860 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20862 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20864 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20866 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20868 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20870 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20872 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20898 unsigned XLen = Subtarget.
getXLen();
20922 unsigned ValWidth =
20927 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20930 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20942 if (Subtarget.hasForcedAtomics())
20946 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20955 unsigned XLen = Subtarget.
getXLen();
20957 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20962 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20968 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20975 EVT DataVT)
const {
20991 return Subtarget.hasStdExtZfhmin();
20993 return Subtarget.hasStdExtF();
20995 return Subtarget.hasStdExtD();
21027 "RVVBitsPerBlock changed, audit needed");
21036 if (!Subtarget.hasVendorXTHeadMemIdx())
21042 Base =
Op->getOperand(0);
21044 int64_t RHSC =
RHS->getSExtValue();
21050 bool isLegalIndexedOffset =
false;
21051 for (
unsigned i = 0; i < 4; i++)
21052 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
21053 isLegalIndexedOffset =
true;
21057 if (!isLegalIndexedOffset)
21074 VT = LD->getMemoryVT();
21075 Ptr = LD->getBasePtr();
21076 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
21077 VT = ST->getMemoryVT();
21078 Ptr = ST->getBasePtr();
21094 if (Subtarget.hasVendorXCVmem()) {
21099 Base = LS->getBasePtr();
21103 if (
Base ==
Op->getOperand(0))
21105 else if (
Base ==
Op->getOperand(1))
21117 VT = LD->getMemoryVT();
21118 Ptr = LD->getBasePtr();
21119 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
21120 VT = ST->getMemoryVT();
21121 Ptr = ST->getBasePtr();
21164 const Constant *PersonalityFn)
const {
21169 const Constant *PersonalityFn)
const {
21193 const bool HasZmmul = Subtarget.hasStdExtZmmul();
21202 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(
C.getNode())) {
21204 const APInt &Imm = ConstNode->getAPIntValue();
21205 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
21206 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
21210 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
21211 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
21212 (Imm - 8).isPowerOf2()))
21217 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
21218 ConstNode->hasOneUse()) {
21219 APInt ImmS = Imm.ashr(Imm.countr_zero());
21220 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
21221 (1 - ImmS).isPowerOf2())
21245 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
21254 unsigned *
Fast)
const {
21257 *
Fast = Subtarget.enableUnalignedScalarMem();
21258 return Subtarget.enableUnalignedScalarMem();
21274 *
Fast = Subtarget.enableUnalignedVectorMem();
21275 return Subtarget.enableUnalignedVectorMem();
21284 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
21296 if (
Op.size() < MinVLenInBytes)
21306 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
21310 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
21312 if (
Op.isFixedDstAlign())
21313 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
21315 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
21323 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
21324 bool IsABIRegCopy =
CC.has_value();
21326 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21327 PartVT == MVT::f32) {
21345 if (PartVTBitSize % ValueVTBitSize == 0) {
21346 assert(PartVTBitSize >= ValueVTBitSize);
21353 if (ValueEltVT != PartEltVT) {
21354 if (PartVTBitSize > ValueVTBitSize) {
21356 assert(Count != 0 &&
"The number of element should not be zero.");
21357 EVT SameEltTypeVT =
21378 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
21379 bool IsABIRegCopy =
CC.has_value();
21380 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
21381 PartVT == MVT::f32) {
21398 if (PartVTBitSize % ValueVTBitSize == 0) {
21399 assert(PartVTBitSize >= ValueVTBitSize);
21400 EVT SameEltTypeVT = ValueVT;
21407 if (ValueEltVT != PartEltVT) {
21409 assert(Count != 0 &&
"The number of element should not be zero.");
21426 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
21433 unsigned Opc =
N->getOpcode();
21463 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
21477 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21482 if (FVTy->getNumElements() < 2)
21497 return Factor * LMUL <= 8;
21501 Align Alignment)
const {
21513 if (!Subtarget.enableUnalignedVectorMem() &&
21521 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
21522 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
21523 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
21524 Intrinsic::riscv_seg8_load};
21543 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21553 {VTy, LI->getPointerOperandType(), XLenTy});
21555 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21560 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21562 Shuffles[i]->replaceAllUsesWith(SubVec);
21569 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
21570 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
21571 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
21572 Intrinsic::riscv_seg8_store};
21592 unsigned Factor)
const {
21594 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
21597 ShuffleVTy->getNumElements() / Factor);
21599 SI->getPointerAddressSpace(),
21600 SI->getDataLayout()))
21607 {VTy, SI->getPointerOperandType(), XLenTy});
21612 for (
unsigned i = 0; i < Factor; i++) {
21621 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21622 Ops.
append({SI->getPointerOperand(), VL});
21638 unsigned Factor = 2;
21653 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21656 {ResVTy, LI->getPointerOperandType(), XLenTy});
21657 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21660 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21661 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21662 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21663 Intrinsic::riscv_vlseg8};
21685 if (
II->getIntrinsicID() != Intrinsic::vector_interleave2)
21688 unsigned Factor = 2;
21691 VectorType *InVTy = cast<VectorType>(
II->getOperand(0)->getType());
21694 SI->getPointerAddressSpace(),
21695 SI->getDataLayout()))
21702 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21705 {InVTy, SI->getPointerOperandType(), XLenTy});
21706 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21709 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21710 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21711 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21712 Intrinsic::riscv_vsseg8};
21719 Builder.
CreateCall(VssegNFunc, {
II->getOperand(0),
II->getOperand(1),
21720 SI->getPointerOperand(), VL});
21730 "Invalid call instruction for a KCFI check");
21732 MBBI->getOpcode()));
21735 Target.setIsRenamable(
false);
21743#define GET_REGISTER_MATCHER
21744#include "RISCVGenAsmMatcher.inc"
21750 if (Reg == RISCV::NoRegister)
21752 if (Reg == RISCV::NoRegister)
21764 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21766 if (NontemporalInfo ==
nullptr)
21774 int NontemporalLevel = 5;
21775 const MDNode *RISCVNontemporalInfo =
21776 I.getMetadata(
"riscv-nontemporal-domain");
21777 if (RISCVNontemporalInfo !=
nullptr)
21780 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21784 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21785 "RISC-V target doesn't support this non-temporal domain.");
21787 NontemporalLevel -= 2;
21789 if (NontemporalLevel & 0b1)
21791 if (NontemporalLevel & 0b10)
21804 return TargetFlags;
21814 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21817 return Subtarget.hasStdExtZbb() &&
21830 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21831 Op == Instruction::And ||
Op == Instruction::Or ||
21832 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21833 Op == Instruction::ShuffleVector ||
Op == Instruction::Load ||
21834 Op == Instruction::Freeze ||
Op == Instruction::Store)
21842 !isa<ReturnInst>(&Inst))
21845 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21846 if (AI->getAllocatedType()->isScalableTy())
21854RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21862 if (!Subtarget.hasShortForwardBranchOpt())
21864 EVT VT =
N->getValueType(0);
21865 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21869 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21874bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21875 EVT VT,
const APInt &AndMask)
const {
21876 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21877 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21881unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21886template <
typename Arg>
21887void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
21890 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
21892 auto It = ArgList.
begin();
21893 MVT FirstArgRegType = It->VT;
21896 if (It == ArgList.
end() || It->Flags.isSplit())
21908 for (; It != ArgList.
end(); ++It)
21909 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
21915 if (isHomogeneousScalableVectorType(ArgList)) {
21917 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
21920 bool FirstVMaskAssigned =
false;
21921 for (
const auto &OutArg : ArgList) {
21922 MVT RegisterVT = OutArg.VT;
21932 RVVArgInfos.push_back({1, RegisterVT,
true});
21933 FirstVMaskAssigned =
true;
21937 RVVArgInfos.push_back({1, RegisterVT,
false});
21944void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
21949 bool FirstVMaskAssigned =
false;
21950 for (
Type *Ty : TypeList) {
21960 RVVArgInfos.push_back(
21966 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
21981 if (!FirstVMaskAssigned &&
21983 RVVArgInfos.push_back({1, RegisterVT,
true});
21984 FirstVMaskAssigned =
true;
21988 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
21994void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
21995 unsigned StartReg) {
21996 assert((StartReg % LMul) == 0 &&
21997 "Start register number should be multiple of lmul");
22016 for (
unsigned i = 0; i < NF; ++i)
22018 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
22020 AllocatedPhysRegs.push_back(
MCPhysReg());
22026void RVVArgDispatcher::compute() {
22028 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
22031 AllocatedPhysRegs.push_back(RISCV::V0);
22037 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
22038 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
22039 StartReg += RegsNeeded) {
22040 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
22041 if ((AssignedMap & Map) == 0) {
22042 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
22043 AssignedMap |=
Map;
22048 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
22051 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
22052 allocate(RVVArgInfos[i]);
22056 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
22057 return AllocatedPhysRegs[CurIdx++];
22064 if (Subtarget.hasStdExtZicfilp()) {
22076#define GET_RISCVVIntrinsicsTable_IMPL
22077#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
uint64_t IntrinsicInst * II
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static unsigned getPACKOpcode(unsigned DestBW, const RISCVSubtarget &Subtarget)
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static SDValue lowerBUILD_VECTORvXf16(SDValue Op, SelectionDAG &DAG)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary operation to its equivalent VW or VW_W form.
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG)
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Double the element size of the build vector to reduce the number of vslide1down in the build vector c...
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
bool isMask(unsigned numBits) const
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
Helper struct to store a base, index and offset that forms an address.
int64_t getOffset() const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
Quantity expandVScale(Quantity X) const
If the ElementCount or TypeSize X is scalable and VScale (VLEN) is exactly known, returns X converted...
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
bool hasVInstructionsBF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool isTargetAndroid() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool shouldExpandCttzElements(EVT VT) const override
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isIdentityMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses elements from exactly one source vector without lane crossin...
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const
Expands target specific indirect branch for the case of JumpTable expansion.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SW_GUARDED_BRIND
Software guarded BRIND node.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ TRUNCATE_VECTOR_VL_USAT
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ TRUNCATE_VECTOR_VL_SSAT
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost, bool FreeZeroes)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)