38#include "llvm/IR/IntrinsicsRISCV.h"
51#define DEBUG_TYPE "riscv-lower"
57 cl::desc(
"Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
63 cl::desc(
"Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
69 cl::desc(
"Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
75 cl::desc(
"Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
81 cl::desc(
"Make i32 a legal type for SelectionDAG on RV64."));
91 !Subtarget.hasStdExtF()) {
92 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
93 "doesn't support the F instruction set extension (ignoring "
97 !Subtarget.hasStdExtD()) {
98 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
99 "doesn't support the D instruction set extension (ignoring "
125 if (Subtarget.hasStdExtZfhmin())
127 if (Subtarget.hasStdExtZfbfmin())
129 if (Subtarget.hasStdExtF())
131 if (Subtarget.hasStdExtD())
133 if (Subtarget.hasStdExtZhinxmin())
135 if (Subtarget.hasStdExtZfinx())
137 if (Subtarget.hasStdExtZdinx()) {
145 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
146 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
148 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
149 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
150 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
151 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
152 MVT::nxv4i64, MVT::nxv8i64};
154 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
155 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
157 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
158 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
160 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
162 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
165 auto addRegClassForRVV = [
this](
MVT VT) {
169 if (VT.getVectorMinNumElements() < MinElts)
172 unsigned Size = VT.getSizeInBits().getKnownMinValue();
175 RC = &RISCV::VRRegClass;
177 RC = &RISCV::VRM2RegClass;
179 RC = &RISCV::VRM4RegClass;
181 RC = &RISCV::VRM8RegClass;
188 for (
MVT VT : BoolVecVTs)
189 addRegClassForRVV(VT);
190 for (
MVT VT : IntVecVTs) {
191 if (VT.getVectorElementType() == MVT::i64 &&
194 addRegClassForRVV(VT);
198 for (
MVT VT : F16VecVTs)
199 addRegClassForRVV(VT);
202 for (
MVT VT : BF16VecVTs)
203 addRegClassForRVV(VT);
206 for (
MVT VT : F32VecVTs)
207 addRegClassForRVV(VT);
210 for (
MVT VT : F64VecVTs)
211 addRegClassForRVV(VT);
214 auto addRegClassForFixedVectors = [
this](
MVT VT) {
221 if (useRVVForFixedLengthVectorVT(VT))
222 addRegClassForFixedVectors(VT);
225 if (useRVVForFixedLengthVectorVT(VT))
226 addRegClassForFixedVectors(VT);
274 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
286 if (!Subtarget.hasStdExtZbb())
290 if (Subtarget.hasStdExtZbb()) {
298 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
303 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) {
307 }
else if (Subtarget.
is64Bit()) {
317 if (!Subtarget.hasStdExtM()) {
323 }
else if (Subtarget.
is64Bit()) {
326 {MVT::i8, MVT::i16, MVT::i32},
Custom);
343 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
346 }
else if (Subtarget.hasVendorXTHeadBb()) {
350 }
else if (Subtarget.hasVendorXCVbitmanip()) {
361 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
362 Subtarget.hasVendorXTHeadBb())
367 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
368 Subtarget.hasVendorXTHeadBb())
373 if (Subtarget.hasVendorXCVbitmanip()) {
381 if (Subtarget.hasStdExtZbb()) {
394 }
else if (!Subtarget.hasVendorXCVbitmanip()) {
400 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
401 Subtarget.hasVendorXCVbitmanip()) {
408 if (!Subtarget.hasStdExtZbb())
420 !Subtarget.hasShortForwardBranchOpt())
424 if (Subtarget.hasShortForwardBranchOpt())
427 if (!Subtarget.hasVendorXTHeadCondMov()) {
433 static const unsigned FPLegalNodeTypes[] = {
446 static const unsigned FPOpToExpand[] = {
450 static const unsigned FPRndMode[] = {
457 static const unsigned ZfhminZfbfminPromoteOps[] = {
468 if (Subtarget.hasStdExtZfbfmin()) {
551 if (Subtarget.hasStdExtZfa()) {
568 if (Subtarget.hasStdExtZfa()) {
645 if (Subtarget.hasStdExtZicbop()) {
649 if (Subtarget.hasStdExtA()) {
651 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
655 }
else if (Subtarget.hasForcedAtomics()) {
676 {MVT::i8, MVT::i16},
Custom);
687 static const unsigned IntegerVPOps[] = {
688 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
689 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
690 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
691 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
692 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
693 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
694 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
695 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
696 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
697 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
698 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
699 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
700 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
701 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF};
703 static const unsigned FloatingPointVPOps[] = {
704 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
705 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
706 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
707 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
708 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
709 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
710 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
711 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
712 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
713 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
714 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
715 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
716 ISD::EXPERIMENTAL_VP_SPLICE};
718 static const unsigned IntegerVecReduceOps[] = {
723 static const unsigned FloatingPointVecReduceOps[] = {
736 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
737 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
738 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
742 for (
MVT VT : BoolVecVTs) {
772 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
796 ISD::VP_TRUNCATE, ISD::VP_SETCC},
812 for (
MVT VT : IntVecVTs) {
823 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
870 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
871 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
895 if (Subtarget.hasStdExtZvkb()) {
903 if (Subtarget.hasStdExtZvbb()) {
907 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
913 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
922 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
942 static const unsigned ZvfhminPromoteOps[] = {
952 static const unsigned ZvfhminPromoteVPOps[] = {
953 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
954 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
955 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
956 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
957 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
958 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
959 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
960 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
964 const auto SetCommonVFPActions = [&](
MVT VT) {
1009 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1010 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1041 const auto SetCommonVFPExtLoadTruncStoreActions =
1043 for (
auto SmallVT : SmallerVTs) {
1050 for (
MVT VT : F16VecVTs) {
1053 SetCommonVFPActions(VT);
1056 for (
MVT VT : F16VecVTs) {
1067 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1078 if (VT == MVT::nxv32f16) {
1091 for (
MVT VT : F32VecVTs) {
1094 SetCommonVFPActions(VT);
1095 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1100 for (
MVT VT : F64VecVTs) {
1103 SetCommonVFPActions(VT);
1104 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1105 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1111 if (!useRVVForFixedLengthVectorVT(VT))
1154 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1181 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1203 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1204 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1241 if (Subtarget.hasStdExtZvkb())
1244 if (Subtarget.hasStdExtZvbb()) {
1266 if (!useRVVForFixedLengthVectorVT(VT))
1287 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1319 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1320 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1371 if (Subtarget.hasStdExtA()) {
1377 if (Subtarget.hasForcedAtomics()) {
1387 if (Subtarget.hasVendorXTHeadMemIdx()) {
1419 if (Subtarget.hasStdExtZbb())
1422 if (Subtarget.hasStdExtZbs() && Subtarget.
is64Bit())
1425 if (Subtarget.hasStdExtZbkb())
1437 ISD::EXPERIMENTAL_VP_REVERSE,
ISD::MUL,
1440 if (Subtarget.hasVendorXTHeadMemPair())
1463MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
1468bool RISCVTargetLowering::shouldExpandGetVectorLength(
EVT TripCountVT,
1470 bool IsScalable)
const {
1477 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.
getXLenVT())
1494bool RISCVTargetLowering::shouldExpandCttzElements(
EVT VT)
const {
1502 unsigned Intrinsic)
const {
1503 auto &
DL =
I.getModule()->getDataLayout();
1505 auto SetRVVLoadStoreInfo = [&](
unsigned PtrOp,
bool IsStore,
1506 bool IsUnitStrided,
bool UsePtrVal =
false) {
1511 Info.ptrVal =
I.getArgOperand(PtrOp);
1513 Info.fallbackAddressSpace =
1514 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1518 MemTy =
I.getArgOperand(0)->getType();
1521 MemTy =
I.getType();
1536 if (
I.hasMetadata(LLVMContext::MD_nontemporal))
1540 switch (Intrinsic) {
1543 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1544 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1545 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1546 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1547 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1548 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1549 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1550 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1551 case Intrinsic::riscv_masked_cmpxchg_i32:
1553 Info.memVT = MVT::i32;
1554 Info.ptrVal =
I.getArgOperand(0);
1560 case Intrinsic::riscv_masked_strided_load:
1561 return SetRVVLoadStoreInfo( 1,
false,
1563 case Intrinsic::riscv_masked_strided_store:
1564 return SetRVVLoadStoreInfo( 1,
true,
1566 case Intrinsic::riscv_seg2_load:
1567 case Intrinsic::riscv_seg3_load:
1568 case Intrinsic::riscv_seg4_load:
1569 case Intrinsic::riscv_seg5_load:
1570 case Intrinsic::riscv_seg6_load:
1571 case Intrinsic::riscv_seg7_load:
1572 case Intrinsic::riscv_seg8_load:
1573 return SetRVVLoadStoreInfo( 0,
false,
1575 case Intrinsic::riscv_seg2_store:
1576 case Intrinsic::riscv_seg3_store:
1577 case Intrinsic::riscv_seg4_store:
1578 case Intrinsic::riscv_seg5_store:
1579 case Intrinsic::riscv_seg6_store:
1580 case Intrinsic::riscv_seg7_store:
1581 case Intrinsic::riscv_seg8_store:
1583 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1586 case Intrinsic::riscv_vle:
1587 case Intrinsic::riscv_vle_mask:
1588 case Intrinsic::riscv_vleff:
1589 case Intrinsic::riscv_vleff_mask:
1590 return SetRVVLoadStoreInfo( 1,
1594 case Intrinsic::riscv_vse:
1595 case Intrinsic::riscv_vse_mask:
1596 return SetRVVLoadStoreInfo( 1,
1600 case Intrinsic::riscv_vlse:
1601 case Intrinsic::riscv_vlse_mask:
1602 case Intrinsic::riscv_vloxei:
1603 case Intrinsic::riscv_vloxei_mask:
1604 case Intrinsic::riscv_vluxei:
1605 case Intrinsic::riscv_vluxei_mask:
1606 return SetRVVLoadStoreInfo( 1,
1609 case Intrinsic::riscv_vsse:
1610 case Intrinsic::riscv_vsse_mask:
1611 case Intrinsic::riscv_vsoxei:
1612 case Intrinsic::riscv_vsoxei_mask:
1613 case Intrinsic::riscv_vsuxei:
1614 case Intrinsic::riscv_vsuxei_mask:
1615 return SetRVVLoadStoreInfo( 1,
1618 case Intrinsic::riscv_vlseg2:
1619 case Intrinsic::riscv_vlseg3:
1620 case Intrinsic::riscv_vlseg4:
1621 case Intrinsic::riscv_vlseg5:
1622 case Intrinsic::riscv_vlseg6:
1623 case Intrinsic::riscv_vlseg7:
1624 case Intrinsic::riscv_vlseg8:
1625 case Intrinsic::riscv_vlseg2ff:
1626 case Intrinsic::riscv_vlseg3ff:
1627 case Intrinsic::riscv_vlseg4ff:
1628 case Intrinsic::riscv_vlseg5ff:
1629 case Intrinsic::riscv_vlseg6ff:
1630 case Intrinsic::riscv_vlseg7ff:
1631 case Intrinsic::riscv_vlseg8ff:
1632 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1635 case Intrinsic::riscv_vlseg2_mask:
1636 case Intrinsic::riscv_vlseg3_mask:
1637 case Intrinsic::riscv_vlseg4_mask:
1638 case Intrinsic::riscv_vlseg5_mask:
1639 case Intrinsic::riscv_vlseg6_mask:
1640 case Intrinsic::riscv_vlseg7_mask:
1641 case Intrinsic::riscv_vlseg8_mask:
1642 case Intrinsic::riscv_vlseg2ff_mask:
1643 case Intrinsic::riscv_vlseg3ff_mask:
1644 case Intrinsic::riscv_vlseg4ff_mask:
1645 case Intrinsic::riscv_vlseg5ff_mask:
1646 case Intrinsic::riscv_vlseg6ff_mask:
1647 case Intrinsic::riscv_vlseg7ff_mask:
1648 case Intrinsic::riscv_vlseg8ff_mask:
1649 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1652 case Intrinsic::riscv_vlsseg2:
1653 case Intrinsic::riscv_vlsseg3:
1654 case Intrinsic::riscv_vlsseg4:
1655 case Intrinsic::riscv_vlsseg5:
1656 case Intrinsic::riscv_vlsseg6:
1657 case Intrinsic::riscv_vlsseg7:
1658 case Intrinsic::riscv_vlsseg8:
1659 case Intrinsic::riscv_vloxseg2:
1660 case Intrinsic::riscv_vloxseg3:
1661 case Intrinsic::riscv_vloxseg4:
1662 case Intrinsic::riscv_vloxseg5:
1663 case Intrinsic::riscv_vloxseg6:
1664 case Intrinsic::riscv_vloxseg7:
1665 case Intrinsic::riscv_vloxseg8:
1666 case Intrinsic::riscv_vluxseg2:
1667 case Intrinsic::riscv_vluxseg3:
1668 case Intrinsic::riscv_vluxseg4:
1669 case Intrinsic::riscv_vluxseg5:
1670 case Intrinsic::riscv_vluxseg6:
1671 case Intrinsic::riscv_vluxseg7:
1672 case Intrinsic::riscv_vluxseg8:
1673 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1676 case Intrinsic::riscv_vlsseg2_mask:
1677 case Intrinsic::riscv_vlsseg3_mask:
1678 case Intrinsic::riscv_vlsseg4_mask:
1679 case Intrinsic::riscv_vlsseg5_mask:
1680 case Intrinsic::riscv_vlsseg6_mask:
1681 case Intrinsic::riscv_vlsseg7_mask:
1682 case Intrinsic::riscv_vlsseg8_mask:
1683 case Intrinsic::riscv_vloxseg2_mask:
1684 case Intrinsic::riscv_vloxseg3_mask:
1685 case Intrinsic::riscv_vloxseg4_mask:
1686 case Intrinsic::riscv_vloxseg5_mask:
1687 case Intrinsic::riscv_vloxseg6_mask:
1688 case Intrinsic::riscv_vloxseg7_mask:
1689 case Intrinsic::riscv_vloxseg8_mask:
1690 case Intrinsic::riscv_vluxseg2_mask:
1691 case Intrinsic::riscv_vluxseg3_mask:
1692 case Intrinsic::riscv_vluxseg4_mask:
1693 case Intrinsic::riscv_vluxseg5_mask:
1694 case Intrinsic::riscv_vluxseg6_mask:
1695 case Intrinsic::riscv_vluxseg7_mask:
1696 case Intrinsic::riscv_vluxseg8_mask:
1697 return SetRVVLoadStoreInfo(
I.arg_size() - 5,
1700 case Intrinsic::riscv_vsseg2:
1701 case Intrinsic::riscv_vsseg3:
1702 case Intrinsic::riscv_vsseg4:
1703 case Intrinsic::riscv_vsseg5:
1704 case Intrinsic::riscv_vsseg6:
1705 case Intrinsic::riscv_vsseg7:
1706 case Intrinsic::riscv_vsseg8:
1707 return SetRVVLoadStoreInfo(
I.arg_size() - 2,
1710 case Intrinsic::riscv_vsseg2_mask:
1711 case Intrinsic::riscv_vsseg3_mask:
1712 case Intrinsic::riscv_vsseg4_mask:
1713 case Intrinsic::riscv_vsseg5_mask:
1714 case Intrinsic::riscv_vsseg6_mask:
1715 case Intrinsic::riscv_vsseg7_mask:
1716 case Intrinsic::riscv_vsseg8_mask:
1717 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1720 case Intrinsic::riscv_vssseg2:
1721 case Intrinsic::riscv_vssseg3:
1722 case Intrinsic::riscv_vssseg4:
1723 case Intrinsic::riscv_vssseg5:
1724 case Intrinsic::riscv_vssseg6:
1725 case Intrinsic::riscv_vssseg7:
1726 case Intrinsic::riscv_vssseg8:
1727 case Intrinsic::riscv_vsoxseg2:
1728 case Intrinsic::riscv_vsoxseg3:
1729 case Intrinsic::riscv_vsoxseg4:
1730 case Intrinsic::riscv_vsoxseg5:
1731 case Intrinsic::riscv_vsoxseg6:
1732 case Intrinsic::riscv_vsoxseg7:
1733 case Intrinsic::riscv_vsoxseg8:
1734 case Intrinsic::riscv_vsuxseg2:
1735 case Intrinsic::riscv_vsuxseg3:
1736 case Intrinsic::riscv_vsuxseg4:
1737 case Intrinsic::riscv_vsuxseg5:
1738 case Intrinsic::riscv_vsuxseg6:
1739 case Intrinsic::riscv_vsuxseg7:
1740 case Intrinsic::riscv_vsuxseg8:
1741 return SetRVVLoadStoreInfo(
I.arg_size() - 3,
1744 case Intrinsic::riscv_vssseg2_mask:
1745 case Intrinsic::riscv_vssseg3_mask:
1746 case Intrinsic::riscv_vssseg4_mask:
1747 case Intrinsic::riscv_vssseg5_mask:
1748 case Intrinsic::riscv_vssseg6_mask:
1749 case Intrinsic::riscv_vssseg7_mask:
1750 case Intrinsic::riscv_vssseg8_mask:
1751 case Intrinsic::riscv_vsoxseg2_mask:
1752 case Intrinsic::riscv_vsoxseg3_mask:
1753 case Intrinsic::riscv_vsoxseg4_mask:
1754 case Intrinsic::riscv_vsoxseg5_mask:
1755 case Intrinsic::riscv_vsoxseg6_mask:
1756 case Intrinsic::riscv_vsoxseg7_mask:
1757 case Intrinsic::riscv_vsoxseg8_mask:
1758 case Intrinsic::riscv_vsuxseg2_mask:
1759 case Intrinsic::riscv_vsuxseg3_mask:
1760 case Intrinsic::riscv_vsuxseg4_mask:
1761 case Intrinsic::riscv_vsuxseg5_mask:
1762 case Intrinsic::riscv_vsuxseg6_mask:
1763 case Intrinsic::riscv_vsuxseg7_mask:
1764 case Intrinsic::riscv_vsuxseg8_mask:
1765 return SetRVVLoadStoreInfo(
I.arg_size() - 4,
1802 return isInt<12>(Imm);
1806 return isInt<12>(Imm);
1819 return (SrcBits == 64 && DestBits == 32);
1830 return (SrcBits == 64 && DestBits == 32);
1837 if (
auto *LD = dyn_cast<LoadSDNode>(Val)) {
1838 EVT MemVT = LD->getMemoryVT();
1839 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1849 return Subtarget.
is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1857 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXCVbitmanip();
1861 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1862 Subtarget.hasVendorXCVbitmanip();
1873 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1878 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1882 EVT VT =
Y.getValueType();
1888 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1889 !isa<ConstantSDNode>(
Y);
1894 if (Subtarget.hasStdExtZbs())
1895 return X.getValueType().isScalarInteger();
1896 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1898 if (Subtarget.hasVendorXTHeadBs())
1899 return C !=
nullptr;
1901 return C &&
C->getAPIntValue().ule(10);
1921 if (BitSize > Subtarget.
getXLen())
1925 int64_t Val = Imm.getSExtValue();
1933 if (!Subtarget.enableUnalignedScalarMem())
1949 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1956 if (XC && OldShiftOpcode ==
ISD::SRL && XC->isOne())
1960 if (NewShiftOpcode ==
ISD::SRL &&
CC->isOne())
1972 case Instruction::Add:
1973 case Instruction::Sub:
1974 case Instruction::Mul:
1975 case Instruction::And:
1976 case Instruction::Or:
1977 case Instruction::Xor:
1978 case Instruction::FAdd:
1979 case Instruction::FSub:
1980 case Instruction::FMul:
1981 case Instruction::FDiv:
1982 case Instruction::ICmp:
1983 case Instruction::FCmp:
1985 case Instruction::Shl:
1986 case Instruction::LShr:
1987 case Instruction::AShr:
1988 case Instruction::UDiv:
1989 case Instruction::SDiv:
1990 case Instruction::URem:
1991 case Instruction::SRem:
1992 return Operand == 1;
2006 auto *II = dyn_cast<IntrinsicInst>(
I);
2010 switch (II->getIntrinsicID()) {
2011 case Intrinsic::fma:
2012 case Intrinsic::vp_fma:
2013 return Operand == 0 || Operand == 1;
2014 case Intrinsic::vp_shl:
2015 case Intrinsic::vp_lshr:
2016 case Intrinsic::vp_ashr:
2017 case Intrinsic::vp_udiv:
2018 case Intrinsic::vp_sdiv:
2019 case Intrinsic::vp_urem:
2020 case Intrinsic::vp_srem:
2021 case Intrinsic::ssub_sat:
2022 case Intrinsic::vp_ssub_sat:
2023 case Intrinsic::usub_sat:
2024 case Intrinsic::vp_usub_sat:
2025 return Operand == 1;
2027 case Intrinsic::vp_add:
2028 case Intrinsic::vp_mul:
2029 case Intrinsic::vp_and:
2030 case Intrinsic::vp_or:
2031 case Intrinsic::vp_xor:
2032 case Intrinsic::vp_fadd:
2033 case Intrinsic::vp_fmul:
2034 case Intrinsic::vp_icmp:
2035 case Intrinsic::vp_fcmp:
2036 case Intrinsic::smin:
2037 case Intrinsic::vp_smin:
2038 case Intrinsic::umin:
2039 case Intrinsic::vp_umin:
2040 case Intrinsic::smax:
2041 case Intrinsic::vp_smax:
2042 case Intrinsic::umax:
2043 case Intrinsic::vp_umax:
2044 case Intrinsic::sadd_sat:
2045 case Intrinsic::vp_sadd_sat:
2046 case Intrinsic::uadd_sat:
2047 case Intrinsic::vp_uadd_sat:
2049 case Intrinsic::vp_sub:
2050 case Intrinsic::vp_fsub:
2051 case Intrinsic::vp_fdiv:
2052 return Operand == 0 || Operand == 1;
2073 if (!Subtarget.sinkSplatOperands())
2076 for (
auto OpIdx :
enumerate(
I->operands())) {
2080 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
2082 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
2091 if (cast<VectorType>(
Op->getType())->getElementType()->isIntegerTy(1))
2096 for (
Use &U :
Op->uses()) {
2148 if (!Subtarget.hasStdExtZfa())
2149 return std::make_pair(-1,
false);
2151 bool IsSupportedVT =
false;
2152 if (VT == MVT::f16) {
2153 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2154 }
else if (VT == MVT::f32) {
2155 IsSupportedVT =
true;
2156 }
else if (VT == MVT::f64) {
2157 assert(Subtarget.hasStdExtD() &&
"Expect D extension");
2158 IsSupportedVT =
true;
2162 return std::make_pair(-1,
false);
2165 if (
Index < 0 && Imm.isNegative())
2169 return std::make_pair(
Index,
false);
2173 bool ForCodeSize)
const {
2174 bool IsLegalVT =
false;
2177 else if (VT == MVT::f32)
2179 else if (VT == MVT::f64)
2181 else if (VT == MVT::bf16)
2182 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2194 return Imm.isZero();
2198 if (Imm.isNegZero())
2211 unsigned Index)
const {
2224 if (EltVT == MVT::i1)
2237 if (
Index + ResElts <= MinVLMAX &&
Index < 31)
2244 if ((ResElts * 2) != SrcElts)
2288 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
2290 Context,
CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2293 IntermediateVT = MVT::i64;
2296 RegisterVT = MVT::i64;
2311 isa<ConstantSDNode>(
LHS.getOperand(1))) {
2317 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
2330 if (
auto *RHSC = dyn_cast<ConstantSDNode>(
RHS)) {
2331 int64_t
C = RHSC->getSExtValue();
2373 switch (KnownSize) {
2401 return RISCV::VRRegClassID;
2403 return RISCV::VRM2RegClassID;
2405 return RISCV::VRM4RegClassID;
2407 return RISCV::VRM8RegClassID;
2417 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2418 "Unexpected subreg numbering");
2419 return RISCV::sub_vrm1_0 +
Index;
2422 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2423 "Unexpected subreg numbering");
2424 return RISCV::sub_vrm2_0 +
Index;
2427 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2428 "Unexpected subreg numbering");
2429 return RISCV::sub_vrm4_0 +
Index;
2436 return RISCV::VRRegClassID;
2445std::pair<unsigned, unsigned>
2447 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
2449 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2450 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2451 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2452 "Register classes not ordered");
2461 unsigned SubRegIdx = RISCV::NoSubRegister;
2462 for (
const unsigned RCID :
2463 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2464 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2468 SubRegIdx =
TRI->composeSubRegIndices(SubRegIdx,
2473 return {SubRegIdx, InsertExtractIdx};
2478bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
2507unsigned RISCVTargetLowering::combineRepeatedFPDivisors()
const {
2514 "Unexpected opcode");
2516 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
2518 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2521 return Op.getOperand(II->
VLOperand + 1 + HasChain);
2591bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
2592 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2601 "Expected legal fixed length vector!");
2604 unsigned MaxELen = Subtarget.
getELen();
2637 return ::getContainerForFixedLengthVector(*
this, VT,
getSubtarget());
2644 "Expected to convert into a scalable vector!");
2645 assert(V.getValueType().isFixedLengthVector() &&
2646 "Expected a fixed length vector operand!");
2656 "Expected to convert into a fixed length vector!");
2657 assert(V.getValueType().isScalableVector() &&
2658 "Expected a scalable vector operand!");
2686 const auto [MinVLMAX, MaxVLMAX] =
2688 if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX)
2694static std::pair<SDValue, SDValue>
2703static std::pair<SDValue, SDValue>
2716static std::pair<SDValue, SDValue>
2733std::pair<unsigned, unsigned>
2749 return std::make_pair(MinVLMAX, MaxVLMAX);
2761 EVT VT,
unsigned DefinedValues)
const {
2775 std::tie(LMul, Fractional) =
2778 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2780 Cost = (LMul * DLenFactor);
2825 MVT DstVT =
Op.getSimpleValueType();
2826 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
2834 Src.getValueType() == MVT::bf16) {
2841 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2849 Opc,
DL, DstVT, Src,
2863 MVT SrcVT = Src.getSimpleValueType();
2869 if (SatVT != DstEltVT)
2873 if (SrcEltSize > (2 * DstEltSize))
2876 MVT DstContainerVT = DstVT;
2877 MVT SrcContainerVT = SrcVT;
2883 "Expected same element count");
2892 {Src, Src, DAG.getCondCode(ISD::SETNE),
2893 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2897 if (DstEltSize > (2 * SrcEltSize)) {
2911 Res, DAG.
getUNDEF(DstContainerVT), VL);
2923 case ISD::VP_FROUNDEVEN:
2927 case ISD::VP_FROUNDTOZERO:
2931 case ISD::VP_FFLOOR:
2939 case ISD::VP_FROUND:
2955 MVT VT =
Op.getSimpleValueType();
2962 MVT ContainerVT = VT;
2969 if (
Op->isVPOpcode()) {
2970 Mask =
Op.getOperand(1);
2974 VL =
Op.getOperand(2);
2996 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3010 switch (
Op.getOpcode()) {
3016 case ISD::VP_FFLOOR:
3019 case ISD::VP_FROUND:
3020 case ISD::VP_FROUNDEVEN:
3021 case ISD::VP_FROUNDTOZERO: {
3037 case ISD::VP_FNEARBYINT:
3050 Src, Src, Mask, VL);
3065 MVT VT =
Op.getSimpleValueType();
3069 MVT ContainerVT = VT;
3081 MVT MaskVT = Mask.getSimpleValueType();
3084 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3085 DAG.getUNDEF(MaskVT), Mask, VL});
3089 {Chain, Src, Src, DAG.getUNDEF(ContainerVT), Unorder, VL});
3090 Chain = Src.getValue(1);
3106 DAG.
getUNDEF(ContainerVT), MaxValNode, VL);
3118 switch (
Op.getOpcode()) {
3129 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3135 DAG.
getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3139 DAG.
getVTList(ContainerVT, MVT::Other), Chain, Src,
3148 DAG.
getVTList(ContainerVT, MVT::Other), Chain,
3149 Truncated, Mask, VL);
3155 Src, Src, Mask, VL);
3165 MVT VT =
Op.getSimpleValueType();
3193 MVT VT =
Op.getSimpleValueType();
3198 MVT ContainerVT = VT;
3220 if (
Merge.isUndef())
3232 if (
Merge.isUndef())
3241 "Unexpected vector MVT");
3269 return std::nullopt;
3284 unsigned EltSizeInBits) {
3287 return std::nullopt;
3288 bool IsInteger =
Op.getValueType().isInteger();
3290 std::optional<unsigned> SeqStepDenom;
3291 std::optional<int64_t> SeqStepNum, SeqAddend;
3292 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
3293 assert(EltSizeInBits >=
Op.getValueType().getScalarSizeInBits());
3298 const unsigned OpSize =
Op.getScalarValueSizeInBits();
3300 if (Elt.isUndef()) {
3301 Elts[
Idx] = std::nullopt;
3305 Elts[
Idx] = Elt->getAsZExtVal() & maskTrailingOnes<uint64_t>(OpSize);
3310 return std::nullopt;
3311 Elts[
Idx] = *ExactInteger;
3324 unsigned IdxDiff =
Idx - PrevElt->second;
3325 int64_t ValDiff =
SignExtend64(*Elt - PrevElt->first, EltSizeInBits);
3333 int64_t Remainder = ValDiff % IdxDiff;
3335 if (Remainder != ValDiff) {
3338 return std::nullopt;
3344 SeqStepNum = ValDiff;
3345 else if (ValDiff != SeqStepNum)
3346 return std::nullopt;
3349 SeqStepDenom = IdxDiff;
3350 else if (IdxDiff != *SeqStepDenom)
3351 return std::nullopt;
3355 if (!PrevElt || PrevElt->first != *Elt)
3356 PrevElt = std::make_pair(*Elt,
Idx);
3360 if (!SeqStepNum || !SeqStepDenom)
3361 return std::nullopt;
3369 (int64_t)(
Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
3370 int64_t Addend =
SignExtend64(*Elt - ExpectedVal, EltSizeInBits);
3373 else if (Addend != SeqAddend)
3374 return std::nullopt;
3377 assert(SeqAddend &&
"Must have an addend if we have a step");
3379 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
3400 MVT ContainerVT = VT;
3428 MVT VT =
Op.getSimpleValueType();
3440 unsigned MostCommonCount = 0;
3442 unsigned NumUndefElts =
3450 unsigned NumScalarLoads = 0;
3456 ValueCounts.
insert(std::make_pair(V, 0));
3457 unsigned &Count = ValueCounts[V];
3459 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3460 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3465 if (++Count >= MostCommonCount) {
3467 MostCommonCount = Count;
3471 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
3472 unsigned NumDefElts = NumElts - NumUndefElts;
3473 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3479 ((MostCommonCount > DominantValueCountThreshold) ||
3492 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3493 LastOp != DominantValue) {
3502 Processed.insert(LastOp);
3507 const SDValue &V = OpIdx.value();
3508 if (V.isUndef() || !Processed.insert(V).second)
3510 if (ValueCounts[V] == 1) {
3519 return DAG.getConstant(V == V1, DL, XLenVT);
3535 MVT VT =
Op.getSimpleValueType();
3565 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.
getXLen());
3566 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.
getELen());
3574 unsigned IntegerViaVecElts =
divideCeil(NumElts, NumViaIntegerBits);
3575 MVT IntegerViaVecVT =
3580 unsigned BitPos = 0, IntegerEltIdx = 0;
3583 for (
unsigned I = 0;
I < NumElts;) {
3585 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3586 Bits |= ((
uint64_t)BitValue << BitPos);
3592 if (
I % NumViaIntegerBits == 0 ||
I == NumElts) {
3593 if (NumViaIntegerBits <= 32)
3594 Bits = SignExtend64<32>(Bits);
3596 Elts[IntegerEltIdx] = Elt;
3605 if (NumElts < NumViaIntegerBits) {
3609 assert(IntegerViaVecVT == MVT::v1i8 &&
"Unexpected mask vector type");
3637 int64_t StepNumerator = SimpleVID->StepNumerator;
3638 unsigned StepDenominator = SimpleVID->StepDenominator;
3639 int64_t Addend = SimpleVID->Addend;
3641 assert(StepNumerator != 0 &&
"Invalid step");
3642 bool Negate =
false;
3643 int64_t SplatStepVal = StepNumerator;
3647 if (StepNumerator != 1 && StepNumerator !=
INT64_MIN &&
3649 Negate = StepNumerator < 0;
3651 SplatStepVal =
Log2_64(std::abs(StepNumerator));
3658 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
3659 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
3661 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3664 MVT VIDContainerVT =
3672 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
3673 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
3675 VID = DAG.
getNode(StepOpcode,
DL, VIDVT, VID, SplatStep);
3677 if (StepDenominator != 1) {
3682 if (Addend != 0 || Negate) {
3701 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3702 "Unexpected sequence type");
3706 unsigned ViaVecLen =
3710 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3713 for (
const auto &OpIdx :
enumerate(
Op->op_values())) {
3714 const auto &SeqV = OpIdx.value();
3715 if (!SeqV.isUndef())
3717 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3722 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3723 SplatValue = SignExtend64<32>(SplatValue);
3745 const auto *BV = cast<BuildVectorSDNode>(
Op);
3748 BV->getRepeatedSequence(Sequence) &&
3749 (Sequence.size() * EltBitSize) <= Subtarget.
getELen()) {
3750 unsigned SeqLen = Sequence.size();
3752 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3753 ViaIntVT == MVT::i64) &&
3754 "Unexpected sequence type");
3759 const unsigned RequiredVL = NumElts / SeqLen;
3760 const unsigned ViaVecLen =
3762 NumElts : RequiredVL;
3765 unsigned EltIdx = 0;
3766 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3770 for (
const auto &SeqV : Sequence) {
3771 if (!SeqV.isUndef())
3773 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3779 if (Subtarget.
is64Bit() && ViaIntVT == MVT::i32)
3780 SplatValue = SignExtend64<32>(SplatValue);
3786 (!Subtarget.
is64Bit() && ViaIntVT == MVT::i64)) &&
3787 "Unexpected bitcast sequence");
3788 if (ViaIntVT.
bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3791 MVT ViaContainerVT =
3798 if (ViaVecLen != RequiredVL)
3815 if (EltBitSize - SignBits < 8) {
3819 Source, DAG, Subtarget);
3836 MVT VT =
Op.getSimpleValueType();
3908 auto OneVRegOfOps =
ArrayRef(BuildVectorOps).
slice(i, ElemsPerVReg);
3912 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
3928 unsigned NumUndefElts =
3930 unsigned NumDefElts = NumElts - NumUndefElts;
3931 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
3938 for (
unsigned i = 0; i < NumElts; i++) {
3940 if (i < NumElts / 2) {
3947 bool SelectMaskVal = (i < NumElts / 2);
3950 assert(SubVecAOps.
size() == NumElts && SubVecBOps.
size() == NumElts &&
3951 MaskVals.
size() == NumElts);
3986 unsigned UndefCount = 0;
3993 LinearBudget -= PerSlideCost;
3996 LinearBudget -= PerSlideCost;
3999 LinearBudget -= PerSlideCost;
4002 if (LinearBudget < 0)
4007 "Illegal type which will result in reserved encoding");
4032 Vec,
Offset, Mask, VL, Policy);
4045 Vec,
Offset, Mask, VL, Policy);
4055 if (isa<ConstantSDNode>(
Lo) && isa<ConstantSDNode>(
Hi)) {
4056 int32_t LoC = cast<ConstantSDNode>(
Lo)->getSExtValue();
4057 int32_t HiC = cast<ConstantSDNode>(
Hi)->getSExtValue();
4060 if ((LoC >> 31) == HiC)
4071 (isa<RegisterSDNode>(VL) &&
4072 cast<RegisterSDNode>(VL)->
getReg() == RISCV::X0))
4074 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->
getAsZExtVal()))
4089 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4090 Hi.getConstantOperandVal(1) == 31)
4109 assert(Scalar.getValueType() == MVT::i64 &&
"Unexpected VT!");
4121 bool HasPassthru = Passthru && !Passthru.
isUndef();
4122 if (!HasPassthru && !Passthru)
4130 if (Scalar.getValueType().bitsLE(XLenVT)) {
4137 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4141 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
4142 "Unexpected scalar for splat lowering!");
4166 SDValue ExtractedVal = Scalar.getOperand(0);
4171 MVT ExtractedContainerVT = ExtractedVT;
4174 DAG, ExtractedContainerVT, Subtarget);
4176 ExtractedVal, DAG, Subtarget);
4178 if (ExtractedContainerVT.
bitsLE(VT))
4193 if (!Scalar.getValueType().bitsLE(XLenVT))
4196 VT,
DL, DAG, Subtarget);
4204 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
4230 if (Src != V2.getOperand(0))
4234 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
4239 V2.getConstantOperandVal(1) != Mask.size())
4243 if (Mask[0] != 0 && Mask[0] != 1)
4248 for (
unsigned i = 1; i != Mask.size(); ++i)
4249 if (Mask[i] != Mask[i - 1] + 2)
4267 int Size = Mask.size();
4269 assert(
Size == (
int)NumElts &&
"Unexpected mask size");
4275 EvenSrc = StartIndexes[0];
4276 OddSrc = StartIndexes[1];
4279 if (EvenSrc != 0 && OddSrc != 0)
4289 int HalfNumElts = NumElts / 2;
4290 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
4306 int Size = Mask.size();
4318 for (
int i = 0; i !=
Size; ++i) {
4324 int StartIdx = i - (M %
Size);
4332 int CandidateRotation = StartIdx < 0 ? -StartIdx :
Size - StartIdx;
4335 Rotation = CandidateRotation;
4336 else if (Rotation != CandidateRotation)
4341 int MaskSrc = M <
Size ? 0 : 1;
4346 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
4351 TargetSrc = MaskSrc;
4352 else if (TargetSrc != MaskSrc)
4359 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
4360 assert((LoSrc >= 0 || HiSrc >= 0) &&
4361 "Failed to find a rotated input vector!");
4376 MVT ContainerVT = VT;
4379 assert(Src.getSimpleValueType().isFixedLengthVector());
4383 MVT SrcContainerVT =
4396 Src = DAG.
getBitcast(WideSrcContainerVT, Src);
4403 unsigned Shift = EvenElts ? 0 : EltBits;
4409 DAG.
getUNDEF(IntContainerVT), TrueMask, VL);
4435 auto findNonEXTRACT_SUBVECTORParent =
4436 [](
SDValue Parent) -> std::pair<SDValue, uint64_t> {
4441 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
4442 Offset += Parent.getConstantOperandVal(1);
4443 Parent = Parent.getOperand(0);
4445 return std::make_pair(Parent,
Offset);
4448 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
4449 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
4458 for (
size_t i = 0; i != NewMask.
size(); ++i) {
4459 if (NewMask[i] == -1)
4462 if (
static_cast<size_t>(NewMask[i]) < NewMask.
size()) {
4463 NewMask[i] = NewMask[i] + V1IndexOffset;
4467 NewMask[i] = NewMask[i] - NewMask.
size() + V2IndexOffset;
4473 if (NewMask[0] <= 0)
4477 for (
unsigned i = 1; i != NewMask.
size(); ++i)
4478 if (NewMask[i - 1] + 1 != NewMask[i])
4482 MVT SrcVT = Src.getSimpleValueType();
4513 int NumSubElts,
Index;
4518 bool OpsSwapped = Mask[
Index] < (int)NumElts;
4519 SDValue InPlace = OpsSwapped ? V2 : V1;
4520 SDValue ToInsert = OpsSwapped ? V1 : V2;
4530 if (NumSubElts +
Index >= (
int)NumElts)
4544 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, InPlace, ToInsert,
4556 bool OpsSwapped =
false;
4557 if (!isa<BuildVectorSDNode>(V1)) {
4558 if (!isa<BuildVectorSDNode>(V2))
4563 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
4571 const unsigned E = Mask.size() - ((
Offset > 0) ?
Offset : 0);
4572 for (
unsigned i = S; i != E; ++i)
4573 if (Mask[i] >= 0 && (
unsigned)Mask[i] !=
Base + i +
Offset)
4579 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
4580 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
4583 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
4585 if (InsertIdx < 0 || InsertIdx / NumElts != (
unsigned)OpsSwapped)
4590 auto OpCode = IsVSlidedown ?
4595 auto Vec = DAG.
getNode(OpCode,
DL, ContainerVT,
4598 Splat, TrueMask, VL);
4609 MVT VecContainerVT = VecVT;
4626 MVT WideContainerVT = WideVT;
4632 EvenV = DAG.
getBitcast(VecContainerVT, EvenV);
4645 }
else if (Subtarget.hasStdExtZvbb()) {
4650 OffsetVec, Passthru, Mask, VL);
4653 Interleaved, EvenV, Passthru, Mask, VL);
4661 Interleaved, OffsetVec, Passthru, Mask, VL);
4669 OddV, Passthru, Mask, VL);
4675 OddV, AllOnesVec, Passthru, Mask, VL);
4683 Interleaved, OddsMul, Passthru, Mask, VL);
4690 Interleaved = DAG.
getBitcast(ResultContainerVT, Interleaved);
4736 if (ViaEltSize > NumElts)
4745 if (ViaEltSize > NumElts)
4751 if (ViaEltSize > NumElts)
4760 MVT &RotateVT,
unsigned &RotateAmt) {
4766 unsigned NumSubElts;
4768 NumElts, NumSubElts, RotateAmt))
4771 NumElts / NumSubElts);
4833 unsigned VRegsPerSrc = NumElts / ElemsPerVReg;
4836 OutMasks(VRegsPerSrc, {-1, {}});
4841 for (
unsigned DstIdx = 0; DstIdx < Mask.size(); DstIdx++) {
4842 int DstVecIdx = DstIdx / ElemsPerVReg;
4843 int DstSubIdx = DstIdx % ElemsPerVReg;
4844 int SrcIdx = Mask[DstIdx];
4845 if (SrcIdx < 0 || (
unsigned)SrcIdx >= 2 * NumElts)
4847 int SrcVecIdx = SrcIdx / ElemsPerVReg;
4848 int SrcSubIdx = SrcIdx % ElemsPerVReg;
4849 if (OutMasks[DstVecIdx].first == -1)
4850 OutMasks[DstVecIdx].first = SrcVecIdx;
4851 if (OutMasks[DstVecIdx].first != SrcVecIdx)
4857 OutMasks[DstVecIdx].second.resize(ElemsPerVReg, -1);
4858 OutMasks[DstVecIdx].second[DstSubIdx] = SrcSubIdx;
4872 for (
unsigned DstVecIdx = 0 ; DstVecIdx < OutMasks.size(); DstVecIdx++) {
4873 auto &[SrcVecIdx, SrcSubMask] = OutMasks[DstVecIdx];
4874 if (SrcVecIdx == -1)
4876 unsigned ExtractIdx = (SrcVecIdx % VRegsPerSrc) * NumOpElts;
4883 unsigned InsertIdx = DstVecIdx * NumOpElts;
4896 MVT VT =
Op.getSimpleValueType();
4911 V2 = V2.isUndef() ? DAG.
getUNDEF(WidenVT)
4935 V.getOperand(0).getSimpleValueType().getVectorNumElements();
4936 V = V.getOperand(
Offset / OpElements);
4942 auto *Ld = cast<LoadSDNode>(V);
4952 SDValue Ops[] = {Ld->getChain(),
4970 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
4971 Ld->getPointerInfo().getWithOffset(
Offset),
4972 Ld->getOriginalAlign(),
4976 Ld->getPointerInfo().getWithOffset(
Offset), SVT,
4977 Ld->getOriginalAlign(),
4978 Ld->getMemOperand()->getFlags());
4989 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
4992 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5014 if (Subtarget.hasStdExtZvkb())
5025 LoV = LoSrc == 0 ? V1 : V2;
5029 HiV = HiSrc == 0 ? V1 : V2;
5035 unsigned InvRotate = NumElts - Rotation;
5045 Res =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Res, LoV,
5065 int EvenSrc, OddSrc;
5070 int Size = Mask.size();
5072 assert(EvenSrc >= 0 &&
"Undef source?");
5073 EvenV = (EvenSrc /
Size) == 0 ? V1 : V2;
5077 assert(OddSrc >= 0 &&
"Undef source?");
5078 OddV = (OddSrc /
Size) == 0 ? V1 : V2;
5087 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
5096 any_of(Mask, [&](
const auto &
Idx) {
return Idx > 255; })) {
5125 MVT IndexContainerVT =
5130 for (
int MaskIndex : Mask) {
5131 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
5140 DAG.
getUNDEF(ContainerVT), TrueMask, VL);
5154 int MaskIndex = MaskIdx.value();
5155 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
5160 for (
int MaskIndex : Mask) {
5161 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ SwapOps;
5168 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5181 for (
int MaskIndex : Mask) {
5182 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
5184 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
5185 ShuffleMaskLHS.
push_back(IsLHSOrUndefIndex && MaskIndex >= 0
5187 ShuffleMaskRHS.
push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
5192 std::swap(ShuffleMaskLHS, ShuffleMaskRHS);
5195 assert(MaskVals.
size() == NumElts &&
"Unexpected select-like shuffle");
5231RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(
SDValue Op,
5233 MVT VT =
Op.getSimpleValueType();
5237 MVT ContainerVT = VT;
5240 if (
Op->isVPOpcode()) {
5241 Mask =
Op.getOperand(1);
5245 VL =
Op.getOperand(2);
5251 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
5253 FloatEltVT = MVT::f32;
5260 "Expected legal float type!");
5267 }
else if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
5270 Src = DAG.
getNode(ISD::VP_AND,
DL, VT, Src, Neg, Mask, VL);
5275 if (FloatVT.
bitsGT(VT)) {
5276 if (
Op->isVPOpcode())
5277 FloatVal = DAG.
getNode(ISD::VP_UINT_TO_FP,
DL, FloatVT, Src, Mask, VL);
5286 if (!
Op->isVPOpcode())
5290 MVT ContainerFloatVT =
5293 Src, Mask, RTZRM, VL);
5300 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
5304 if (
Op->isVPOpcode()) {
5313 else if (IntVT.
bitsGT(VT))
5318 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
5323 if (
Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
5324 return DAG.
getNode(ISD::VP_SUB,
DL, VT, Exp,
5329 unsigned Adjust = ExponentBias + (EltSize - 1);
5331 if (
Op->isVPOpcode())
5341 else if (
Op.getOpcode() == ISD::VP_CTLZ)
5342 Res = DAG.
getNode(ISD::VP_UMIN,
DL, VT, Res,
5352 MVT SrcVT =
Source.getSimpleValueType();
5361 SrcVT = ContainerVT;
5374 if (
Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
5391 auto *
Load = cast<LoadSDNode>(
Op);
5392 assert(Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
5395 Load->getMemoryVT(),
5396 *
Load->getMemOperand()))
5400 MVT VT =
Op.getSimpleValueType();
5402 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5403 "Unexpected unaligned RVV load type");
5407 "Expecting equally-sized RVV vector types to be legal");
5409 Load->getPointerInfo(),
Load->getOriginalAlign(),
5410 Load->getMemOperand()->getFlags());
5420 auto *
Store = cast<StoreSDNode>(
Op);
5421 assert(Store &&
Store->getValue().getValueType().isVector() &&
5422 "Expected vector store");
5425 Store->getMemoryVT(),
5426 *
Store->getMemOperand()))
5433 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
5434 "Unexpected unaligned RVV store type");
5438 "Expecting equally-sized RVV vector types to be legal");
5439 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
5441 Store->getPointerInfo(),
Store->getOriginalAlign(),
5442 Store->getMemOperand()->getFlags());
5447 assert(
Op.getValueType() == MVT::i64 &&
"Unexpected VT");
5449 int64_t Imm = cast<ConstantSDNode>(
Op)->getSExtValue();
5476 unsigned ShiftAmt, AddOpc;
5493 if (Subtarget.hasStdExtZtso()) {
5516 "Unexpected custom legalisation");
5537 "Unexpected custom legalisation");
5552 "Unexpected custom legalisation");
5553 if (isa<ConstantSDNode>(
Op.getOperand(1)))
5573 "Unexpected custom legalisation");
5589 MVT VT =
Op.getSimpleValueType();
5591 unsigned Check =
Op.getConstantOperandVal(1);
5592 unsigned TDCMask = 0;
5620 MVT VT0 =
Op.getOperand(0).getSimpleValueType();
5625 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5627 VL =
Op.getOperand(3);
5630 VL,
Op->getFlags());
5645 if (
Op.getOpcode() == ISD::VP_IS_FPCLASS) {
5647 MVT MaskContainerVT =
5650 VL =
Op.getOperand(3);
5655 Mask, VL,
Op->getFlags());
5658 DAG.
getUNDEF(ContainerDstVT), TDCMaskV, VL);
5663 DAG.
getUNDEF(ContainerVT), Mask, VL});
5667 TDCMaskV, DAG.
getUNDEF(ContainerDstVT), Mask, VL);
5671 DAG.
getUNDEF(ContainerDstVT), SplatZero, VL);
5675 DAG.
getUNDEF(ContainerVT), Mask, VL});
5691 MVT VT =
Op.getSimpleValueType();
5718 return DAG.
getNode(Opc,
DL, VT, NewX, NewY);
5725 MVT ContainerVT = VT;
5733 if (
Op->isVPOpcode()) {
5734 Mask =
Op.getOperand(2);
5738 VL =
Op.getOperand(3);
5746 {X, X, DAG.getCondCode(ISD::SETOEQ),
5747 DAG.getUNDEF(ContainerVT), Mask, VL});
5755 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
5756 DAG.getUNDEF(ContainerVT), Mask, VL});
5766 DAG.
getUNDEF(ContainerVT), Mask, VL);
5774#define OP_CASE(NODE) \
5776 return RISCVISD::NODE##_VL;
5777#define VP_CASE(NODE) \
5778 case ISD::VP_##NODE: \
5779 return RISCVISD::NODE##_VL;
5781 switch (
Op.getOpcode()) {
5857 case ISD::VP_CTLZ_ZERO_UNDEF:
5860 case ISD::VP_CTTZ_ZERO_UNDEF:
5869 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5874 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5879 if (
Op.getSimpleValueType().getVectorElementType() == MVT::i1)
5882 case ISD::VP_SELECT:
5891 case ISD::VP_SIGN_EXTEND:
5893 case ISD::VP_ZERO_EXTEND:
5895 case ISD::VP_FP_TO_SINT:
5897 case ISD::VP_FP_TO_UINT:
5900 case ISD::VP_FMINNUM:
5903 case ISD::VP_FMAXNUM:
5908 case ISD::VP_LLRINT:
5920 "not a RISC-V target specific op");
5926 "adding target specific op should update this function");
5946 "not a RISC-V target specific op");
5952 "adding target specific op should update this function");
5971 if (!
Op.getOperand(j).getValueType().isVector()) {
5972 LoOperands[j] =
Op.getOperand(j);
5973 HiOperands[j] =
Op.getOperand(j);
5976 std::tie(LoOperands[j], HiOperands[j]) =
5981 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
5983 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
5998 std::tie(LoOperands[j], HiOperands[j]) =
6002 if (!
Op.getOperand(j).getValueType().isVector()) {
6003 LoOperands[j] =
Op.getOperand(j);
6004 HiOperands[j] =
Op.getOperand(j);
6007 std::tie(LoOperands[j], HiOperands[j]) =
6012 DAG.
getNode(
Op.getOpcode(),
DL, LoVT, LoOperands,
Op->getFlags());
6014 DAG.
getNode(
Op.getOpcode(),
DL, HiVT, HiOperands,
Op->getFlags());
6024 auto [EVLLo, EVLHi] =
6025 DAG.
SplitEVL(
Op.getOperand(3),
Op.getOperand(1).getValueType(),
DL);
6029 {Op.getOperand(0), Lo, MaskLo, EVLLo},
Op->getFlags());
6031 {ResLo, Hi, MaskHi, EVLHi},
Op->getFlags());
6049 if (!
Op.getOperand(j).getValueType().isVector()) {
6050 LoOperands[j] =
Op.getOperand(j);
6051 HiOperands[j] =
Op.getOperand(j);
6054 std::tie(LoOperands[j], HiOperands[j]) =
6059 DAG.
getNode(
Op.getOpcode(),
DL, LoVTs, LoOperands,
Op->getFlags());
6062 DAG.
getNode(
Op.getOpcode(),
DL, HiVTs, HiOperands,
Op->getFlags());
6071 switch (
Op.getOpcode()) {
6077 return lowerGlobalAddress(
Op, DAG);
6079 return lowerBlockAddress(
Op, DAG);
6081 return lowerConstantPool(
Op, DAG);
6083 return lowerJumpTable(
Op, DAG);
6085 return lowerGlobalTLSAddress(
Op, DAG);
6089 return lowerSELECT(
Op, DAG);
6091 return lowerBRCOND(
Op, DAG);
6093 return lowerVASTART(
Op, DAG);
6095 return lowerFRAMEADDR(
Op, DAG);
6097 return lowerRETURNADDR(
Op, DAG);
6104 return lowerShiftLeftParts(
Op, DAG);
6106 return lowerShiftRightParts(
Op, DAG,
true);
6108 return lowerShiftRightParts(
Op, DAG,
false);
6111 if (
Op.getValueType().isFixedLengthVector()) {
6112 assert(Subtarget.hasStdExtZvkb());
6113 return lowerToScalableOp(
Op, DAG);
6115 assert(Subtarget.hasVendorXTHeadBb() &&
6116 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
6117 "Unexpected custom legalization");
6119 if (!isa<ConstantSDNode>(
Op.getOperand(1)))
6124 EVT VT =
Op.getValueType();
6128 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
6134 if (VT == MVT::bf16 && Op0VT == MVT::i16 &&
6135 Subtarget.hasStdExtZfbfmin()) {
6140 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.
is64Bit() &&
6147 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32) {
6164 "Unexpected types");
6198 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
6200 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
6202 return LowerINTRINSIC_VOID(
Op, DAG);
6204 return LowerIS_FPCLASS(
Op, DAG);
6206 MVT VT =
Op.getSimpleValueType();
6208 assert(Subtarget.hasStdExtZvbb());
6209 return lowerToScalableOp(
Op, DAG);
6212 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected custom legalization");
6220 if (!
Op.getSimpleValueType().isVector())
6222 return lowerVectorTruncLike(
Op, DAG);
6225 if (
Op.getOperand(0).getValueType().isVector() &&
6226 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6227 return lowerVectorMaskExt(
Op, DAG, 1);
6230 if (
Op.getOperand(0).getValueType().isVector() &&
6231 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6232 return lowerVectorMaskExt(
Op, DAG, -1);
6235 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
6237 return lowerINSERT_VECTOR_ELT(
Op, DAG);
6239 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
6241 MVT VT =
Op.getSimpleValueType();
6249 MVT ContainerVT = VT;
6255 DAG.
getUNDEF(ContainerVT), Scalar, VL);
6262 MVT VT =
Op.getSimpleValueType();
6282 }
else if ((Val % 8) == 0) {
6298 if (
Op.getValueType() == MVT::f16 && Subtarget.
is64Bit() &&
6299 Op.getOperand(1).getValueType() == MVT::i32) {
6311 if (
Op.getValueType() == MVT::nxv32f16 &&
6318 EVT VT =
Op.getValueType();
6321 if (VT == MVT::f32 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin())
6323 if (VT == MVT::f64 && Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
6329 if (!
Op.getValueType().isVector())
6331 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6335 EVT VT =
Op.getValueType();
6338 if (VT == MVT::bf16 && Op0VT == MVT::f32 && Subtarget.hasStdExtZfbfmin())
6340 if (VT == MVT::bf16 && Op0VT == MVT::f64 && Subtarget.hasStdExtZfbfmin() &&
6348 if (!
Op.getValueType().isVector())
6350 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6354 return lowerStrictFPExtendOrRoundLike(
Op, DAG);
6357 if (
Op.getValueType().isVector() &&
6358 Op.getValueType().getScalarType() == MVT::f16 &&
6361 if (
Op.getValueType() == MVT::nxv32f16)
6376 Op1.getValueType().isVector() &&
6377 Op1.getValueType().getScalarType() == MVT::f16 &&
6380 if (Op1.getValueType() == MVT::nxv32f16)
6385 Op1.getValueType().getVectorElementCount());
6388 return DAG.
getNode(
Op.getOpcode(),
DL,
Op.getValueType(), WidenVec);
6398 MVT VT =
Op.getSimpleValueType();
6402 bool IsStrict =
Op->isStrictFPOpcode();
6403 SDValue Src =
Op.getOperand(0 + IsStrict);
6405 MVT SrcVT = Src.getSimpleValueType();
6410 "Unexpected vector element types");
6414 if (EltSize > (2 * SrcEltSize)) {
6426 Op.getOperand(0), Ext);
6430 assert(SrcEltVT == MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
6435 auto [FExt, Chain] =
6437 return DAG.
getNode(
Op.getOpcode(),
DL,
Op->getVTList(), Chain, FExt);
6444 if (SrcEltSize > (2 * EltSize)) {
6447 assert(EltVT == MVT::f16 &&
"Unexpected [US]_TO_FP lowering");
6452 Op.getOperand(0), Src);
6467 Op.getOperand(0), Src);
6481 unsigned RVVOpc = 0;
6482 switch (
Op.getOpcode()) {
6514 "Expected same element count");
6521 Op.getOperand(0), Src, Mask, VL);
6525 Src = DAG.
getNode(RVVOpc,
DL, ContainerVT, Src, Mask, VL);
6540 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6547 MVT VT =
Op.getSimpleValueType();
6569 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
6585 makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg, CallOptions,
DL)
6605 return lowerVECREDUCE(
Op, DAG);
6609 if (
Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
6610 return lowerVectorMaskVecReduction(
Op, DAG,
false);
6611 return lowerVECREDUCE(
Op, DAG);
6618 return lowerFPVECREDUCE(
Op, DAG);
6619 case ISD::VP_REDUCE_ADD:
6620 case ISD::VP_REDUCE_UMAX:
6621 case ISD::VP_REDUCE_SMAX:
6622 case ISD::VP_REDUCE_UMIN:
6623 case ISD::VP_REDUCE_SMIN:
6624 case ISD::VP_REDUCE_FADD:
6625 case ISD::VP_REDUCE_SEQ_FADD:
6626 case ISD::VP_REDUCE_FMIN:
6627 case ISD::VP_REDUCE_FMAX:
6628 if (
Op.getOperand(1).getValueType() == MVT::nxv32f16 &&
6632 return lowerVPREDUCE(
Op, DAG);
6633 case ISD::VP_REDUCE_AND:
6634 case ISD::VP_REDUCE_OR:
6635 case ISD::VP_REDUCE_XOR:
6636 if (
Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
6637 return lowerVectorMaskVecReduction(
Op, DAG,
true);
6638 return lowerVPREDUCE(
Op, DAG);
6639 case ISD::VP_CTTZ_ELTS:
6640 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
6641 return lowerVPCttzElements(
Op, DAG);
6645 DAG.
getUNDEF(ContainerVT), DAG, Subtarget);
6648 return lowerINSERT_SUBVECTOR(
Op, DAG);
6650 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
6652 return lowerVECTOR_DEINTERLEAVE(
Op, DAG);
6654 return lowerVECTOR_INTERLEAVE(
Op, DAG);
6656 return lowerSTEP_VECTOR(
Op, DAG);
6658 return lowerVECTOR_REVERSE(
Op, DAG);
6660 return lowerVECTOR_SPLICE(
Op, DAG);
6664 if (
Op.getValueType().getScalarType() == MVT::f16 &&
6667 if (
Op.getValueType() == MVT::nxv32f16)
6679 if (
Op.getValueType().getVectorElementType() == MVT::i1)
6680 return lowerVectorMaskSplat(
Op, DAG);
6688 MVT VT =
Op.getSimpleValueType();
6689 MVT ContainerVT = VT;
6707 Op->ops().take_front(HalfNumOps));
6709 Op->ops().drop_front(HalfNumOps));
6713 unsigned NumOpElts =
6714 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
6717 SDValue SubVec = OpIdx.value();
6728 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
6730 if (
Op.getValueType().isFixedLengthVector())
6731 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
6734 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
6736 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
6737 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
6741 return lowerMaskedLoad(
Op, DAG);
6744 return lowerMaskedStore(
Op, DAG);
6753 EVT VT =
Op.getValueType();
6764 MVT OpVT =
Op.getOperand(0).getSimpleValueType();
6766 MVT VT =
Op.getSimpleValueType();
6771 "Unexpected CondCode");
6779 if (isa<ConstantSDNode>(
RHS)) {
6780 int64_t Imm = cast<ConstantSDNode>(
RHS)->getSExtValue();
6781 if (Imm != 0 && isInt<12>((
uint64_t)Imm + 1)) {
6800 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
6805 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
6821 return lowerToScalableOp(
Op, DAG);
6825 if (
Op.getSimpleValueType().isFixedLengthVector())
6826 return lowerToScalableOp(
Op, DAG);
6828 assert(
Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.
is64Bit() &&
6829 "Unexpected custom legalisation");
6841 if (
Op.getValueType() == MVT::nxv32f16 &&
6852 return lowerToScalableOp(
Op, DAG);
6855 if (!
Op.getValueType().isVector())
6857 return lowerToScalableOp(
Op, DAG);
6860 if (!
Op.getValueType().isVector())
6862 return lowerToScalableOp(
Op, DAG);
6866 EVT VT =
Op->getValueType(0);
6881 return lowerABS(
Op, DAG);
6886 if (Subtarget.hasStdExtZvbb())
6887 return lowerToScalableOp(
Op, DAG);
6889 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
6891 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
6893 if (
Op.getValueType() == MVT::nxv32f16 &&
6897 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
6904 if (
Op.getValueType() == MVT::nxv32f16 &&
6908 return lowerToScalableOp(
Op, DAG);
6911 return lowerVectorStrictFSetcc(
Op, DAG);
6921 case ISD::VP_GATHER:
6922 return lowerMaskedGather(
Op, DAG);
6924 case ISD::VP_SCATTER:
6925 return lowerMaskedScatter(
Op, DAG);
6927 return lowerGET_ROUNDING(
Op, DAG);
6929 return lowerSET_ROUNDING(
Op, DAG);
6931 return lowerEH_DWARF_CFA(
Op, DAG);
6932 case ISD::VP_SELECT:
6941 case ISD::VP_UADDSAT:
6942 case ISD::VP_USUBSAT:
6943 case ISD::VP_SADDSAT:
6944 case ISD::VP_SSUBSAT:
6946 case ISD::VP_LLRINT:
6947 return lowerVPOp(
Op, DAG);
6951 return lowerLogicVPOp(
Op, DAG);
6960 case ISD::VP_FMINNUM:
6961 case ISD::VP_FMAXNUM:
6962 case ISD::VP_FCOPYSIGN:
6963 if (
Op.getValueType() == MVT::nxv32f16 &&
6971 return lowerVPOp(
Op, DAG);
6972 case ISD::VP_IS_FPCLASS:
6973 return LowerIS_FPCLASS(
Op, DAG);
6974 case ISD::VP_SIGN_EXTEND:
6975 case ISD::VP_ZERO_EXTEND:
6976 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
6977 return lowerVPExtMaskOp(
Op, DAG);
6978 return lowerVPOp(
Op, DAG);
6979 case ISD::VP_TRUNCATE:
6980 return lowerVectorTruncLike(
Op, DAG);
6981 case ISD::VP_FP_EXTEND:
6982 case ISD::VP_FP_ROUND:
6983 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
6984 case ISD::VP_SINT_TO_FP:
6985 case ISD::VP_UINT_TO_FP:
6986 if (
Op.getValueType().isVector() &&
6987 Op.getValueType().getScalarType() == MVT::f16 &&
6990 if (
Op.getValueType() == MVT::nxv32f16)
7002 case ISD::VP_FP_TO_SINT:
7003 case ISD::VP_FP_TO_UINT:
7005 Op1.getValueType().isVector() &&
7006 Op1.getValueType().getScalarType() == MVT::f16 &&
7009 if (Op1.getValueType() == MVT::nxv32f16)
7014 Op1.getValueType().getVectorElementCount());
7018 {WidenVec, Op.getOperand(1), Op.getOperand(2)});
7020 return lowerVPFPIntConvOp(
Op, DAG);
7022 if (
Op.getOperand(0).getSimpleValueType() == MVT::nxv32f16 &&
7026 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() == MVT::i1)
7027 return lowerVPSetCCMaskOp(
Op, DAG);
7033 case ISD::VP_BITREVERSE:
7035 return lowerVPOp(
Op, DAG);
7037 case ISD::VP_CTLZ_ZERO_UNDEF:
7038 if (Subtarget.hasStdExtZvbb())
7039 return lowerVPOp(
Op, DAG);
7040 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7042 case ISD::VP_CTTZ_ZERO_UNDEF:
7043 if (Subtarget.hasStdExtZvbb())
7044 return lowerVPOp(
Op, DAG);
7045 return lowerCTLZ_CTTZ_ZERO_UNDEF(
Op, DAG);
7047 return lowerVPOp(
Op, DAG);
7048 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
7049 return lowerVPStridedLoad(
Op, DAG);
7050 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
7051 return lowerVPStridedStore(
Op, DAG);
7053 case ISD::VP_FFLOOR:
7055 case ISD::VP_FNEARBYINT:
7056 case ISD::VP_FROUND:
7057 case ISD::VP_FROUNDEVEN:
7058 case ISD::VP_FROUNDTOZERO:
7059 if (
Op.getValueType() == MVT::nxv32f16 &&
7064 case ISD::VP_FMAXIMUM:
7065 case ISD::VP_FMINIMUM:
7066 if (
Op.getValueType() == MVT::nxv32f16 &&
7071 case ISD::EXPERIMENTAL_VP_SPLICE:
7072 return lowerVPSpliceExperimental(
Op, DAG);
7073 case ISD::EXPERIMENTAL_VP_REVERSE:
7074 return lowerVPReverseExperimental(
Op, DAG);
7092 N->getOffset(), Flags);
7100template <
class NodeTy>
7102 bool IsLocal,
bool IsExternWeak)
const {
7112 if (IsLocal && !Subtarget.allowTaggedGlobals())
7174 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7183 return getAddr(
N, DAG);
7190 return getAddr(
N, DAG);
7197 return getAddr(
N, DAG);
7202 bool UseGOT)
const {
7266 Args.push_back(Entry);
7299 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
7313 Addr = getStaticTLSAddr(
N, DAG,
false);
7316 Addr = getStaticTLSAddr(
N, DAG,
true);
7321 : getDynamicTLSAddr(
N, DAG);
7338 if (
LHS == LHS2 &&
RHS == RHS2) {
7343 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
7351 return std::nullopt;
7359 MVT VT =
N->getSimpleValueType(0);
7389 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7392 if (~TrueVal == FalseVal) {
7432 if (Subtarget.hasShortForwardBranchOpt())
7435 unsigned SelOpNo = 0;
7445 unsigned ConstSelOpNo = 1;
7446 unsigned OtherSelOpNo = 2;
7447 if (!dyn_cast<ConstantSDNode>(Sel->
getOperand(ConstSelOpNo))) {
7452 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
7453 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
7457 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
7458 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
7464 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
7466 std::swap(NewConstOps[0], NewConstOps[1]);
7478 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
7480 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
7483 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
7484 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
7493 MVT VT =
Op.getSimpleValueType();
7507 if ((Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps()) &&
7535 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
7539 TrueVal, Subtarget.
getXLen(), Subtarget,
true);
7541 FalseVal, Subtarget.
getXLen(), Subtarget,
true);
7542 bool IsCZERO_NEZ = TrueValCost <= FalseValCost;
7544 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal,
DL, VT);
7549 DL, VT, LHSVal, CondV);
7565 if (
Op.hasOneUse()) {
7566 unsigned UseOpc =
Op->use_begin()->getOpcode();
7572 return lowerSELECT(NewSel, DAG);
7600 SDValue Ops[] = {CondV,
Zero, SetNE, TrueV, FalseV};
7621 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
7625 if (TrueVal - 1 == FalseVal)
7627 if (TrueVal + 1 == FalseVal)
7634 RHS == TrueV && LHS == FalseV) {
7651 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
7677 LHS, RHS, TargetCC,
Op.getOperand(2));
7695 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
7707 int XLenInBytes = Subtarget.
getXLen() / 8;
7709 EVT VT =
Op.getValueType();
7712 unsigned Depth =
Op.getConstantOperandVal(0);
7714 int Offset = -(XLenInBytes * 2);
7730 int XLenInBytes = Subtarget.
getXLen() / 8;
7735 EVT VT =
Op.getValueType();
7737 unsigned Depth =
Op.getConstantOperandVal(0);
7739 int Off = -XLenInBytes;
7740 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
7759 EVT VT =
Lo.getValueType();
7798 EVT VT =
Lo.getValueType();
7849 MVT VT =
Op.getSimpleValueType();
7875 MVT VecVT =
Op.getSimpleValueType();
7877 "Unexpected SPLAT_VECTOR_PARTS lowering");
7883 MVT ContainerVT = VecVT;
7903 int64_t ExtTrueVal)
const {
7905 MVT VecVT =
Op.getSimpleValueType();
7908 assert(Src.getValueType().isVector() &&
7909 Src.getValueType().getVectorElementType() == MVT::i1);
7930 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
7932 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
7935 SplatZero, DAG.
getUNDEF(ContainerVT), VL);
7940SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
7942 MVT ExtVT =
Op.getSimpleValueType();
7946 MVT VT =
Op.getOperand(0).getSimpleValueType();
7972 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
7974 EVT MaskVT =
Op.getValueType();
7977 "Unexpected type for vector mask lowering");
7979 MVT VecVT = Src.getSimpleValueType();
7983 VL =
Op.getOperand(2);
7986 MVT ContainerVT = VecVT;
7992 MVT MaskContainerVT =
7999 std::tie(Mask, VL) =
8007 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
8009 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
8013 DAG.
getUNDEF(ContainerVT), Mask, VL);
8016 DAG.
getUNDEF(MaskContainerVT), Mask, VL});
8024 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
8027 MVT VT =
Op.getSimpleValueType();
8029 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8033 return lowerVectorMaskTruncLike(
Op, DAG);
8041 MVT SrcVT = Src.getSimpleValueType();
8046 "Unexpected vector truncate lowering");
8048 MVT ContainerVT = SrcVT;
8052 VL =
Op.getOperand(2);
8065 std::tie(Mask, VL) =
8076 }
while (SrcEltVT != DstEltVT);
8085RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(
SDValue Op,
8090 MVT VT =
Op.getSimpleValueType();
8091 MVT SrcVT = Src.getSimpleValueType();
8092 MVT ContainerVT = VT;
8113 Chain, Src, Mask, VL);
8114 Chain = Src.getValue(1);
8121 Chain, Src, Mask, VL);
8132RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(
SDValue Op,
8135 Op.getOpcode() == ISD::VP_FP_ROUND ||
Op.getOpcode() == ISD::VP_FP_EXTEND;
8142 MVT VT =
Op.getSimpleValueType();
8144 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
8147 MVT SrcVT = Src.getSimpleValueType();
8154 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
8157 MVT ContainerVT = VT;
8161 VL =
Op.getOperand(2);
8175 std::tie(Mask, VL) =
8181 Src = DAG.
getNode(ConvOpc,
DL, ContainerVT, Src, Mask, VL);
8187 unsigned InterConvOpc =
8192 DAG.
getNode(InterConvOpc,
DL, InterVT, Src, Mask, VL);
8194 DAG.
getNode(ConvOpc,
DL, ContainerVT, IntermediateConv, Mask, VL);
8205static std::optional<MVT>
8211 const unsigned MinVLMAX = VectorBitsMin / EltSize;
8213 if (MaxIdx < MinVLMAX)
8215 else if (MaxIdx < MinVLMAX * 2)
8217 else if (MaxIdx < MinVLMAX * 4)
8222 return std::nullopt;
8235 MVT VecVT =
Op.getSimpleValueType();
8249 MVT ContainerVT = VecVT;
8258 MVT OrigContainerVT = ContainerVT;
8261 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx)) {
8262 const unsigned OrigIdx = IdxC->getZExtValue();
8265 DL, DAG, Subtarget)) {
8266 ContainerVT = *ShrunkVT;
8275 VLEN && ContainerVT.
bitsGT(M1VT)) {
8278 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8279 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8280 unsigned ExtractIdx =
8299 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
8300 const auto *CVal = cast<ConstantSDNode>(Val);
8301 if (isInt<32>(CVal->getSExtValue())) {
8302 IsLegalInsert =
true;
8311 if (IsLegalInsert) {
8317 Vec = DAG.
getNode(Opc,
DL, ContainerVT, Vec, Val, VL);
8333 std::tie(ValLo, ValHi) = DAG.
SplitScalar(Val,
DL, MVT::i32, MVT::i32);
8334 MVT I32ContainerVT =
8345 Vec, Vec, ValLo, I32Mask, InsertI64VL);
8350 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
8352 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8357 ValInVec, AlignedIdx);
8367 DAG.
getUNDEF(I32ContainerVT), ValLo,
8368 I32Mask, InsertI64VL);
8370 DAG.
getUNDEF(I32ContainerVT), ValInVec, ValHi,
8371 I32Mask, InsertI64VL);
8373 ValInVec = DAG.
getBitcast(ContainerVT, ValInVec);
8386 Idx, Mask, InsertVL, Policy);
8390 Slideup, AlignedIdx);
8405 EVT EltVT =
Op.getValueType();
8412 MVT ContainerVT = VecVT;
8428 unsigned WidenVecLen;
8431 unsigned MaxEEW = Subtarget.
getELen();
8436 "the number of elements should be power of 2");
8440 ExtractBitIdx =
Idx;
8442 WideEltVT = LargestEltVT;
8445 ExtractElementIdx = DAG.
getNode(
8456 Vec, ExtractElementIdx);
8472 MVT ContainerVT = VecVT;
8483 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx);
8484 IdxC && VLen && VecVT.
getSizeInBits().getKnownMinValue() > *VLen) {
8486 unsigned OrigIdx = IdxC->getZExtValue();
8489 unsigned RemIdx = OrigIdx % ElemsPerVReg;
8490 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
8491 unsigned ExtractIdx =
8501 std::optional<uint64_t> MaxIdx;
8504 if (
auto *IdxC = dyn_cast<ConstantSDNode>(
Idx))
8505 MaxIdx = IdxC->getZExtValue();
8507 if (
auto SmallerVT =
8509 ContainerVT = *SmallerVT;
8556 "Unexpected opcode");
8563 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8568 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8585 if (OpVT.
bitsLT(XLenVT)) {
8592 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8603 MVT VT =
Op.getOperand(SplatOp - 1).getSimpleValueType();
8606 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
8617 case Intrinsic::riscv_vslide1up:
8618 case Intrinsic::riscv_vslide1down:
8619 case Intrinsic::riscv_vslide1up_mask:
8620 case Intrinsic::riscv_vslide1down_mask: {
8623 bool IsMasked = NumOps == 7;
8629 std::tie(ScalarLo, ScalarHi) =
8637 if (isa<ConstantSDNode>(AVL)) {
8638 const auto [MinVLMAX, MaxVLMAX] =
8642 if (AVLInt <= MinVLMAX) {
8644 }
else if (AVLInt >= 2 * MaxVLMAX) {
8651 Intrinsic::riscv_vsetvlimax,
DL, MVT::i32);
8685 if (IntNo == Intrinsic::riscv_vslide1up ||
8686 IntNo == Intrinsic::riscv_vslide1up_mask) {
8688 ScalarHi, I32Mask, I32VL);
8690 ScalarLo, I32Mask, I32VL);
8693 ScalarLo, I32Mask, I32VL);
8695 ScalarHi, I32Mask, I32VL);
8744 const unsigned ElementWidth = 8;
8749 [[maybe_unused]]
unsigned MinVF =
8752 [[maybe_unused]]
unsigned VF =
N->getConstantOperandVal(2);
8756 bool Fractional = VF < LMul1VF;
8757 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
8778 MVT ContainerVT = OpVT;
8805 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
8809 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
8826 if (OpVT.
bitsLT(XLenVT)) {
8829 ScalarOp = DAG.
getNode(ExtOpc,
DL, XLenVT, ScalarOp);
8842 EVT ValType = V.getValueType();
8843 if (ValType.isVector() && ValType.isFloatingPoint()) {
8846 ValType.getVectorElementCount());
8849 if (ValType.isFixedLengthVector()) {
8851 DAG, V.getSimpleValueType(), Subtarget);
8867 unsigned IntNo =
Op.getConstantOperandVal(0);
8874 case Intrinsic::thread_pointer: {
8878 case Intrinsic::riscv_orc_b:
8879 case Intrinsic::riscv_brev8:
8880 case Intrinsic::riscv_sha256sig0:
8881 case Intrinsic::riscv_sha256sig1:
8882 case Intrinsic::riscv_sha256sum0:
8883 case Intrinsic::riscv_sha256sum1:
8884 case Intrinsic::riscv_sm3p0:
8885 case Intrinsic::riscv_sm3p1: {
8905 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8907 case Intrinsic::riscv_sm4ks:
8908 case Intrinsic::riscv_sm4ed: {
8918 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
Op.getOperand(3));
8922 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2),
8925 case Intrinsic::riscv_zip:
8926 case Intrinsic::riscv_unzip: {
8929 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1));
8931 case Intrinsic::riscv_mopr: {
8944 case Intrinsic::riscv_moprr: {
8956 Op.getOperand(2),
Op.getOperand(3));
8958 case Intrinsic::riscv_clmul:
8969 case Intrinsic::riscv_clmulh:
8970 case Intrinsic::riscv_clmulr: {
8988 return DAG.
getNode(Opc,
DL, XLenVT,
Op.getOperand(1),
Op.getOperand(2));
8990 case Intrinsic::experimental_get_vector_length:
8992 case Intrinsic::experimental_cttz_elts:
8994 case Intrinsic::riscv_vmv_x_s: {
8998 case Intrinsic::riscv_vfmv_f_s:
9001 case Intrinsic::riscv_vmv_v_x:
9003 Op.getOperand(3),
Op.getSimpleValueType(),
DL, DAG,
9005 case Intrinsic::riscv_vfmv_v_f:
9007 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9008 case Intrinsic::riscv_vmv_s_x: {
9011 if (
Scalar.getValueType().bitsLE(XLenVT)) {
9014 Op.getOperand(1), Scalar,
Op.getOperand(3));
9017 assert(
Scalar.getValueType() == MVT::i64 &&
"Unexpected scalar VT!");
9034 MVT VT =
Op.getSimpleValueType();
9039 if (
Op.getOperand(1).isUndef())
9055 case Intrinsic::riscv_vfmv_s_f:
9057 Op.getOperand(1),
Op.getOperand(2),
Op.getOperand(3));
9059 case Intrinsic::riscv_vaesdf_vv:
9060 case Intrinsic::riscv_vaesdf_vs:
9061 case Intrinsic::riscv_vaesdm_vv:
9062 case Intrinsic::riscv_vaesdm_vs:
9063 case Intrinsic::riscv_vaesef_vv:
9064 case Intrinsic::riscv_vaesef_vs:
9065 case Intrinsic::riscv_vaesem_vv:
9066 case Intrinsic::riscv_vaesem_vs:
9067 case Intrinsic::riscv_vaeskf1:
9068 case Intrinsic::riscv_vaeskf2:
9069 case Intrinsic::riscv_vaesz_vs:
9070 case Intrinsic::riscv_vsm4k:
9071 case Intrinsic::riscv_vsm4r_vv:
9072 case Intrinsic::riscv_vsm4r_vs: {
9073 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9074 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9075 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9080 case Intrinsic::riscv_vsm3c:
9081 case Intrinsic::riscv_vsm3me: {
9082 if (!
isValidEGW(8,
Op.getSimpleValueType(), Subtarget) ||
9083 !
isValidEGW(8,
Op->getOperand(1).getSimpleValueType(), Subtarget))
9088 case Intrinsic::riscv_vsha2ch:
9089 case Intrinsic::riscv_vsha2cl:
9090 case Intrinsic::riscv_vsha2ms: {
9091 if (
Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
9092 !Subtarget.hasStdExtZvknhb())
9094 if (!
isValidEGW(4,
Op.getSimpleValueType(), Subtarget) ||
9095 !
isValidEGW(4,
Op->getOperand(1).getSimpleValueType(), Subtarget) ||
9096 !
isValidEGW(4,
Op->getOperand(2).getSimpleValueType(), Subtarget))
9100 case Intrinsic::riscv_sf_vc_v_x:
9101 case Intrinsic::riscv_sf_vc_v_i:
9102 case Intrinsic::riscv_sf_vc_v_xv:
9103 case Intrinsic::riscv_sf_vc_v_iv:
9104 case Intrinsic::riscv_sf_vc_v_vv:
9105 case Intrinsic::riscv_sf_vc_v_fv:
9106 case Intrinsic::riscv_sf_vc_v_xvv:
9107 case Intrinsic::riscv_sf_vc_v_ivv:
9108 case Intrinsic::riscv_sf_vc_v_vvv:
9109 case Intrinsic::riscv_sf_vc_v_fvv:
9110 case Intrinsic::riscv_sf_vc_v_xvw:
9111 case Intrinsic::riscv_sf_vc_v_ivw:
9112 case Intrinsic::riscv_sf_vc_v_vvw:
9113 case Intrinsic::riscv_sf_vc_v_fvw: {
9114 MVT VT =
Op.getSimpleValueType();
9151 MVT VT =
Op.getSimpleValueType();
9155 if (VT.isFloatingPoint()) {
9160 if (VT.isFixedLengthVector())
9170 if (VT.isFixedLengthVector())
9172 if (VT.isFloatingPoint())
9191 unsigned IntNo =
Op.getConstantOperandVal(1);
9195 case Intrinsic::riscv_masked_strided_load: {
9204 MVT VT =
Op->getSimpleValueType(0);
9205 MVT ContainerVT = VT;
9218 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9230 ScalarVT,
Load->getMemOperand());
9236 Load->getMemOperand());
9241 IsUnmasked ? Intrinsic::riscv_vlse : Intrinsic::riscv_vlse_mask,
DL,
9246 Ops.push_back(DAG.
getUNDEF(ContainerVT));
9248 Ops.push_back(PassThru);
9250 Ops.push_back(Stride);
9252 Ops.push_back(Mask);
9257 Ops.push_back(Policy);
9263 Load->getMemoryVT(),
Load->getMemOperand());
9264 Chain =
Result.getValue(1);
9270 case Intrinsic::riscv_seg2_load:
9271 case Intrinsic::riscv_seg3_load:
9272 case Intrinsic::riscv_seg4_load:
9273 case Intrinsic::riscv_seg5_load:
9274 case Intrinsic::riscv_seg6_load:
9275 case Intrinsic::riscv_seg7_load:
9276 case Intrinsic::riscv_seg8_load: {
9279 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
9280 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
9281 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
9282 Intrinsic::riscv_vlseg8};
9283 unsigned NF =
Op->getNumValues() - 1;
9284 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9286 MVT VT =
Op->getSimpleValueType(0);
9292 auto *
Load = cast<MemIntrinsicSDNode>(
Op);
9294 ContainerVTs.push_back(MVT::Other);
9302 Load->getMemoryVT(),
Load->getMemOperand());
9304 for (
unsigned int RetIdx = 0; RetIdx < NF; RetIdx++)
9310 case Intrinsic::riscv_sf_vc_v_x_se:
9312 case Intrinsic::riscv_sf_vc_v_i_se:
9314 case Intrinsic::riscv_sf_vc_v_xv_se:
9316 case Intrinsic::riscv_sf_vc_v_iv_se:
9318 case Intrinsic::riscv_sf_vc_v_vv_se:
9320 case Intrinsic::riscv_sf_vc_v_fv_se:
9322 case Intrinsic::riscv_sf_vc_v_xvv_se:
9324 case Intrinsic::riscv_sf_vc_v_ivv_se:
9326 case Intrinsic::riscv_sf_vc_v_vvv_se:
9328 case Intrinsic::riscv_sf_vc_v_fvv_se:
9330 case Intrinsic::riscv_sf_vc_v_xvw_se:
9332 case Intrinsic::riscv_sf_vc_v_ivw_se:
9334 case Intrinsic::riscv_sf_vc_v_vvw_se:
9336 case Intrinsic::riscv_sf_vc_v_fvw_se:
9345 unsigned IntNo =
Op.getConstantOperandVal(1);
9349 case Intrinsic::riscv_masked_strided_store: {
9360 MVT ContainerVT = VT;
9374 IsUnmasked ? Intrinsic::riscv_vsse : Intrinsic::riscv_vsse_mask,
DL,
9377 auto *
Store = cast<MemIntrinsicSDNode>(
Op);
9387 Ops,
Store->getMemoryVT(),
9388 Store->getMemOperand());
9390 case Intrinsic::riscv_seg2_store:
9391 case Intrinsic::riscv_seg3_store:
9392 case Intrinsic::riscv_seg4_store:
9393 case Intrinsic::riscv_seg5_store:
9394 case Intrinsic::riscv_seg6_store:
9395 case Intrinsic::riscv_seg7_store:
9396 case Intrinsic::riscv_seg8_store: {
9399 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
9400 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
9401 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
9402 Intrinsic::riscv_vsseg8};
9405 assert(NF >= 2 && NF <= 8 &&
"Unexpected seg number");
9407 MVT VT =
Op->getOperand(2).getSimpleValueType();
9415 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(
Op);
9417 for (
unsigned i = 0; i < NF; i++)
9419 ContainerVT, FixedIntrinsic->getOperand(2 + i), DAG, Subtarget));
9424 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
9426 case Intrinsic::riscv_sf_vc_xv_se:
9428 case Intrinsic::riscv_sf_vc_iv_se:
9430 case Intrinsic::riscv_sf_vc_vv_se:
9432 case Intrinsic::riscv_sf_vc_fv_se:
9434 case Intrinsic::riscv_sf_vc_xvv_se:
9436 case Intrinsic::riscv_sf_vc_ivv_se:
9438 case Intrinsic::riscv_sf_vc_vvv_se:
9440 case Intrinsic::riscv_sf_vc_fvv_se:
9442 case Intrinsic::riscv_sf_vc_xvw_se:
9444 case Intrinsic::riscv_sf_vc_ivw_se:
9446 case Intrinsic::riscv_sf_vc_vvw_se:
9448 case Intrinsic::riscv_sf_vc_fvw_se:
9456 switch (ISDOpcode) {
9459 case ISD::VP_REDUCE_ADD:
9462 case ISD::VP_REDUCE_UMAX:
9465 case ISD::VP_REDUCE_SMAX:
9468 case ISD::VP_REDUCE_UMIN:
9471 case ISD::VP_REDUCE_SMIN:
9474 case ISD::VP_REDUCE_AND:
9477 case ISD::VP_REDUCE_OR:
9480 case ISD::VP_REDUCE_XOR:
9483 case ISD::VP_REDUCE_FADD:
9485 case ISD::VP_REDUCE_SEQ_FADD:
9487 case ISD::VP_REDUCE_FMAX:
9489 case ISD::VP_REDUCE_FMIN:
9499 SDValue Vec =
Op.getOperand(IsVP ? 1 : 0);
9504 Op.getOpcode() == ISD::VP_REDUCE_AND ||
9505 Op.getOpcode() == ISD::VP_REDUCE_OR ||
9506 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
9507 "Unexpected reduction lowering");
9511 MVT ContainerVT = VecVT;
9520 VL =
Op.getOperand(3);
9522 std::tie(Mask, VL) =
9530 switch (
Op.getOpcode()) {
9534 case ISD::VP_REDUCE_AND: {
9544 case ISD::VP_REDUCE_OR:
9551 case ISD::VP_REDUCE_XOR: {
9574 return DAG.
getNode(BaseOpc,
DL,
Op.getValueType(), SetCC,
Op.getOperand(0));
9578 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
9579 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
9580 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
9581 (ImmAVL && ImmAVL->getZExtValue() >= 1);
9597 auto InnerVT = VecVT.
bitsLE(M1VT) ? VecVT : M1VT;
9601 auto InnerVL = NonZeroAVL ? VL : DAG.
getConstant(1,
DL, XLenVT);
9604 if (M1VT != InnerVT)
9610 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
9629 VecEVT =
Lo.getValueType();
9642 MVT ContainerVT = VecVT;
9662 Mask, VL,
DL, DAG, Subtarget);
9668static std::tuple<unsigned, SDValue, SDValue>
9672 auto Flags =
Op->getFlags();
9673 unsigned Opcode =
Op.getOpcode();
9697 return std::make_tuple(RVVOpc,
Op.getOperand(0), Front);
9705 MVT VecEltVT =
Op.getSimpleValueType();
9709 std::tie(RVVOpcode, VectorVal, ScalarVal) =
9713 MVT ContainerVT = VecVT;
9719 MVT ResVT =
Op.getSimpleValueType();
9722 VL,
DL, DAG, Subtarget);
9727 if (
Op->getFlags().hasNoNaNs())
9733 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
9734 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
9740 DL, ResVT, NoNaNs, Res,
9767 Vec, Mask, VL,
DL, DAG, Subtarget);
9779 unsigned OrigIdx =
Op.getConstantOperandVal(2);
9788 (OrigIdx != 0 || !Vec.
isUndef())) {
9791 assert(OrigIdx % 8 == 0 &&
"Invalid index");
9794 "Unexpected mask vector lowering");
9828 MVT ContainerVT = VecVT;
9864 SubVec =
getVSlideup(DAG, Subtarget,
DL, ContainerVT, Vec, SubVec,
9865 SlideupAmt, Mask, VL, Policy);
9873 MVT ContainerVecVT = VecVT;
9879 MVT ContainerSubVecVT = SubVecVT;
9895 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
9896 SubRegIdx = Decompose.first;
9898 (OrigIdx % Vscale));
9902 ContainerVecVT, ContainerSubVecVT, OrigIdx,
TRI);
9903 SubRegIdx = Decompose.first;
9910 bool ExactlyVecRegSized =
9912 .isKnownMultipleOf(Subtarget.
expandVScale(VecRegSize));
9927 if (RemIdx.
isZero() && (ExactlyVecRegSized || Vec.
isUndef())) {
9931 if (SubRegIdx == RISCV::NoSubRegister) {
9950 MVT InterSubVT = ContainerVecVT;
9989 SubVec =
getVSlideup(DAG, Subtarget,
DL, InterSubVT, AlignedExtract, SubVec,
9990 SlideupAmt, Mask, VL, Policy);
9995 if (ContainerVecVT.
bitsGT(InterSubVT))
10004 return DAG.
getBitcast(
Op.getSimpleValueType(), SubVec);
10010 MVT SubVecVT =
Op.getSimpleValueType();
10015 unsigned OrigIdx =
Op.getConstantOperandVal(1);
10026 assert(OrigIdx % 8 == 0 &&
"Invalid index");
10029 "Unexpected mask vector lowering");
10068 MVT ContainerVT = VecVT;
10076 if (
auto ShrunkVT =
10078 ContainerVT = *ShrunkVT;
10092 DAG.
getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
10104 MVT ContainerSubVecVT = SubVecVT;
10108 unsigned SubRegIdx;
10118 VecVT, ContainerSubVecVT, OrigIdx / Vscale,
TRI);
10119 SubRegIdx = Decompose.first;
10121 (OrigIdx % Vscale));
10125 VecVT, ContainerSubVecVT, OrigIdx,
TRI);
10126 SubRegIdx = Decompose.first;
10149 MVT InterSubVT = VecVT;
10153 assert(SubRegIdx != RISCV::NoSubRegister);
10167 Vec, SlidedownAmt, Mask, VL);
10176 return DAG.
getBitcast(
Op.getSimpleValueType(), Slidedown);
10183 MVT VT =
N.getSimpleValueType();
10187 assert(
Op.getSimpleValueType() == VT &&
10188 "Operands and result must be same type");
10192 unsigned NumVals =
N->getNumValues();
10195 NumVals,
N.getValueType().changeVectorElementType(MVT::i8)));
10198 for (
unsigned I = 0;
I < NumVals;
I++) {
10204 if (TruncVals.
size() > 1)
10206 return TruncVals.
front();
10212 MVT VecVT =
Op.getSimpleValueType();
10215 "vector_interleave on non-scalable vector!");
10226 EVT SplitVT = Op0Lo.getValueType();
10229 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op0Hi);
10231 DAG.
getVTList(SplitVT, SplitVT), Op1Lo, Op1Hi);
10245 Op.getOperand(0),
Op.getOperand(1));
10272 Concat, EvenIdx, Passthru, Mask, VL);
10274 Concat, OddIdx, Passthru, Mask, VL);
10288 MVT VecVT =
Op.getSimpleValueType();
10291 "vector_interleave on non-scalable vector!");
10304 EVT SplitVT = Op0Lo.getValueType();
10307 DAG.
getVTList(SplitVT, SplitVT), Op0Lo, Op1Lo);
10309 DAG.
getVTList(SplitVT, SplitVT), Op0Hi, Op1Hi);
10331 Op.getOperand(0),
Op.getOperand(1));
10379 MVT VT =
Op.getSimpleValueType();
10384 uint64_t StepValImm =
Op.getConstantOperandVal(0);
10385 if (StepValImm != 1) {
10394 VL, VT,
DL, DAG, Subtarget);
10409 MVT VecVT =
Op.getSimpleValueType();
10419 unsigned MaxVLMAX =
10429 if (MaxVLMAX > 256 && EltSize == 8) {
10476 return DAG.
getNode(GatherOpc,
DL, VecVT,
Op.getOperand(0), Indices,
10486 MVT VecVT =
Op.getSimpleValueType();
10490 int64_t ImmValue = cast<ConstantSDNode>(
Op.getOperand(2))->getSExtValue();
10491 SDValue DownOffset, UpOffset;
10492 if (ImmValue >= 0) {
10508 DownOffset, TrueMask, UpOffset);
10509 return getVSlideup(DAG, Subtarget,
DL, VecVT, SlideDown, V2, UpOffset,
10515RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(
SDValue Op,
10518 auto *
Load = cast<LoadSDNode>(
Op);
10521 Load->getMemoryVT(),
10522 *
Load->getMemOperand()) &&
10523 "Expecting a correctly-aligned load");
10525 MVT VT =
Op.getSimpleValueType();
10531 const auto [MinVLMAX, MaxVLMAX] =
10534 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10548 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle,
DL, XLenVT);
10557 Load->getMemoryVT(),
Load->getMemOperand());
10564RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(
SDValue Op,
10567 auto *
Store = cast<StoreSDNode>(
Op);
10570 Store->getMemoryVT(),
10571 *
Store->getMemOperand()) &&
10572 "Expecting a correctly-aligned store");
10594 const auto [MinVLMAX, MaxVLMAX] =
10597 getLMUL1VT(ContainerVT).bitsLE(ContainerVT)) {
10609 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse,
DL, XLenVT);
10612 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
10613 Store->getMemoryVT(),
Store->getMemOperand());
10619 MVT VT =
Op.getSimpleValueType();
10621 const auto *MemSD = cast<MemSDNode>(
Op);
10622 EVT MemVT = MemSD->getMemoryVT();
10624 SDValue Chain = MemSD->getChain();
10628 if (
const auto *VPLoad = dyn_cast<VPLoadSDNode>(
Op)) {
10629 Mask = VPLoad->getMask();
10631 VL = VPLoad->getVectorLength();
10633 const auto *MLoad = cast<MaskedLoadSDNode>(
Op);
10634 Mask = MLoad->getMask();
10635 PassThru = MLoad->getPassThru();
10642 MVT ContainerVT = VT;
10656 IsUnmasked ? Intrinsic::riscv_vle : Intrinsic::riscv_vle_mask;
10673 Chain =
Result.getValue(1);
10685 const auto *MemSD = cast<MemSDNode>(
Op);
10686 EVT MemVT = MemSD->getMemoryVT();
10688 SDValue Chain = MemSD->getChain();
10692 bool IsCompressingStore =
false;
10693 if (
const auto *VPStore = dyn_cast<VPStoreSDNode>(
Op)) {
10694 Val = VPStore->getValue();
10695 Mask = VPStore->getMask();
10696 VL = VPStore->getVectorLength();
10698 const auto *MStore = cast<MaskedStoreSDNode>(
Op);
10699 Val = MStore->getValue();
10700 Mask = MStore->getMask();
10701 IsCompressingStore = MStore->isCompressingStore();
10710 MVT ContainerVT = VT;
10715 if (!IsUnmasked || IsCompressingStore) {
10724 if (IsCompressingStore) {
10727 DAG.
getUNDEF(ContainerVT), Val, Mask, VL);
10734 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
10743 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
10747RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(
SDValue Op,
10749 MVT InVT =
Op.getOperand(0).getSimpleValueType();
10752 MVT VT =
Op.getSimpleValueType();
10766 {Op1, Op2,
Op.getOperand(2), DAG.
getUNDEF(MaskVT), Mask, VL});
10773 unsigned Opc =
Op.getOpcode();
10780 MVT VT =
Op.getSimpleValueType();
10813 MVT ContainerInVT = InVT;
10832 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10836 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
10844 {Chain, Op1, Op2, CC, Mask, Mask, VL});
10849 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
10862 MVT VT =
Op.getSimpleValueType();
10866 "Unexpected type for ISD::ABS");
10868 MVT ContainerVT = VT;
10875 if (
Op->getOpcode() == ISD::VP_ABS) {
10876 Mask =
Op->getOperand(1);
10880 VL =
Op->getOperand(2);
10888 DAG.
getUNDEF(ContainerVT), Mask, VL);
10890 DAG.
getUNDEF(ContainerVT), Mask, VL);
10897SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
10900 MVT VT =
Op.getSimpleValueType();
10904 "Can only handle COPYSIGN with matching types.");
10913 Sign, DAG.
getUNDEF(ContainerVT), Mask, VL);
10918SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
10920 MVT VT =
Op.getSimpleValueType();
10923 MVT I1ContainerVT =
10937 Op2, DAG.
getUNDEF(ContainerVT), VL);
10948 MVT VT =
Op.getSimpleValueType();
10953 for (
const SDValue &V :
Op->op_values()) {
10954 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
10957 if (!
V.getValueType().isVector()) {
10963 assert(useRVVForFixedLengthVectorVT(
V.getSimpleValueType()) &&
10964 "Only fixed length vectors are supported!");
10978 if (
Op->isStrictFPOpcode()) {
10987 DAG.
getNode(NewOpc,
DL, ContainerVT, Ops,
Op->getFlags());
11001 MVT VT =
Op.getSimpleValueType();
11004 MVT ContainerVT = VT;
11010 assert(!isa<VTSDNode>(V) &&
"Unexpected VTSDNode node!");
11016 if (*MaskIdx == OpIdx.index())
11020 if (
Op.getOpcode() == ISD::VP_MERGE) {
11024 assert(
Op.getOpcode() == ISD::VP_SELECT);
11031 if (!
V.getValueType().isFixedLengthVector()) {
11036 MVT OpVT =
V.getSimpleValueType();
11038 assert(useRVVForFixedLengthVectorVT(OpVT) &&
11039 "Only fixed length vectors are supported!");
11044 return DAG.
getNode(RISCVISDOpc,
DL, VT, Ops,
Op->getFlags());
11054 MVT VT =
Op.getSimpleValueType();
11060 MVT ContainerVT = VT;
11070 DAG.
getUNDEF(ContainerVT), Zero, VL);
11073 Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1,
DL, XLenVT);
11075 DAG.
getUNDEF(ContainerVT), SplatValue, VL);
11078 ZeroSplat, DAG.
getUNDEF(ContainerVT), VL);
11087 MVT VT =
Op.getSimpleValueType();
11091 ISD::CondCode Condition = cast<CondCodeSDNode>(
Op.getOperand(2))->get();
11095 MVT ContainerVT = VT;
11105 switch (Condition) {
11173 MVT DstVT =
Op.getSimpleValueType();
11174 MVT SrcVT = Src.getSimpleValueType();
11187 if (DstEltSize >= SrcEltSize) {
11196 if (SrcEltSize == 1) {
11207 ZeroSplat, DAG.
getUNDEF(IntVT), VL);
11208 }
else if (DstEltSize > (2 * SrcEltSize)) {
11212 Src = DAG.
getNode(RISCVISDExtOpc,
DL, IntVT, Src, Mask, VL);
11218 "Wrong input/output vector types");
11221 if (DstEltSize > (2 * SrcEltSize)) {
11237 MVT InterimFVT = DstVT;
11238 if (SrcEltSize > (2 * DstEltSize)) {
11239 assert(SrcEltSize == (4 * DstEltSize) &&
"Unexpected types!");
11246 if (InterimFVT != DstVT) {
11252 "Wrong input/output vector types");
11256 if (DstEltSize == 1) {
11259 assert(SrcEltSize >= 16 &&
"Unexpected FP type!");
11269 DAG.
getUNDEF(InterimIVT), SplatZero, VL);
11279 while (InterimIVT != DstVT) {
11291 MVT VT =
Op.getSimpleValueType();
11298RISCVTargetLowering::lowerVPSpliceExperimental(
SDValue Op,
11310 MVT VT =
Op.getSimpleValueType();
11311 MVT ContainerVT = VT;
11324 if (IsMaskVector) {
11335 SplatZeroOp1, DAG.
getUNDEF(ContainerVT), EVL1);
11344 SplatZeroOp2, DAG.
getUNDEF(ContainerVT), EVL2);
11347 int64_t ImmValue = cast<ConstantSDNode>(
Offset)->getSExtValue();
11348 SDValue DownOffset, UpOffset;
11349 if (ImmValue >= 0) {
11363 Op1, DownOffset, Mask, UpOffset);
11367 if (IsMaskVector) {
11371 {Result, DAG.getConstant(0, DL, ContainerVT),
11372 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
11382RISCVTargetLowering::lowerVPReverseExperimental(
SDValue Op,
11385 MVT VT =
Op.getSimpleValueType();
11392 MVT ContainerVT = VT;
11400 MVT GatherVT = ContainerVT;
11404 if (IsMaskVector) {
11415 SplatZero, DAG.
getUNDEF(IndicesVT), EVL);
11421 unsigned MaxVLMAX =
11430 if (MaxVLMAX > 256 && EltSize == 8) {
11458 DAG.
getUNDEF(GatherVT), Result, Diff, Mask, EVL);
11460 if (IsMaskVector) {
11483 DAG.
getUNDEF(IndicesVT), VecLen, EVL);
11485 DAG.
getUNDEF(IndicesVT), Mask, EVL);
11487 DAG.
getUNDEF(GatherVT), Mask, EVL);
11489 if (IsMaskVector) {
11504 MVT VT =
Op.getSimpleValueType();
11506 return lowerVPOp(
Op, DAG);
11513 MVT ContainerVT = VT;
11532 MVT VT =
Op.getSimpleValueType();
11533 MVT ContainerVT = VT;
11539 auto *VPNode = cast<VPStridedLoadSDNode>(
Op);
11545 : Intrinsic::riscv_vlse_mask,
11548 DAG.
getUNDEF(ContainerVT), VPNode->getBasePtr(),
11549 VPNode->getStride()};
11557 Ops.
push_back(VPNode->getVectorLength());
11565 VPNode->getMemoryVT(), VPNode->getMemOperand());
11579 auto *VPNode = cast<VPStridedStoreSDNode>(
Op);
11580 SDValue StoreVal = VPNode->getValue();
11582 MVT ContainerVT = VT;
11593 : Intrinsic::riscv_vsse_mask,
11596 VPNode->getBasePtr(), VPNode->getStride()};
11604 Ops.
push_back(VPNode->getVectorLength());
11607 Ops, VPNode->getMemoryVT(),
11608 VPNode->getMemOperand());
11620 MVT VT =
Op.getSimpleValueType();
11622 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11623 EVT MemVT = MemSD->getMemoryVT();
11625 SDValue Chain = MemSD->getChain();
11631 if (
auto *VPGN = dyn_cast<VPGatherSDNode>(
Op.getNode())) {
11632 Index = VPGN->getIndex();
11633 Mask = VPGN->getMask();
11635 VL = VPGN->getVectorLength();
11640 auto *MGN = cast<MaskedGatherSDNode>(
Op.getNode());
11641 Index = MGN->getIndex();
11642 Mask = MGN->getMask();
11643 PassThru = MGN->getPassThru();
11647 MVT IndexVT =
Index.getSimpleValueType();
11651 "Unexpected VTs!");
11652 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11655 "Unexpected extending MGATHER/VP_GATHER");
11661 MVT ContainerVT = VT;
11685 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
11702 Chain =
Result.getValue(1);
11719 const auto *MemSD = cast<MemSDNode>(
Op.getNode());
11720 EVT MemVT = MemSD->getMemoryVT();
11722 SDValue Chain = MemSD->getChain();
11725 [[maybe_unused]]
bool IsTruncatingStore =
false;
11728 if (
auto *VPSN = dyn_cast<VPScatterSDNode>(
Op.getNode())) {
11729 Index = VPSN->getIndex();
11730 Mask = VPSN->getMask();
11731 Val = VPSN->getValue();
11732 VL = VPSN->getVectorLength();
11734 IsTruncatingStore =
false;
11737 auto *MSN = cast<MaskedScatterSDNode>(
Op.getNode());
11738 Index = MSN->getIndex();
11739 Mask = MSN->getMask();
11740 Val = MSN->getValue();
11741 IsTruncatingStore = MSN->isTruncatingStore();
11745 MVT IndexVT =
Index.getSimpleValueType();
11749 "Unexpected VTs!");
11750 assert(
BasePtr.getSimpleValueType() == XLenVT &&
"Unexpected pointer type");
11753 assert(!IsTruncatingStore &&
"Unexpected truncating MSCATTER/VP_SCATTER");
11759 MVT ContainerVT = VT;
11783 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
11793 DAG.
getVTList(MVT::Other), Ops, MemVT, MMO);
11802 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11810 static const int Table =
11834 RISCVSysReg::lookupSysRegByName(
"FRM")->Encoding,
DL, XLenVT);
11840 static const unsigned Table =
11863 bool isRISCV64 = Subtarget.
is64Bit();
11927 switch (
N->getOpcode()) {
11929 llvm_unreachable(
"Don't know how to custom type legalize this operation!");
11935 "Unexpected custom legalisation");
11936 bool IsStrict =
N->isStrictFPOpcode();
11939 SDValue Op0 = IsStrict ?
N->getOperand(1) :
N->getOperand(0);
11957 Opc,
DL, VTs, Chain, Op0,
11991 std::tie(Result, Chain) =
11992 makeLibCall(DAG, LC,
N->getValueType(0), Op0, CallOptions,
DL, Chain);
12020 Op0.
getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
12031 assert(!Subtarget.
is64Bit() &&
"READCYCLECOUNTER/READSTEADYCOUNTER only "
12032 "has custom type legalization on riscv32");
12034 SDValue LoCounter, HiCounter;
12038 RISCVSysReg::lookupSysRegByName(
"CYCLE")->Encoding,
DL, XLenVT);
12040 RISCVSysReg::lookupSysRegByName(
"CYCLEH")->Encoding,
DL, XLenVT);
12043 RISCVSysReg::lookupSysRegByName(
"TIME")->Encoding,
DL, XLenVT);
12045 RISCVSysReg::lookupSysRegByName(
"TIMEH")->Encoding,
DL, XLenVT);
12049 N->getOperand(0), LoCounter, HiCounter);
12073 unsigned Size =
N->getSimpleValueType(0).getSizeInBits();
12074 unsigned XLen = Subtarget.
getXLen();
12077 assert(
Size == (XLen * 2) &&
"Unexpected custom legalisation");
12085 if (LHSIsU == RHSIsU)
12102 if (RHSIsU && LHSIsS && !RHSIsS)
12104 else if (LHSIsU && RHSIsS && !LHSIsS)
12114 "Unexpected custom legalisation");
12121 "Unexpected custom legalisation");
12124 if (
N->getOpcode() ==
ISD::SHL && Subtarget.hasStdExtZbs() &&
12150 "Unexpected custom legalisation");
12151 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
12152 Subtarget.hasVendorXTHeadBb()) &&
12153 "Unexpected custom legalization");
12154 if (!isa<ConstantSDNode>(
N->getOperand(1)) &&
12155 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
12164 "Unexpected custom legalisation");
12178 MVT VT =
N->getSimpleValueType(0);
12179 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
12180 Subtarget.
is64Bit() && Subtarget.hasStdExtM() &&
12181 "Unexpected custom legalisation");
12193 if (VT != MVT::i32)
12202 "Unexpected custom legalisation");
12206 if (!isa<ConstantSDNode>(
N->getOperand(1)))
12223 EVT OType =
N->getValueType(1);
12236 "Unexpected custom legalisation");
12253 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1), Res,
12257 Overflow = DAG.
getSetCC(
DL,
N->getValueType(1),
N->getOperand(0),
12275 "Unexpected custom legalisation");
12276 if (Subtarget.hasStdExtZbb()) {
12297 "Unexpected custom legalisation");
12303 "Unexpected custom legalisation");
12305 if (Subtarget.hasStdExtZbb()) {
12339 EVT VT =
N->getValueType(0);
12344 if (VT == MVT::i16 && Op0VT == MVT::f16 &&
12348 }
else if (VT == MVT::i16 && Op0VT == MVT::bf16 &&
12349 Subtarget.hasStdExtZfbfmin()) {
12352 }
else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.
is64Bit() &&
12357 }
else if (VT == MVT::i64 && Op0VT == MVT::f64 && XLenVT == MVT::i32) {
12359 DAG.
getVTList(MVT::i32, MVT::i32), Op0);
12379 MVT VT =
N->getSimpleValueType(0);
12381 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.
is64Bit())) &&
12382 "Unexpected custom legalisation");
12383 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
12409 assert(!Subtarget.
is64Bit() &&
N->getValueType(0) == MVT::i64 &&
12411 "Unexpected EXTRACT_VECTOR_ELT legalization");
12414 MVT ContainerVT = VecVT;
12442 DAG.
getUNDEF(ContainerVT), Mask, VL);
12450 unsigned IntNo =
N->getConstantOperandVal(0);
12454 "Don't know how to custom type legalize this intrinsic!");
12455 case Intrinsic::experimental_get_vector_length: {
12460 case Intrinsic::experimental_cttz_elts: {
12466 case Intrinsic::riscv_orc_b:
12467 case Intrinsic::riscv_brev8:
12468 case Intrinsic::riscv_sha256sig0:
12469 case Intrinsic::riscv_sha256sig1:
12470 case Intrinsic::riscv_sha256sum0:
12471 case Intrinsic::riscv_sha256sum1:
12472 case Intrinsic::riscv_sm3p0:
12473 case Intrinsic::riscv_sm3p1: {
12474 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12494 case Intrinsic::riscv_sm4ks:
12495 case Intrinsic::riscv_sm4ed: {
12503 DAG.
getNode(Opc,
DL, MVT::i64, NewOp0, NewOp1,
N->getOperand(3));
12507 case Intrinsic::riscv_mopr: {
12508 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12518 case Intrinsic::riscv_moprr: {
12519 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12531 case Intrinsic::riscv_clmul: {
12532 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12543 case Intrinsic::riscv_clmulh:
12544 case Intrinsic::riscv_clmulr: {
12545 if (!Subtarget.
is64Bit() ||
N->getValueType(0) != MVT::i32)
12573 case Intrinsic::riscv_vmv_x_s: {
12574 EVT VT =
N->getValueType(0);
12576 if (VT.
bitsLT(XLenVT)) {
12585 "Unexpected custom legalization");
12623 case ISD::VP_REDUCE_ADD:
12624 case ISD::VP_REDUCE_AND:
12625 case ISD::VP_REDUCE_OR:
12626 case ISD::VP_REDUCE_XOR:
12627 case ISD::VP_REDUCE_SMAX:
12628 case ISD::VP_REDUCE_UMAX:
12629 case ISD::VP_REDUCE_SMIN:
12630 case ISD::VP_REDUCE_UMIN:
12694 const EVT VT =
N->getValueType(0);
12695 const unsigned Opc =
N->getOpcode();
12702 (Opc !=
ISD::FADD || !
N->getFlags().hasAllowReassociation()))
12707 "Inconsistent mappings");
12718 !isa<ConstantSDNode>(
RHS.getOperand(1)))
12721 uint64_t RHSIdx = cast<ConstantSDNode>(
RHS.getOperand(1))->getLimitedValue();
12736 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(
LHS.getOperand(1))) {
12738 cast<ConstantSDNode>(
LHS.getOperand(1))->getLimitedValue();
12739 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
12743 return DAG.
getNode(ReduceOpc,
DL, VT, Vec,
N->getFlags());
12750 if (
LHS.getOpcode() != ReduceOpc)
12765 auto Flags = ReduceVec->
getFlags();
12766 Flags.intersectWith(
N->getFlags());
12767 return DAG.
getNode(ReduceOpc,
DL, VT, Vec, Flags);
12777 auto BinOpToRVVReduce = [](
unsigned Opc) {
12806 auto IsReduction = [&BinOpToRVVReduce](
SDValue V,
unsigned Opc) {
12809 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
12812 unsigned Opc =
N->getOpcode();
12813 unsigned ReduceIdx;
12814 if (IsReduction(
N->getOperand(0), Opc))
12816 else if (IsReduction(
N->getOperand(1), Opc))
12822 if (Opc ==
ISD::FADD && !
N->getFlags().hasAllowReassociation())
12825 SDValue Extract =
N->getOperand(ReduceIdx);
12857 SDValue NewStart =
N->getOperand(1 - ReduceIdx);
12884 if (!Subtarget.hasStdExtZba())
12888 EVT VT =
N->getValueType(0);
12900 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
12901 auto *N1C = dyn_cast<ConstantSDNode>(N1->
getOperand(1));
12904 int64_t C0 = N0C->getSExtValue();
12905 int64_t C1 = N1C->getSExtValue();
12906 if (C0 <= 0 || C1 <= 0)
12910 int64_t Bits = std::min(C0, C1);
12911 int64_t Diff = std::abs(C0 - C1);
12912 if (Diff != 1 && Diff != 2 && Diff != 3)
12940 EVT VT =
N->getValueType(0);
12948 if ((!Subtarget.hasStdExtZicond() &&
12949 !Subtarget.hasVendorXVentanaCondOps()) ||
12971 bool SwapSelectOps;
12977 SwapSelectOps =
false;
12978 NonConstantVal = FalseVal;
12980 SwapSelectOps =
true;
12981 NonConstantVal = TrueVal;
12987 FalseVal = DAG.
getNode(
N->getOpcode(),
SDLoc(
N), VT, OtherOp, NonConstantVal);
13035 EVT VT =
N->getValueType(0);
13043 auto *N0C = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
13044 auto *N1C = dyn_cast<ConstantSDNode>(
N->getOperand(1));
13050 if (!N0C->hasOneUse())
13052 int64_t C0 = N0C->getSExtValue();
13053 int64_t C1 = N1C->getSExtValue();
13055 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(C1))
13058 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
13059 !isInt<12>(C0 * (C1 / C0))) {
13062 }
else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
13063 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
13066 }
else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
13067 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
13094 EVT VT =
N->getValueType(0);
13125 unsigned OuterExtend =
13129 OuterExtend,
SDLoc(
N), VT,
13137 EVT VT =
N->getValueType(0);
13182 EVT VT =
N->getValueType(0);
13186 auto *N0C = dyn_cast<ConstantSDNode>(N0);
13192 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
13202 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.
isInteger())
13224 EVT VT =
N->getValueType(0);
13253 bool IsAnd =
N->getOpcode() ==
ISD::AND;
13277 EVT VT =
N->getValueType(0);
13297 EVT VT =
N->getValueType(0);
13383 EVT VT =
N->getValueType(0);
13454 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.
getOperand(0));
13459 const APInt &Imm = ConstN00->getAPIntValue();
13460 if ((Imm + 1).isSignedIntN(12))
13477 LHS.getValueType());
13498 EVT VT =
N->getValueType(0);
13510 if (!Subtarget.hasStdExtZba() && !Subtarget.hasVendorXTHeadBa())
13524 for (
uint64_t Divisor : {3, 5, 9}) {
13525 if (MulAmt % Divisor != 0)
13527 uint64_t MulAmt2 = MulAmt / Divisor;
13534 if (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9) {
13550 if (ScaleShift >= 1 && ScaleShift < 4) {
13551 unsigned ShiftAmt =
Log2_64((MulAmt & (MulAmt - 1)));
13565 for (
uint64_t Divisor : {3, 5, 9}) {
13570 if ((
C >> TZ) == Divisor && (TZ == 1 || TZ == 2 || TZ == 3)) {
13581 if (MulAmt > 2 &&
isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
13583 if (ScaleShift >= 1 && ScaleShift < 4) {
13584 unsigned ShiftAmt =
Log2_64(((MulAmt - 1) & (MulAmt - 2)));
13615 EVT VT =
N->getValueType(0);
13623 unsigned AddSubOpc;
13629 auto IsAddSubWith1 = [&](
SDValue V) ->
bool {
13630 AddSubOpc = V->getOpcode();
13632 SDValue Opnd = V->getOperand(1);
13633 MulOper = V->getOperand(0);
13642 if (IsAddSubWith1(N0)) {
13644 return DAG.
getNode(AddSubOpc,
DL, VT, N1, MulVal);
13647 if (IsAddSubWith1(N1)) {
13649 return DAG.
getNode(AddSubOpc,
DL, VT, N0, MulVal);
13661 if (isIndexTypeSigned(IndexType))
13664 if (!
N->hasOneUse())
13667 EVT VT =
N.getValueType();
13706 EVT SrcVT = Src.getValueType();
13710 NewElen = std::max(NewElen, 8U);
13733 EVT VT =
N->getValueType(0);
13736 if (OpVT != MVT::i64 || !Subtarget.
is64Bit())
13740 auto *N1C = dyn_cast<ConstantSDNode>(N1);
13752 if (!isIntEqualitySetCC(
Cond))
13761 const APInt &C1 = N1C->getAPIntValue();
13779 EVT VT =
N->getValueType(0);
13783 cast<VTSDNode>(
N->getOperand(1))->getVT().bitsGE(MVT::i16))
13785 Src.getOperand(0));
13793struct CombineResult;
13795enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
13822struct NodeExtensionHelper {
13831 bool SupportsFPExt;
13834 bool EnforceOneUse;
13849 return OrigOperand;
13860 unsigned getExtOpc(ExtKind SupportsExt)
const {
13861 switch (SupportsExt) {
13862 case ExtKind::SExt:
13864 case ExtKind::ZExt:
13866 case ExtKind::FPExt:
13877 std::optional<ExtKind> SupportsExt)
const {
13878 if (!SupportsExt.has_value())
13879 return OrigOperand;
13881 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
13885 if (
Source.getValueType() == NarrowVT)
13888 unsigned ExtOpc = getExtOpc(*SupportsExt);
13892 auto [
Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
13899 return DAG.
getNode(ExtOpc,
DL, NarrowVT, Source, Mask, VL);
13917 static MVT getNarrowType(
const SDNode *Root, ExtKind SupportsExt) {
13923 MVT EltVT = SupportsExt == ExtKind::FPExt
13925 :
MVT::getIntegerVT(NarrowSize);
13927 assert((
int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
13928 "Trying to extend something we can't represent");
13935 static unsigned getSExtOpcode(
unsigned Opcode) {
13958 static unsigned getZExtOpcode(
unsigned Opcode) {
13984 static unsigned getFPExtOpcode(
unsigned Opcode) {
14001 static unsigned getSUOpcode(
unsigned Opcode) {
14003 "SU is only supported for MUL");
14009 static unsigned getWOpcode(
unsigned Opcode, ExtKind SupportsExt) {
14029 using CombineToTry = std::function<std::optional<CombineResult>(
14030 SDNode * ,
const NodeExtensionHelper & ,
14035 bool needToPromoteOtherUsers()
const {
return EnforceOneUse; }
14039 unsigned Opc = OrigOperand.
getOpcode();
14043 "Unexpected Opcode");
14056 unsigned ScalarBits =
Op.getValueSizeInBits();
14059 if (ScalarBits < EltBits)
14065 if (NarrowSize < 8)
14069 SupportsSExt =
true;
14073 SupportsZExt =
true;
14075 EnforceOneUse =
false;
14082 SupportsZExt =
false;
14083 SupportsSExt =
false;
14084 SupportsFPExt =
false;
14085 EnforceOneUse =
true;
14086 unsigned Opc = OrigOperand.
getOpcode();
14108 SupportsZExt =
true;
14111 SupportsSExt =
true;
14114 SupportsFPExt =
true;
14118 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
14126 static bool isSupportedRoot(
const SDNode *Root,
14155 Subtarget.hasStdExtZvbb();
14157 return Subtarget.hasStdExtZvbb();
14166 assert(isSupportedRoot(Root, Subtarget) &&
14167 "Trying to build an helper with an "
14168 "unsupported root");
14169 assert(OperandIdx < 2 &&
"Requesting something else than LHS or RHS");
14185 if (OperandIdx == 1) {
14194 EnforceOneUse =
false;
14199 fillUpExtensionSupport(Root, DAG, Subtarget);
14205 static std::pair<SDValue, SDValue>
14208 assert(isSupportedRoot(Root, Subtarget) &&
"Unexpected root");
14227 switch (
N->getOpcode()) {
14265struct CombineResult {
14267 unsigned TargetOpcode;
14269 std::optional<ExtKind> LHSExt;
14270 std::optional<ExtKind> RHSExt;
14274 NodeExtensionHelper
LHS;
14276 NodeExtensionHelper
RHS;
14278 CombineResult(
unsigned TargetOpcode,
SDNode *Root,
14279 const NodeExtensionHelper &
LHS, std::optional<ExtKind> LHSExt,
14280 const NodeExtensionHelper &
RHS, std::optional<ExtKind> RHSExt)
14281 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
14290 std::tie(Mask, VL) =
14291 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
14305 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
14306 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
14321static std::optional<CombineResult>
14322canFoldToVWWithSameExtensionImpl(
SDNode *Root,
const NodeExtensionHelper &LHS,
14323 const NodeExtensionHelper &RHS,
14326 if ((AllowExtMask & ExtKind::ZExt) &&
LHS.SupportsZExt &&
RHS.SupportsZExt)
14327 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->
getOpcode()),
14328 Root, LHS, {ExtKind::ZExt}, RHS,
14330 if ((AllowExtMask & ExtKind::SExt) &&
LHS.SupportsSExt &&
RHS.SupportsSExt)
14331 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->
getOpcode()),
14332 Root, LHS, {ExtKind::SExt}, RHS,
14334 if ((AllowExtMask & ExtKind::FPExt) &&
LHS.SupportsFPExt &&
RHS.SupportsFPExt)
14335 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->
getOpcode()),
14336 Root, LHS, {ExtKind::FPExt}, RHS,
14338 return std::nullopt;
14347static std::optional<CombineResult>
14348canFoldToVWWithSameExtension(
SDNode *Root,
const NodeExtensionHelper &LHS,
14351 return canFoldToVWWithSameExtensionImpl(
14352 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
14360static std::optional<CombineResult>
14361canFoldToVW_W(
SDNode *Root,
const NodeExtensionHelper &LHS,
14364 if (
RHS.SupportsFPExt)
14365 return CombineResult(
14366 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::FPExt),
14367 Root, LHS, std::nullopt, RHS, {ExtKind::FPExt});
14374 return CombineResult(
14375 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::ZExt), Root,
14376 LHS, std::nullopt, RHS, {ExtKind::ZExt});
14378 return CombineResult(
14379 NodeExtensionHelper::getWOpcode(Root->
getOpcode(), ExtKind::SExt), Root,
14380 LHS, std::nullopt, RHS, {ExtKind::SExt});
14381 return std::nullopt;
14388static std::optional<CombineResult>
14389canFoldToVWWithSEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14392 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::SExt, DAG,
14400static std::optional<CombineResult>
14401canFoldToVWWithZEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14404 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
14412static std::optional<CombineResult>
14413canFoldToVWWithFPEXT(
SDNode *Root,
const NodeExtensionHelper &LHS,
14416 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::FPExt, DAG,
14424static std::optional<CombineResult>
14425canFoldToVW_SU(
SDNode *Root,
const NodeExtensionHelper &LHS,
14429 if (!
LHS.SupportsSExt || !
RHS.SupportsZExt)
14430 return std::nullopt;
14431 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->
getOpcode()),
14432 Root, LHS, {ExtKind::SExt}, RHS,
14437NodeExtensionHelper::getSupportedFoldings(
const SDNode *Root) {
14448 Strategies.
push_back(canFoldToVWWithSameExtension);
14453 Strategies.
push_back(canFoldToVWWithSameExtension);
14458 Strategies.
push_back(canFoldToVWWithSameExtension);
14465 Strategies.
push_back(canFoldToVWWithZEXT);
14470 Strategies.
push_back(canFoldToVWWithSEXT);
14475 Strategies.
push_back(canFoldToVWWithZEXT);
14480 Strategies.
push_back(canFoldToVWWithFPEXT);
14509 if (!NodeExtensionHelper::isSupportedRoot(
N, Subtarget))
14515 Inserted.insert(
N);
14518 while (!Worklist.
empty()) {
14520 if (!NodeExtensionHelper::isSupportedRoot(Root, Subtarget))
14523 NodeExtensionHelper
LHS(
N, 0, DAG, Subtarget);
14524 NodeExtensionHelper
RHS(
N, 1, DAG, Subtarget);
14525 auto AppendUsersIfNeeded = [&Worklist,
14526 &Inserted](
const NodeExtensionHelper &
Op) {
14527 if (
Op.needToPromoteOtherUsers()) {
14528 for (
SDNode *TheUse :
Op.OrigOperand->uses()) {
14529 if (Inserted.insert(TheUse).second)
14541 NodeExtensionHelper::getSupportedFoldings(
N);
14543 assert(!FoldingStrategies.
empty() &&
"Nothing to be folded");
14544 bool Matched =
false;
14545 for (
int Attempt = 0;
14546 (Attempt != 1 + NodeExtensionHelper::isCommutative(
N)) && !Matched;
14549 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
14550 FoldingStrategies) {
14551 std::optional<CombineResult> Res =
14552 FoldingStrategy(
N,
LHS,
RHS, DAG, Subtarget);
14559 if (Res->LHSExt.has_value())
14560 AppendUsersIfNeeded(
LHS);
14561 if (Res->RHSExt.has_value())
14562 AppendUsersIfNeeded(
RHS);
14573 SDValue InputRootReplacement;
14580 for (CombineResult Res : CombinesToApply) {
14581 SDValue NewValue = Res.materialize(DAG, Subtarget);
14582 if (!InputRootReplacement) {
14584 "First element is expected to be the current node");
14585 InputRootReplacement = NewValue;
14590 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
14594 return InputRootReplacement;
14601 unsigned Opc =
N->getOpcode();
14606 SDValue MergeOp =
N->getOperand(1);
14607 unsigned MergeOpc = MergeOp.
getOpcode();
14618 SDValue Passthru =
N->getOperand(2);
14632 Z = Z.getOperand(1);
14638 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
14645 [[maybe_unused]]
unsigned Opc =
N->getOpcode();
14674 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
14680 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
14682 if (MemVT == MVT::i32)
14688 Opcode,
SDLoc(LSNode1), DAG.
getVTList({XLenVT, XLenVT, MVT::Other}),
14723 if (!Subtarget.hasVendorXTHeadMemPair())
14735 auto ExtractBaseAndOffset = [](
SDValue Ptr) -> std::pair<SDValue, uint64_t> {
14737 if (
auto *C1 = dyn_cast<ConstantSDNode>(
Ptr->getOperand(1)))
14738 return {
Ptr->getOperand(0), C1->getZExtValue()};
14742 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->
getOperand(OpNum));
14765 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->
getOperand(OpNum));
14768 if (Base1 != Base2)
14772 bool Valid =
false;
14773 if (MemVT == MVT::i32) {
14775 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
14777 }
else if (MemVT == MVT::i64) {
14779 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
14813 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14821 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14831 EVT VT =
N->getValueType(0);
14834 MVT SrcVT = Src.getSimpleValueType();
14835 MVT SrcContainerVT = SrcVT;
14837 SDValue XVal = Src.getOperand(0);
14864 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14868 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask, VL);
14872 FpToInt = DAG.
getNode(Opc,
DL, ContainerVT, XVal, Mask,
14885 if (VT != MVT::i32 && VT != XLenVT)
14915 EVT DstVT =
N->getValueType(0);
14916 if (DstVT != XLenVT)
14922 if (Src->isStrictFPOpcode() || Src->isTargetStrictFPOpcode())
14930 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
14933 EVT SatVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
14942 if (SatVT == DstVT)
14944 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
14950 Src = Src.getOperand(0);
14971 assert(Subtarget.hasStdExtZbkb() &&
"Unexpected extension");
14977 EVT VT =
N->getValueType(0);
15030 unsigned Offset =
N->isTargetStrictFPOpcode();
15037 auto invertIfNegative = [&Mask, &VL](
SDValue &V) {
15039 V.getOperand(2) == VL) {
15041 V = V.getOperand(0);
15048 bool NegA = invertIfNegative(
A);
15049 bool NegB = invertIfNegative(
B);
15050 bool NegC = invertIfNegative(
C);
15053 if (!NegA && !NegB && !NegC)
15057 if (
N->isTargetStrictFPOpcode())
15059 {N->getOperand(0), A, B, C, Mask, VL});
15069 if (
N->getValueType(0).isScalableVector() &&
15070 N->getValueType(0).getVectorElementType() == MVT::f32 &&
15077 if (
N->isTargetStrictFPOpcode())
15102 switch (
N->getOpcode()) {
15122 return DAG.
getNode(NewOpc,
SDLoc(
N),
N->getValueType(0), Op0, Op1,
15123 N->getOperand(2), Mask, VL);
15130 if (
N->getValueType(0) != MVT::i64 || !Subtarget.
is64Bit())
15133 if (!isa<ConstantSDNode>(
N->getOperand(1)))
15135 uint64_t ShAmt =
N->getConstantOperandVal(1);
15146 cast<VTSDNode>(N0.
getOperand(1))->getVT() == MVT::i32 &&
15174 AddC = dyn_cast<ConstantSDNode>(N0.
getOperand(IsAdd ? 1 : 0));
15187 !isa<ConstantSDNode>(U->getOperand(1)) ||
15188 U->getConstantOperandVal(1) > 32)
15243 if (!
Cond.hasOneUse())
15262 EVT VT =
Cond.getValueType();
15307 LHS =
LHS.getOperand(0);
15317 LHS.getOperand(0).getValueType() == Subtarget.
getXLenVT()) {
15321 CCVal = cast<CondCodeSDNode>(
LHS.getOperand(2))->get();
15325 RHS =
LHS.getOperand(1);
15326 LHS =
LHS.getOperand(0);
15335 RHS =
LHS.getOperand(1);
15336 LHS =
LHS.getOperand(0);
15352 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
15393 bool Commutative =
true;
15394 unsigned Opc = TrueVal.getOpcode();
15402 Commutative =
false;
15410 if (!TrueVal.hasOneUse() || isa<ConstantSDNode>(FalseVal))
15414 if (FalseVal == TrueVal.getOperand(0))
15416 else if (Commutative && FalseVal == TrueVal.getOperand(1))
15421 EVT VT =
N->getValueType(0);
15423 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
15429 assert(IdentityOperand &&
"No identity operand!");
15434 DAG.
getSelect(
DL, OtherOpVT,
N->getOperand(0), OtherOp, IdentityOperand);
15435 return DAG.
getNode(TrueVal.getOpcode(),
DL, VT, FalseVal, NewSel);
15456 CountZeroes =
N->getOperand(2);
15457 ValOnZero =
N->getOperand(1);
15459 CountZeroes =
N->getOperand(1);
15460 ValOnZero =
N->getOperand(2);
15479 if (
Cond->getOperand(0) != CountZeroesArgument)
15495 CountZeroes, BitWidthMinusOne);
15505 EVT VT =
N->getValueType(0);
15506 EVT CondVT =
Cond.getValueType();
15514 (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())) {
15520 const APInt &MaskVal =
LHS.getConstantOperandAPInt(1);
15541 SDValue TrueVal =
N->getOperand(1);
15542 SDValue FalseVal =
N->getOperand(2);
15557 EVT VT =
N->getValueType(0);
15564 const unsigned Opcode =
N->op_begin()->getNode()->getOpcode();
15579 if (
Op.isUndef()) {
15592 if (
Op.getOpcode() != Opcode || !
Op.hasOneUse())
15596 if (!isa<ConstantSDNode>(
Op.getOperand(1)) &&
15597 !isa<ConstantFPSDNode>(
Op.getOperand(1)))
15601 if (
Op.getOperand(0).getValueType() !=
Op.getOperand(1).getValueType())
15629 const unsigned InVecOpcode = InVec->
getOpcode();
15639 if (!isa<ConstantSDNode>(InValRHS) && !isa<ConstantFPSDNode>(InValRHS))
15646 InVecLHS, InValLHS, EltNo);
15648 InVecRHS, InValRHS, EltNo);
15657 auto *IndexC = dyn_cast<ConstantSDNode>(EltNo);
15660 unsigned Elt = IndexC->getZExtValue();
15668 unsigned ConcatOpIdx = Elt / ConcatNumElts;
15671 ConcatOp, InVal, NewIdx);
15675 ConcatOps[ConcatOpIdx] = ConcatOp;
15687 EVT VT =
N->getValueType(0);
15697 auto *BaseLd = dyn_cast<LoadSDNode>(
N->getOperand(0));
15699 !
SDValue(BaseLd, 0).hasOneUse())
15702 EVT BaseLdVT = BaseLd->getValueType(0);
15709 auto *Ld = dyn_cast<LoadSDNode>(
Op);
15710 if (!Ld || !Ld->isSimple() || !
Op.hasOneUse() ||
15712 Ld->getValueType(0) != BaseLdVT)
15721 using PtrDiff = std::pair<std::variant<int64_t, SDValue>,
bool>;
15723 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
15728 if (BIO1.equalBaseIndex(BIO2, DAG))
15733 SDValue P2 = Ld2->getBasePtr();
15739 return std::nullopt;
15743 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
15748 for (
auto *It = Lds.
begin() + 1; It != Lds.
end() - 1; It++)
15749 if (GetPtrDiff(*It, *std::next(It)) != BaseDiff)
15757 unsigned WideScalarBitWidth =
15770 auto [StrideVariant, MustNegateStride] = *BaseDiff;
15771 SDValue Stride = std::holds_alternative<SDValue>(StrideVariant)
15772 ? std::get<SDValue>(StrideVariant)
15775 if (MustNegateStride)
15788 BaseLd->getBasePtr(), Stride, AllOneMask};
15791 if (
auto *ConstStride = dyn_cast<ConstantSDNode>(Stride);
15792 ConstStride && ConstStride->getSExtValue() >= 0)
15796 ConstStride->getSExtValue() * (
N->getNumOperands() - 1);
15802 BaseLd->getPointerInfo(), BaseLd->getMemOperand()->getFlags(), MemSize,
15806 Ops, WideVecVT, MMO);
15818 if (
N->getValueType(0).isFixedLengthVector())
15821 SDValue Addend =
N->getOperand(0);
15825 SDValue AddMergeOp =
N->getOperand(2);
15830 auto IsVWMulOpc = [](
unsigned Opc) {
15859 return std::make_pair(
N->getOperand(3),
N->getOperand(4));
15860 }(
N, DAG, Subtarget);
15865 if (AddMask != MulMask || AddVL != MulVL)
15870 "Unexpected opcode after VWMACC_VL");
15872 "Unexpected opcode after VWMACC_VL!");
15874 "Unexpected opcode after VWMUL_VL!");
15876 "Unexpected opcode after VWMUL_VL!");
15879 EVT VT =
N->getValueType(0);
15895 const EVT IndexVT =
Index.getValueType();
15899 if (!isIndexTypeSigned(IndexType))
15931 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
15934 if (
Index->getOperand(i)->isUndef())
15937 if (
C % ElementSize != 0)
15939 C =
C / ElementSize;
15943 ActiveLanes.
set(
C);
15945 return ActiveLanes.
all();
15963 if (NumElems % 2 != 0)
15967 const unsigned WiderElementSize = ElementSize * 2;
15968 if (WiderElementSize > ST.getELen()/8)
15971 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
15974 for (
unsigned i = 0; i <
Index->getNumOperands(); i++) {
15977 if (
Index->getOperand(i)->isUndef())
15983 if (
C % WiderElementSize != 0)
15988 if (
C !=
Last + ElementSize)
16004 auto SimplifyDemandedLowBitsHelper = [&](
unsigned OpNo,
unsigned LowBits) {
16015 switch (
N->getOpcode()) {
16035 APInt V =
C->getValueAPF().bitcastToAPInt();
16070 if (SimplifyDemandedLowBitsHelper(0, 32) ||
16071 SimplifyDemandedLowBitsHelper(1, 5))
16079 if (SimplifyDemandedLowBitsHelper(0, 32))
16096 MVT VT =
N->getSimpleValueType(0);
16105 "Unexpected value type!");
16128 EVT VT =
N->getValueType(0);
16193 if (
N->getValueType(0) == MVT::i64 && Subtarget.
is64Bit()) {
16198 Src.getOperand(0));
16203 Src.getOperand(0), Src.getOperand(1));
16218 auto IsTruncNode = [](
SDValue V) {
16221 SDValue VL = V.getOperand(2);
16222 auto *
C = dyn_cast<ConstantSDNode>(VL);
16224 bool IsVLMAXForVMSET = (
C &&
C->isAllOnes()) ||
16225 (isa<RegisterSDNode>(VL) &&
16226 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0);
16235 while (IsTruncNode(
Op)) {
16236 if (!
Op.hasOneUse())
16238 Op =
Op.getOperand(0);
16270 unsigned Opc =
N->getOpcode();
16285 return DAG.
getNode(InvOpc,
SDLoc(
N),
N->getValueType(0), Val, NewCond);
16295 N->getValueType(0), Val,
Cond.getOperand(0));
16306 SDValue FalseV =
N->getOperand(4);
16308 EVT VT =
N->getValueType(0);
16311 if (TrueV == FalseV)
16316 if (!Subtarget.hasShortForwardBranchOpt() && isa<ConstantSDNode>(TrueV) &&
16322 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
16323 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
16326 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
16327 isInt<12>(TrueSImm - FalseSImm)) {
16343 {LHS, RHS, CC, TrueV, FalseV});
16410 N->getOperand(0),
LHS,
RHS,
CC,
N->getOperand(4));
16423 EVT VT =
N->getValueType(0);
16447 const auto *MGN = dyn_cast<MaskedGatherSDNode>(
N);
16448 const EVT VT =
N->getValueType(0);
16450 SDValue ScaleOp = MGN->getScale();
16452 assert(!MGN->isIndexScaled() &&
16453 "Scaled gather/scatter should not be formed");
16458 N->getVTList(), MGN->getMemoryVT(),
DL,
16459 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16460 MGN->getBasePtr(), Index, ScaleOp},
16461 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16465 N->getVTList(), MGN->getMemoryVT(),
DL,
16466 {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
16467 MGN->getBasePtr(), Index, ScaleOp},
16468 MGN->getMemOperand(), IndexType, MGN->getExtensionType());
16474 if (std::optional<VIDSequence> SimpleVID =
16476 SimpleVID && SimpleVID->StepDenominator == 1) {
16477 const int64_t StepNumerator = SimpleVID->StepNumerator;
16478 const int64_t Addend = SimpleVID->Addend;
16485 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
16495 {MGN->getChain(), IntID, MGN->getPassThru(), BasePtr,
16496 DAG.
getConstant(StepNumerator,
DL, XLenVT), MGN->getMask()};
16498 Ops, VT, MGN->getMemOperand());
16506 MGN->getBasePtr(), DAG.
getUNDEF(XLenVT),
16508 MGN->getMemoryVT(), MGN->getMemOperand(),
16517 MGN->getMemOperand()->getBaseAlign(), Subtarget)) {
16519 for (
unsigned i = 0; i <
Index->getNumOperands(); i += 2)
16521 EVT IndexVT =
Index.getValueType()
16528 assert(EltCnt.isKnownEven() &&
"Splitting vector, but not in half!");
16530 EltCnt.divideCoefficientBy(2));
16533 EltCnt.divideCoefficientBy(2));
16538 {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
16547 const auto *MSN = dyn_cast<MaskedScatterSDNode>(
N);
16549 SDValue ScaleOp = MSN->getScale();
16551 assert(!MSN->isIndexScaled() &&
16552 "Scaled gather/scatter should not be formed");
16557 N->getVTList(), MSN->getMemoryVT(),
DL,
16558 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16560 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16564 N->getVTList(), MSN->getMemoryVT(),
DL,
16565 {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
16567 MSN->getMemOperand(), IndexType, MSN->isTruncatingStore());
16569 EVT VT = MSN->getValue()->getValueType(0);
16571 if (!MSN->isTruncatingStore() &&
16575 return DAG.
getMaskedStore(MSN->getChain(),
DL, Shuffle, MSN->getBasePtr(),
16576 DAG.
getUNDEF(XLenVT), MSN->getMask(),
16577 MSN->getMemoryVT(), MSN->getMemOperand(),
16582 case ISD::VP_GATHER: {
16583 const auto *VPGN = dyn_cast<VPGatherSDNode>(
N);
16585 SDValue ScaleOp = VPGN->getScale();
16587 assert(!VPGN->isIndexScaled() &&
16588 "Scaled gather/scatter should not be formed");
16593 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16594 ScaleOp, VPGN->getMask(),
16595 VPGN->getVectorLength()},
16596 VPGN->getMemOperand(), IndexType);
16600 {VPGN->getChain(), VPGN->getBasePtr(), Index,
16601 ScaleOp, VPGN->getMask(),
16602 VPGN->getVectorLength()},
16603 VPGN->getMemOperand(), IndexType);
16607 case ISD::VP_SCATTER: {
16608 const auto *VPSN = dyn_cast<VPScatterSDNode>(
N);
16610 SDValue ScaleOp = VPSN->getScale();
16612 assert(!VPSN->isIndexScaled() &&
16613 "Scaled gather/scatter should not be formed");
16618 {VPSN->getChain(), VPSN->getValue(),
16619 VPSN->getBasePtr(), Index, ScaleOp,
16620 VPSN->getMask(), VPSN->getVectorLength()},
16621 VPSN->getMemOperand(), IndexType);
16625 {VPSN->getChain(), VPSN->getValue(),
16626 VPSN->getBasePtr(), Index, ScaleOp,
16627 VPSN->getMask(), VPSN->getVectorLength()},
16628 VPSN->getMemOperand(), IndexType);
16642 EVT VT =
N->getValueType(0);
16645 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt,
16646 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
16664 EVT VT =
N->getValueType(0);
16668 return DAG.
getNode(
N->getOpcode(),
DL, VT,
N->getOperand(0), ShAmt);
16698 if (
N->getValueType(0).isScalableVector() &&
16699 N->getValueType(0).getVectorElementType() == MVT::f32 &&
16714 auto *Store = cast<StoreSDNode>(
N);
16715 SDValue Chain = Store->getChain();
16716 EVT MemVT = Store->getMemoryVT();
16717 SDValue Val = Store->getValue();
16720 bool IsScalarizable =
16722 Store->isSimple() &&
16752 NewVT, *Store->getMemOperand())) {
16754 return DAG.
getStore(Chain,
DL, NewV, Store->getBasePtr(),
16755 Store->getPointerInfo(), Store->getOriginalAlign(),
16756 Store->getMemOperand()->getFlags());
16764 if (
auto *L = dyn_cast<LoadSDNode>(Val);
16766 L->hasNUsesOfValue(1, 0) && L->hasNUsesOfValue(1, 1) &&
16768 L->getMemoryVT() == MemVT) {
16771 NewVT, *Store->getMemOperand()) &&
16773 NewVT, *L->getMemOperand())) {
16775 L->getPointerInfo(), L->getOriginalAlign(),
16776 L->getMemOperand()->getFlags());
16777 return DAG.
getStore(Chain,
DL, NewL, Store->getBasePtr(),
16778 Store->getPointerInfo(), Store->getOriginalAlign(),
16779 Store->getMemOperand()->getFlags());
16791 MVT VecVT = Src.getSimpleValueType();
16798 Store->getChain(),
DL, Src, Store->getBasePtr(), Store->getOffset(),
16801 Store->getMemOperand(), Store->getAddressingMode(),
16802 Store->isTruncatingStore(),
false);
16809 EVT VT =
N->getValueType(0);
16831 const MVT VT =
N->getSimpleValueType(0);
16832 SDValue Passthru =
N->getOperand(0);
16833 SDValue Scalar =
N->getOperand(1);
16842 const MVT VT =
N->getSimpleValueType(0);
16843 SDValue Passthru =
N->getOperand(0);
16844 SDValue Scalar =
N->getOperand(1);
16849 unsigned ScalarSize = Scalar.getValueSizeInBits();
16851 if (ScalarSize > EltWidth && Passthru.
isUndef())
16852 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
16859 (!Const || Const->isZero() ||
16860 !Const->getAPIntValue().sextOrTrunc(EltWidth).isSignedIntN(5)))
16870 if (
N->getOperand(0).isUndef() &&
16873 Src.getOperand(0).getValueType().isScalableVector()) {
16874 EVT VT =
N->getValueType(0);
16875 EVT SrcVT = Src.getOperand(0).getValueType();
16879 return Src.getOperand(0);
16885 const MVT VT =
N->getSimpleValueType(0);
16886 SDValue Passthru =
N->getOperand(0);
16887 SDValue Scalar =
N->getOperand(1);
16891 Scalar.getOperand(0).getValueType() ==
N->getValueType(0))
16892 return Scalar.getOperand(0);
16901 DAG.
getNode(
N->getOpcode(),
DL, M1VT, M1Passthru, Scalar, VL);
16911 Const && !Const->isZero() && isInt<5>(Const->getSExtValue()) &&
16919 MVT VecVT =
N->getOperand(0).getSimpleValueType();
16921 if (M1VT.
bitsLT(VecVT)) {
16932 unsigned IntNo =
N->getConstantOperandVal(IntOpNo);
16937 case Intrinsic::riscv_masked_strided_load: {
16938 MVT VT =
N->getSimpleValueType(0);
16939 auto *Load = cast<MemIntrinsicSDNode>(
N);
16940 SDValue PassThru =
N->getOperand(2);
16942 SDValue Stride =
N->getOperand(4);
16948 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
16949 StrideC && StrideC->getZExtValue() == ElementSize)
16951 DAG.
getUNDEF(XLenVT), Mask, PassThru,
16952 Load->getMemoryVT(), Load->getMemOperand(),
16956 case Intrinsic::riscv_masked_strided_store: {
16957 auto *Store = cast<MemIntrinsicSDNode>(
N);
16960 SDValue Stride =
N->getOperand(4);
16965 const unsigned ElementSize =
Value.getValueType().getScalarStoreSize();
16966 if (
auto *StrideC = dyn_cast<ConstantSDNode>(Stride);
16967 StrideC && StrideC->getZExtValue() == ElementSize)
16970 Value.getValueType(), Store->getMemOperand(),
16974 case Intrinsic::riscv_vcpop:
16975 case Intrinsic::riscv_vcpop_mask:
16976 case Intrinsic::riscv_vfirst:
16977 case Intrinsic::riscv_vfirst_mask: {
16979 if (IntNo == Intrinsic::riscv_vcpop_mask ||
16980 IntNo == Intrinsic::riscv_vfirst_mask)
16981 VL =
N->getOperand(3);
16986 EVT VT =
N->getValueType(0);
16987 if (IntNo == Intrinsic::riscv_vfirst ||
16988 IntNo == Intrinsic::riscv_vfirst_mask)
16997 EVT VT =
N->getValueType(0);
17001 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
17020 EVT XVT,
unsigned KeptBits)
const {
17025 if (XVT != MVT::i32 && XVT != MVT::i64)
17029 if (KeptBits == 32 || KeptBits == 64)
17033 return Subtarget.hasStdExtZbb() &&
17034 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.
is64Bit()) ||
17042 "Expected shift op");
17053 auto *C1 = dyn_cast<ConstantSDNode>(N0->
getOperand(1));
17054 auto *C2 = dyn_cast<ConstantSDNode>(
N->getOperand(1));
17056 const APInt &C1Int = C1->getAPIntValue();
17057 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
17083 if (C1Cost < ShiftedC1Cost)
17097 EVT VT =
Op.getValueType();
17101 unsigned Opcode =
Op.getOpcode();
17109 const APInt &Mask =
C->getAPIntValue();
17118 auto IsLegalMask = [ShrunkMask, ExpandedMask](
const APInt &Mask) ->
bool {
17119 return ShrunkMask.
isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
17121 auto UseMask = [Mask,
Op, &TLO](
const APInt &NewMask) ->
bool {
17122 if (NewMask == Mask)
17127 Op.getOperand(0), NewC);
17140 APInt NewMask =
APInt(Mask.getBitWidth(), 0xffff);
17141 if (IsLegalMask(NewMask))
17142 return UseMask(NewMask);
17145 if (VT == MVT::i64) {
17147 if (IsLegalMask(NewMask))
17148 return UseMask(NewMask);
17163 APInt NewMask = ShrunkMask;
17164 if (MinSignedBits <= 12)
17166 else if (!
C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.
isSignedIntN(32))
17172 assert(IsLegalMask(NewMask));
17173 return UseMask(NewMask);
17177 static const uint64_t GREVMasks[] = {
17178 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
17179 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
17181 for (
unsigned Stage = 0; Stage != 6; ++Stage) {
17182 unsigned Shift = 1 << Stage;
17183 if (ShAmt & Shift) {
17185 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
17197 const APInt &DemandedElts,
17199 unsigned Depth)
const {
17201 unsigned Opc =
Op.getOpcode();
17206 "Should use MaskedValueIsZero if you don't know whether Op"
17207 " is a target node!");
17290 assert(MinVLenB > 0 &&
"READ_VLENB without vector extension enabled?");
17293 if (MaxVLenB == MinVLenB)
17310 case Intrinsic::riscv_vsetvli:
17311 case Intrinsic::riscv_vsetvlimax: {
17312 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
17313 unsigned VSEW =
Op.getConstantOperandVal(HasAVL + 1);
17319 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
17322 if (HasAVL && isa<ConstantSDNode>(
Op.getOperand(1)))
17323 MaxVL = std::min(MaxVL,
Op.getConstantOperandVal(1));
17325 unsigned KnownZeroFirstBit =
Log2_32(MaxVL) + 1;
17338 unsigned Depth)
const {
17339 switch (
Op.getOpcode()) {
17345 if (Tmp == 1)
return 1;
17348 return std::min(Tmp, Tmp2);
17360 if (Tmp < 33)
return 1;
17385 unsigned XLen = Subtarget.
getXLen();
17386 unsigned EltBits =
Op.getOperand(0).getScalarValueSizeInBits();
17387 if (EltBits <= XLen)
17388 return XLen - EltBits + 1;
17392 unsigned IntNo =
Op.getConstantOperandVal(1);
17396 case Intrinsic::riscv_masked_atomicrmw_xchg_i64:
17397 case Intrinsic::riscv_masked_atomicrmw_add_i64:
17398 case Intrinsic::riscv_masked_atomicrmw_sub_i64:
17399 case Intrinsic::riscv_masked_atomicrmw_nand_i64:
17400 case Intrinsic::riscv_masked_atomicrmw_max_i64:
17401 case Intrinsic::riscv_masked_atomicrmw_min_i64:
17402 case Intrinsic::riscv_masked_atomicrmw_umax_i64:
17403 case Intrinsic::riscv_masked_atomicrmw_umin_i64:
17404 case Intrinsic::riscv_masked_cmpxchg_i64:
17412 assert(Subtarget.hasStdExtA());
17427 switch (
Op.getOpcode()) {
17433 return !
Op.getValueType().isInteger();
17441 assert(Ld &&
"Unexpected null LoadSDNode");
17449 auto *CNode = dyn_cast<ConstantPoolSDNode>(
Ptr);
17450 if (!CNode || CNode->isMachineConstantPoolEntry() ||
17451 CNode->getOffset() != 0)
17459 auto *CNode = GetSupportedConstantPool(
Ptr);
17460 if (!CNode || CNode->getTargetFlags() != 0)
17463 return CNode->getConstVal();
17471 auto *CNodeLo = GetSupportedConstantPool(
Ptr.getOperand(1));
17472 auto *CNodeHi = GetSupportedConstantPool(
Ptr.getOperand(0).getOperand(0));
17478 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
17481 return CNodeLo->getConstVal();
17486 assert(
MI.getOpcode() == RISCV::ReadCounterWide &&
"Unexpected instruction");
17518 Register ReadAgainReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
17521 int64_t LoCounter =
MI.getOperand(2).getImm();
17522 int64_t HiCounter =
MI.getOperand(3).getImm();
17532 BuildMI(LoopMBB,
DL,
TII->get(RISCV::CSRRS), ReadAgainReg)
17544 MI.eraseFromParent();
17552 assert(
MI.getOpcode() == RISCV::SplitF64Pseudo &&
"Unexpected instruction");
17560 Register SrcReg =
MI.getOperand(2).getReg();
17580 MI.eraseFromParent();
17587 assert(
MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
17588 "Unexpected instruction");
17594 Register DstReg =
MI.getOperand(0).getReg();
17617 MI.eraseFromParent();
17622 switch (
MI.getOpcode()) {
17625 case RISCV::Select_GPR_Using_CC_GPR:
17626 case RISCV::Select_FPR16_Using_CC_GPR:
17627 case RISCV::Select_FPR16INX_Using_CC_GPR:
17628 case RISCV::Select_FPR32_Using_CC_GPR:
17629 case RISCV::Select_FPR32INX_Using_CC_GPR:
17630 case RISCV::Select_FPR64_Using_CC_GPR:
17631 case RISCV::Select_FPR64INX_Using_CC_GPR:
17632 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
17638 unsigned RelOpcode,
unsigned EqOpcode,
17641 Register DstReg =
MI.getOperand(0).getReg();
17642 Register Src1Reg =
MI.getOperand(1).getReg();
17643 Register Src2Reg =
MI.getOperand(2).getReg();
17645 Register SavedFFlags =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17669 MI.eraseFromParent();
17720 F->insert(It, FirstMBB);
17721 F->insert(It, SecondMBB);
17722 F->insert(It, SinkMBB);
17771 First.eraseFromParent();
17814 SelectDests.
insert(
MI.getOperand(0).getReg());
17818 if (
MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR && Next != BB->
end() &&
17819 Next->getOpcode() ==
MI.getOpcode() &&
17820 Next->getOperand(5).getReg() ==
MI.getOperand(0).getReg() &&
17821 Next->getOperand(5).isKill()) {
17826 SequenceMBBI != E; ++SequenceMBBI) {
17827 if (SequenceMBBI->isDebugInstr())
17830 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
17831 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
17832 SequenceMBBI->getOperand(3).getImm() !=
CC ||
17833 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
17834 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
17836 LastSelectPseudo = &*SequenceMBBI;
17838 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
17841 if (SequenceMBBI->hasUnmodeledSideEffects() ||
17842 SequenceMBBI->mayLoadOrStore() ||
17843 SequenceMBBI->usesCustomInsertionHook())
17846 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
17861 F->insert(
I, IfFalseMBB);
17862 F->insert(
I, TailMBB);
17866 TailMBB->
push_back(DebugInstr->removeFromParent());
17870 TailMBB->
splice(TailMBB->
end(), HeadMBB,
17889 auto SelectMBBI =
MI.getIterator();
17890 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
17891 auto InsertionPoint = TailMBB->
begin();
17892 while (SelectMBBI != SelectEnd) {
17893 auto Next = std::next(SelectMBBI);
17896 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
17897 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
17898 .
addReg(SelectMBBI->getOperand(4).getReg())
17900 .
addReg(SelectMBBI->getOperand(5).getReg())
17915 RISCVVInversePseudosTable::getBaseInfo(MCOpcode, LMul, SEW);
17916 assert(
Inverse &&
"Unexpected LMUL and SEW pair for instruction");
17918 RISCV::lookupMaskedIntrinsicByUnmasked(
Inverse->Pseudo);
17919 assert(Masked &&
"Could not find masked instruction for LMUL and SEW pair");
17925 unsigned CVTXOpc) {
17931 Register SavedFFLAGS =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
17944 .
add(
MI.getOperand(1))
17945 .
add(
MI.getOperand(2))
17946 .
add(
MI.getOperand(3))
17948 .
add(
MI.getOperand(4))
17949 .
add(
MI.getOperand(5))
17950 .
add(
MI.getOperand(6))
17965 .
add(
MI.getOperand(0))
17966 .
add(
MI.getOperand(1))
17968 .
add(
MI.getOperand(3))
17970 .
add(
MI.getOperand(4))
17971 .
add(
MI.getOperand(5))
17972 .
add(
MI.getOperand(6))
17982 MI.eraseFromParent();
17988 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
17990 switch (
MI.getOpcode()) {
17993 case RISCV::PseudoFROUND_H:
17994 CmpOpc = RISCV::FLT_H;
17995 F2IOpc = RISCV::FCVT_W_H;
17996 I2FOpc = RISCV::FCVT_H_W;
17997 FSGNJOpc = RISCV::FSGNJ_H;
17998 FSGNJXOpc = RISCV::FSGNJX_H;
17999 RC = &RISCV::FPR16RegClass;
18001 case RISCV::PseudoFROUND_H_INX:
18002 CmpOpc = RISCV::FLT_H_INX;
18003 F2IOpc = RISCV::FCVT_W_H_INX;
18004 I2FOpc = RISCV::FCVT_H_W_INX;
18005 FSGNJOpc = RISCV::FSGNJ_H_INX;
18006 FSGNJXOpc = RISCV::FSGNJX_H_INX;
18007 RC = &RISCV::GPRF16RegClass;
18009 case RISCV::PseudoFROUND_S:
18010 CmpOpc = RISCV::FLT_S;
18011 F2IOpc = RISCV::FCVT_W_S;
18012 I2FOpc = RISCV::FCVT_S_W;
18013 FSGNJOpc = RISCV::FSGNJ_S;
18014 FSGNJXOpc = RISCV::FSGNJX_S;
18015 RC = &RISCV::FPR32RegClass;
18017 case RISCV::PseudoFROUND_S_INX:
18018 CmpOpc = RISCV::FLT_S_INX;
18019 F2IOpc = RISCV::FCVT_W_S_INX;
18020 I2FOpc = RISCV::FCVT_S_W_INX;
18021 FSGNJOpc = RISCV::FSGNJ_S_INX;
18022 FSGNJXOpc = RISCV::FSGNJX_S_INX;
18023 RC = &RISCV::GPRF32RegClass;
18025 case RISCV::PseudoFROUND_D:
18027 CmpOpc = RISCV::FLT_D;
18028 F2IOpc = RISCV::FCVT_L_D;
18029 I2FOpc = RISCV::FCVT_D_L;
18030 FSGNJOpc = RISCV::FSGNJ_D;
18031 FSGNJXOpc = RISCV::FSGNJX_D;
18032 RC = &RISCV::FPR64RegClass;
18034 case RISCV::PseudoFROUND_D_INX:
18036 CmpOpc = RISCV::FLT_D_INX;
18037 F2IOpc = RISCV::FCVT_L_D_INX;
18038 I2FOpc = RISCV::FCVT_D_L_INX;
18039 FSGNJOpc = RISCV::FSGNJ_D_INX;
18040 FSGNJXOpc = RISCV::FSGNJX_D_INX;
18041 RC = &RISCV::GPRRegClass;
18053 F->insert(
I, CvtMBB);
18054 F->insert(
I, DoneMBB);
18065 Register DstReg =
MI.getOperand(0).getReg();
18066 Register SrcReg =
MI.getOperand(1).getReg();
18067 Register MaxReg =
MI.getOperand(2).getReg();
18068 int64_t FRM =
MI.getOperand(3).getImm();
18073 Register FabsReg =
MRI.createVirtualRegister(RC);
18077 Register CmpReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18092 Register F2IReg =
MRI.createVirtualRegister(&RISCV::GPRRegClass);
18114 MI.eraseFromParent();
18121 switch (
MI.getOpcode()) {
18124 case RISCV::ReadCounterWide:
18126 "ReadCounterWide is only to be used on riscv32");
18128 case RISCV::Select_GPR_Using_CC_GPR:
18129 case RISCV::Select_FPR16_Using_CC_GPR:
18130 case RISCV::Select_FPR16INX_Using_CC_GPR:
18131 case RISCV::Select_FPR32_Using_CC_GPR:
18132 case RISCV::Select_FPR32INX_Using_CC_GPR:
18133 case RISCV::Select_FPR64_Using_CC_GPR:
18134 case RISCV::Select_FPR64INX_Using_CC_GPR:
18135 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
18137 case RISCV::BuildPairF64Pseudo:
18139 case RISCV::SplitF64Pseudo:
18141 case RISCV::PseudoQuietFLE_H:
18143 case RISCV::PseudoQuietFLE_H_INX:
18144 return emitQuietFCMP(
MI, BB, RISCV::FLE_H_INX, RISCV::FEQ_H_INX, Subtarget);
18145 case RISCV::PseudoQuietFLT_H:
18147 case RISCV::PseudoQuietFLT_H_INX:
18148 return emitQuietFCMP(
MI, BB, RISCV::FLT_H_INX, RISCV::FEQ_H_INX, Subtarget);
18149 case RISCV::PseudoQuietFLE_S:
18151 case RISCV::PseudoQuietFLE_S_INX:
18152 return emitQuietFCMP(
MI, BB, RISCV::FLE_S_INX, RISCV::FEQ_S_INX, Subtarget);
18153 case RISCV::PseudoQuietFLT_S:
18155 case RISCV::PseudoQuietFLT_S_INX:
18156 return emitQuietFCMP(
MI, BB, RISCV::FLT_S_INX, RISCV::FEQ_S_INX, Subtarget);
18157 case RISCV::PseudoQuietFLE_D:
18159 case RISCV::PseudoQuietFLE_D_INX:
18160 return emitQuietFCMP(
MI, BB, RISCV::FLE_D_INX, RISCV::FEQ_D_INX, Subtarget);
18161 case RISCV::PseudoQuietFLE_D_IN32X:
18164 case RISCV::PseudoQuietFLT_D:
18166 case RISCV::PseudoQuietFLT_D_INX:
18167 return emitQuietFCMP(
MI, BB, RISCV::FLT_D_INX, RISCV::FEQ_D_INX, Subtarget);
18168 case RISCV::PseudoQuietFLT_D_IN32X:
18172 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
18174 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
18176 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
18178 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
18180 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
18182 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
18184 case RISCV::PseudoFROUND_H:
18185 case RISCV::PseudoFROUND_H_INX:
18186 case RISCV::PseudoFROUND_S:
18187 case RISCV::PseudoFROUND_S_INX:
18188 case RISCV::PseudoFROUND_D:
18189 case RISCV::PseudoFROUND_D_INX:
18190 case RISCV::PseudoFROUND_D_IN32X:
18192 case TargetOpcode::STATEPOINT:
18198 MI.addOperand(*
MI.getMF(),
18204 case TargetOpcode::STACKMAP:
18205 case TargetOpcode::PATCHPOINT:
18208 "supported on 64-bit targets");
18226 if (
MI.readsRegister(RISCV::FRM,
nullptr))
18258 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
18259 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
18262 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
18263 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
18266 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
18267 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
18271 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
18272 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
18273 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
18275 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
18276 RISCV::V20M2, RISCV::V22M2};
18284 static const MCPhysReg ArgIGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18285 RISCV::X13, RISCV::X14, RISCV::X15,
18286 RISCV::X16, RISCV::X17};
18288 static const MCPhysReg ArgEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18289 RISCV::X13, RISCV::X14, RISCV::X15};
18300 static const MCPhysReg FastCCIGPRs[] = {
18301 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
18302 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
18303 RISCV::X29, RISCV::X30, RISCV::X31};
18306 static const MCPhysReg FastCCEGPRs[] = {RISCV::X10, RISCV::X11, RISCV::X12,
18307 RISCV::X13, RISCV::X14, RISCV::X15,
18322 unsigned XLenInBytes = XLen / 8;
18335 Align StackAlign(XLenInBytes);
18336 if (!
EABI || XLen != 32)
18368 unsigned XLen =
DL.getLargestLegalIntTypeSizeInBits();
18369 assert(XLen == 32 || XLen == 64);
18370 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
18374 if (ArgFlags.
isNest()) {
18375 if (
unsigned Reg = State.
AllocateReg(RISCV::X7)) {
18383 if (!LocVT.
isVector() && IsRet && ValNo > 1)
18388 bool UseGPRForF16_F32 =
true;
18391 bool UseGPRForF64 =
true;
18403 UseGPRForF16_F32 = !IsFixed;
18407 UseGPRForF16_F32 = !IsFixed;
18408 UseGPRForF64 = !IsFixed;
18414 UseGPRForF16_F32 =
true;
18415 UseGPRForF64 =
true;
18422 if (UseGPRForF16_F32 &&
18423 (ValVT == MVT::f16 || ValVT == MVT::bf16 || ValVT == MVT::f32)) {
18426 }
else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
18443 unsigned TwoXLenInBytes = (2 * XLen) / 8;
18445 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes &&
18449 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
18458 "PendingLocs and PendingArgFlags out of sync");
18462 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
18463 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
18511 PendingLocs.
size() <= 2) {
18512 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
18517 PendingLocs.
clear();
18518 PendingArgFlags.
clear();
18520 XLen, State, VA, AF, ValNo, ValVT, LocVT, ArgFlags,
18526 unsigned StoreSizeBytes = XLen / 8;
18529 if ((ValVT == MVT::f16 || ValVT == MVT::bf16) && !UseGPRForF16_F32)
18531 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
18533 else if (ValVT == MVT::f64 && !UseGPRForF64)
18569 if (!PendingLocs.
empty()) {
18571 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
18573 for (
auto &It : PendingLocs) {
18575 It.convertToReg(Reg);
18580 PendingLocs.clear();
18581 PendingArgFlags.
clear();
18585 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
18587 "Expected an XLenVT or vector types at this stage");
18605template <
typename ArgTy>
18607 for (
const auto &ArgIdx :
enumerate(Args)) {
18608 MVT ArgVT = ArgIdx.value().VT;
18610 return ArgIdx.index();
18612 return std::nullopt;
18615void RISCVTargetLowering::analyzeInputArgs(
18618 RISCVCCAssignFn Fn)
const {
18619 unsigned NumArgs =
Ins.size();
18632 for (
unsigned i = 0; i != NumArgs; ++i) {
18636 Type *ArgTy =
nullptr;
18638 ArgTy = FType->getReturnType();
18639 else if (Ins[i].isOrigArg())
18640 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
18644 ArgFlags, CCInfo,
true, IsRet, ArgTy, *
this,
18646 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type "
18653void RISCVTargetLowering::analyzeOutputArgs(
18656 CallLoweringInfo *CLI, RISCVCCAssignFn Fn)
const {
18657 unsigned NumArgs = Outs.
size();
18667 for (
unsigned i = 0; i != NumArgs; i++) {
18668 MVT ArgVT = Outs[i].VT;
18670 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
18674 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *
this,
18676 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type "
18730 if (In.isOrigArg()) {
18735 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
18736 (
BitWidth < 32 && In.Flags.isZExt())) {
18765 }
else if (LocVT == MVT::i64 && VA.
getValVT() == MVT::f32) {
18811 ExtType,
DL, LocVT, Chain, FIN,
18828 Register LoVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18841 Register HiVReg =
RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
18851 unsigned ValNo,
MVT ValVT,
MVT LocVT,
18854 bool IsFixed,
bool IsRet,
Type *OrigTy,
18857 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18866 if (LocVT == MVT::f16 &&
18867 (Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZfhmin())) {
18869 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
18870 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
18871 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
18872 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
18873 if (
unsigned Reg = State.
AllocateReg(FPR16List)) {
18879 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18881 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
18882 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
18883 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
18884 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
18885 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
18891 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
18893 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
18894 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
18895 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
18896 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
18897 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
18904 if ((LocVT == MVT::f16 &&
18905 (Subtarget.hasStdExtZhinx() || Subtarget.hasStdExtZhinxmin())) ||
18906 (LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
18907 (LocVT == MVT::f64 && Subtarget.
is64Bit() &&
18908 Subtarget.hasStdExtZdinx())) {
18915 if (LocVT == MVT::f16) {
18921 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
18927 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
18935 if (AllocatedVReg) {
18970 if (ArgFlags.
isNest()) {
18972 "Attribute 'nest' is not supported in GHC calling convention");
18976 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
18977 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
18979 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
18991 if (LocVT == MVT::f32 && Subtarget.hasStdExtF()) {
18994 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
18995 RISCV::F18_F, RISCV::F19_F,
18996 RISCV::F20_F, RISCV::F21_F};
18997 if (
unsigned Reg = State.
AllocateReg(FPR32List)) {
19003 if (LocVT == MVT::f64 && Subtarget.hasStdExtD()) {
19006 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
19007 RISCV::F24_D, RISCV::F25_D,
19008 RISCV::F26_D, RISCV::F27_D};
19009 if (
unsigned Reg = State.
AllocateReg(FPR64List)) {
19015 if ((LocVT == MVT::f32 && Subtarget.hasStdExtZfinx()) ||
19016 (LocVT == MVT::f64 && Subtarget.hasStdExtZdinx() &&
19036 switch (CallConv) {
19046 if (Subtarget.hasStdExtE())
19050 "(Zdinx/D) instruction set extensions");
19054 if (Func.hasFnAttribute(
"interrupt")) {
19055 if (!Func.arg_empty())
19057 "Functions with the interrupt attribute cannot have arguments!");
19062 if (!(Kind ==
"user" || Kind ==
"supervisor" || Kind ==
"machine"))
19064 "Function interrupt attribute argument not supported!");
19069 unsigned XLenInBytes = Subtarget.
getXLen() / 8;
19071 std::vector<SDValue> OutChains;
19080 analyzeInputArgs(MF, CCInfo, Ins,
false,
19084 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
19105 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
19106 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
19108 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
19110 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
19139 int VarArgsSaveSize = XLenInBytes * (ArgRegs.
size() -
Idx);
19144 if (VarArgsSaveSize == 0) {
19148 int VaArgOffset = -VarArgsSaveSize;
19156 XLenInBytes, VaArgOffset -
static_cast<int>(XLenInBytes),
true);
19157 VarArgsSaveSize += XLenInBytes;
19164 for (
unsigned I =
Idx;
I < ArgRegs.
size(); ++
I) {
19169 Chain,
DL, ArgValue, FIN,
19171 OutChains.push_back(Store);
19185 if (!OutChains.empty()) {
19186 OutChains.push_back(Chain);
19196bool RISCVTargetLowering::isEligibleForTailCallOptimization(
19200 auto CalleeCC = CLI.CallConv;
19201 auto &Outs = CLI.Outs;
19203 auto CallerCC = Caller.getCallingConv();
19210 if (Caller.hasFnAttribute(
"interrupt"))
19225 for (
auto &VA : ArgLocs)
19231 auto IsCallerStructRet = Caller.hasStructRetAttr();
19232 auto IsCalleeStructRet = Outs.
empty() ?
false : Outs[0].Flags.isSRet();
19233 if (IsCallerStructRet || IsCalleeStructRet)
19238 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
19239 if (CalleeCC != CallerCC) {
19240 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
19241 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
19248 for (
auto &Arg : Outs)
19249 if (Arg.Flags.isByVal())
19284 if (Subtarget.hasStdExtE())
19288 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
19294 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
19300 "site marked musttail");
19307 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19309 if (!Flags.isByVal())
19313 unsigned Size = Flags.getByValSize();
19314 Align Alignment = Flags.getNonZeroByValAlign();
19321 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment,
19335 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
19338 SDValue ArgValue = OutVals[OutIdx];
19358 if (!StackPtr.getNode())
19369 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
19387 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
19388 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
19394 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
19395 SDValue PartValue = OutVals[OutIdx + 1];
19396 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
19408 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
19410 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
19412 for (
const auto &Part : Parts) {
19413 SDValue PartValue = Part.first;
19414 SDValue PartOffset = Part.second;
19421 ArgValue = SpillSlot;
19427 if (Flags.isByVal())
19428 ArgValue = ByValArgs[j++];
19435 assert(!IsTailCall &&
"Tail call not allowed if stack is used "
19436 "for passing parameters");
19439 if (!StackPtr.getNode())
19452 if (!MemOpChains.
empty())
19458 for (
auto &Reg : RegsToPass) {
19459 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
19466 validateCCReservedRegs(RegsToPass, MF);
19471 "Return address register required, but has been reserved."});
19490 for (
auto &Reg : RegsToPass)
19496 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
19497 assert(Mask &&
"Missing call preserved mask for calling convention");
19506 "Unexpected CFI type for a direct call");
19536 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
19537 auto &VA = RVLocs[i];
19545 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
19546 assert(VA.needsCustom());
19571 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
19572 MVT VT = Outs[i].VT;
19576 ArgFlags, CCInfo,
true,
true,
19577 nullptr, *
this, Dispatcher))
19609 for (
unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
19610 SDValue Val = OutVals[OutIdx];
19619 DAG.
getVTList(MVT::i32, MVT::i32), Val);
19623 Register RegHi = RVLocs[++i].getLocReg();
19629 "Return value register required, but has been reserved."});
19645 "Return value register required, but has been reserved."});
19667 if (Func.hasFnAttribute(
"interrupt")) {
19668 if (!Func.getReturnType()->isVoidTy())
19670 "Functions with the interrupt attribute must have void return type!");
19676 if (Kind ==
"supervisor")
19682 return DAG.
getNode(RetOpc,
DL, MVT::Other, RetOps);
19685void RISCVTargetLowering::validateCCReservedRegs(
19686 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
19695 F,
"Argument register required, but has been reserved."});
19701 if (
N->getNumValues() != 1)
19703 if (!
N->hasNUsesOfValue(1, 0))
19706 SDNode *Copy = *
N->use_begin();
19720 if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
19724 bool HasRet =
false;
19725 for (
SDNode *Node : Copy->uses()) {
19733 Chain = Copy->getOperand(0);
19742#define NODE_NAME_CASE(NODE) \
19743 case RISCVISD::NODE: \
19744 return "RISCVISD::" #NODE;
19997#undef NODE_NAME_CASE
20004 if (Constraint.
size() == 1) {
20005 switch (Constraint[0]) {
20021 if (Constraint ==
"vr" || Constraint ==
"vm")
20027std::pair<unsigned, const TargetRegisterClass *>
20033 if (Constraint.
size() == 1) {
20034 switch (Constraint[0]) {
20039 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
20040 return std::make_pair(0U, &RISCV::GPRF16RegClass);
20041 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
20042 return std::make_pair(0U, &RISCV::GPRF32RegClass);
20043 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.
is64Bit())
20044 return std::make_pair(0U, &RISCV::GPRPairRegClass);
20045 return std::make_pair(0U, &RISCV::GPRNoX0RegClass);
20047 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16)
20048 return std::make_pair(0U, &RISCV::FPR16RegClass);
20049 if (Subtarget.hasStdExtF() && VT == MVT::f32)
20050 return std::make_pair(0U, &RISCV::FPR32RegClass);
20051 if (Subtarget.hasStdExtD() && VT == MVT::f64)
20052 return std::make_pair(0U, &RISCV::FPR64RegClass);
20057 }
else if (Constraint ==
"vr") {
20058 for (
const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
20059 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20061 return std::make_pair(0U, RC);
20063 }
else if (Constraint ==
"vm") {
20064 if (
TRI->isTypeLegalForClass(RISCV::VMV0RegClass, VT.
SimpleTy))
20065 return std::make_pair(0U, &RISCV::VMV0RegClass);
20073 .
Case(
"{zero}", RISCV::X0)
20074 .
Case(
"{ra}", RISCV::X1)
20075 .
Case(
"{sp}", RISCV::X2)
20076 .
Case(
"{gp}", RISCV::X3)
20077 .
Case(
"{tp}", RISCV::X4)
20078 .
Case(
"{t0}", RISCV::X5)
20079 .
Case(
"{t1}", RISCV::X6)
20080 .
Case(
"{t2}", RISCV::X7)
20081 .
Cases(
"{s0}",
"{fp}", RISCV::X8)
20082 .
Case(
"{s1}", RISCV::X9)
20083 .
Case(
"{a0}", RISCV::X10)
20084 .
Case(
"{a1}", RISCV::X11)
20085 .
Case(
"{a2}", RISCV::X12)
20086 .
Case(
"{a3}", RISCV::X13)
20087 .
Case(
"{a4}", RISCV::X14)
20088 .
Case(
"{a5}", RISCV::X15)
20089 .
Case(
"{a6}", RISCV::X16)
20090 .
Case(
"{a7}", RISCV::X17)
20091 .
Case(
"{s2}", RISCV::X18)
20092 .
Case(
"{s3}", RISCV::X19)
20093 .
Case(
"{s4}", RISCV::X20)
20094 .
Case(
"{s5}", RISCV::X21)
20095 .
Case(
"{s6}", RISCV::X22)
20096 .
Case(
"{s7}", RISCV::X23)
20097 .
Case(
"{s8}", RISCV::X24)
20098 .
Case(
"{s9}", RISCV::X25)
20099 .
Case(
"{s10}", RISCV::X26)
20100 .
Case(
"{s11}", RISCV::X27)
20101 .
Case(
"{t3}", RISCV::X28)
20102 .
Case(
"{t4}", RISCV::X29)
20103 .
Case(
"{t5}", RISCV::X30)
20104 .
Case(
"{t6}", RISCV::X31)
20106 if (XRegFromAlias != RISCV::NoRegister)
20107 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
20116 if (Subtarget.hasStdExtF()) {
20118 .
Cases(
"{f0}",
"{ft0}", RISCV::F0_F)
20119 .
Cases(
"{f1}",
"{ft1}", RISCV::F1_F)
20120 .
Cases(
"{f2}",
"{ft2}", RISCV::F2_F)
20121 .
Cases(
"{f3}",
"{ft3}", RISCV::F3_F)
20122 .
Cases(
"{f4}",
"{ft4}", RISCV::F4_F)
20123 .
Cases(
"{f5}",
"{ft5}", RISCV::F5_F)
20124 .
Cases(
"{f6}",
"{ft6}", RISCV::F6_F)
20125 .
Cases(
"{f7}",
"{ft7}", RISCV::F7_F)
20126 .
Cases(
"{f8}",
"{fs0}", RISCV::F8_F)
20127 .
Cases(
"{f9}",
"{fs1}", RISCV::F9_F)
20128 .
Cases(
"{f10}",
"{fa0}", RISCV::F10_F)
20129 .
Cases(
"{f11}",
"{fa1}", RISCV::F11_F)
20130 .
Cases(
"{f12}",
"{fa2}", RISCV::F12_F)
20131 .
Cases(
"{f13}",
"{fa3}", RISCV::F13_F)
20132 .
Cases(
"{f14}",
"{fa4}", RISCV::F14_F)
20133 .
Cases(
"{f15}",
"{fa5}", RISCV::F15_F)
20134 .
Cases(
"{f16}",
"{fa6}", RISCV::F16_F)
20135 .
Cases(
"{f17}",
"{fa7}", RISCV::F17_F)
20136 .
Cases(
"{f18}",
"{fs2}", RISCV::F18_F)
20137 .
Cases(
"{f19}",
"{fs3}", RISCV::F19_F)
20138 .
Cases(
"{f20}",
"{fs4}", RISCV::F20_F)
20139 .
Cases(
"{f21}",
"{fs5}", RISCV::F21_F)
20140 .
Cases(
"{f22}",
"{fs6}", RISCV::F22_F)
20141 .
Cases(
"{f23}",
"{fs7}", RISCV::F23_F)
20142 .
Cases(
"{f24}",
"{fs8}", RISCV::F24_F)
20143 .
Cases(
"{f25}",
"{fs9}", RISCV::F25_F)
20144 .
Cases(
"{f26}",
"{fs10}", RISCV::F26_F)
20145 .
Cases(
"{f27}",
"{fs11}", RISCV::F27_F)
20146 .
Cases(
"{f28}",
"{ft8}", RISCV::F28_F)
20147 .
Cases(
"{f29}",
"{ft9}", RISCV::F29_F)
20148 .
Cases(
"{f30}",
"{ft10}", RISCV::F30_F)
20149 .
Cases(
"{f31}",
"{ft11}", RISCV::F31_F)
20151 if (FReg != RISCV::NoRegister) {
20152 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F &&
"Unknown fp-reg");
20153 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
20154 unsigned RegNo = FReg - RISCV::F0_F;
20155 unsigned DReg = RISCV::F0_D + RegNo;
20156 return std::make_pair(DReg, &RISCV::FPR64RegClass);
20158 if (VT == MVT::f32 || VT == MVT::Other)
20159 return std::make_pair(FReg, &RISCV::FPR32RegClass);
20160 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
20161 unsigned RegNo = FReg - RISCV::F0_F;
20162 unsigned HReg = RISCV::F0_H + RegNo;
20163 return std::make_pair(HReg, &RISCV::FPR16RegClass);
20170 .
Case(
"{v0}", RISCV::V0)
20171 .
Case(
"{v1}", RISCV::V1)
20172 .
Case(
"{v2}", RISCV::V2)
20173 .
Case(
"{v3}", RISCV::V3)
20174 .
Case(
"{v4}", RISCV::V4)
20175 .
Case(
"{v5}", RISCV::V5)
20176 .
Case(
"{v6}", RISCV::V6)
20177 .
Case(
"{v7}", RISCV::V7)
20178 .
Case(
"{v8}", RISCV::V8)
20179 .
Case(
"{v9}", RISCV::V9)
20180 .
Case(
"{v10}", RISCV::V10)
20181 .
Case(
"{v11}", RISCV::V11)
20182 .
Case(
"{v12}", RISCV::V12)
20183 .
Case(
"{v13}", RISCV::V13)
20184 .
Case(
"{v14}", RISCV::V14)
20185 .
Case(
"{v15}", RISCV::V15)
20186 .
Case(
"{v16}", RISCV::V16)
20187 .
Case(
"{v17}", RISCV::V17)
20188 .
Case(
"{v18}", RISCV::V18)
20189 .
Case(
"{v19}", RISCV::V19)
20190 .
Case(
"{v20}", RISCV::V20)
20191 .
Case(
"{v21}", RISCV::V21)
20192 .
Case(
"{v22}", RISCV::V22)
20193 .
Case(
"{v23}", RISCV::V23)
20194 .
Case(
"{v24}", RISCV::V24)
20195 .
Case(
"{v25}", RISCV::V25)
20196 .
Case(
"{v26}", RISCV::V26)
20197 .
Case(
"{v27}", RISCV::V27)
20198 .
Case(
"{v28}", RISCV::V28)
20199 .
Case(
"{v29}", RISCV::V29)
20200 .
Case(
"{v30}", RISCV::V30)
20201 .
Case(
"{v31}", RISCV::V31)
20203 if (VReg != RISCV::NoRegister) {
20204 if (
TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.
SimpleTy))
20205 return std::make_pair(VReg, &RISCV::VMRegClass);
20206 if (
TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.
SimpleTy))
20207 return std::make_pair(VReg, &RISCV::VRRegClass);
20208 for (
const auto *RC :
20209 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
20210 if (
TRI->isTypeLegalForClass(*RC, VT.
SimpleTy)) {
20211 VReg =
TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
20212 return std::make_pair(VReg, RC);
20218 std::pair<Register, const TargetRegisterClass *> Res =
20224 if (Res.second == &RISCV::GPRF16RegClass ||
20225 Res.second == &RISCV::GPRF32RegClass ||
20226 Res.second == &RISCV::GPRPairRegClass)
20227 return std::make_pair(Res.first, &RISCV::GPRRegClass);
20235 if (ConstraintCode.
size() == 1) {
20236 switch (ConstraintCode[0]) {
20251 if (Constraint.
size() == 1) {
20252 switch (Constraint[0]) {
20255 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20257 if (isInt<12>(CVal))
20270 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op)) {
20272 if (isUInt<5>(CVal))
20290 if (Subtarget.hasStdExtZtso()) {
20306 if (Subtarget.hasStdExtZtso()) {
20314 if (Subtarget.enableSeqCstTrailingFence() && isa<StoreInst>(Inst) &&
20331 if (Subtarget.hasForcedAtomics())
20336 if (Subtarget.hasStdExtZacas() &&
20337 (
Size >= 32 || Subtarget.hasStdExtZabha()))
20343 if (
Size < 32 && !Subtarget.hasStdExtZabha())
20356 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
20358 return Intrinsic::riscv_masked_atomicrmw_add_i32;
20360 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
20362 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
20364 return Intrinsic::riscv_masked_atomicrmw_max_i32;
20366 return Intrinsic::riscv_masked_atomicrmw_min_i32;
20368 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
20370 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
20379 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
20381 return Intrinsic::riscv_masked_atomicrmw_add_i64;
20383 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
20385 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
20387 return Intrinsic::riscv_masked_atomicrmw_max_i64;
20389 return Intrinsic::riscv_masked_atomicrmw_min_i64;
20391 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
20393 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
20419 unsigned XLen = Subtarget.
getXLen();
20443 unsigned ValWidth =
20448 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
20451 Builder.
CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
20463 if (Subtarget.hasForcedAtomics())
20467 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
20476 unsigned XLen = Subtarget.
getXLen();
20478 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
20483 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
20489 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
20496 EVT DataVT)
const {
20512 return Subtarget.hasStdExtZfhmin();
20514 return Subtarget.hasStdExtF();
20516 return Subtarget.hasStdExtD();
20548 "RVVBitsPerBlock changed, audit needed");
20557 if (!Subtarget.hasVendorXTHeadMemIdx())
20563 Base =
Op->getOperand(0);
20565 int64_t RHSC =
RHS->getSExtValue();
20571 bool isLegalIndexedOffset =
false;
20572 for (
unsigned i = 0; i < 4; i++)
20573 if (isInt<5>(RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
20574 isLegalIndexedOffset =
true;
20578 if (!isLegalIndexedOffset)
20595 VT = LD->getMemoryVT();
20596 Ptr = LD->getBasePtr();
20597 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20598 VT = ST->getMemoryVT();
20599 Ptr = ST->getBasePtr();
20618 VT = LD->getMemoryVT();
20619 Ptr = LD->getBasePtr();
20620 }
else if (
StoreSDNode *ST = dyn_cast<StoreSDNode>(
N)) {
20621 VT = ST->getMemoryVT();
20622 Ptr = ST->getBasePtr();
20665 const Constant *PersonalityFn)
const {
20670 const Constant *PersonalityFn)
const {
20694 const bool HasExtMOrZmmul =
20695 Subtarget.hasStdExtM() || Subtarget.hasStdExtZmmul();
20704 if (
auto *ConstNode = dyn_cast<ConstantSDNode>(
C.getNode())) {
20706 const APInt &Imm = ConstNode->getAPIntValue();
20707 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
20708 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
20712 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
20713 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
20714 (Imm - 8).isPowerOf2()))
20719 if (!Imm.isSignedIntN(12) && Imm.countr_zero() < 12 &&
20720 ConstNode->hasOneUse()) {
20721 APInt ImmS = Imm.ashr(Imm.countr_zero());
20722 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
20723 (1 - ImmS).isPowerOf2())
20747 if (C1.
isSignedIntN(12) && !(C1 * C2).isSignedIntN(12))
20756 unsigned *
Fast)
const {
20759 *
Fast = Subtarget.enableUnalignedScalarMem();
20760 return Subtarget.enableUnalignedScalarMem();
20776 *
Fast = Subtarget.enableUnalignedVectorMem();
20777 return Subtarget.enableUnalignedVectorMem();
20786 if (FuncAttributes.
hasFnAttr(Attribute::NoImplicitFloat))
20798 if (
Op.size() < MinVLenInBytes)
20808 MVT PreferredVT = (
Op.isMemset() && !
Op.isZeroMemset()) ? MVT::i8 : ELenVT;
20812 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
20814 if (
Op.isFixedDstAlign())
20815 RequiredAlign = std::min(RequiredAlign,
Op.getDstAlign());
20817 RequiredAlign = std::min(RequiredAlign,
Op.getSrcAlign());
20825 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
20826 bool IsABIRegCopy =
CC.has_value();
20828 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20829 PartVT == MVT::f32) {
20847 if (PartVTBitSize % ValueVTBitSize == 0) {
20848 assert(PartVTBitSize >= ValueVTBitSize);
20855 if (ValueEltVT != PartEltVT) {
20856 if (PartVTBitSize > ValueVTBitSize) {
20858 assert(Count != 0 &&
"The number of element should not be zero.");
20859 EVT SameEltTypeVT =
20880 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID>
CC)
const {
20881 bool IsABIRegCopy =
CC.has_value();
20882 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
20883 PartVT == MVT::f32) {
20900 if (PartVTBitSize % ValueVTBitSize == 0) {
20901 assert(PartVTBitSize >= ValueVTBitSize);
20902 EVT SameEltTypeVT = ValueVT;
20909 if (ValueEltVT != PartEltVT) {
20911 assert(Count != 0 &&
"The number of element should not be zero.");
20928 bool OptSize = Attr.
hasFnAttr(Attribute::MinSize);
20935 unsigned Opc =
N->getOpcode();
20959 VectorType *VTy,
unsigned Factor,
Align Alignment,
unsigned AddrSpace,
20973 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
20978 if (FVTy->getNumElements() < 2)
20988 return Factor * LMUL <= 8;
20992 Align Alignment)
const {
21004 if (!Subtarget.enableUnalignedVectorMem() &&
21012 Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
21013 Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
21014 Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
21015 Intrinsic::riscv_seg8_load};
21034 auto *VTy = cast<FixedVectorType>(Shuffles[0]->
getType());
21044 {VTy, LI->getPointerOperandType(), XLenTy});
21046 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21051 for (
unsigned i = 0; i < Shuffles.
size(); i++) {
21053 Shuffles[i]->replaceAllUsesWith(SubVec);
21060 Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
21061 Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
21062 Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
21063 Intrinsic::riscv_seg8_store};
21083 unsigned Factor)
const {
21085 auto *ShuffleVTy = cast<FixedVectorType>(SVI->
getType());
21088 ShuffleVTy->getNumElements() / Factor);
21090 SI->getPointerAddressSpace(),
21091 SI->getModule()->getDataLayout()))
21098 {VTy, SI->getPointerOperandType(), XLenTy});
21103 for (
unsigned i = 0; i < Factor; i++) {
21112 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
21113 Ops.
append({SI->getPointerOperand(), VL});
21129 unsigned Factor = 2;
21144 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21147 {ResVTy, LI->getPointerOperandType(), XLenTy});
21148 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21151 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
21152 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
21153 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
21154 Intrinsic::riscv_vlseg8};
21179 unsigned Factor = 2;
21185 SI->getPointerAddressSpace(),
21186 SI->getModule()->getDataLayout()))
21193 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
21196 {InVTy, SI->getPointerOperandType(), XLenTy});
21197 VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
21200 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
21201 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
21202 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
21203 Intrinsic::riscv_vsseg8};
21211 SI->getPointerOperand(), VL});
21221 "Invalid call instruction for a KCFI check");
21223 MBBI->getOpcode()));
21226 Target.setIsRenamable(
false);
21234#define GET_REGISTER_MATCHER
21235#include "RISCVGenAsmMatcher.inc"
21241 if (Reg == RISCV::NoRegister)
21243 if (Reg == RISCV::NoRegister)
21255 const MDNode *NontemporalInfo =
I.getMetadata(LLVMContext::MD_nontemporal);
21257 if (NontemporalInfo ==
nullptr)
21265 int NontemporalLevel = 5;
21266 const MDNode *RISCVNontemporalInfo =
21267 I.getMetadata(
"riscv-nontemporal-domain");
21268 if (RISCVNontemporalInfo !=
nullptr)
21271 cast<ConstantAsMetadata>(RISCVNontemporalInfo->
getOperand(0))
21275 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
21276 "RISC-V target doesn't support this non-temporal domain.");
21278 NontemporalLevel -= 2;
21280 if (NontemporalLevel & 0b1)
21282 if (NontemporalLevel & 0b10)
21295 return TargetFlags;
21305 return isTypeLegal(VT) && Subtarget.hasStdExtZvbb();
21308 return Subtarget.hasStdExtZbb() &&
21321 if (
Op == Instruction::Add ||
Op == Instruction::Sub ||
21322 Op == Instruction::And ||
Op == Instruction::Or ||
21323 Op == Instruction::Xor ||
Op == Instruction::InsertElement ||
21324 Op == Instruction::ShuffleVector ||
Op == Instruction::Load)
21332 !isa<ReturnInst>(&Inst))
21335 if (
const AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) {
21336 if (AI->getAllocatedType()->isScalableTy())
21344RISCVTargetLowering::BuildSDIVPow2(
SDNode *
N,
const APInt &Divisor,
21352 if (!Subtarget.hasShortForwardBranchOpt())
21354 EVT VT =
N->getValueType(0);
21355 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.
is64Bit())))
21359 if (Divisor.
sgt(2048) || Divisor.
slt(-2048))
21364bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
21365 EVT VT,
const APInt &AndMask)
const {
21366 if (Subtarget.hasStdExtZicond() || Subtarget.hasVendorXVentanaCondOps())
21367 return !Subtarget.hasStdExtZbs() && AndMask.
ugt(1024);
21371unsigned RISCVTargetLowering::getMinimumJumpTableEntries()
const {
21376template <
typename Arg>
21377void RVVArgDispatcher::constructArgInfos(
ArrayRef<Arg> ArgList) {
21380 auto isHomogeneousScalableVectorType = [](
ArrayRef<Arg> ArgList) {
21382 auto It = ArgList.
begin();
21383 MVT FirstArgRegType = It->VT;
21386 if (It == ArgList.
end() || It->Flags.isSplit())
21398 for (; It != ArgList.
end(); ++It)
21399 if (It->Flags.isSplit() || It->VT != FirstArgRegType)
21405 if (isHomogeneousScalableVectorType(ArgList)) {
21407 RVVArgInfos.push_back({(
unsigned)ArgList.
size(), ArgList[0].VT,
false});
21410 bool FirstVMaskAssigned =
false;
21411 for (
const auto &OutArg : ArgList) {
21412 MVT RegisterVT = OutArg.VT;
21422 RVVArgInfos.push_back({1, RegisterVT,
true});
21423 FirstVMaskAssigned =
true;
21427 RVVArgInfos.push_back({1, RegisterVT,
false});
21434void RVVArgDispatcher::constructArgInfos<Type *>(
ArrayRef<Type *> TypeList) {
21439 bool FirstVMaskAssigned =
false;
21440 for (
Type *Ty : TypeList) {
21450 RVVArgInfos.push_back(
21456 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
Value != NumValues;
21471 if (!FirstVMaskAssigned &&
21473 RVVArgInfos.push_back({1, RegisterVT,
true});
21474 FirstVMaskAssigned =
true;
21478 RVVArgInfos.insert(RVVArgInfos.end(), NumRegs, {1, RegisterVT, false});
21484void RVVArgDispatcher::allocatePhysReg(
unsigned NF,
unsigned LMul,
21485 unsigned StartReg) {
21486 assert((StartReg % LMul) == 0 &&
21487 "Start register number should be multiple of lmul");
21506 for (
unsigned i = 0; i < NF; ++i)
21508 AllocatedPhysRegs.push_back(VRArrays[(StartReg - 8) / LMul + i]);
21510 AllocatedPhysRegs.push_back(
MCPhysReg());
21516void RVVArgDispatcher::compute() {
21518 auto allocate = [&](
const RVVArgInfo &
ArgInfo) {
21521 AllocatedPhysRegs.push_back(RISCV::V0);
21527 unsigned TotalRegsNeeded =
ArgInfo.NF * RegsNeeded;
21528 for (
unsigned StartReg = 0; StartReg + TotalRegsNeeded <=
NumArgVRs;
21529 StartReg += RegsNeeded) {
21530 uint32_t Map = ((1 << TotalRegsNeeded) - 1) << StartReg;
21531 if ((AssignedMap & Map) == 0) {
21532 allocatePhysReg(
ArgInfo.NF, RegsNeeded, StartReg + 8);
21533 AssignedMap |=
Map;
21538 allocatePhysReg(
ArgInfo.NF, RegsNeeded, 0);
21541 for (
unsigned i = 0; i < RVVArgInfos.size(); ++i)
21542 allocate(RVVArgInfos[i]);
21546 assert(CurIdx < AllocatedPhysRegs.size() &&
"Index out of range");
21547 return AllocatedPhysRegs[CurIdx++];
21552#define GET_RISCVVIntrinsicsTable_IMPL
21553#include "RISCVGenSearchableTables.inc"
unsigned const MachineRegisterInfo * MRI
static MCRegister MatchRegisterName(StringRef Name)
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget, const AArch64TargetLowering &TLI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V)
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define NODE_NAME_CASE(node)
static bool isConstant(const MachineInstr &MI)
amdgpu AMDGPU Register Bank Select
static bool isZeroOrAllOnes(SDValue N, bool AllOnes)
static SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes, TargetLowering::DAGCombinerInfo &DCI)
static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp, TargetLowering::DAGCombinerInfo &DCI, bool AllOnes=false)
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
const MCPhysReg ArgFPR32s[]
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
const MCPhysReg ArgFPR64s[]
const MCPhysReg ArgGPRs[]
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
loop Loop Strength Reduction
static bool isSplat(Value *V)
Return true if V is a splat of a value (which is used when multiplying a matrix with a scalar).
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, const TargetLowering::DAGCombinerInfo &DCI, const MipsSETargetLowering *TL, const MipsSubtarget &Subtarget)
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const MipsSubtarget &Subtarget)
static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const MipsSubtarget &Subtarget)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static bool IsSelect(MachineInstr &MI)
const char LLVMTargetMachineRef TM
static StringRef getExtensionType(StringRef Ext)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG)
static SDValue lowerSADDO_SSUBO(SDValue Op, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original ...
static const Intrinsic::ID FixedVlsegIntrIds[]
static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT getLMUL1VT(MVT VT)
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2, bool EABI)
static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
Match v(f)slide1up/down idioms.
static const MCPhysReg ArgVRM2s[]
static bool isInterleaveShuffle(ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguou...
static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
According to the property that indexed load/store instructions zero-extend their indices,...
static void promoteVCIXScalar(const SDValue &Op, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static const RISCV::RISCVMaskedPseudoInfo * lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVII::VLMUL LMul, unsigned SEW)
static SDValue expandMul(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue performVWADDSUBW_VLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
Match the index of a gather or scatter operation as an operation with twice the element width and hal...
static bool isLegalBitRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, MVT &RotateVT, unsigned &RotateAmt)
static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG)
static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG)
static std::optional< uint64_t > getExactInteger(const APFloat &APF, uint32_t BitWidth)
static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG)
static SDValue performMemPairCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG)
static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static unsigned getRVVReductionOp(unsigned ISDOpcode)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue getVCIXISDNodeVOID(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG)
static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool hasMaskOp(unsigned Opcode)
Return true if a RISC-V target specified op has a mask operand.
static bool legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static unsigned getRISCVVLOp(SDValue Op)
Get a RISC-V target specified VL op for a given SDNode.
static unsigned getVecReduceOpcode(unsigned Opc)
Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static std::pair< SDValue, SDValue > getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INT_SATCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue > getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< unsigned > preAssignMask(const ArgTy &Args)
static SDValue getVLOperand(SDValue Op)
static MachineBasicBlock * emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static SDValue lowerCttzElts(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerVectorXRINT(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG)
static SDValue getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static bool isSelectPseudo(MachineInstr &MI)
static std::optional< MVT > getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool useRVVForFixedLengthVectorVT(MVT VT, const RISCVSubtarget &Subtarget)
static Value * useTpOffset(IRBuilderBase &IRB, unsigned Offset)
static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc)
static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG)
static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerSMULO(SDValue Op, SelectionDAG &DAG)
static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangeme...
static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG)
static bool hasMergeOp(unsigned Opcode)
Return true if a RISC-V target specified op has a merge operand.
static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void processVCIXOperands(SDValue &OrigOp, SmallVectorImpl< SDValue > &Operands, SelectionDAG &DAG)
static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< VIDSequence > isSimpleVIDSequence(SDValue Op, unsigned EltSizeInBits)
static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue lowerUADDSAT_USUBSAT(SDValue Op, SelectionDAG &DAG)
static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC)
static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc)
static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isValidEGW(int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool isNonZeroAVL(SDValue AVL)
static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue combineBinOp_VLToVWBinOp_VL(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
Combine a binary operation to its equivalent VW or VW_W form.
static SDValue getVCIXISDNodeWCHAIN(SDValue &Op, SelectionDAG &DAG, unsigned Type)
static ArrayRef< MCPhysReg > getFastCCArgGPRs(const RISCVABI::ABI ABI)
static const MCPhysReg ArgVRM8s[]
static MachineBasicBlock * emitReadCounterWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
static const MCPhysReg ArgVRM4s[]
static cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static SDValue lowerSADDSAT_SSUBSAT(SDValue Op, SelectionDAG &DAG)
static SDValue getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static const MCPhysReg ArgFPR16s[]
static SDValue combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed...
static SDValue performVFMADD_VLCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large pro...
static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily t...
static const Intrinsic::ID FixedVssegIntrIds[]
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool isCommutative(Instruction *I)
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static SymbolRef::Type getType(const Symbol *Sym)
static constexpr int Concat[]
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool sgt(const APInt &RHS) const
Signed greater than comparison.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool isNegative() const
Determine sign of this APInt.
void clearAllBits()
Set every bit to 0.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned getSignificantBits() const
Get the minimum bit size for this signed APInt.
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
bool slt(const APInt &RHS) const
Signed less than comparison.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
An arbitrary precision integer that knows its signedness.
an instruction to allocate memory on the stack
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
bool isFloatingPointOperation() const
BinOp getOperation() const
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
bool hasFnAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the function.
StringRef getValueAsString() const
Return the attribute's value as a string.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
LLVM Basic Block Representation.
const Function * getParent() const
Return the enclosing method, or null if none.
bool test(unsigned Idx) const
bool all() const
all - Returns true if all bits are set.
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
MCRegister AllocateReg(MCPhysReg Reg)
AllocateReg - Attempt to allocate one register.
int64_t AllocateStack(unsigned Size, Align Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
SmallVectorImpl< CCValAssign > & getPendingLocs()
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
void addLoc(const CCValAssign &V)
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
bool isIndirectCall() const
Return true if the callsite is an indirect call.
This class represents a function call, abstracting a target machine's calling convention.
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Diagnostic information for unsupported feature in backend.
static constexpr ElementCount getScalable(ScalarTy MinVal)
static constexpr ElementCount getFixed(ScalarTy MinVal)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
AttributeList getAttributes() const
Return the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Argument * getArg(unsigned i) const
Helper struct to store a base, index and offset that forms an address.
int64_t getOffset() const
bool hasExternalWeakLinkage() const
Module * getParent()
Get the module that this global value is contained inside of...
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Store the specified register of the given register class to the specified stack frame index.
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
Load the specified register of the given register class from the specified stack frame index.
Common base class shared among various IRBuilders.
Value * CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
BasicBlock * GetInsertBlock() const
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
static InstructionCost getInvalid(CostType Val=0)
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Class to represent integer types.
A wrapper class for inspecting calls to intrinsic functions.
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Base class for LoadSDNode and StoreSDNode.
bool isIndexed() const
Return true if this is a pre/post inc/dec load/store.
An instruction for reading from memory.
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Value * getPointerOperand()
Align getAlign() const
Return the alignment of the access that is being performed.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
Context object for machine code objects.
Base class for the full range of assembler expressions which are needed for parsing.
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
const MDOperand & getOperand(unsigned I) const
static MVT getFloatingPointVT(unsigned BitWidth)
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
static MVT getScalableVectorVT(MVT VT, unsigned NumElements)
MVT changeTypeToInteger()
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
uint64_t getScalarStoreSize() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool bitsGT(MVT VT) const
Return true if this has more bits than VT.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool bitsGE(MVT VT) const
Return true if this has no less bits than VT.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
bool isValid() const
Return true if this is a valid simple valuetype.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
static auto integer_scalable_vector_valuetypes()
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static auto fp_fixedlen_vector_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
void push_back(MachineInstr *MI)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Instructions::iterator instr_iterator
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
void setFlag(MIFlag Flag)
Set a MI flag.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
A description of a memory reference used in the backend.
const MDNode * getRanges() const
Return the range tag for the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
Align getBaseAlign() const
Return the minimum known alignment in bytes of the base address, without the offset.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
This is an abstract virtual class for memory operations.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
void setVarArgsFrameIndex(int Index)
int getVarArgsFrameIndex() const
void setVarArgsSaveSize(int Size)
void addSExt32Register(Register Reg)
RISCVABI::ABI getTargetABI() const
unsigned getMinimumJumpTableEntries() const
bool hasStdExtCOrZca() const
unsigned getMaxLMULForFixedLengthVectors() const
bool hasVInstructionsI64() const
bool hasVInstructionsF64() const
bool hasStdExtDOrZdinx() const
bool hasStdExtZfhOrZhinx() const
unsigned getRealMinVLen() const
Quantity expandVScale(Quantity X) const
If the ElementCount or TypeSize X is scalable and VScale (VLEN) is exactly known, returns X converted...
bool useRVVForFixedLengthVectors() const
bool isTargetFuchsia() const
unsigned getDLenFactor() const
bool hasVInstructionsF16Minimal() const
bool hasConditionalMoveFusion() const
bool isRegisterReservedByUser(Register i) const
bool hasVInstructionsF16() const
bool hasVInstructionsBF16() const
unsigned getMaxBuildIntsCost() const
Align getPrefLoopAlignment() const
bool hasVInstructions() const
std::optional< unsigned > getRealVLen() const
bool useConstantPoolForLargeInts() const
Align getPrefFunctionAlignment() const
bool hasStdExtZfhminOrZhinxmin() const
unsigned getRealMaxVLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVInstrInfo * getInstrInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
bool hasVInstructionsF32() const
bool hasStdExtFOrZfinx() const
static std::pair< unsigned, unsigned > computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
InstructionCost getVRGatherVVCost(MVT VT) const
Return the cost of a vrgather.vv instruction for the type VT.
bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
std::pair< int, bool > getLegalZfaFPImm(const APFloat &Imm, EVT VT) const
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Instruction * emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
const RISCVSubtarget & getSubtarget() const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool preferScalarizeSplat(SDNode *N) const override
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool canSplatOperand(Instruction *I, int Operand) const
Return true if the (vector) instruction I will be lowered to an instruction with a scalar splat opera...
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
InstructionCost getVRGatherVICost(MVT VT) const
Return the cost of a vrgather.vi (or vx) instruction for the type VT.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Return true if it is beneficial to convert a load of a constant to just the constant itself.
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the register type for a given MVT, ensuring vectors are treated as a series of gpr sized integ...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
bool areTwoSDNodeTargetMMOFlagsMergeable(const MemSDNode &NodeX, const MemSDNode &NodeY) const override
Return true if it is valid to merge the TargetMMOFlags in two SDNodes.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
static unsigned computeVLMAX(unsigned VectorBits, unsigned EltSize, unsigned MinSize)
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
InstructionCost getLMULCost(MVT VT) const
Return the cost of LMUL for linear operations.
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
InstructionCost getVSlideVICost(MVT VT) const
Return the cost of a vslidedown.vi or vslideup.vi instruction for the type VT.
bool fallBackToDAGISel(const Instruction &Inst) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a vlsegN intrinsic.
bool isCtpopFast(EVT VT) const override
Return true if ctpop instruction is fast.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
MVT getContainerForFixedLengthVector(MVT VT) const
static unsigned getRegClassIDForVecVT(MVT VT)
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
SDValue computeVLMax(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG) const
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Returns the register with the specified architectural or ABI name.
InstructionCost getVSlideVXCost(MVT VT) const
Return the cost of a vslidedown.vx or vslideup.vx instruction for the type VT.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul)
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getCustomCtpopCost(EVT VT, ISD::CondCode Cond) const override
Return the maximum number of "x & (x - 1)" operations that can be done instead of deferring to a cust...
void AdjustInstrPostInstrSelection(MachineInstr &MI, SDNode *Node) const override
This method should be implemented by targets that mark instructions with the 'hasPostISelHook' flag.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a vssegN intrinsic.
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isLegalElementTypeForRVV(EVT ScalarTy) const
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *II, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
static RISCVII::VLMUL getLMUL(MVT VT)
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, unsigned NumParts, MVT PartVT, std::optional< CallingConv::ID > CC) const override
Target-specific splitting of values into parts that fit a register storing a legal type.
Instruction * emitTrailingFence(IRBuilderBase &Builder, Instruction *Inst, AtomicOrdering Ord) const override
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Return the number of registers for a given MVT, ensuring vectors are treated as a series of gpr sized...
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const
Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if this node ...
bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override
Returns true by value, base pointer and offset pointer and addressing mode by reference if the node's...
SDValue joinRegisterPartsIntoValue(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts, MVT PartVT, EVT ValueVT, std::optional< CallingConv::ID > CC) const override
Target-specific combining of register parts into its original value.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool isLegalStridedLoadStore(EVT DataType, Align Alignment) const
Return true if a stride load store of the given result type and alignment is legal.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
As per the spec, the rules for passing vector arguments are as follows:
static constexpr unsigned NumArgVRs
MCPhysReg getNextPhysReg()
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setCFIType(uint32_t Type)
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
bool shouldOptForSize() const
std::pair< SDValue, SDValue > SplitVectorOperand(const SDNode *N, unsigned OpNo)
Split the node's operand with EXTRACT_SUBVECTOR and return the low/high part.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This instruction constructs a fixed permutation of two input vectors.
static bool isBitRotateMask(ArrayRef< int > Mask, unsigned EltSizeInBits, unsigned MinSubElts, unsigned MaxSubElts, unsigned &NumSubElts, unsigned &RotateAmt)
Checks if the shuffle is a bit rotation of the first operand across multiple subelements,...
VectorType * getType() const
Overload to return most specific vector type.
static void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)
Convert the input shuffle mask operand to a vector of integers.
static bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
static bool isInsertSubvectorMask(ArrayRef< int > Mask, int NumSrcElts, int &NumSubElts, int &Index)
Return true if this shuffle mask is an insert subvector mask.
static bool isInterleaveMask(ArrayRef< int > Mask, unsigned Factor, unsigned NumInputElts, SmallVectorImpl< unsigned > &StartIndexes)
Return true if the mask interleaves one or more input vectors together.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getSplatIndex() const
ArrayRef< int > getMask() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
An instruction for storing to memory.
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
std::string lower() const
A switch()-like statement whose cases are string literals.
StringSwitch & Case(StringLiteral S, T Value)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Class to represent struct types.
bool containsHomogeneousScalableVectorTypes() const
Returns true if this struct contains homogeneous scalable vector types.
unsigned getNumElements() const
Random access to the elements.
Type * getTypeAtIndex(const Value *V) const
Given an index value into the type, return the type of the element.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
Convenience method to set an operation to Promote and specify the type in a single call.
unsigned getMinCmpXchgSizeInBits() const
Returns the size of the smallest cmpxchg or ll/sc instruction the backend supports.
void setIndexedLoadAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed load does or does not work with the specified type and indicate w...
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
bool isOperationCustom(unsigned Op, EVT VT) const
Return true if the operation uses custom lowering, regardless of whether the type is legal or not.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual bool shouldFoldSelectWithSingleBitTest(EVT VT, const APInt &AndMask) const
virtual Value * getIRStackGuard(IRBuilderBase &IRB) const
If the target has a standard location for the stack protector guard, returns the address of that loca...
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setIndexedStoreAction(ArrayRef< unsigned > IdxModes, MVT VT, LegalizeAction Action)
Indicate that the specified indexed store does or does not work with the specified type and indicate ...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setLibcallName(RTLIB::Libcall Call, const char *Name)
Rename the default libcall routine name for the specified libcall.
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
SDValue buildSDIVPow2WithCMov(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl< SDNode * > &Created) const
Build sdiv by power-of-2 with conditional move instructions Ref: "Hacker's Delight" by Henry Warren 1...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual SDValue LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, SelectionDAG &DAG) const
Lower TLS global address SDNode for target independent emulated TLS model.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
virtual unsigned getJumpTableEncoding() const
Return the entry encoding for a jump table in the current function.
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
bool isStructTy() const
True if this is an instance of StructType.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isScalableTy() const
Return true if this is a type whose size is a known multiple of vscale.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Type * getContainedType(unsigned i) const
This method is used to implement the type iterator (defined at the end of the file).
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
A Use represents the edge between a Value definition and its users.
User * getUser() const
Returns the User that contains this Use.
Value * getOperand(unsigned i) const
unsigned getNumOperands() const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
bool hasOneUse() const
Return true if there is exactly one use of this value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
constexpr ScalarTy getFixedValue() const
constexpr LeafTy multiplyCoefficientBy(ScalarTy RHS) const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr bool isZero() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ RISCV_VectorCall
Calling convention used for RISC-V V-extension.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
@ GRAAL
Used by GraalVM. Two additional registers are reserved.
@ C
The default llvm calling convention, compatible with C.
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
ABI getTargetABI(StringRef ABIName)
bool match(Val *V, const Pattern &P)
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
auto m_Undef()
Match an arbitrary undef constant.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
@ TAIL_UNDISTURBED_MASK_UNDISTURBED
static VLMUL getLMul(uint64_t TSFlags)
static int getFRMOpNum(const MCInstrDesc &Desc)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #3 and #4) ...
@ STRICT_VFCVT_RTZ_XU_F_VL
@ LAST_RISCV_STRICTFP_OPCODE
@ STRICT_VFROUND_NOEXCEPT_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
@ STRICT_VFCVT_RTZ_X_F_VL
int getLoadFPImm(APFloat FPImm)
getLoadFPImm - Return a 5-bit binary encoding of the floating-point immediate value.
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static RISCVII::VLMUL encodeLMUL(unsigned LMUL, bool Fractional)
static unsigned encodeSEW(unsigned SEW)
static constexpr unsigned FPMASK_Negative_Zero
static constexpr unsigned FPMASK_Positive_Subnormal
static constexpr unsigned FPMASK_Positive_Normal
static constexpr unsigned FPMASK_Negative_Subnormal
static constexpr unsigned FPMASK_Negative_Normal
bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI, RVVArgDispatcher &RVVDispatcher)
static constexpr unsigned FPMASK_Positive_Infinity
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
static constexpr unsigned FPMASK_Negative_Infinity
static constexpr unsigned FPMASK_Quiet_NaN
ArrayRef< MCPhysReg > getArgGPRs(const RISCVABI::ABI ABI)
bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
static constexpr unsigned FPMASK_Signaling_NaN
static constexpr unsigned FPMASK_Positive_Zero
static constexpr unsigned RVVBitsPerBlock
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ Kill
The last use of a register.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator)
Returns the integer ceil(Numerator / Denominator).
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
bool isReleaseOrStronger(AtomicOrdering AO)
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ SMin
Signed integer min implemented in terms of select(cmp()).
unsigned getKillRegState(bool B)
DWARFExpression::Operation Op
RoundingMode
Rounding mode.
@ TowardZero
roundTowardZero.
@ NearestTiesToEven
roundTiesToEven.
@ TowardPositive
roundTowardPositive.
@ NearestTiesToAway
roundTiesToAway.
@ TowardNegative
roundTowardNegative.
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
bool isAcquireOrStronger(AtomicOrdering AO)
constexpr unsigned BitWidth
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
unsigned Log2(Align A)
Returns the log2 of the alignment.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static unsigned int semanticsPrecision(const fltSemantics &)
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
Helper struct shared between Function Specialization and SCCP Solver.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
uint64_t getScalarStoreSize() const
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
bool isFixedLengthVector() const
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
bool isUnknown() const
Returns true if we don't know any bits.
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Align valueOrOne() const
For convenience, returns a valid alignment or 1 if undefined.
BitVector getReservedRegs(const MachineFunction &MF) const override
Register getFrameRegister(const MachineFunction &MF) const override
bool hasScalarOperand() const
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
const ConstantInt * CFIType
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isAfterLegalizeDAG() const
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool recursivelyDeleteUnusedNodes(SDNode *N)
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)