34 #include "llvm/IR/IntrinsicsRISCV.h"
44 #define DEBUG_TYPE "riscv-lower"
46 STATISTIC(NumTailCalls,
"Number of tail calls");
60 errs() <<
"Hard-float 'f' ABI can't be used for a target that "
61 "doesn't support the F instruction set extension (ignoring "
66 errs() <<
"Hard-float 'd' ABI can't be used for a target that "
67 "doesn't support the D instruction set extension (ignoring "
114 auto addRegClassForRVV = [
this](
MVT VT) {
115 unsigned Size = VT.getSizeInBits().getKnownMinValue();
119 RC = &RISCV::VRRegClass;
120 else if (Size == 128)
121 RC = &RISCV::VRM2RegClass;
122 else if (Size == 256)
123 RC = &RISCV::VRM4RegClass;
125 RC = &RISCV::VRM8RegClass;
130 for (
MVT VT : BoolVecVTs)
131 addRegClassForRVV(VT);
132 for (
MVT VT : IntVecVTs) {
133 if (VT.getVectorElementType() ==
MVT::i64 &&
136 addRegClassForRVV(VT);
140 for (
MVT VT : F16VecVTs)
141 addRegClassForRVV(VT);
144 for (
MVT VT : F32VecVTs)
145 addRegClassForRVV(VT);
148 for (
MVT VT : F64VecVTs)
149 addRegClassForRVV(VT);
152 auto addRegClassForFixedVectors = [
this](
MVT VT) {
159 if (useRVVForFixedLengthVectorVT(VT))
160 addRegClassForFixedVectors(VT);
163 if (useRVVForFixedLengthVectorVT(VT))
164 addRegClassForFixedVectors(VT);
201 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
306 for (
auto NT : FPLegalNodeTypes)
331 for (
auto NT : FPLegalNodeTypes)
337 for (
auto Op : FPOpToExpand)
347 for (
auto NT : FPLegalNodeTypes)
357 for (
auto Op : FPOpToExpand)
423 static const unsigned IntegerVPOps[] = {
424 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
425 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
426 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
427 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
428 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
429 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
430 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
431 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FPTOSI,
432 ISD::VP_FPTOUI, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
433 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE};
435 static const unsigned FloatingPointVPOps[] = {
436 ISD::VP_FADD, ISD::VP_FSUB,
437 ISD::VP_FMUL, ISD::VP_FDIV,
438 ISD::VP_FNEG, ISD::VP_FMA,
439 ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
440 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX,
441 ISD::VP_MERGE, ISD::VP_SELECT,
442 ISD::VP_SITOFP, ISD::VP_UITOFP,
443 ISD::VP_SETCC, ISD::VP_FP_ROUND,
459 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
460 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
461 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
465 for (
MVT VT : BoolVecVTs) {
488 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
508 {ISD::VP_FPTOSI, ISD::VP_FPTOUI, ISD::VP_TRUNCATE, ISD::VP_SETCC}, VT,
512 for (
MVT VT : IntVecVTs) {
513 if (VT.getVectorElementType() ==
MVT::i64 &&
576 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT,
599 if (VT.getVectorElementType() !=
MVT::i64) {
624 const auto SetCommonVFPActions = [&](
MVT VT) {
654 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT,
671 const auto SetCommonVFPExtLoadTruncStoreActions =
673 for (
auto SmallVT : SmallerVTs) {
680 for (
MVT VT : F16VecVTs)
681 SetCommonVFPActions(VT);
683 for (
MVT VT : F32VecVTs) {
685 SetCommonVFPActions(VT);
686 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
689 for (
MVT VT : F64VecVTs) {
691 SetCommonVFPActions(VT);
692 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
693 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
698 if (!useRVVForFixedLengthVectorVT(VT))
735 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
749 {ISD::VP_FPTOSI, ISD::VP_FPTOUI, ISD::VP_SETCC, ISD::VP_TRUNCATE},
771 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT,
819 if (!useRVVForFixedLengthVectorVT(VT))
844 {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER, ISD::VP_SCATTER}, VT,
857 for (
auto CC : VFPCCToExpand)
933 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy()
const {
940 unsigned Intrinsic)
const {
941 auto &
DL =
I.getModule()->getDataLayout();
945 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
946 case Intrinsic::riscv_masked_atomicrmw_add_i32:
947 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
948 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
949 case Intrinsic::riscv_masked_atomicrmw_max_i32:
950 case Intrinsic::riscv_masked_atomicrmw_min_i32:
951 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
952 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
953 case Intrinsic::riscv_masked_cmpxchg_i32:
956 Info.ptrVal =
I.getArgOperand(0);
962 case Intrinsic::riscv_masked_strided_load:
964 Info.ptrVal =
I.getArgOperand(1);
966 Info.align =
Align(
DL.getTypeSizeInBits(
I.getType()->getScalarType()) / 8);
970 case Intrinsic::riscv_masked_strided_store:
972 Info.ptrVal =
I.getArgOperand(1);
976 DL.getTypeSizeInBits(
I.getArgOperand(0)->getType()->getScalarType()) /
981 case Intrinsic::riscv_seg2_load:
982 case Intrinsic::riscv_seg3_load:
983 case Intrinsic::riscv_seg4_load:
984 case Intrinsic::riscv_seg5_load:
985 case Intrinsic::riscv_seg6_load:
986 case Intrinsic::riscv_seg7_load:
987 case Intrinsic::riscv_seg8_load:
989 Info.ptrVal =
I.getArgOperand(0);
991 getValueType(
DL,
I.getType()->getStructElementType(0)->getScalarType());
994 I.getType()->getStructElementType(0)->getScalarType()) /
1033 return isInt<12>(
Imm);
1037 return isInt<12>(
Imm);
1048 return (SrcBits == 64 && DestBits == 32);
1057 return (SrcBits == 64 && DestBits == 32);
1064 if (
auto *
LD = dyn_cast<LoadSDNode>(Val)) {
1065 EVT MemVT =
LD->getMemoryVT();
1092 EVT VT =
Y.getValueType();
1100 !isa<ConstantSDNode>(
Y);
1105 auto *
C = dyn_cast<ConstantSDNode>(
Y);
1106 return C &&
C->getAPIntValue().ule(10);
1112 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
1140 if (!
I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1144 switch (
I->getOpcode()) {
1146 case Instruction::Sub:
1148 case Instruction::And:
1149 case Instruction::Or:
1150 case Instruction::Xor:
1151 case Instruction::FAdd:
1152 case Instruction::FSub:
1153 case Instruction::FMul:
1154 case Instruction::FDiv:
1155 case Instruction::ICmp:
1156 case Instruction::FCmp:
1158 case Instruction::Shl:
1159 case Instruction::LShr:
1160 case Instruction::AShr:
1161 case Instruction::UDiv:
1162 case Instruction::SDiv:
1163 case Instruction::URem:
1164 case Instruction::SRem:
1165 return Operand == 1;
1167 if (
auto *II = dyn_cast<IntrinsicInst>(
I)) {
1168 switch (II->getIntrinsicID()) {
1169 case Intrinsic::fma:
1170 case Intrinsic::vp_fma:
1171 return Operand == 0 || Operand == 1;
1175 case Intrinsic::vp_add:
1176 case Intrinsic::vp_mul:
1177 case Intrinsic::vp_and:
1178 case Intrinsic::vp_or:
1179 case Intrinsic::vp_xor:
1180 case Intrinsic::vp_fadd:
1181 case Intrinsic::vp_fmul:
1182 case Intrinsic::vp_shl:
1183 case Intrinsic::vp_lshr:
1184 case Intrinsic::vp_ashr:
1185 case Intrinsic::vp_udiv:
1186 case Intrinsic::vp_sdiv:
1187 case Intrinsic::vp_urem:
1188 case Intrinsic::vp_srem:
1189 return Operand == 1;
1192 case Intrinsic::vp_sub:
1193 case Intrinsic::vp_fsub:
1194 case Intrinsic::vp_fdiv:
1195 return Operand == 0 || Operand == 1;
1206 for (
auto OpIdx :
enumerate(
I->operands())) {
1207 if (!IsSinker(
I, OpIdx.index()))
1210 Instruction *
Op = dyn_cast<Instruction>(OpIdx.value().get());
1212 if (!
Op ||
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op; }))
1222 for (
Use &U :
Op->uses()) {
1228 Ops.push_back(&
Op->getOperandUse(0));
1229 Ops.push_back(&OpIdx.value());
1244 bool ForCodeSize)
const {
1252 return Imm.isZero();
1324 switch (KnownSize) {
1352 return RISCV::VRRegClassID;
1354 return RISCV::VRM2RegClassID;
1356 return RISCV::VRM4RegClassID;
1358 return RISCV::VRM8RegClassID;
1368 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1369 "Unexpected subreg numbering");
1370 return RISCV::sub_vrm1_0 + Index;
1373 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1374 "Unexpected subreg numbering");
1375 return RISCV::sub_vrm2_0 + Index;
1378 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1379 "Unexpected subreg numbering");
1380 return RISCV::sub_vrm4_0 + Index;
1387 return RISCV::VRRegClassID;
1396 std::pair<unsigned, unsigned>
1398 MVT VecVT,
MVT SubVecVT,
unsigned InsertExtractIdx,
1400 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1401 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1402 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1403 "Register classes not ordered");
1412 unsigned SubRegIdx = RISCV::NoSubRegister;
1413 for (
const unsigned RCID :
1414 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1415 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1424 return {SubRegIdx, InsertExtractIdx};
1429 bool RISCVTargetLowering::mergeStoresAfterLegalization(
EVT VT)
const {
1458 "Unexpected opcode");
1460 unsigned IntNo =
Op.getConstantOperandVal(HasChain ? 1 : 0);
1462 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
1465 return Op.getOperand(II->
VLOperand + 1 + HasChain);
1468 static bool useRVVForFixedLengthVectorVT(
MVT VT,
1535 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(
MVT VT)
const {
1544 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1545 "Expected legal fixed length vector!");
1548 unsigned MaxELen = Subtarget.
getELEN();
1588 "Expected to convert into a scalable vector!");
1590 "Expected a fixed length vector operand!");
1600 "Expected to convert into a fixed length vector!");
1602 "Expected a scalable vector operand!");
1628 static std::pair<SDValue, SDValue>
1641 static std::pair<SDValue, SDValue>
1657 EVT VT,
unsigned DefinedValues)
const {
1668 EVT DstVT =
Op.getValueType();
1669 EVT SatVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
1683 Opc,
DL, DstVT, Src,
1696 MVT VT =
Op.getSimpleValueType();
1719 Truncated = DAG.
getSelect(
DL, VT, NeedAdjust, Adjust, Truncated);
1728 Truncated = DAG.
getSelect(
DL, VT, NeedAdjust, Adjust, Truncated);
1747 return DAG.
getSelect(
DL, VT, Setcc, Truncated, Src);
1758 MVT VT =
Op.getSimpleValueType();
1773 Point5Pred.
next(
true);
1799 return DAG.
getSelect(
DL, VT, Setcc, Truncated, Src);
1819 unsigned NumElts =
Op.getNumOperands();
1821 if (!
Op.getValueType().isInteger())
1827 unsigned EltSizeInBits =
Op.getValueType().getScalarSizeInBits();
1828 for (
unsigned Idx = 0; Idx < NumElts; Idx++) {
1831 if (
Op.getOperand(Idx).isUndef())
1834 if (!isa<ConstantSDNode>(
Op.getOperand(Idx)))
1837 uint64_t Val =
Op.getConstantOperandVal(Idx) &
1838 maskTrailingOnes<uint64_t>(EltSizeInBits);
1843 unsigned IdxDiff = Idx - PrevElt->second;
1844 int64_t ValDiff =
SignExtend64(Val - PrevElt->first, EltSizeInBits);
1852 int64_t Remainder = ValDiff % IdxDiff;
1854 if (Remainder != ValDiff) {
1863 SeqStepNum = ValDiff;
1864 else if (ValDiff != SeqStepNum)
1868 SeqStepDenom = IdxDiff;
1869 else if (IdxDiff != *SeqStepDenom)
1874 if (!PrevElt || PrevElt->first != Val)
1875 PrevElt = std::make_pair(Val, Idx);
1879 if (!SeqStepNum || !SeqStepDenom)
1884 for (
unsigned Idx = 0; Idx < NumElts; Idx++) {
1885 if (
Op.getOperand(Idx).isUndef())
1887 uint64_t Val =
Op.getConstantOperandVal(Idx) &
1888 maskTrailingOnes<uint64_t>(EltSizeInBits);
1890 (int64_t)(Idx * (
uint64_t)*SeqStepNum) / *SeqStepDenom;
1891 int64_t Addend =
SignExtend64(Val - ExpectedVal, EltSizeInBits);
1894 else if (Addend != SeqAddend)
1898 assert(SeqAddend &&
"Must have an addend if we have a step");
1900 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
1919 MVT ContainerVT = VT;
1939 MVT VT =
Op.getSimpleValueType();
1949 unsigned NumElts =
Op.getNumOperands();
1970 unsigned NumViaIntegerBits =
1981 MVT IntegerViaVecVT =
1986 unsigned BitPos = 0, IntegerEltIdx = 0;
1989 for (
unsigned I = 0;
I < NumElts;
I++, BitPos++) {
1992 if (
I != 0 &&
I % NumViaIntegerBits == 0) {
1993 if (NumViaIntegerBits <= 32)
2003 bool BitValue = !V.
isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
2009 if (NumViaIntegerBits <= 32)
2015 if (NumElts < NumViaIntegerBits) {
2019 assert(IntegerViaVecVT ==
MVT::v1i8 &&
"Unexpected mask vector type");
2043 assert(Splat.getValueType() == XLenVT &&
2044 "Unexpected type for i1 splat value");
2072 int64_t StepNumerator = SimpleVID->StepNumerator;
2073 unsigned StepDenominator = SimpleVID->StepDenominator;
2074 int64_t Addend = SimpleVID->Addend;
2076 assert(StepNumerator != 0 &&
"Invalid step");
2077 bool Negate =
false;
2078 int64_t SplatStepVal = StepNumerator;
2080 if (StepNumerator != 1) {
2082 Negate = StepNumerator < 0;
2092 if (((StepOpcode ==
ISD::MUL && isInt<12>(SplatStepVal)) ||
2093 (StepOpcode ==
ISD::SHL && isUInt<5>(SplatStepVal))) &&
2095 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
2102 if ((StepOpcode ==
ISD::MUL && SplatStepVal != 1) ||
2103 (StepOpcode ==
ISD::SHL && SplatStepVal != 0)) {
2106 VID = DAG.
getNode(StepOpcode,
DL, VT, VID, SplatStep);
2108 if (StepDenominator != 1) {
2113 if (Addend != 0 || Negate) {
2131 const auto *BV = cast<BuildVectorSDNode>(
Op);
2132 if (VT.
isInteger() && EltBitSize < 64 &&
2134 BV->getRepeatedSequence(
Sequence) &&
2135 (
Sequence.size() * EltBitSize) <= 64) {
2141 "Unexpected sequence type");
2143 unsigned EltIdx = 0;
2144 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2148 for (
const auto &SeqV :
Sequence) {
2149 if (!SeqV.isUndef())
2150 SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2151 << (EltIdx * EltBitSize));
2158 SplatValue = SignExtend64<32>(SplatValue);
2165 "Unexpected bitcast sequence");
2169 MVT ViaContainerVT =
2189 unsigned MostCommonCount = 0;
2191 unsigned NumUndefElts =
2199 unsigned NumScalarLoads = 0;
2205 ValueCounts.
insert(std::make_pair(V, 0));
2206 unsigned &Count = ValueCounts[V];
2208 if (
auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2209 NumScalarLoads += !CFP->isExactlyValue(+0.0);
2214 if (++Count >= MostCommonCount) {
2216 MostCommonCount = Count;
2220 assert(DominantValue &&
"Not expecting an all-undef BUILD_VECTOR");
2221 unsigned NumDefElts = NumElts - NumUndefElts;
2222 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2227 ((MostCommonCount > DominantValueCountThreshold) ||
2235 const SDValue &V = OpIdx.value();
2236 if (V.
isUndef() || !Processed.insert(V).second)
2238 if (ValueCounts[V] == 1) {
2247 return DAG.getConstant(V == V1, DL, XLenVT);
2266 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2267 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2268 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2271 if ((LoC >> 31) == HiC)
2276 auto *Const = dyn_cast<ConstantSDNode>(VL);
2277 if (LoC == HiC && Const && Const->isAllOnesValue()) {
2313 bool HasPassthru = Passthru && !Passthru.
isUndef();
2314 if (!HasPassthru && !Passthru)
2326 if (Scalar.getValueType().bitsLE(XLenVT)) {
2333 Scalar = DAG.
getNode(ExtOpc,
DL, XLenVT, Scalar);
2338 (!Const ||
isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2344 "Unexpected scalar for splat lowering!");
2360 int Size =
Mask.size();
2363 int Srcs[] = {-1, -1};
2364 for (
int i = 0;
i != Size; ++
i) {
2373 int Src =
Mask[
i] / Size;
2376 if (Srcs[Pol] != Src)
2381 int Elt =
Mask[
i] % Size;
2387 if (Srcs[0] < 0 || Srcs[1] < 0 || Srcs[0] == Srcs[1])
2391 SwapSources = Srcs[0] > Srcs[1];
2409 int Size =
Mask.size();
2421 for (
int i = 0;
i != Size; ++
i) {
2427 int StartIdx =
i - (
M % Size);
2435 int CandidateRotation = StartIdx < 0 ? -StartIdx : Size - StartIdx;
2438 Rotation = CandidateRotation;
2439 else if (Rotation != CandidateRotation)
2444 int MaskSrc =
M < Size ? 0 : 1;
2449 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
2454 TargetSrc = MaskSrc;
2455 else if (TargetSrc != MaskSrc)
2462 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
2463 assert((LoSrc >= 0 || HiSrc >= 0) &&
2464 "Failed to find a rotated input vector!");
2475 MVT VT =
Op.getSimpleValueType();
2499 Offset %= OpElements;
2504 auto *Ld = cast<LoadSDNode>(V);
2514 SDValue Ops[] = {Ld->getChain(),
2532 V = DAG.
getLoad(SVT,
DL, Ld->getChain(), NewAddr,
2533 Ld->getPointerInfo().getWithOffset(Offset),
2534 Ld->getOriginalAlign(),
2538 Ld->getPointerInfo().getWithOffset(Offset), SVT,
2539 Ld->getOriginalAlign(),
2540 Ld->getMemOperand()->getFlags());
2551 assert(Lane < (
int)NumElts &&
"Unexpected lane!");
2568 LoV = LoSrc == 0 ? V1 :
V2;
2572 HiV = HiSrc == 0 ? V1 :
V2;
2578 unsigned InvRotate = NumElts - Rotation;
2622 MVT WideIntContainerVT =
2654 V2, Multiplier, TrueMask, VL);
2674 int MaskIndex = MaskIdx.value();
2675 return MaskIndex < 0 || MaskIdx.index() == (
unsigned)MaskIndex % NumElts;
2678 assert(!V1.
isUndef() &&
"Unexpected shuffle canonicalization");
2690 bool InvertMask = IsSelect == SwapOps;
2699 for (
int MaskIndex :
Mask) {
2700 bool SelectMaskVal = (MaskIndex < (
int)NumElts) ^ InvertMask;
2701 MaskVals.push_back(DAG.
getConstant(SelectMaskVal,
DL, XLenVT));
2703 bool IsLHSOrUndefIndex = MaskIndex < (
int)NumElts;
2704 GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
2707 GatherIndicesRHS.push_back(
2708 IsLHSOrUndefIndex ? DAG.
getUNDEF(XLenVT)
2710 if (IsLHSOrUndefIndex && MaskIndex >= 0)
2711 ++LHSIndexCounts[MaskIndex];
2712 if (!IsLHSOrUndefIndex)
2713 ++RHSIndexCounts[MaskIndex - NumElts];
2719 std::swap(GatherIndicesLHS, GatherIndicesRHS);
2722 assert(MaskVals.size() == NumElts &&
"Unexpected select-like shuffle");
2748 MVT IndexContainerVT =
2762 if (LHSIndexCounts.
size() == 1) {
2763 int SplatIndex = LHSIndexCounts.
begin()->getFirst();
2765 DAG.
getNode(GatherVXOpc,
DL, ContainerVT, V1,
2772 Gather = DAG.
getNode(GatherVVOpc,
DL, ContainerVT, V1, LHSIndices,
2779 if (!
V2.isUndef()) {
2784 if (RHSIndexCounts.
size() == 1) {
2785 int SplatIndex = RHSIndexCounts.
begin()->getFirst();
2792 V2 = DAG.
getNode(GatherVVOpc,
DL, ContainerVT,
V2, RHSIndices, TrueMask,
2827 MVT VT =
Op.getSimpleValueType();
2841 "Expected legal float type!");
2856 unsigned ShiftAmt = FloatEltVT ==
MVT::f64 ? 52 : 23;
2862 unsigned ExponentBias = FloatEltVT ==
MVT::f64 ? 1023 : 127;
2871 unsigned Adjust = ExponentBias + (EltSize - 1);
2881 auto *
Load = cast<LoadSDNode>(
Op);
2882 assert(
Load &&
Load->getMemoryVT().isVector() &&
"Expected vector load");
2885 Load->getMemoryVT(),
2886 *
Load->getMemOperand()))
2890 MVT VT =
Op.getSimpleValueType();
2892 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2893 "Unexpected unaligned RVV load type");
2897 "Expecting equally-sized RVV vector types to be legal");
2899 Load->getPointerInfo(),
Load->getOriginalAlign(),
2900 Load->getMemOperand()->getFlags());
2910 auto *
Store = cast<StoreSDNode>(
Op);
2912 "Expected vector store");
2915 Store->getMemoryVT(),
2916 *
Store->getMemOperand()))
2923 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2924 "Unexpected unaligned RVV store type");
2928 "Expecting equally-sized RVV vector types to be legal");
2929 StoredVal = DAG.
getBitcast(NewVT, StoredVal);
2931 Store->getPointerInfo(),
Store->getOriginalAlign(),
2932 Store->getMemOperand()->getFlags());
2937 switch (
Op.getOpcode()) {
2941 return lowerGlobalAddress(
Op, DAG);
2943 return lowerBlockAddress(
Op, DAG);
2945 return lowerConstantPool(
Op, DAG);
2947 return lowerJumpTable(
Op, DAG);
2949 return lowerGlobalTLSAddress(
Op, DAG);
2951 return lowerSELECT(
Op, DAG);
2953 return lowerBRCOND(
Op, DAG);
2955 return lowerVASTART(
Op, DAG);
2957 return lowerFRAMEADDR(
Op, DAG);
2959 return lowerRETURNADDR(
Op, DAG);
2961 return lowerShiftLeftParts(
Op, DAG);
2963 return lowerShiftRightParts(
Op, DAG,
true);
2965 return lowerShiftRightParts(
Op, DAG,
false);
2968 EVT VT =
Op.getValueType();
3016 return LowerINTRINSIC_WO_CHAIN(
Op, DAG);
3018 return LowerINTRINSIC_W_CHAIN(
Op, DAG);
3020 return LowerINTRINSIC_VOID(
Op, DAG);
3023 MVT VT =
Op.getSimpleValueType();
3046 MVT VT =
Op.getSimpleValueType();
3047 assert(VT == Subtarget.
getXLenVT() &&
"Unexpected custom legalization");
3052 unsigned ShAmtWidth = Subtarget.
getXLen() - 1;
3067 return DAG.
getNode(Opc,
DL, VT, Op0, Op1, ShAmt);
3071 if (!
Op.getSimpleValueType().isVector())
3073 return lowerVectorTruncLike(
Op, DAG);
3076 if (
Op.getOperand(0).getValueType().isVector() &&
3077 Op.getOperand(0).getValueType().getVectorElementType() ==
MVT::i1)
3078 return lowerVectorMaskExt(
Op, DAG, 1);
3081 if (
Op.getOperand(0).getValueType().isVector() &&
3082 Op.getOperand(0).getValueType().getVectorElementType() ==
MVT::i1)
3083 return lowerVectorMaskExt(
Op, DAG, -1);
3086 return lowerSPLAT_VECTOR_PARTS(
Op, DAG);
3088 return lowerINSERT_VECTOR_ELT(
Op, DAG);
3090 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
3092 MVT VT =
Op.getSimpleValueType();
3128 Op.getOperand(1).getValueType() ==
MVT::i32) {
3140 if (!
Op.getValueType().isVector())
3142 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
3150 MVT VT =
Op.getSimpleValueType();
3156 MVT SrcVT = Src.getSimpleValueType();
3161 "Unexpected vector element types");
3165 if (EltSize > (2 * SrcEltSize)) {
3177 assert(SrcEltVT ==
MVT::f16 &&
"Unexpected FP_TO_[US]INT lowering");
3186 if (SrcEltSize > (2 * EltSize)) {
3209 unsigned RVVOpc = 0;
3210 switch (
Op.getOpcode()) {
3227 MVT ContainerVT, SrcContainerVT;
3229 if (SrcEltSize > EltSize) {
3259 return lowerVECREDUCE(
Op, DAG);
3263 if (
Op.getOperand(0).getValueType().getVectorElementType() ==
MVT::i1)
3264 return lowerVectorMaskVecReduction(
Op, DAG,
false);
3265 return lowerVECREDUCE(
Op, DAG);
3270 return lowerFPVECREDUCE(
Op, DAG);
3271 case ISD::VP_REDUCE_ADD:
3272 case ISD::VP_REDUCE_UMAX:
3273 case ISD::VP_REDUCE_SMAX:
3274 case ISD::VP_REDUCE_UMIN:
3275 case ISD::VP_REDUCE_SMIN:
3276 case ISD::VP_REDUCE_FADD:
3277 case ISD::VP_REDUCE_SEQ_FADD:
3278 case ISD::VP_REDUCE_FMIN:
3279 case ISD::VP_REDUCE_FMAX:
3280 return lowerVPREDUCE(
Op, DAG);
3281 case ISD::VP_REDUCE_AND:
3282 case ISD::VP_REDUCE_OR:
3283 case ISD::VP_REDUCE_XOR:
3284 if (
Op.getOperand(1).getValueType().getVectorElementType() ==
MVT::i1)
3285 return lowerVectorMaskVecReduction(
Op, DAG,
true);
3286 return lowerVPREDUCE(
Op, DAG);
3288 return lowerINSERT_SUBVECTOR(
Op, DAG);
3290 return lowerEXTRACT_SUBVECTOR(
Op, DAG);
3292 return lowerSTEP_VECTOR(
Op, DAG);
3294 return lowerVECTOR_REVERSE(
Op, DAG);
3296 return lowerVECTOR_SPLICE(
Op, DAG);
3300 if (
Op.getValueType().getVectorElementType() ==
MVT::i1)
3301 return lowerVectorMaskSplat(
Op, DAG);
3309 MVT VT =
Op.getSimpleValueType();
3310 unsigned NumOpElts =
3311 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
3314 SDValue SubVec = OpIdx.value();
3324 if (
auto V = expandUnalignedRVVLoad(
Op, DAG))
3326 if (
Op.getValueType().isFixedLengthVector())
3327 return lowerFixedLengthVectorLoadToRVV(
Op, DAG);
3330 if (
auto V = expandUnalignedRVVStore(
Op, DAG))
3332 if (
Op.getOperand(1).getValueType().isFixedLengthVector())
3333 return lowerFixedLengthVectorStoreToRVV(
Op, DAG);
3337 return lowerMaskedLoad(
Op, DAG);
3340 return lowerMaskedStore(
Op, DAG);
3342 return lowerFixedLengthVectorSetccToRVV(
Op, DAG);
3373 if (
Op.getSimpleValueType().isFixedLengthVector())
3374 return lowerFixedLengthVectorShiftToRVV(
Op, DAG);
3377 "Unexpected custom legalisation");
3416 return lowerABS(
Op, DAG);
3421 return lowerFixedLengthVectorSelectToRVV(
Op, DAG);
3423 return lowerFixedLengthVectorFCOPYSIGNToRVV(
Op, DAG);
3425 case ISD::VP_GATHER:
3426 return lowerMaskedGather(
Op, DAG);
3428 case ISD::VP_SCATTER:
3429 return lowerMaskedScatter(
Op, DAG);
3431 return lowerGET_ROUNDING(
Op, DAG);
3433 return lowerSET_ROUNDING(
Op, DAG);
3434 case ISD::VP_SELECT:
3476 case ISD::VP_SIGN_EXTEND:
3477 case ISD::VP_ZERO_EXTEND:
3478 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() ==
MVT::i1)
3479 return lowerVPExtMaskOp(
Op, DAG);
3480 return lowerVPOp(
Op, DAG,
3481 Op.getOpcode() == ISD::VP_SIGN_EXTEND
3484 case ISD::VP_TRUNCATE:
3485 return lowerVectorTruncLike(
Op, DAG);
3486 case ISD::VP_FP_EXTEND:
3487 case ISD::VP_FP_ROUND:
3488 return lowerVectorFPExtendOrRoundLike(
Op, DAG);
3489 case ISD::VP_FPTOSI:
3491 case ISD::VP_FPTOUI:
3493 case ISD::VP_SITOFP:
3495 case ISD::VP_UITOFP:
3498 if (
Op.getOperand(0).getSimpleValueType().getVectorElementType() ==
MVT::i1)
3499 return lowerVPSetCCMaskOp(
Op, DAG);
3518 N->getOffset(), Flags);
3526 template <
class NodeTy>
3528 bool IsLocal)
const {
3576 template SDValue RISCVTargetLowering::getAddr<GlobalAddressSDNode>(
3578 template SDValue RISCVTargetLowering::getAddr<BlockAddressSDNode>(
3580 template SDValue RISCVTargetLowering::getAddr<ConstantPoolSDNode>(
3582 template SDValue RISCVTargetLowering::getAddr<JumpTableSDNode>(
3589 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
3619 bool UseGOT)
const {
3684 Args.push_back(Entry);
3701 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
3712 Addr = getStaticTLSAddr(
N, DAG,
false);
3715 Addr = getStaticTLSAddr(
N, DAG,
true);
3719 Addr = getDynamicTLSAddr(
N, DAG);
3731 MVT VT =
Op.getSimpleValueType();
3753 const auto *CC = cast<CondCodeSDNode>(CondV.
getOperand(2));
3763 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3765 const APInt &
TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3766 const APInt &
FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3786 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
3806 LHS,
RHS, TargetCC,
Op.getOperand(2));
3824 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
3836 int XLenInBytes = Subtarget.
getXLen() / 8;
3838 EVT VT =
Op.getValueType();
3841 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
3843 int Offset = -(XLenInBytes * 2);
3859 int XLenInBytes = Subtarget.
getXLen() / 8;
3864 EVT VT =
Op.getValueType();
3866 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
3868 int Off = -XLenInBytes;
3869 SDValue FrameAddr = lowerFRAMEADDR(
Op, DAG);
3888 EVT VT =
Lo.getValueType();
3927 EVT VT =
Lo.getValueType();
3978 MVT VT =
Op.getSimpleValueType();
3991 "Unexpected type for i1 splat value");
4007 MVT VecVT =
Op.getSimpleValueType();
4009 "Unexpected SPLAT_VECTOR_PARTS lowering");
4011 assert(
Op.getNumOperands() == 2 &&
"Unexpected number of operands!");
4019 std::tie(
Mask, VL) =
4027 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
4028 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4029 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4032 if ((LoC >> 31) == HiC)
4038 if (
Hi.getOpcode() ==
ISD::SRA &&
Hi.getOperand(0) == Lo &&
4039 isa<ConstantSDNode>(
Hi.getOperand(1)) &&
4040 Hi.getConstantOperandVal(1) == 31)
4055 int64_t ExtTrueVal)
const {
4057 MVT VecVT =
Op.getSimpleValueType();
4060 assert(Src.getValueType().isVector() &&
4061 Src.getValueType().getVectorElementType() ==
MVT::i1);
4083 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
4085 DAG.
getUNDEF(ContainerVT), SplatTrueVal, VL);
4087 SplatTrueVal, SplatZero, VL);
4092 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
4094 MVT ExtVT =
Op.getSimpleValueType();
4098 MVT VT =
Op.getOperand(0).getSimpleValueType();
4125 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
4127 EVT MaskVT =
Op.getValueType();
4130 "Unexpected type for vector mask lowering");
4132 MVT VecVT = Src.getSimpleValueType();
4136 VL =
Op.getOperand(2);
4139 MVT ContainerVT = VecVT;
4145 MVT MaskContainerVT =
4152 std::tie(
Mask, VL) =
4160 DAG.
getUNDEF(ContainerVT), SplatOne, VL);
4162 DAG.
getUNDEF(ContainerVT), SplatZero, VL);
4176 bool IsVPTrunc =
Op.getOpcode() == ISD::VP_TRUNCATE;
4179 MVT VT =
Op.getSimpleValueType();
4181 assert(VT.
isVector() &&
"Unexpected type for vector truncate lowering");
4185 return lowerVectorMaskTruncLike(
Op, DAG);
4193 MVT SrcVT = Src.getSimpleValueType();