32#include "llvm/IR/IntrinsicsLoongArch.h"
42#define DEBUG_TYPE "loongarch-isel-lowering"
57 cl::desc(
"Maximum number of instructions used (including code sequence "
58 "to generate the value and moving the value to FPR) when "
59 "materializing floating-point immediates (default = 3)"),
63 "Materialize FP immediate within 2 instructions"),
65 "Materialize FP immediate within 3 instructions"),
67 "Materialize FP immediate within 4 instructions"),
69 "Materialize FP immediate within 5 instructions"),
71 "Materialize FP immediate within 6 instructions "
72 "(behaves same as 5 on loongarch64)")));
75 cl::desc(
"Trap on integer division by zero."),
82 MVT GRLenVT = Subtarget.getGRLenVT();
87 if (Subtarget.hasBasicF())
89 if (Subtarget.hasBasicD())
93 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64};
95 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v8f32, MVT::v4f64};
97 if (Subtarget.hasExtLSX())
101 if (Subtarget.hasExtLASX())
102 for (
MVT VT : LASXVTs)
170 if (Subtarget.is64Bit()) {
198 if (!Subtarget.is64Bit()) {
204 if (Subtarget.hasBasicD())
216 if (Subtarget.hasBasicF()) {
248 if (Subtarget.is64Bit())
251 if (!Subtarget.hasBasicD()) {
253 if (Subtarget.is64Bit()) {
262 if (Subtarget.hasBasicD()) {
295 if (Subtarget.is64Bit())
301 if (Subtarget.hasExtLSX()) {
316 for (
MVT VT : LSXVTs) {
330 for (
MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
358 for (
MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
360 for (
MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
362 for (
MVT VT : {MVT::v4i32, MVT::v2i64}) {
367 for (
MVT VT : {MVT::v4f32, MVT::v2f64}) {
391 {MVT::v16i8, MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v8i16, MVT::v4i16,
392 MVT::v2i16, MVT::v4i32, MVT::v2i32, MVT::v2i64}) {
407 for (
MVT VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16})
409 for (
MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16i32, MVT::v8i64,
416 if (Subtarget.hasExtLASX()) {
417 for (
MVT VT : LASXVTs) {
432 for (
MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) {
461 for (
MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
463 for (
MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})
465 for (
MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {
469 for (
MVT VT : {MVT::v8f32, MVT::v4f64}) {
488 for (
MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16}) {
493 {MVT::v2i64, MVT::v4i32, MVT::v4i64, MVT::v8i16, MVT::v8i32}) {
500 if (Subtarget.hasBasicF()) {
511 if (Subtarget.hasExtLSX()) {
523 if (Subtarget.hasExtLASX()) {
550 if (Subtarget.hasLAMCAS())
553 if (Subtarget.hasSCQ()) {
573 switch (
Op.getOpcode()) {
575 return lowerATOMIC_FENCE(
Op, DAG);
577 return lowerEH_DWARF_CFA(
Op, DAG);
579 return lowerGlobalAddress(
Op, DAG);
581 return lowerGlobalTLSAddress(
Op, DAG);
583 return lowerINTRINSIC_WO_CHAIN(
Op, DAG);
585 return lowerINTRINSIC_W_CHAIN(
Op, DAG);
587 return lowerINTRINSIC_VOID(
Op, DAG);
589 return lowerBlockAddress(
Op, DAG);
591 return lowerJumpTable(
Op, DAG);
593 return lowerShiftLeftParts(
Op, DAG);
595 return lowerShiftRightParts(
Op, DAG,
true);
597 return lowerShiftRightParts(
Op, DAG,
false);
599 return lowerConstantPool(
Op, DAG);
601 return lowerFP_TO_SINT(
Op, DAG);
603 return lowerFP_TO_UINT(
Op, DAG);
605 return lowerBITCAST(
Op, DAG);
607 return lowerUINT_TO_FP(
Op, DAG);
609 return lowerSINT_TO_FP(
Op, DAG);
611 return lowerVASTART(
Op, DAG);
613 return lowerFRAMEADDR(
Op, DAG);
615 return lowerRETURNADDR(
Op, DAG);
617 return lowerWRITE_REGISTER(
Op, DAG);
619 return lowerINSERT_VECTOR_ELT(
Op, DAG);
621 return lowerEXTRACT_VECTOR_ELT(
Op, DAG);
623 return lowerBUILD_VECTOR(
Op, DAG);
625 return lowerCONCAT_VECTORS(
Op, DAG);
627 return lowerVECTOR_SHUFFLE(
Op, DAG);
629 return lowerBITREVERSE(
Op, DAG);
631 return lowerSCALAR_TO_VECTOR(
Op, DAG);
633 return lowerPREFETCH(
Op, DAG);
635 return lowerSELECT(
Op, DAG);
637 return lowerBRCOND(
Op, DAG);
639 return lowerFP_TO_FP16(
Op, DAG);
641 return lowerFP16_TO_FP(
Op, DAG);
643 return lowerFP_TO_BF16(
Op, DAG);
645 return lowerBF16_TO_FP(
Op, DAG);
647 return lowerVECREDUCE_ADD(
Op, DAG);
650 return lowerRotate(
Op, DAG);
658 return lowerVECREDUCE(
Op, DAG);
660 return lowerConstantFP(
Op, DAG);
662 return lowerSETCC(
Op, DAG);
664 return lowerFP_ROUND(
Op, DAG);
666 return lowerFP_EXTEND(
Op, DAG);
668 return lowerSIGN_EXTEND_VECTOR_INREG(
Op, DAG);
670 return lowerDYNAMIC_STACKALLOC(
Op, DAG);
679 EVT VT = V.getValueType();
685 return V.getOperand(0);
689 (
isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) {
691 Not = DAG.
getBitcast(V.getOperand(0).getValueType(), Not);
701 if (!V->isOnlyUserOf(SplatValue.getNode()))
705 Not = DAG.
getBitcast(V.getOperand(0).getValueType(), Not);
713 V.getOperand(0).hasOneUse() && V.getOperand(1).hasOneUse()) {
741 (
N->getOpcode() == LoongArchISD::VPACKEV) ||
742 (
N->getOpcode() == LoongArchISD::VPERMI)) &&
749 if (Opcode0 != Opcode1)
752 if (Opcode0 !=
ISD::FP_ROUND && Opcode0 != LoongArchISD::VFCVT)
759 EVT VT =
N.getValueType();
773 if (Subtarget.hasExtLASX() && VT.
is256BitVector() && SVT0 == MVT::v4f32 &&
774 SSVT0 == MVT::v4f64) {
793 if ((
N->getOpcode() == LoongArchISD::VPACKEV ||
794 N->getOpcode() == LoongArchISD::VPERMI) &&
795 Opcode0 == LoongArchISD::VFCVT) {
800 if (!Subtarget.hasExtLSX() || SVT0 != MVT::v4f32 || SSVT0 != MVT::v2f64)
803 if (
N->getOpcode() == LoongArchISD::VPACKEV &&
804 (VT == MVT::v2i64 || VT == MVT::v2f64)) {
810 if (
N->getOpcode() == LoongArchISD::VPERMI && VT == MVT::v4f32) {
826 MVT VT =
Op.getSimpleValueType();
827 MVT SVT =
In.getSimpleValueType();
829 if (VT == MVT::v4f32 && SVT == MVT::v4f64) {
842 EVT VT =
Op.getValueType();
844 EVT SVT = Src.getValueType();
847 VT == MVT::v2f64 && SVT == MVT::v2f32 && Subtarget.hasExtLSX();
849 VT == MVT::v4f64 && SVT == MVT::v4f32 && Subtarget.hasExtLASX();
850 if (!V2F32ToV2F64 && !V4F32ToV4F64)
862 const uint64_t
Imm =
Op.getConstantOperandVal(1);
877 if (
SDValue V = CheckVecHighPart(Src)) {
879 "Unexpected wide vector");
880 Opcode = LoongArchISD::VFCVTH;
883 Opcode = LoongArchISD::VFCVTL;
885 DAG.
getUNDEF(WideOpVT), Src, ZeroIdx);
890 return DAG.
getNode(Opcode,
DL, VT, VFCVTOp);
896 SmallVector<int, 8>
Mask = {0, 1, 4, 5, 2, 3, 6, 7};
908 EVT VT =
Op.getValueType();
913 assert((VT == MVT::f32 && Subtarget.hasBasicF()) ||
914 (VT == MVT::f64 && Subtarget.hasBasicD()));
931 int InsNum = Seq.size() + ((VT == MVT::f64 && !Subtarget.is64Bit()) ? 2 : 1);
941 if (Subtarget.is64Bit())
943 return DAG.
getNode(Subtarget.is64Bit() ? LoongArchISD::MOVGR2FR_W_LA64
944 : LoongArchISD::MOVGR2FR_W,
948 if (Subtarget.is64Bit()) {
950 return DAG.
getNode(LoongArchISD::MOVGR2FR_D,
DL, VT, NewVal);
954 return DAG.
getNode(LoongArchISD::MOVGR2FR_D_LO_HI,
DL, VT,
Lo,
Hi);
966 EVT ResultVT =
Op.getValueType();
967 EVT OperandVT =
Op.getOperand(0).getValueType();
972 if (ResultVT == SetCCResultVT)
975 assert(
Op.getOperand(0).getValueType() ==
Op.getOperand(1).getValueType() &&
976 "SETCC operands must have the same type!");
980 Op.getOperand(1),
Op.getOperand(2));
982 if (ResultVT.
bitsGT(SetCCResultVT))
984 else if (ResultVT.
bitsLT(SetCCResultVT))
996SDValue LoongArchTargetLowering::lowerSIGN_EXTEND_VECTOR_INREG(
1000 MVT SrcVT = Src.getSimpleValueType();
1001 MVT DstVT =
Op.getSimpleValueType();
1012 DAG.
getNode(LoongArchISD::VILVL,
DL, SrcVT, Mask, Src);
1032 MVT OpVT =
Op.getSimpleValueType();
1039 unsigned LegalVecSize = 128;
1040 bool isLASX256Vector =
1050 if (isLASX256Vector) {
1056 for (
unsigned i = 1; i < NumEles; i *= 2, EleBits *= 2) {
1057 EleBits = std::min(EleBits, 64u);
1060 Val = DAG.
getNode(LoongArchISD::VHADDW,
DL, VecTy, Val, Val);
1063 if (isLASX256Vector) {
1089 MVT OpVT =
Op.getSimpleValueType();
1102 MVT GRLenVT = Subtarget.getGRLenVT();
1104 for (
int i = NumEles; i > 1; i /= 2) {
1107 Val = DAG.
getNode(Opcode,
DL, VecTy, Tmp, Val);
1116 unsigned IsData =
Op.getConstantOperandVal(4);
1121 return Op.getOperand(0);
1128 MVT VT =
Op.getSimpleValueType();
1134 unsigned Opcode =
Op.getOpcode();
1137 auto checkCstSplat = [](
SDValue V, APInt &CstSplatValue) {
1143 CstSplatValue =
C->getAPIntValue();
1151 APInt CstSplatValue;
1152 bool IsCstSplat = checkCstSplat(Amt, CstSplatValue);
1156 if (IsCstSplat && CstSplatValue.
urem(EltSizeInBits) == 0)
1172 return DAG.
getNode(Opcode,
DL, VT, R, Urem);
1188 if (
LHS == LHS2 &&
RHS == RHS2) {
1193 }
else if (
LHS == RHS2 &&
RHS == LHS2) {
1201 return std::nullopt;
1209 MVT VT =
N->getSimpleValueType(0);
1240 if (~TrueVal == FalseVal) {
1280 unsigned SelOpNo = 0;
1290 unsigned ConstSelOpNo = 1;
1291 unsigned OtherSelOpNo = 2;
1298 if (!ConstSelOpNode || ConstSelOpNode->
isOpaque())
1303 if (!ConstBinOpNode || ConstBinOpNode->
isOpaque())
1309 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
1311 std::swap(NewConstOps[0], NewConstOps[1]);
1323 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
1325 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
1328 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
1329 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
1349 ShAmt =
LHS.getValueSizeInBits() - 1 -
Log2_64(Mask);
1363 int64_t
C = RHSC->getSExtValue();
1406 MVT VT =
Op.getSimpleValueType();
1407 MVT GRLenVT = Subtarget.getGRLenVT();
1412 if (
Op.hasOneUse()) {
1413 unsigned UseOpc =
Op->user_begin()->getOpcode();
1415 SDNode *BinOp = *
Op->user_begin();
1422 return lowerSELECT(NewSel, DAG);
1439 return DAG.
getNode(LoongArchISD::SELECT_CC,
DL, VT,
Ops);
1462 if (TrueVal - 1 == FalseVal)
1464 if (TrueVal + 1 == FalseVal)
1471 RHS == TrueV &&
LHS == FalseV) {
1496 return DAG.
getNode(LoongArchISD::SELECT_CC,
DL, VT,
Ops);
1503 MVT GRLenVT = Subtarget.getGRLenVT();
1514 return DAG.
getNode(LoongArchISD::BR_CC,
DL,
Op.getValueType(),
1515 Op.getOperand(0),
LHS,
RHS, TargetCC,
1518 return DAG.
getNode(LoongArchISD::BRCOND,
DL,
Op.getValueType(),
1519 Op.getOperand(0), CondV,
Op.getOperand(2));
1523 return DAG.
getNode(LoongArchISD::BR_CC,
DL,
Op.getValueType(),
1529LoongArchTargetLowering::lowerSCALAR_TO_VECTOR(
SDValue Op,
1532 MVT OpVT =
Op.getSimpleValueType();
1543 EVT ResTy =
Op->getValueType(0);
1548 if (!Subtarget.is64Bit() && (ResTy == MVT::v16i8 || ResTy == MVT::v32i8))
1558 for (
unsigned int i = 0; i < NewEltNum; i++) {
1561 unsigned RevOp = (ResTy == MVT::v16i8 || ResTy == MVT::v32i8)
1562 ? (
unsigned)LoongArchISD::BITREV_8B
1580 for (
unsigned int i = 0; i < NewEltNum; i++)
1581 for (
int j = OrigEltNum / NewEltNum - 1;
j >= 0;
j--)
1582 Mask.push_back(j + (OrigEltNum / NewEltNum) * i);
1600 if (EltBits > 32 || EltBits == 1)
1628 int MaskOffset,
const APInt &Zeroable) {
1629 int Size = Mask.size();
1630 unsigned SizeInBits =
Size * ScalarSizeInBits;
1632 auto CheckZeros = [&](
int Shift,
int Scale,
bool Left) {
1633 for (
int i = 0; i <
Size; i += Scale)
1634 for (
int j = 0; j < Shift; ++j)
1635 if (!Zeroable[i + j + (
Left ? 0 : (Scale - Shift))])
1643 for (
unsigned i = Pos, e = Pos +
Size; i != e; ++i,
Low += Step)
1644 if (!(Mask[i] == -1 || Mask[i] ==
Low))
1649 auto MatchShift = [&](
int Shift,
int Scale,
bool Left) {
1650 for (
int i = 0; i !=
Size; i += Scale) {
1651 unsigned Pos =
Left ? i + Shift : i;
1652 unsigned Low =
Left ? i : i + Shift;
1653 unsigned Len = Scale - Shift;
1658 int ShiftEltBits = ScalarSizeInBits * Scale;
1659 bool ByteShift = ShiftEltBits > 64;
1660 Opcode =
Left ? (ByteShift ? LoongArchISD::VBSLL : LoongArchISD::VSLLI)
1661 : (ByteShift ? LoongArchISD::VBSRL : LoongArchISD::VSRLI);
1662 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
1666 Scale = ByteShift ? Scale / 2 : Scale;
1672 return (
int)ShiftAmt;
1675 unsigned MaxWidth = 128;
1676 for (
int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
1677 for (
int Shift = 1; Shift != Scale; ++Shift)
1678 for (
bool Left : {
true,
false})
1679 if (CheckZeros(Shift, Scale,
Left)) {
1680 int ShiftAmt = MatchShift(Shift, Scale,
Left);
1705 const APInt &Zeroable) {
1706 int Size = Mask.size();
1720 Mask,
Size, Zeroable);
1728 "Illegal integer vector type");
1737template <
typename ValType>
1740 unsigned CheckStride,
1742 ValType ExpectedIndex,
unsigned ExpectedIndexStride) {
1746 if (*
I != -1 && *
I != ExpectedIndex)
1748 ExpectedIndex += ExpectedIndexStride;
1752 for (
unsigned n = 0; n < CheckStride &&
I != End; ++n, ++
I)
1764 int Size = Mask.size();
1773 int VectorSizeInBits =
V1.getValueSizeInBits();
1774 int ScalarSizeInBits = VectorSizeInBits /
Size;
1775 assert(!(VectorSizeInBits % ScalarSizeInBits) &&
"Illegal shuffle mask size");
1776 (void)ScalarSizeInBits;
1778 for (
int i = 0; i <
Size; ++i) {
1784 if ((M >= 0 && M <
Size && V1IsZero) || (M >=
Size && V2IsZero)) {
1801 RepeatedMask.
assign(LaneSize, -1);
1802 int Size = Mask.size();
1803 for (
int i = 0; i <
Size; ++i) {
1804 assert(Mask[i] == -1 || Mask[i] >= 0);
1807 if ((Mask[i] %
Size) / LaneSize != i / LaneSize)
1814 Mask[i] <
Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
1815 if (RepeatedMask[i % LaneSize] < 0)
1817 RepeatedMask[i % LaneSize] = LocalM;
1818 else if (RepeatedMask[i % LaneSize] != LocalM)
1835 int NumElts = RepeatedMask.
size();
1837 int Scale = 16 / NumElts;
1839 for (
int i = 0; i < NumElts; ++i) {
1840 int M = RepeatedMask[i];
1841 assert((M == -1 || (0 <= M && M < (2 * NumElts))) &&
1842 "Unexpected mask index.");
1847 int StartIdx = i - (M % NumElts);
1854 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
1857 Rotation = CandidateRotation;
1858 else if (Rotation != CandidateRotation)
1873 else if (TargetV != MaskV)
1878 assert(Rotation != 0 &&
"Failed to locate a viable rotation!");
1879 assert((
Lo ||
Hi) &&
"Failed to find a rotated input vector!");
1888 return Rotation * Scale;
1907 if (ByteRotation <= 0)
1914 int LoByteShift = 16 - ByteRotation;
1915 int HiByteShift = ByteRotation;
1938 const APInt &Zeroable) {
1952 for (
int i = 0; i < NumElements; i++) {
1956 if (i % Scale != 0) {
1968 M = M % NumElements;
1971 Offset = M - (i / Scale);
1974 if (
Offset % (NumElements / Scale))
1976 }
else if (InputV != V)
1979 if (M != (
Offset + (i / Scale)))
1989 unsigned VilVLoHi = LoongArchISD::VILVL;
1990 if (
Offset >= (NumElements / 2)) {
1991 VilVLoHi = LoongArchISD::VILVH;
1992 Offset -= (NumElements / 2);
1999 InputV = DAG.
getNode(VilVLoHi,
DL, InputVT, Ext, InputV);
2003 }
while (Scale > 1);
2009 for (
int NumExtElements = Bits / 64; NumExtElements < NumElements;
2010 NumExtElements *= 2) {
2030 int SplatIndex = -1;
2031 for (
const auto &M : Mask) {
2038 if (SplatIndex == -1)
2041 assert(SplatIndex < (
int)Mask.size() &&
"Out of bounds mask index");
2043 return DAG.
getNode(LoongArchISD::VREPLVEI,
DL, VT,
V1,
2073 unsigned SubVecSize = 4;
2074 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2077 int SubMask[4] = {-1, -1, -1, -1};
2078 for (
unsigned i = 0; i < SubVecSize; ++i) {
2079 for (
unsigned j = i; j < Mask.size(); j += SubVecSize) {
2085 M -= 4 * (j / SubVecSize);
2086 if (M < 0 || M >= 4)
2092 if (SubMask[i] == -1)
2096 else if (M != -1 && M != SubMask[i])
2103 for (
int i = SubVecSize - 1; i >= 0; --i) {
2116 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2117 return DAG.
getNode(LoongArchISD::VSHUF4I_D,
DL, VT,
V1, V2,
2120 return DAG.
getNode(LoongArchISD::VSHUF4I,
DL, VT,
V1,
2138 if (VT != MVT::v16i8 && VT != MVT::v8i16 && VT != MVT::v32i8 &&
2147 for (
int i = 0; i < WidenNumElts; ++i)
2148 WidenMask[i] = WidenNumElts - 1 - i;
2156 return DAG.
getNode(LoongArchISD::VSHUF4I,
DL, VT,
2180 const auto &Begin = Mask.begin();
2181 const auto &End = Mask.end();
2198 return DAG.
getNode(LoongArchISD::VPACKEV,
DL, VT, V2,
V1);
2220 const auto &Begin = Mask.begin();
2221 const auto &End = Mask.end();
2238 return DAG.
getNode(LoongArchISD::VPACKOD,
DL, VT, V2,
V1);
2261 const auto &Begin = Mask.begin();
2262 const auto &End = Mask.end();
2263 unsigned HalfSize = Mask.size() / 2;
2281 return DAG.
getNode(LoongArchISD::VILVH,
DL, VT, V2,
V1);
2304 const auto &Begin = Mask.begin();
2305 const auto &End = Mask.end();
2322 return DAG.
getNode(LoongArchISD::VILVL,
DL, VT, V2,
V1);
2344 const auto &Begin = Mask.begin();
2345 const auto &Mid = Mask.begin() + Mask.size() / 2;
2346 const auto &End = Mask.end();
2364 return DAG.
getNode(LoongArchISD::VPICKEV,
DL, VT, V2,
V1);
2386 const auto &Begin = Mask.begin();
2387 const auto &Mid = Mask.begin() + Mask.size() / 2;
2388 const auto &End = Mask.end();
2405 return DAG.
getNode(LoongArchISD::VPICKOD,
DL, VT, V2,
V1);
2431 if (Mask.size() != NumElts)
2434 auto tryLowerToExtrAndIns = [&](
unsigned Base) ->
SDValue {
2437 for (
unsigned i = 0; i < NumElts; ++i) {
2440 if (Mask[i] !=
int(
Base + i)) {
2453 int DiffMask = Mask[DiffPos];
2454 if (DiffMask < 0 || DiffMask >=
int(2 * NumElts))
2460 if (
unsigned(DiffMask) < NumElts) {
2465 SrcIdx =
unsigned(DiffMask) - NumElts;
2481 if (
SDValue Result = tryLowerToExtrAndIns(0))
2483 return tryLowerToExtrAndIns(NumElts);
2491 unsigned &MaskImm) {
2492 unsigned MaskSize = Mask.size();
2494 auto isValid = [&](
int M,
int Off) {
2495 return (M == -1) || (M >= Off && M < Off + 4);
2498 auto buildImm = [&](
int MLo,
int MHi,
unsigned Off,
unsigned I) {
2499 auto immPart = [&](
int M,
unsigned Off) {
2500 return (M == -1 ? 0 : (M - Off)) & 0x3;
2502 MaskImm |= immPart(MLo, Off) << (
I * 2);
2503 MaskImm |= immPart(MHi, Off) << ((
I + 1) * 2);
2506 for (
unsigned i = 0; i < 4; i += 2) {
2508 int MHi = Mask[i + 1];
2510 if (MaskSize == 8) {
2511 int M2Lo = Mask[i + 4];
2512 int M2Hi = Mask[i + 5];
2513 if (M2Lo != MLo + 4 || M2Hi != MHi + 4)
2519 buildImm(MLo, MHi, 0, i);
2522 buildImm(MLo, MHi, MaskSize, i);
2552 if ((VT != MVT::v4i32 && VT != MVT::v4f32) ||
2557 unsigned MaskImm = 0;
2561 return DAG.
getNode(LoongArchISD::VPERMI,
DL, VT, SrcVec[1], SrcVec[0],
2588 return DAG.
getNode(LoongArchISD::VSHUF,
DL, VT, MaskVec, V2,
V1);
2601 "Vector type is unsupported for lsx!");
2603 "Two operands have different types!");
2605 "Unexpected mask size for shuffle!");
2606 assert(Mask.size() % 2 == 0 &&
"Expected even mask size.");
2608 APInt KnownUndef, KnownZero;
2610 APInt Zeroable = KnownUndef | KnownZero;
2683 int SplatIndex = -1;
2684 for (
const auto &M : Mask) {
2691 if (SplatIndex == -1)
2694 const auto &Begin = Mask.begin();
2695 const auto &End = Mask.end();
2696 int HalfSize = Mask.size() / 2;
2698 if (SplatIndex >= HalfSize)
2701 assert(SplatIndex < (
int)Mask.size() &&
"Out of bounds mask index");
2705 return DAG.
getNode(LoongArchISD::VREPLVEI,
DL, VT,
V1,
2719 if (Mask.size() == 4) {
2720 unsigned MaskImm = 0;
2721 for (
int i = 1; i >= 0; --i) {
2723 int MHi = Mask[i + 2];
2724 if (!(MLo == -1 || (MLo >= 0 && MLo <= 1) || (MLo >= 4 && MLo <= 5)) ||
2725 !(MHi == -1 || (MHi >= 2 && MHi <= 3) || (MHi >= 6 && MHi <= 7)))
2727 if (MHi != -1 && MLo != -1 && MHi != MLo + 2)
2732 MaskImm |= ((MLo <= 1) ? MLo : (MLo - 2)) & 0x3;
2734 MaskImm |= ((MHi <= 3) ? (MHi - 2) : (MHi - 4)) & 0x3;
2737 return DAG.
getNode(LoongArchISD::VSHUF4I_D,
DL, VT,
V1, V2,
2750 unsigned MaskSize = Mask.size();
2755 if (VT == MVT::v8i32 || VT == MVT::v8f32) {
2757 unsigned MaskImm = 0;
2761 return DAG.
getNode(LoongArchISD::VPERMI,
DL, VT, SrcVec[1], SrcVec[0],
2766 if (VT == MVT::v4i64 || VT == MVT::v4f64) {
2767 unsigned MaskImm = 0;
2768 for (
unsigned i = 0; i < MaskSize; ++i) {
2771 if (Mask[i] >= (
int)MaskSize)
2773 MaskImm |= Mask[i] << (i * 2);
2776 return DAG.
getNode(LoongArchISD::XVPERMI,
DL, VT,
V1,
2788 if (Mask.size() != 8 || (VT != MVT::v8i32 && VT != MVT::v8f32))
2792 unsigned HalfSize = NumElts / 2;
2793 bool FrontLo =
true, FrontHi =
true;
2794 bool BackLo =
true, BackHi =
true;
2796 auto inRange = [](
int val,
int low,
int high) {
2797 return (val == -1) || (val >= low && val < high);
2800 for (
unsigned i = 0; i < HalfSize; ++i) {
2801 int Fronti = Mask[i];
2802 int Backi = Mask[i + HalfSize];
2804 FrontLo &=
inRange(Fronti, 0, HalfSize);
2805 FrontHi &=
inRange(Fronti, HalfSize, NumElts);
2806 BackLo &=
inRange(Backi, 0, HalfSize);
2807 BackHi &=
inRange(Backi, HalfSize, NumElts);
2813 if ((FrontLo || FrontHi) && (BackLo || BackHi))
2818 for (
unsigned i = 0; i < NumElts; ++i)
2823 return DAG.
getNode(LoongArchISD::XVPERM,
DL, VT,
V1, MaskVec);
2845 const auto &Begin = Mask.begin();
2846 const auto &End = Mask.end();
2847 unsigned HalfSize = Mask.size() / 2;
2848 unsigned LeftSize = HalfSize / 2;
2856 Mask.size() + HalfSize - LeftSize, 1) &&
2858 Mask.size() + HalfSize + LeftSize, 1))
2869 Mask.size() + HalfSize - LeftSize, 1) &&
2871 Mask.size() + HalfSize + LeftSize, 1))
2876 return DAG.
getNode(LoongArchISD::VILVH,
DL, VT, V2,
V1);
2884 const auto &Begin = Mask.begin();
2885 const auto &End = Mask.end();
2886 unsigned HalfSize = Mask.size() / 2;
2894 Mask.size() + HalfSize, 1))
2905 Mask.size() + HalfSize, 1))
2910 return DAG.
getNode(LoongArchISD::VILVL,
DL, VT, V2,
V1);
2918 const auto &Begin = Mask.begin();
2919 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
2920 const auto &Mid = Mask.begin() + Mask.size() / 2;
2921 const auto &RightMid = Mask.end() - Mask.size() / 4;
2922 const auto &End = Mask.end();
2923 unsigned HalfSize = Mask.size() / 2;
2945 return DAG.
getNode(LoongArchISD::VPICKEV,
DL, VT, V2,
V1);
2953 const auto &Begin = Mask.begin();
2954 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
2955 const auto &Mid = Mask.begin() + Mask.size() / 2;
2956 const auto &RightMid = Mask.end() - Mask.size() / 4;
2957 const auto &End = Mask.end();
2958 unsigned HalfSize = Mask.size() / 2;
2981 return DAG.
getNode(LoongArchISD::VPICKOD,
DL, VT, V2,
V1);
2990 int HalfSize = NumElts / 2;
2994 if ((
int)Mask.size() != NumElts)
2997 auto tryLowerToExtrAndIns = [&](
int Base) ->
SDValue {
2999 for (
int i = 0; i < NumElts; ++i) {
3002 if (Mask[i] !=
Base + i) {
3004 if (DiffPos.
size() > 2)
3012 if (DiffPos.
size() == 1) {
3013 if (DiffPos[0] < HalfSize && Mask[DiffPos[0] + HalfSize] == -1)
3014 DiffPos.
push_back(DiffPos[0] + HalfSize);
3015 else if (DiffPos[0] >= HalfSize && Mask[DiffPos[0] - HalfSize] == -1)
3016 DiffPos.
insert(DiffPos.
begin(), DiffPos[0] - HalfSize);
3020 if (DiffPos.
size() != 2 || DiffPos[1] != DiffPos[0] + HalfSize)
3024 int DiffMaskLo = Mask[DiffPos[0]];
3025 int DiffMaskHi = Mask[DiffPos[1]];
3026 DiffMaskLo = DiffMaskLo == -1 ? DiffMaskHi - HalfSize : DiffMaskLo;
3027 DiffMaskHi = DiffMaskHi == -1 ? DiffMaskLo + HalfSize : DiffMaskHi;
3028 if (!(DiffMaskLo >= 0 && DiffMaskLo < HalfSize) &&
3029 !(DiffMaskLo >= NumElts && DiffMaskLo < NumElts + HalfSize))
3031 if (!(DiffMaskHi >= HalfSize && DiffMaskHi < NumElts) &&
3032 !(DiffMaskHi >= NumElts + HalfSize && DiffMaskHi < 2 * NumElts))
3034 if (DiffMaskHi != DiffMaskLo + HalfSize)
3038 SDValue SrcVec = (DiffMaskLo < HalfSize) ?
V1 : V2;
3040 (DiffMaskLo < HalfSize) ? DiffMaskLo : (DiffMaskLo - NumElts);
3061 if (
SDValue Result = tryLowerToExtrAndIns(0))
3063 return tryLowerToExtrAndIns(NumElts);
3072 if (VT != MVT::v8i32 && VT != MVT::v8f32 && VT != MVT::v4i64 &&
3077 int MaskSize = Mask.size();
3083 auto checkReplaceOne = [&](
int Base,
int Replaced) ->
int {
3085 for (
int i = 0; i < MaskSize; ++i) {
3086 if (Mask[i] ==
Base + i || Mask[i] == -1)
3088 if (Mask[i] != Replaced)
3099 int Idx = checkReplaceOne(0, MaskSize);
3101 return DAG.
getNode(LoongArchISD::XVINSVE0,
DL, VT,
V1, V2,
3105 Idx = checkReplaceOne(MaskSize, 0);
3107 return DAG.
getNode(LoongArchISD::XVINSVE0,
DL, VT, V2,
V1,
3118 int MaskSize = Mask.size();
3119 int HalfSize = Mask.size() / 2;
3120 const auto &Begin = Mask.begin();
3121 const auto &Mid = Mask.begin() + HalfSize;
3122 const auto &End = Mask.end();
3134 for (
auto it = Begin; it < Mid; it++) {
3137 else if ((*it >= 0 && *it < HalfSize) ||
3138 (*it >= MaskSize && *it < MaskSize + HalfSize)) {
3139 int M = *it < HalfSize ? *it : *it - HalfSize;
3144 assert((
int)MaskAlloc.
size() == HalfSize &&
"xvshuf convert failed!");
3146 for (
auto it = Mid; it < End; it++) {
3149 else if ((*it >= HalfSize && *it < MaskSize) ||
3150 (*it >= MaskSize + HalfSize && *it < MaskSize * 2)) {
3151 int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize;
3156 assert((
int)MaskAlloc.
size() == MaskSize &&
"xvshuf convert failed!");
3160 return DAG.
getNode(LoongArchISD::VSHUF,
DL, VT, MaskVec, V2,
V1);
3188 enum HalfMaskType { HighLaneTy, LowLaneTy,
None };
3190 int MaskSize = Mask.size();
3191 int HalfSize = Mask.size() / 2;
3194 HalfMaskType preMask =
None, postMask =
None;
3196 if (std::all_of(Mask.begin(), Mask.begin() + HalfSize, [&](
int M) {
3197 return M < 0 || (M >= 0 && M < HalfSize) ||
3198 (M >= MaskSize && M < MaskSize + HalfSize);
3200 preMask = HighLaneTy;
3201 else if (std::all_of(Mask.begin(), Mask.begin() + HalfSize, [&](
int M) {
3202 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3203 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3205 preMask = LowLaneTy;
3207 if (std::all_of(Mask.begin() + HalfSize, Mask.end(), [&](
int M) {
3208 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3209 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3211 postMask = LowLaneTy;
3212 else if (std::all_of(Mask.begin() + HalfSize, Mask.end(), [&](
int M) {
3213 return M < 0 || (M >= 0 && M < HalfSize) ||
3214 (M >= MaskSize && M < MaskSize + HalfSize);
3216 postMask = HighLaneTy;
3224 if (preMask == HighLaneTy && postMask == LowLaneTy) {
3227 if (preMask == LowLaneTy && postMask == HighLaneTy) {
3235 V2 = DAG.
getNode(LoongArchISD::XVPERMI,
DL, MVT::v4i64, V2,
3240 for (
auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3241 *it = *it < 0 ? *it : *it - HalfSize;
3243 for (
auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3244 *it = *it < 0 ? *it : *it + HalfSize;
3246 }
else if (preMask == LowLaneTy && postMask == LowLaneTy) {
3254 V2 = DAG.
getNode(LoongArchISD::XVPERMI,
DL, MVT::v4i64, V2,
3259 for (
auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3260 *it = *it < 0 ? *it : *it - HalfSize;
3262 }
else if (preMask == HighLaneTy && postMask == HighLaneTy) {
3270 V2 = DAG.
getNode(LoongArchISD::XVPERMI,
DL, MVT::v4i64, V2,
3275 for (
auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3276 *it = *it < 0 ? *it : *it + HalfSize;
3301 int Size = Mask.size();
3302 int LaneSize =
Size / 2;
3304 bool LaneCrossing[2] = {
false,
false};
3305 for (
int i = 0; i <
Size; ++i)
3306 if (Mask[i] >= 0 && ((Mask[i] %
Size) / LaneSize) != (i / LaneSize))
3307 LaneCrossing[(Mask[i] %
Size) / LaneSize] =
true;
3310 if (!LaneCrossing[0] && !LaneCrossing[1])
3314 InLaneMask.
assign(Mask.begin(), Mask.end());
3315 for (
int i = 0; i <
Size; ++i) {
3316 int &M = InLaneMask[i];
3319 if (((M %
Size) / LaneSize) != (i / LaneSize))
3320 M = (M % LaneSize) + ((i / LaneSize) * LaneSize) +
Size;
3325 DAG.
getUNDEF(MVT::v4i64), {2, 3, 0, 1});
3340 "Vector type is unsupported for lasx!");
3342 "Two operands have different types!");
3344 "Unexpected mask size for shuffle!");
3345 assert(Mask.size() % 2 == 0 &&
"Expected even mask size.");
3346 assert(Mask.size() >= 4 &&
"Mask size is less than 4.");
3348 APInt KnownUndef, KnownZero;
3350 APInt Zeroable = KnownUndef | KnownZero;
3437 ArrayRef<int> OrigMask = SVOp->
getMask();
3440 MVT VT =
Op.getSimpleValueType();
3444 bool V1IsUndef =
V1.isUndef();
3445 bool V2IsUndef = V2.
isUndef();
3446 if (V1IsUndef && V2IsUndef)
3459 any_of(OrigMask, [NumElements](
int M) {
return M >= NumElements; })) {
3460 SmallVector<int, 8> NewMask(OrigMask);
3461 for (
int &M : NewMask)
3462 if (M >= NumElements)
3468 int MaskUpperLimit = OrigMask.
size() * (V2IsUndef ? 1 : 2);
3469 (void)MaskUpperLimit;
3471 [&](
int M) {
return -1 <=
M &&
M < MaskUpperLimit; }) &&
3472 "Out of bounds shuffle index");
3494 std::tie(Res, Chain) =
3495 makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions,
DL, Chain);
3496 if (Subtarget.is64Bit())
3497 return DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Res);
3509 SDValue Arg = Subtarget.is64Bit() ? DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
3513 std::tie(Res, Chain) =
makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
3514 CallOptions,
DL, Chain);
3520 assert(Subtarget.hasBasicF() &&
"Unexpected custom legalization");
3526 makeLibCall(DAG, LC, MVT::f32,
Op.getOperand(0), CallOptions,
DL).first;
3527 if (Subtarget.is64Bit())
3528 return DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Res);
3534 assert(Subtarget.hasBasicF() &&
"Unexpected custom legalization");
3535 MVT VT =
Op.getSimpleValueType();
3540 SDValue Res = Subtarget.is64Bit() ? DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
3561 "Unsupported vector type for broadcast.");
3564 bool IsIdeneity =
true;
3566 for (
int i = 0; i !=
NumOps; i++) {
3568 if (
Op.getOpcode() !=
ISD::LOAD || (IdentitySrc &&
Op != IdentitySrc)) {
3580 auto ExtType = LN->getExtensionType();
3585 assert(LN->isUnindexed() &&
"Unexpected indexed load.");
3590 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
3608 for (
unsigned i = 1; i <
Ops.size(); ++i) {
3622 EVT ResTy,
unsigned first) {
3625 assert(first + NumElts <= Node->getSimpleValueType(0).getVectorNumElements());
3628 Node->op_begin() + first + NumElts);
3637 MVT VT =
Node->getSimpleValueType(0);
3638 EVT ResTy =
Op->getValueType(0);
3641 APInt SplatValue, SplatUndef;
3642 unsigned SplatBitSize;
3645 bool UseSameConstant =
true;
3650 if ((!Subtarget.hasExtLSX() || !Is128Vec) &&
3651 (!Subtarget.hasExtLASX() || !Is256Vec))
3657 if (
Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
3659 SplatBitSize <= 64) {
3661 if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
3665 if (SplatBitSize == 64 && !Subtarget.is64Bit()) {
3672 if ((Is128Vec && ResTy == MVT::v4i32) ||
3673 (Is256Vec && ResTy == MVT::v8i32))
3679 switch (SplatBitSize) {
3683 ViaVecTy = Is128Vec ? MVT::v16i8 : MVT::v32i8;
3686 ViaVecTy = Is128Vec ? MVT::v8i16 : MVT::v16i16;
3689 ViaVecTy = Is128Vec ? MVT::v4i32 : MVT::v8i32;
3692 ViaVecTy = Is128Vec ? MVT::v2i64 : MVT::v4i64;
3700 if (ViaVecTy != ResTy)
3709 for (
unsigned i = 0; i < NumElts; ++i) {
3714 ConstantValue = Opi;
3715 else if (ConstantValue != Opi)
3716 UseSameConstant =
false;
3721 if (IsConstant && UseSameConstant && ResTy != MVT::v2f64) {
3723 for (
unsigned i = 0; i < NumElts; ++i) {
3741 BitVector UndefElements;
3742 if (
Node->getRepeatedSequence(Sequence, &UndefElements) &&
3743 UndefElements.
count() == 0) {
3747 EVT FillTy = Is256Vec
3753 fillVector(Sequence, DAG,
DL, Subtarget, FillVec, FillTy);
3756 unsigned SplatLen = NumElts / SeqLen;
3762 if (SplatEltTy == MVT::i128)
3763 SplatTy = MVT::v4i64;
3771 DAG.
getNode((SplatEltTy == MVT::i128) ? LoongArchISD::XVREPLVE0Q
3772 : LoongArchISD::XVREPLVE0,
3773 DL, SplatTy, SrcVec);
3775 SplatVec = DAG.
getNode(LoongArchISD::VREPLVEI,
DL, SplatTy, SrcVec,
3788 if (ResTy == MVT::v8i32 || ResTy == MVT::v8f32 || ResTy == MVT::v4i64 ||
3789 ResTy == MVT::v4f64) {
3790 unsigned NonUndefCount = 0;
3791 for (
unsigned i = NumElts / 2; i < NumElts; ++i) {
3792 if (!
Node->getOperand(i).isUndef()) {
3794 if (NonUndefCount > 1)
3798 if (NonUndefCount == 1)
3811 VecTy, NumElts / 2);
3822 MVT ResVT =
Op.getSimpleValueType();
3826 unsigned NumFreezeUndef = 0;
3827 unsigned NumZero = 0;
3828 unsigned NumNonZero = 0;
3829 unsigned NonZeros = 0;
3830 SmallSet<SDValue, 4> Undefs;
3831 for (
unsigned i = 0; i != NumOperands; ++i) {
3846 assert(i <
sizeof(NonZeros) * CHAR_BIT);
3853 if (NumNonZero > 2) {
3857 Ops.slice(0, NumOperands / 2));
3859 Ops.slice(NumOperands / 2));
3872 MVT SubVT =
Op.getOperand(0).getSimpleValueType();
3874 for (
unsigned i = 0; i != NumOperands; ++i) {
3875 if ((NonZeros & (1 << i)) == 0)
3886LoongArchTargetLowering::lowerEXTRACT_VECTOR_ELT(
SDValue Op,
3888 MVT EltVT =
Op.getSimpleValueType();
3893 MVT GRLenVT = Subtarget.getGRLenVT();
3921 ? DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
DL, MVT::f32, Idx)
3925 DAG.
getBitcast((VecTy == MVT::v4f64) ? MVT::v4i64 : VecTy, IdxVec);
3927 DAG.
getNode(LoongArchISD::VSHUF,
DL, VecTy, MaskVec, TmpVec, Vec);
3936 DAG.
getNode(LoongArchISD::XVPERM,
DL, VecTy, Vec, SplatIdx);
3945LoongArchTargetLowering::lowerINSERT_VECTOR_ELT(
SDValue Op,
3947 MVT VT =
Op.getSimpleValueType();
3970 if (!Subtarget.is64Bit() && IdxTy == MVT::i64) {
3972 for (
unsigned i = 0; i < NumElts; ++i) {
3980 for (
unsigned i = 0; i < NumElts; ++i) {
3989 for (
unsigned i = 0; i < NumElts; ++i)
4021 if (Subtarget.is64Bit() &&
Op.getOperand(2).getValueType() == MVT::i32) {
4023 "On LA64, only 64-bit registers can be written.");
4024 return Op.getOperand(0);
4027 if (!Subtarget.is64Bit() &&
Op.getOperand(2).getValueType() == MVT::i64) {
4029 "On LA32, only 32-bit registers can be written.");
4030 return Op.getOperand(0);
4040 "be a constant integer");
4046 Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF);
4047 EVT VT =
Op.getValueType();
4050 unsigned Depth =
Op.getConstantOperandVal(0);
4051 int GRLenInBytes = Subtarget.getGRLen() / 8;
4054 int Offset = -(GRLenInBytes * 2);
4066 if (
Op.getConstantOperandVal(0) != 0) {
4068 "return address can only be determined for the current frame");
4074 MVT GRLenVT = Subtarget.getGRLenVT();
4086 auto Size = Subtarget.getGRLen() / 8;
4094 auto *FuncInfo = MF.
getInfo<LoongArchMachineFunctionInfo>();
4104 MachinePointerInfo(SV));
4111 EVT VT =
Op.getValueType();
4119 if (Subtarget.hasExtLSX() && Op0VT == MVT::i64 &&
4120 (VT == MVT::f32 || VT == MVT::f64)) {
4130 if (!Subtarget.is64Bit() || !Subtarget.hasBasicF() || Subtarget.hasBasicD())
4133 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4134 !Subtarget.hasBasicD() &&
"unexpected target features");
4138 if (
C &&
C->getZExtValue() < UINT64_C(0xFFFFFFFF))
4142 if (Op0->
getOpcode() == LoongArchISD::BSTRPICK &&
4152 EVT RetVT =
Op.getValueType();
4158 std::tie(Result, Chain) =
4165 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4166 !Subtarget.hasBasicD() &&
"unexpected target features");
4177 EVT RetVT =
Op.getValueType();
4183 std::tie(Result, Chain) =
4192 EVT VT =
Op.getValueType();
4196 if (
Op.getValueType() == MVT::f32 && Op0VT == MVT::i32 &&
4197 Subtarget.is64Bit() && Subtarget.hasBasicF()) {
4199 return DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
DL, MVT::f32, NewOp0);
4201 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit()) {
4204 return DAG.
getNode(LoongArchISD::BUILD_PAIR_F64,
DL, MVT::f64,
Lo,
Hi);
4218 if (
Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&
4219 !Subtarget.hasBasicD()) {
4221 return DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Dst);
4231 if (!Subtarget.hasExtLSX())
4236 EVT VT =
Op.getValueType();
4237 EVT SrcVT = Src.getValueType();
4242 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4245 if (SrcVT == MVT::f32)
4267 N->getOffset(), Flags);
4275template <
class NodeTy>
4278 bool IsLocal)
const {
4289 assert(Subtarget.is64Bit() &&
"Large code model requires LA64");
4370 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
4372 const GlobalValue *GV =
N->getGlobal();
4384 unsigned Opc,
bool UseGOT,
4388 MVT GRLenVT = Subtarget.getGRLenVT();
4402 if (
Opc == LoongArch::PseudoLA_TLS_LE && !Large)
4440 Args.emplace_back(Load, CallTy);
4443 TargetLowering::CallLoweringInfo CLI(DAG);
4458 const GlobalValue *GV =
N->getGlobal();
4472LoongArchTargetLowering::lowerGlobalTLSAddress(
SDValue Op,
4479 assert((!Large || Subtarget.is64Bit()) &&
"Large code model requires LA64");
4482 assert(
N->getOffset() == 0 &&
"unexpected offset in global node");
4495 return getDynamicTLSAddr(
N, DAG,
4496 Large ? LoongArch::PseudoLA_TLS_GD_LARGE
4497 : LoongArch::PseudoLA_TLS_GD,
4504 return getDynamicTLSAddr(
N, DAG,
4505 Large ? LoongArch::PseudoLA_TLS_LD_LARGE
4506 : LoongArch::PseudoLA_TLS_LD,
4511 return getStaticTLSAddr(
N, DAG,
4512 Large ? LoongArch::PseudoLA_TLS_IE_LARGE
4513 : LoongArch::PseudoLA_TLS_IE,
4520 return getStaticTLSAddr(
N, DAG, LoongArch::PseudoLA_TLS_LE,
4524 return getTLSDescAddr(
N, DAG,
4525 Large ? LoongArch::PseudoLA_TLS_DESC_LARGE
4526 : LoongArch::PseudoLA_TLS_DESC,
4530template <
unsigned N>
4535 if ((IsSigned && !
isInt<N>(CImm->getSExtValue())) ||
4536 (!IsSigned && !
isUInt<N>(CImm->getZExtValue()))) {
4538 ": argument out of range.");
4545LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(
SDValue Op,
4547 switch (
Op.getConstantOperandVal(0)) {
4550 case Intrinsic::thread_pointer: {
4554 case Intrinsic::loongarch_lsx_vpickve2gr_d:
4555 case Intrinsic::loongarch_lsx_vpickve2gr_du:
4556 case Intrinsic::loongarch_lsx_vreplvei_d:
4557 case Intrinsic::loongarch_lasx_xvrepl128vei_d:
4559 case Intrinsic::loongarch_lsx_vreplvei_w:
4560 case Intrinsic::loongarch_lasx_xvrepl128vei_w:
4561 case Intrinsic::loongarch_lasx_xvpickve2gr_d:
4562 case Intrinsic::loongarch_lasx_xvpickve2gr_du:
4563 case Intrinsic::loongarch_lasx_xvpickve_d:
4564 case Intrinsic::loongarch_lasx_xvpickve_d_f:
4566 case Intrinsic::loongarch_lasx_xvinsve0_d:
4568 case Intrinsic::loongarch_lsx_vsat_b:
4569 case Intrinsic::loongarch_lsx_vsat_bu:
4570 case Intrinsic::loongarch_lsx_vrotri_b:
4571 case Intrinsic::loongarch_lsx_vsllwil_h_b:
4572 case Intrinsic::loongarch_lsx_vsllwil_hu_bu:
4573 case Intrinsic::loongarch_lsx_vsrlri_b:
4574 case Intrinsic::loongarch_lsx_vsrari_b:
4575 case Intrinsic::loongarch_lsx_vreplvei_h:
4576 case Intrinsic::loongarch_lasx_xvsat_b:
4577 case Intrinsic::loongarch_lasx_xvsat_bu:
4578 case Intrinsic::loongarch_lasx_xvrotri_b:
4579 case Intrinsic::loongarch_lasx_xvsllwil_h_b:
4580 case Intrinsic::loongarch_lasx_xvsllwil_hu_bu:
4581 case Intrinsic::loongarch_lasx_xvsrlri_b:
4582 case Intrinsic::loongarch_lasx_xvsrari_b:
4583 case Intrinsic::loongarch_lasx_xvrepl128vei_h:
4584 case Intrinsic::loongarch_lasx_xvpickve_w:
4585 case Intrinsic::loongarch_lasx_xvpickve_w_f:
4587 case Intrinsic::loongarch_lasx_xvinsve0_w:
4589 case Intrinsic::loongarch_lsx_vsat_h:
4590 case Intrinsic::loongarch_lsx_vsat_hu:
4591 case Intrinsic::loongarch_lsx_vrotri_h:
4592 case Intrinsic::loongarch_lsx_vsllwil_w_h:
4593 case Intrinsic::loongarch_lsx_vsllwil_wu_hu:
4594 case Intrinsic::loongarch_lsx_vsrlri_h:
4595 case Intrinsic::loongarch_lsx_vsrari_h:
4596 case Intrinsic::loongarch_lsx_vreplvei_b:
4597 case Intrinsic::loongarch_lasx_xvsat_h:
4598 case Intrinsic::loongarch_lasx_xvsat_hu:
4599 case Intrinsic::loongarch_lasx_xvrotri_h:
4600 case Intrinsic::loongarch_lasx_xvsllwil_w_h:
4601 case Intrinsic::loongarch_lasx_xvsllwil_wu_hu:
4602 case Intrinsic::loongarch_lasx_xvsrlri_h:
4603 case Intrinsic::loongarch_lasx_xvsrari_h:
4604 case Intrinsic::loongarch_lasx_xvrepl128vei_b:
4606 case Intrinsic::loongarch_lsx_vsrlni_b_h:
4607 case Intrinsic::loongarch_lsx_vsrani_b_h:
4608 case Intrinsic::loongarch_lsx_vsrlrni_b_h:
4609 case Intrinsic::loongarch_lsx_vsrarni_b_h:
4610 case Intrinsic::loongarch_lsx_vssrlni_b_h:
4611 case Intrinsic::loongarch_lsx_vssrani_b_h:
4612 case Intrinsic::loongarch_lsx_vssrlni_bu_h:
4613 case Intrinsic::loongarch_lsx_vssrani_bu_h:
4614 case Intrinsic::loongarch_lsx_vssrlrni_b_h:
4615 case Intrinsic::loongarch_lsx_vssrarni_b_h:
4616 case Intrinsic::loongarch_lsx_vssrlrni_bu_h:
4617 case Intrinsic::loongarch_lsx_vssrarni_bu_h:
4618 case Intrinsic::loongarch_lasx_xvsrlni_b_h:
4619 case Intrinsic::loongarch_lasx_xvsrani_b_h:
4620 case Intrinsic::loongarch_lasx_xvsrlrni_b_h:
4621 case Intrinsic::loongarch_lasx_xvsrarni_b_h:
4622 case Intrinsic::loongarch_lasx_xvssrlni_b_h:
4623 case Intrinsic::loongarch_lasx_xvssrani_b_h:
4624 case Intrinsic::loongarch_lasx_xvssrlni_bu_h:
4625 case Intrinsic::loongarch_lasx_xvssrani_bu_h:
4626 case Intrinsic::loongarch_lasx_xvssrlrni_b_h:
4627 case Intrinsic::loongarch_lasx_xvssrarni_b_h:
4628 case Intrinsic::loongarch_lasx_xvssrlrni_bu_h:
4629 case Intrinsic::loongarch_lasx_xvssrarni_bu_h:
4631 case Intrinsic::loongarch_lsx_vsat_w:
4632 case Intrinsic::loongarch_lsx_vsat_wu:
4633 case Intrinsic::loongarch_lsx_vrotri_w:
4634 case Intrinsic::loongarch_lsx_vsllwil_d_w:
4635 case Intrinsic::loongarch_lsx_vsllwil_du_wu:
4636 case Intrinsic::loongarch_lsx_vsrlri_w:
4637 case Intrinsic::loongarch_lsx_vsrari_w:
4638 case Intrinsic::loongarch_lsx_vslei_bu:
4639 case Intrinsic::loongarch_lsx_vslei_hu:
4640 case Intrinsic::loongarch_lsx_vslei_wu:
4641 case Intrinsic::loongarch_lsx_vslei_du:
4642 case Intrinsic::loongarch_lsx_vslti_bu:
4643 case Intrinsic::loongarch_lsx_vslti_hu:
4644 case Intrinsic::loongarch_lsx_vslti_wu:
4645 case Intrinsic::loongarch_lsx_vslti_du:
4646 case Intrinsic::loongarch_lsx_vbsll_v:
4647 case Intrinsic::loongarch_lsx_vbsrl_v:
4648 case Intrinsic::loongarch_lasx_xvsat_w:
4649 case Intrinsic::loongarch_lasx_xvsat_wu:
4650 case Intrinsic::loongarch_lasx_xvrotri_w:
4651 case Intrinsic::loongarch_lasx_xvsllwil_d_w:
4652 case Intrinsic::loongarch_lasx_xvsllwil_du_wu:
4653 case Intrinsic::loongarch_lasx_xvsrlri_w:
4654 case Intrinsic::loongarch_lasx_xvsrari_w:
4655 case Intrinsic::loongarch_lasx_xvslei_bu:
4656 case Intrinsic::loongarch_lasx_xvslei_hu:
4657 case Intrinsic::loongarch_lasx_xvslei_wu:
4658 case Intrinsic::loongarch_lasx_xvslei_du:
4659 case Intrinsic::loongarch_lasx_xvslti_bu:
4660 case Intrinsic::loongarch_lasx_xvslti_hu:
4661 case Intrinsic::loongarch_lasx_xvslti_wu:
4662 case Intrinsic::loongarch_lasx_xvslti_du:
4663 case Intrinsic::loongarch_lasx_xvbsll_v:
4664 case Intrinsic::loongarch_lasx_xvbsrl_v:
4666 case Intrinsic::loongarch_lsx_vseqi_b:
4667 case Intrinsic::loongarch_lsx_vseqi_h:
4668 case Intrinsic::loongarch_lsx_vseqi_w:
4669 case Intrinsic::loongarch_lsx_vseqi_d:
4670 case Intrinsic::loongarch_lsx_vslei_b:
4671 case Intrinsic::loongarch_lsx_vslei_h:
4672 case Intrinsic::loongarch_lsx_vslei_w:
4673 case Intrinsic::loongarch_lsx_vslei_d:
4674 case Intrinsic::loongarch_lsx_vslti_b:
4675 case Intrinsic::loongarch_lsx_vslti_h:
4676 case Intrinsic::loongarch_lsx_vslti_w:
4677 case Intrinsic::loongarch_lsx_vslti_d:
4678 case Intrinsic::loongarch_lasx_xvseqi_b:
4679 case Intrinsic::loongarch_lasx_xvseqi_h:
4680 case Intrinsic::loongarch_lasx_xvseqi_w:
4681 case Intrinsic::loongarch_lasx_xvseqi_d:
4682 case Intrinsic::loongarch_lasx_xvslei_b:
4683 case Intrinsic::loongarch_lasx_xvslei_h:
4684 case Intrinsic::loongarch_lasx_xvslei_w:
4685 case Intrinsic::loongarch_lasx_xvslei_d:
4686 case Intrinsic::loongarch_lasx_xvslti_b:
4687 case Intrinsic::loongarch_lasx_xvslti_h:
4688 case Intrinsic::loongarch_lasx_xvslti_w:
4689 case Intrinsic::loongarch_lasx_xvslti_d:
4691 case Intrinsic::loongarch_lsx_vsrlni_h_w:
4692 case Intrinsic::loongarch_lsx_vsrani_h_w:
4693 case Intrinsic::loongarch_lsx_vsrlrni_h_w:
4694 case Intrinsic::loongarch_lsx_vsrarni_h_w:
4695 case Intrinsic::loongarch_lsx_vssrlni_h_w:
4696 case Intrinsic::loongarch_lsx_vssrani_h_w:
4697 case Intrinsic::loongarch_lsx_vssrlni_hu_w:
4698 case Intrinsic::loongarch_lsx_vssrani_hu_w:
4699 case Intrinsic::loongarch_lsx_vssrlrni_h_w:
4700 case Intrinsic::loongarch_lsx_vssrarni_h_w:
4701 case Intrinsic::loongarch_lsx_vssrlrni_hu_w:
4702 case Intrinsic::loongarch_lsx_vssrarni_hu_w:
4703 case Intrinsic::loongarch_lsx_vfrstpi_b:
4704 case Intrinsic::loongarch_lsx_vfrstpi_h:
4705 case Intrinsic::loongarch_lasx_xvsrlni_h_w:
4706 case Intrinsic::loongarch_lasx_xvsrani_h_w:
4707 case Intrinsic::loongarch_lasx_xvsrlrni_h_w:
4708 case Intrinsic::loongarch_lasx_xvsrarni_h_w:
4709 case Intrinsic::loongarch_lasx_xvssrlni_h_w:
4710 case Intrinsic::loongarch_lasx_xvssrani_h_w:
4711 case Intrinsic::loongarch_lasx_xvssrlni_hu_w:
4712 case Intrinsic::loongarch_lasx_xvssrani_hu_w:
4713 case Intrinsic::loongarch_lasx_xvssrlrni_h_w:
4714 case Intrinsic::loongarch_lasx_xvssrarni_h_w:
4715 case Intrinsic::loongarch_lasx_xvssrlrni_hu_w:
4716 case Intrinsic::loongarch_lasx_xvssrarni_hu_w:
4717 case Intrinsic::loongarch_lasx_xvfrstpi_b:
4718 case Intrinsic::loongarch_lasx_xvfrstpi_h:
4720 case Intrinsic::loongarch_lsx_vsat_d:
4721 case Intrinsic::loongarch_lsx_vsat_du:
4722 case Intrinsic::loongarch_lsx_vrotri_d:
4723 case Intrinsic::loongarch_lsx_vsrlri_d:
4724 case Intrinsic::loongarch_lsx_vsrari_d:
4725 case Intrinsic::loongarch_lasx_xvsat_d:
4726 case Intrinsic::loongarch_lasx_xvsat_du:
4727 case Intrinsic::loongarch_lasx_xvrotri_d:
4728 case Intrinsic::loongarch_lasx_xvsrlri_d:
4729 case Intrinsic::loongarch_lasx_xvsrari_d:
4731 case Intrinsic::loongarch_lsx_vsrlni_w_d:
4732 case Intrinsic::loongarch_lsx_vsrani_w_d:
4733 case Intrinsic::loongarch_lsx_vsrlrni_w_d:
4734 case Intrinsic::loongarch_lsx_vsrarni_w_d:
4735 case Intrinsic::loongarch_lsx_vssrlni_w_d:
4736 case Intrinsic::loongarch_lsx_vssrani_w_d:
4737 case Intrinsic::loongarch_lsx_vssrlni_wu_d:
4738 case Intrinsic::loongarch_lsx_vssrani_wu_d:
4739 case Intrinsic::loongarch_lsx_vssrlrni_w_d:
4740 case Intrinsic::loongarch_lsx_vssrarni_w_d:
4741 case Intrinsic::loongarch_lsx_vssrlrni_wu_d:
4742 case Intrinsic::loongarch_lsx_vssrarni_wu_d:
4743 case Intrinsic::loongarch_lasx_xvsrlni_w_d:
4744 case Intrinsic::loongarch_lasx_xvsrani_w_d:
4745 case Intrinsic::loongarch_lasx_xvsrlrni_w_d:
4746 case Intrinsic::loongarch_lasx_xvsrarni_w_d:
4747 case Intrinsic::loongarch_lasx_xvssrlni_w_d:
4748 case Intrinsic::loongarch_lasx_xvssrani_w_d:
4749 case Intrinsic::loongarch_lasx_xvssrlni_wu_d:
4750 case Intrinsic::loongarch_lasx_xvssrani_wu_d:
4751 case Intrinsic::loongarch_lasx_xvssrlrni_w_d:
4752 case Intrinsic::loongarch_lasx_xvssrarni_w_d:
4753 case Intrinsic::loongarch_lasx_xvssrlrni_wu_d:
4754 case Intrinsic::loongarch_lasx_xvssrarni_wu_d:
4756 case Intrinsic::loongarch_lsx_vsrlni_d_q:
4757 case Intrinsic::loongarch_lsx_vsrani_d_q:
4758 case Intrinsic::loongarch_lsx_vsrlrni_d_q:
4759 case Intrinsic::loongarch_lsx_vsrarni_d_q:
4760 case Intrinsic::loongarch_lsx_vssrlni_d_q:
4761 case Intrinsic::loongarch_lsx_vssrani_d_q:
4762 case Intrinsic::loongarch_lsx_vssrlni_du_q:
4763 case Intrinsic::loongarch_lsx_vssrani_du_q:
4764 case Intrinsic::loongarch_lsx_vssrlrni_d_q:
4765 case Intrinsic::loongarch_lsx_vssrarni_d_q:
4766 case Intrinsic::loongarch_lsx_vssrlrni_du_q:
4767 case Intrinsic::loongarch_lsx_vssrarni_du_q:
4768 case Intrinsic::loongarch_lasx_xvsrlni_d_q:
4769 case Intrinsic::loongarch_lasx_xvsrani_d_q:
4770 case Intrinsic::loongarch_lasx_xvsrlrni_d_q:
4771 case Intrinsic::loongarch_lasx_xvsrarni_d_q:
4772 case Intrinsic::loongarch_lasx_xvssrlni_d_q:
4773 case Intrinsic::loongarch_lasx_xvssrani_d_q:
4774 case Intrinsic::loongarch_lasx_xvssrlni_du_q:
4775 case Intrinsic::loongarch_lasx_xvssrani_du_q:
4776 case Intrinsic::loongarch_lasx_xvssrlrni_d_q:
4777 case Intrinsic::loongarch_lasx_xvssrarni_d_q:
4778 case Intrinsic::loongarch_lasx_xvssrlrni_du_q:
4779 case Intrinsic::loongarch_lasx_xvssrarni_du_q:
4781 case Intrinsic::loongarch_lsx_vnori_b:
4782 case Intrinsic::loongarch_lsx_vshuf4i_b:
4783 case Intrinsic::loongarch_lsx_vshuf4i_h:
4784 case Intrinsic::loongarch_lsx_vshuf4i_w:
4785 case Intrinsic::loongarch_lasx_xvnori_b:
4786 case Intrinsic::loongarch_lasx_xvshuf4i_b:
4787 case Intrinsic::loongarch_lasx_xvshuf4i_h:
4788 case Intrinsic::loongarch_lasx_xvshuf4i_w:
4789 case Intrinsic::loongarch_lasx_xvpermi_d:
4791 case Intrinsic::loongarch_lsx_vshuf4i_d:
4792 case Intrinsic::loongarch_lsx_vpermi_w:
4793 case Intrinsic::loongarch_lsx_vbitseli_b:
4794 case Intrinsic::loongarch_lsx_vextrins_b:
4795 case Intrinsic::loongarch_lsx_vextrins_h:
4796 case Intrinsic::loongarch_lsx_vextrins_w:
4797 case Intrinsic::loongarch_lsx_vextrins_d:
4798 case Intrinsic::loongarch_lasx_xvshuf4i_d:
4799 case Intrinsic::loongarch_lasx_xvpermi_w:
4800 case Intrinsic::loongarch_lasx_xvpermi_q:
4801 case Intrinsic::loongarch_lasx_xvbitseli_b:
4802 case Intrinsic::loongarch_lasx_xvextrins_b:
4803 case Intrinsic::loongarch_lasx_xvextrins_h:
4804 case Intrinsic::loongarch_lasx_xvextrins_w:
4805 case Intrinsic::loongarch_lasx_xvextrins_d:
4807 case Intrinsic::loongarch_lsx_vrepli_b:
4808 case Intrinsic::loongarch_lsx_vrepli_h:
4809 case Intrinsic::loongarch_lsx_vrepli_w:
4810 case Intrinsic::loongarch_lsx_vrepli_d:
4811 case Intrinsic::loongarch_lasx_xvrepli_b:
4812 case Intrinsic::loongarch_lasx_xvrepli_h:
4813 case Intrinsic::loongarch_lasx_xvrepli_w:
4814 case Intrinsic::loongarch_lasx_xvrepli_d:
4816 case Intrinsic::loongarch_lsx_vldi:
4817 case Intrinsic::loongarch_lasx_xvldi:
4833LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(
SDValue Op,
4836 MVT GRLenVT = Subtarget.getGRLenVT();
4837 EVT VT =
Op.getValueType();
4839 const StringRef ErrorMsgOOR =
"argument out of range";
4840 const StringRef ErrorMsgReqLA64 =
"requires loongarch64";
4841 const StringRef ErrorMsgReqF =
"requires basic 'f' target feature";
4843 switch (
Op.getConstantOperandVal(1)) {
4846 case Intrinsic::loongarch_crc_w_b_w:
4847 case Intrinsic::loongarch_crc_w_h_w:
4848 case Intrinsic::loongarch_crc_w_w_w:
4849 case Intrinsic::loongarch_crc_w_d_w:
4850 case Intrinsic::loongarch_crcc_w_b_w:
4851 case Intrinsic::loongarch_crcc_w_h_w:
4852 case Intrinsic::loongarch_crcc_w_w_w:
4853 case Intrinsic::loongarch_crcc_w_d_w:
4855 case Intrinsic::loongarch_csrrd_w:
4856 case Intrinsic::loongarch_csrrd_d: {
4857 unsigned Imm =
Op.getConstantOperandVal(2);
4860 : DAG.
getNode(LoongArchISD::CSRRD,
DL, {GRLenVT, MVT::Other},
4863 case Intrinsic::loongarch_csrwr_w:
4864 case Intrinsic::loongarch_csrwr_d: {
4865 unsigned Imm =
Op.getConstantOperandVal(3);
4868 : DAG.
getNode(LoongArchISD::CSRWR,
DL, {GRLenVT, MVT::Other},
4869 {Chain,
Op.getOperand(2),
4872 case Intrinsic::loongarch_csrxchg_w:
4873 case Intrinsic::loongarch_csrxchg_d: {
4874 unsigned Imm =
Op.getConstantOperandVal(4);
4877 : DAG.
getNode(LoongArchISD::CSRXCHG,
DL, {GRLenVT, MVT::Other},
4878 {Chain,
Op.getOperand(2),
Op.getOperand(3),
4881 case Intrinsic::loongarch_iocsrrd_d: {
4883 LoongArchISD::IOCSRRD_D,
DL, {GRLenVT, MVT::Other},
4886#define IOCSRRD_CASE(NAME, NODE) \
4887 case Intrinsic::loongarch_##NAME: { \
4888 return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other}, \
4889 {Chain, Op.getOperand(2)}); \
4895 case Intrinsic::loongarch_cpucfg: {
4896 return DAG.
getNode(LoongArchISD::CPUCFG,
DL, {GRLenVT, MVT::Other},
4897 {Chain,
Op.getOperand(2)});
4899 case Intrinsic::loongarch_lddir_d: {
4900 unsigned Imm =
Op.getConstantOperandVal(3);
4905 case Intrinsic::loongarch_movfcsr2gr: {
4906 if (!Subtarget.hasBasicF())
4908 unsigned Imm =
Op.getConstantOperandVal(2);
4911 : DAG.
getNode(LoongArchISD::MOVFCSR2GR,
DL, {VT, MVT::Other},
4914 case Intrinsic::loongarch_lsx_vld:
4915 case Intrinsic::loongarch_lsx_vldrepl_b:
4916 case Intrinsic::loongarch_lasx_xvld:
4917 case Intrinsic::loongarch_lasx_xvldrepl_b:
4921 case Intrinsic::loongarch_lsx_vldrepl_h:
4922 case Intrinsic::loongarch_lasx_xvldrepl_h:
4926 Op,
"argument out of range or not a multiple of 2", DAG)
4928 case Intrinsic::loongarch_lsx_vldrepl_w:
4929 case Intrinsic::loongarch_lasx_xvldrepl_w:
4933 Op,
"argument out of range or not a multiple of 4", DAG)
4935 case Intrinsic::loongarch_lsx_vldrepl_d:
4936 case Intrinsic::loongarch_lasx_xvldrepl_d:
4940 Op,
"argument out of range or not a multiple of 8", DAG)
4951 return Op.getOperand(0);
4957 MVT GRLenVT = Subtarget.getGRLenVT();
4959 uint64_t IntrinsicEnum =
Op.getConstantOperandVal(1);
4961 const StringRef ErrorMsgOOR =
"argument out of range";
4962 const StringRef ErrorMsgReqLA64 =
"requires loongarch64";
4963 const StringRef ErrorMsgReqLA32 =
"requires loongarch32";
4964 const StringRef ErrorMsgReqF =
"requires basic 'f' target feature";
4966 switch (IntrinsicEnum) {
4970 case Intrinsic::loongarch_cacop_d:
4971 case Intrinsic::loongarch_cacop_w: {
4972 if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit())
4974 if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit())
4983 case Intrinsic::loongarch_dbar: {
4990 case Intrinsic::loongarch_ibar: {
4997 case Intrinsic::loongarch_break: {
5004 case Intrinsic::loongarch_movgr2fcsr: {
5005 if (!Subtarget.hasBasicF())
5015 case Intrinsic::loongarch_syscall: {
5022#define IOCSRWR_CASE(NAME, NODE) \
5023 case Intrinsic::loongarch_##NAME: { \
5024 SDValue Op3 = Op.getOperand(3); \
5025 return Subtarget.is64Bit() \
5026 ? DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, \
5027 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5028 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3)) \
5029 : DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, Op2, \
5036 case Intrinsic::loongarch_iocsrwr_d: {
5037 return !Subtarget.is64Bit()
5044#define ASRT_LE_GT_CASE(NAME) \
5045 case Intrinsic::loongarch_##NAME: { \
5046 return !Subtarget.is64Bit() \
5047 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG) \
5052#undef ASRT_LE_GT_CASE
5053 case Intrinsic::loongarch_ldpte_d: {
5054 unsigned Imm =
Op.getConstantOperandVal(3);
5055 return !Subtarget.is64Bit()
5060 case Intrinsic::loongarch_lsx_vst:
5061 case Intrinsic::loongarch_lasx_xvst:
5065 case Intrinsic::loongarch_lasx_xvstelm_b:
5070 case Intrinsic::loongarch_lsx_vstelm_b:
5075 case Intrinsic::loongarch_lasx_xvstelm_h:
5080 Op,
"argument out of range or not a multiple of 2", DAG)
5082 case Intrinsic::loongarch_lsx_vstelm_h:
5087 Op,
"argument out of range or not a multiple of 2", DAG)
5089 case Intrinsic::loongarch_lasx_xvstelm_w:
5094 Op,
"argument out of range or not a multiple of 4", DAG)
5096 case Intrinsic::loongarch_lsx_vstelm_w:
5101 Op,
"argument out of range or not a multiple of 4", DAG)
5103 case Intrinsic::loongarch_lasx_xvstelm_d:
5108 Op,
"argument out of range or not a multiple of 8", DAG)
5110 case Intrinsic::loongarch_lsx_vstelm_d:
5115 Op,
"argument out of range or not a multiple of 8", DAG)
5126 EVT VT =
Lo.getValueType();
5167 EVT VT =
Lo.getValueType();
5221 return LoongArchISD::DIV_W;
5223 return LoongArchISD::DIV_WU;
5225 return LoongArchISD::MOD_W;
5227 return LoongArchISD::MOD_WU;
5229 return LoongArchISD::SLL_W;
5231 return LoongArchISD::SRA_W;
5233 return LoongArchISD::SRL_W;
5236 return LoongArchISD::ROTR_W;
5238 return LoongArchISD::CTZ_W;
5240 return LoongArchISD::CLZ_W;
5259 NewOp0 = DAG.
getNode(ExtOpc,
DL, MVT::i64,
N->getOperand(0));
5260 NewRes = DAG.
getNode(WOpcode,
DL, MVT::i64, NewOp0);
5264 NewOp0 = DAG.
getNode(ExtOpc,
DL, MVT::i64,
N->getOperand(0));
5270 NewRes = DAG.
getNode(WOpcode,
DL, MVT::i64, NewOp0, NewOp1);
5297 StringRef ErrorMsg,
bool WithChain =
true) {
5302 Results.push_back(
N->getOperand(0));
5305template <
unsigned N>
5310 const StringRef ErrorMsgOOR =
"argument out of range";
5311 unsigned Imm =
Node->getConstantOperandVal(2);
5345 switch (
N->getConstantOperandVal(0)) {
5348 case Intrinsic::loongarch_lsx_vpickve2gr_b:
5350 LoongArchISD::VPICK_SEXT_ELT);
5352 case Intrinsic::loongarch_lsx_vpickve2gr_h:
5353 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
5355 LoongArchISD::VPICK_SEXT_ELT);
5357 case Intrinsic::loongarch_lsx_vpickve2gr_w:
5359 LoongArchISD::VPICK_SEXT_ELT);
5361 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
5363 LoongArchISD::VPICK_ZEXT_ELT);
5365 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
5366 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
5368 LoongArchISD::VPICK_ZEXT_ELT);
5370 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
5372 LoongArchISD::VPICK_ZEXT_ELT);
5374 case Intrinsic::loongarch_lsx_bz_b:
5375 case Intrinsic::loongarch_lsx_bz_h:
5376 case Intrinsic::loongarch_lsx_bz_w:
5377 case Intrinsic::loongarch_lsx_bz_d:
5378 case Intrinsic::loongarch_lasx_xbz_b:
5379 case Intrinsic::loongarch_lasx_xbz_h:
5380 case Intrinsic::loongarch_lasx_xbz_w:
5381 case Intrinsic::loongarch_lasx_xbz_d:
5383 LoongArchISD::VALL_ZERO);
5385 case Intrinsic::loongarch_lsx_bz_v:
5386 case Intrinsic::loongarch_lasx_xbz_v:
5388 LoongArchISD::VANY_ZERO);
5390 case Intrinsic::loongarch_lsx_bnz_b:
5391 case Intrinsic::loongarch_lsx_bnz_h:
5392 case Intrinsic::loongarch_lsx_bnz_w:
5393 case Intrinsic::loongarch_lsx_bnz_d:
5394 case Intrinsic::loongarch_lasx_xbnz_b:
5395 case Intrinsic::loongarch_lasx_xbnz_h:
5396 case Intrinsic::loongarch_lasx_xbnz_w:
5397 case Intrinsic::loongarch_lasx_xbnz_d:
5399 LoongArchISD::VALL_NONZERO);
5401 case Intrinsic::loongarch_lsx_bnz_v:
5402 case Intrinsic::loongarch_lasx_xbnz_v:
5404 LoongArchISD::VANY_NONZERO);
5412 assert(
N->getValueType(0) == MVT::i128 &&
5413 "AtomicCmpSwap on types less than 128 should be legal");
5417 switch (
MemOp->getMergedOrdering()) {
5421 Opcode = LoongArch::PseudoCmpXchg128Acquire;
5425 Opcode = LoongArch::PseudoCmpXchg128;
5432 auto CmpVal = DAG.
SplitScalar(
N->getOperand(2),
DL, MVT::i64, MVT::i64);
5433 auto NewVal = DAG.
SplitScalar(
N->getOperand(3),
DL, MVT::i64, MVT::i64);
5434 SDValue Ops[] = {
N->getOperand(1), CmpVal.first, CmpVal.second,
5435 NewVal.first, NewVal.second,
N->getOperand(0)};
5438 Opcode,
SDLoc(
N), DAG.
getVTList(MVT::i64, MVT::i64, MVT::i64, MVT::Other),
5449 EVT VT =
N->getValueType(0);
5450 switch (
N->getOpcode()) {
5455 assert(
N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5456 "Unexpected custom legalisation");
5463 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5464 "Unexpected custom legalisation");
5466 Subtarget.hasDiv32() && VT == MVT::i32
5473 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5474 "Unexpected custom legalisation");
5482 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5483 "Unexpected custom legalisation");
5490 MVT VT =
N->getSimpleValueType(0);
5491 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5492 "Unexpected custom legalisation");
5494 "Unexpected type action!");
5499 Ld->getPointerInfo(), Ld->getBaseAlign(),
5500 Ld->getMemOperand()->getFlags());
5511 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5512 "Unexpected custom legalisation");
5519 if (Src.getValueType() == MVT::f16)
5530 EVT OpVT = Src.getValueType();
5534 std::tie(Result, Chain) =
5541 EVT SrcVT = Src.getValueType();
5542 if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() &&
5543 Subtarget.hasBasicF()) {
5545 DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Src);
5547 }
else if (VT == MVT::i64 && SrcVT == MVT::f64 && !Subtarget.is64Bit()) {
5549 DAG.
getVTList(MVT::i32, MVT::i32), Src);
5557 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5558 "Unexpected custom legalisation");
5561 TLI.expandFP_TO_UINT(
N, Tmp1, Tmp2, DAG);
5566 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5567 "Unexpected custom legalisation");
5573 if (OpVT == MVT::v2f64) {
5583 assert((VT == MVT::i16 || VT == MVT::i32) &&
5584 "Unexpected custom legalization");
5585 MVT GRLenVT = Subtarget.getGRLenVT();
5592 Tmp = DAG.
getNode(LoongArchISD::REVB_2H,
DL, GRLenVT, NewSrc);
5597 Tmp = DAG.
getNode(LoongArchISD::REVB_2W,
DL, GRLenVT, NewSrc);
5605 assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
5606 "Unexpected custom legalization");
5607 MVT GRLenVT = Subtarget.getGRLenVT();
5614 Tmp = DAG.
getNode(LoongArchISD::BITREV_4B,
DL, GRLenVT, NewSrc);
5617 Tmp = DAG.
getNode(LoongArchISD::BITREV_W,
DL, GRLenVT, NewSrc);
5625 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5626 "Unexpected custom legalisation");
5633 MVT GRLenVT = Subtarget.getGRLenVT();
5634 const StringRef ErrorMsgOOR =
"argument out of range";
5635 const StringRef ErrorMsgReqLA64 =
"requires loongarch64";
5636 const StringRef ErrorMsgReqF =
"requires basic 'f' target feature";
5638 switch (
N->getConstantOperandVal(1)) {
5641 case Intrinsic::loongarch_movfcsr2gr: {
5642 if (!Subtarget.hasBasicF()) {
5652 LoongArchISD::MOVFCSR2GR,
SDLoc(
N), {MVT::i64, MVT::Other},
5659#define CRC_CASE_EXT_BINARYOP(NAME, NODE) \
5660 case Intrinsic::loongarch_##NAME: { \
5661 SDValue NODE = DAG.getNode( \
5662 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5663 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5664 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
5665 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
5666 Results.push_back(NODE.getValue(1)); \
5675#undef CRC_CASE_EXT_BINARYOP
5677#define CRC_CASE_EXT_UNARYOP(NAME, NODE) \
5678 case Intrinsic::loongarch_##NAME: { \
5679 SDValue NODE = DAG.getNode( \
5680 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5682 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
5683 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
5684 Results.push_back(NODE.getValue(1)); \
5689#undef CRC_CASE_EXT_UNARYOP
5690#define CSR_CASE(ID) \
5691 case Intrinsic::loongarch_##ID: { \
5692 if (!Subtarget.is64Bit()) \
5693 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64); \
5701 case Intrinsic::loongarch_csrrd_w: {
5708 DAG.
getNode(LoongArchISD::CSRRD,
DL, {GRLenVT, MVT::Other},
5715 case Intrinsic::loongarch_csrwr_w: {
5716 unsigned Imm =
N->getConstantOperandVal(3);
5722 DAG.
getNode(LoongArchISD::CSRWR,
DL, {GRLenVT, MVT::Other},
5730 case Intrinsic::loongarch_csrxchg_w: {
5731 unsigned Imm =
N->getConstantOperandVal(4);
5737 LoongArchISD::CSRXCHG,
DL, {GRLenVT, MVT::Other},
5746#define IOCSRRD_CASE(NAME, NODE) \
5747 case Intrinsic::loongarch_##NAME: { \
5748 SDValue IOCSRRDResults = \
5749 DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
5750 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \
5751 Results.push_back( \
5752 DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0))); \
5753 Results.push_back(IOCSRRDResults.getValue(1)); \
5760 case Intrinsic::loongarch_cpucfg: {
5762 DAG.
getNode(LoongArchISD::CPUCFG,
DL, {GRLenVT, MVT::Other},
5769 case Intrinsic::loongarch_lddir_d: {
5770 if (!Subtarget.is64Bit()) {
5780 if (Subtarget.is64Bit())
5782 "On LA64, only 64-bit registers can be read.");
5785 "On LA32, only 32-bit registers can be read.");
5787 Results.push_back(
N->getOperand(0));
5798 OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
5811 MVT VT =
N->getSimpleValueType(0);
5817 EVT InVT = In.getValueType();
5828 for (
unsigned I = 0;
I < MinElts; ++
I)
5829 TruncMask[
I] = Scale *
I;
5831 unsigned WidenNumElts = 128 / In.getScalarValueSizeInBits();
5832 MVT SVT = In.getSimpleValueType().getScalarType();
5838 "Illegal vector type in truncation");
5850 if (!Subtarget.hasExtLSX() || Subtarget.hasExtLASX())
5853 EVT DstVT =
N->getValueType(0);
5855 MVT SrcVT = Src.getSimpleValueType();
5869 unsigned WidenSrcElts = 128 / SrcEltBits;
5876 unsigned FirstStageEltBits = 128 / NumElts;
5880 SrcVT = FirstStageVT;
5881 SrcEltBits = FirstStageEltBits;
5888 while (SrcEltBits < DstEltBits) {
5889 unsigned NextEltBits = SrcEltBits * 2;
5892 unsigned NextEltsPerBlock = CurEltsPerBlock / 2;
5910 Blocks = std::move(NextBlocks);
5911 SrcVT = NextBlockVT;
5912 SrcEltBits = NextEltBits;
5929 assert(
N->getOpcode() ==
ISD::AND &&
"Unexpected opcode combine into ANDN");
5931 MVT VT =
N->getSimpleValueType(0);
5950 return DAG.
getNode(LoongArchISD::VANDN,
DL, VT,
X,
Y);
5954 unsigned MinSizeInBits) {
5962 unsigned SplatBitSize;
5965 return Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
5966 HasAnyUndefs, MinSizeInBits,
5978 for (
unsigned i = 0, NumElts = BV->getNumOperands(); i < NumElts; ++i) {
5989 unsigned EltIdx = IdxC->getZExtValue();
5991 Start = (int)EltIdx - (
int)(i * 2);
5992 if (Start < 0 || Start > 1 || EltIdx != (
unsigned)(Start + (
int)(i * 2)))
5998 else if (Src != CurSrc)
6002 if (!Src || Start < 0)
6012 if (!Subtarget.hasExtLSX())
6015 unsigned Opc =
N->getOpcode();
6018 EVT VT =
N->getValueType(0);
6025 unsigned ExtOpc =
LHS.getOpcode();
6033 if (ExtOpc !=
RHS.getOpcode())
6036 if (!
LHS.hasOneUse() || !
RHS.hasOneUse())
6039 unsigned OddIdx, EvenIdx;
6043 if (!LHSVec || !RHSVec)
6045 if (OddIdx != 1 || EvenIdx != 0)
6055 if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(SrcVT))
6068 TargetOpc =
isSigned ? LoongArchISD::VHADDW : LoongArchISD::VHADDW_U;
6070 TargetOpc =
isSigned ? LoongArchISD::VHSUBW : LoongArchISD::VHSUBW_U;
6072 return DAG.
getNode(TargetOpc,
DL, VT, LHSVec, RHSVec);
6084 EVT VT =
N->getValueType(0);
6150 if (
And.getOperand(0) ==
X) {
6179 if (ShiftVal != (SplatVal + 1))
6188 : LoongArchISD::VSRAR,
6198 SDValue FirstOperand =
N->getOperand(0);
6199 SDValue SecondOperand =
N->getOperand(1);
6200 unsigned FirstOperandOpc = FirstOperand.
getOpcode();
6201 EVT ValTy =
N->getValueType(0);
6204 unsigned SMIdx, SMLen;
6213 if (!Subtarget.has32S())
6269 NewOperand = FirstOperand;
6272 msb = lsb + SMLen - 1;
6276 if (FirstOperandOpc ==
ISD::SRA || FirstOperandOpc ==
ISD::SRL || lsb == 0)
6290 EVT DstVT =
N.getValueType();
6297 EVT SrcVT = Src.getValueType();
6301 if (
N.getConstantOperandVal(1) != 0)
6317 if (NumElts % 2 != 0)
6323 for (
unsigned I = 0;
I != NumElts; ++
I) {
6336 SrcVT = Src.getValueType();
6346 }
else if (ThisSrc != Src) {
6350 unsigned Half = NumElts / 2;
6351 unsigned ExpectedIdx = (
I < Half) ?
I : (
I + Half);
6352 if (CI->getZExtValue() != ExpectedIdx)
6362 if (!Subtarget.hasExtLSX())
6367 EVT VT =
N->getValueType(0);
6374 unsigned ExtOpc =
LHS.getOpcode();
6382 if (!
LHS.hasOneUse())
6408 unsigned Opc =
isSigned ? LoongArchISD::VSLLWIL : LoongArchISD::VSLLWIL_U;
6417 if (!Subtarget.has32S())
6429 SDValue FirstOperand =
N->getOperand(0);
6431 EVT ValTy =
N->getValueType(0);
6434 unsigned MaskIdx, MaskLen;
6449 if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1)
6450 return DAG.
getNode(LoongArchISD::BSTRPICK,
DL, ValTy,
6474 switch (Src.getOpcode()) {
6477 return Src.getOperand(0).getValueSizeInBits() ==
Size;
6487 return Src.getOperand(0).getScalarValueSizeInBits() == 1 &&
6500 switch (Src.getOpcode()) {
6510 Src.getOpcode(),
DL, SExtVT,
6516 DL, SExtVT, Src.getOperand(0),
6528 EVT VT =
N->getValueType(0);
6530 EVT SrcVT = Src.getValueType();
6532 if (Src.getOpcode() !=
ISD::SETCC || !Src.hasOneUse())
6537 EVT CmpVT = Src.getOperand(0).getValueType();
6542 else if (Subtarget.has32S() && Subtarget.hasExtLASX() &&
6555 Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ;
6560 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
6565 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
6570 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
6572 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6577 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
6579 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6584 Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ;
6601 EVT VT =
N->getValueType(0);
6603 EVT SrcVT = Src.getValueType();
6620 bool UseLASX =
false;
6621 bool PropagateSExt =
false;
6623 if (Src.getOpcode() ==
ISD::SETCC && Src.hasOneUse()) {
6624 EVT CmpVT = Src.getOperand(0).getValueType();
6633 SExtVT = MVT::v2i64;
6636 SExtVT = MVT::v4i32;
6638 SExtVT = MVT::v4i64;
6640 PropagateSExt =
true;
6644 SExtVT = MVT::v8i16;
6646 SExtVT = MVT::v8i32;
6648 PropagateSExt =
true;
6652 SExtVT = MVT::v16i8;
6654 SExtVT = MVT::v16i16;
6656 PropagateSExt =
true;
6660 SExtVT = MVT::v32i8;
6668 if (!Subtarget.has32S() || !Subtarget.hasExtLASX()) {
6669 if (Src.getSimpleValueType() == MVT::v32i8) {
6677 }
else if (UseLASX) {
6683 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
6696 EVT ValTy =
N->getValueType(0);
6697 SDValue N0 =
N->getOperand(0), N1 =
N->getOperand(1);
6701 unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1;
6703 bool SwapAndRetried =
false;
6706 if (!Subtarget.has32S())
6712 if (ValBits != 32 && ValBits != 64)
6727 MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 &&
6730 (MaskIdx0 + MaskLen0 <= ValBits)) {
6751 MaskLen0 == MaskLen1 && MaskIdx1 == 0 &&
6752 (MaskIdx0 + MaskLen0 <= ValBits)) {
6769 (MaskIdx0 + MaskLen0 <= 64) &&
6777 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
6778 : (MaskIdx0 + MaskLen0 - 1),
6794 (MaskIdx0 + MaskLen0 <= ValBits)) {
6817 DAG.
getConstant(ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
6818 : (MaskIdx0 + MaskLen0 - 1),
6833 unsigned MaskIdx, MaskLen;
6834 if (N1.getOpcode() ==
ISD::SHL && N1.getOperand(0).getOpcode() ==
ISD::AND &&
6843 return DAG.
getNode(LoongArchISD::BSTRINS,
DL, ValTy, N0,
6861 N1.getOperand(0).getOpcode() ==
ISD::SHL &&
6867 return DAG.
getNode(LoongArchISD::BSTRINS,
DL, ValTy, N0,
6875 if (!SwapAndRetried) {
6877 SwapAndRetried =
true;
6881 SwapAndRetried =
false;
6898 return DAG.
getNode(LoongArchISD::BSTRINS,
DL, ValTy, N0,
6907 if (!SwapAndRetried) {
6909 SwapAndRetried =
true;
6919 switch (V.getNode()->getOpcode()) {
6931 if ((TypeNode->
getVT() == MVT::i8) || (TypeNode->
getVT() == MVT::i16)) {
6939 if ((TypeNode->
getVT() == MVT::i8) || (TypeNode->
getVT() == MVT::i16)) {
7016 SDNode *AndNode =
N->getOperand(0).getNode();
7024 SDValue CmpInputValue =
N->getOperand(1);
7033 if (!CN || !CN->
isZero())
7035 AndInputValue1 = AndInputValue1.
getOperand(0);
7039 if (AndInputValue2 != CmpInputValue)
7072 TruncInputValue1, TruncInputValue2);
7074 DAG.
getSetCC(
SDLoc(
N),
N->getValueType(0), NewAnd, TruncInputValue2, CC);
7087 if (Src.getOpcode() != LoongArchISD::REVB_2W)
7090 return DAG.
getNode(LoongArchISD::BITREV_4B,
SDLoc(
N),
N->getValueType(0),
7115 LHS.getOperand(0).getValueType() == Subtarget.
getGRLenVT()) {
7143 ShAmt =
LHS.getValueSizeInBits() - 1 - ShAmt;
7176 return DAG.
getNode(LoongArchISD::BR_CC,
DL,
N->getValueType(0),
7177 N->getOperand(0),
LHS,
RHS, CC,
N->getOperand(4));
7193 EVT VT =
N->getValueType(0);
7196 if (TrueV == FalseV)
7227 return DAG.
getNode(LoongArchISD::SELECT_CC,
DL,
N->getValueType(0),
7228 {LHS, RHS, CC, TrueV, FalseV});
7233template <
unsigned N>
7237 bool IsSigned =
false) {
7241 if ((IsSigned && !
isInt<N>(CImm->getSExtValue())) ||
7242 (!IsSigned && !
isUInt<N>(CImm->getZExtValue()))) {
7244 ": argument out of range.");
7250template <
unsigned N>
7254 EVT ResTy =
Node->getValueType(0);
7258 if ((IsSigned && !
isInt<N>(CImm->getSExtValue())) ||
7259 (!IsSigned && !
isUInt<N>(CImm->getZExtValue()))) {
7261 ": argument out of range.");
7266 IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned),
7272 EVT ResTy =
Node->getValueType(0);
7280 EVT ResTy =
Node->getValueType(0);
7289template <
unsigned N>
7292 EVT ResTy =
Node->getValueType(0);
7297 ": argument out of range.");
7307template <
unsigned N>
7310 EVT ResTy =
Node->getValueType(0);
7315 ": argument out of range.");
7324template <
unsigned N>
7327 EVT ResTy =
Node->getValueType(0);
7332 ": argument out of range.");
7341template <
unsigned W>
7344 unsigned Imm =
N->getConstantOperandVal(2);
7346 const StringRef ErrorMsg =
"argument out of range";
7348 return DAG.
getUNDEF(
N->getValueType(0));
7354 return DAG.
getNode(ResOp,
DL,
N->getValueType(0), Vec, Idx, EltVT);
7362 switch (
N->getConstantOperandVal(0)) {
7365 case Intrinsic::loongarch_lsx_vadd_b:
7366 case Intrinsic::loongarch_lsx_vadd_h:
7367 case Intrinsic::loongarch_lsx_vadd_w:
7368 case Intrinsic::loongarch_lsx_vadd_d:
7369 case Intrinsic::loongarch_lasx_xvadd_b:
7370 case Intrinsic::loongarch_lasx_xvadd_h:
7371 case Intrinsic::loongarch_lasx_xvadd_w:
7372 case Intrinsic::loongarch_lasx_xvadd_d:
7375 case Intrinsic::loongarch_lsx_vaddi_bu:
7376 case Intrinsic::loongarch_lsx_vaddi_hu:
7377 case Intrinsic::loongarch_lsx_vaddi_wu:
7378 case Intrinsic::loongarch_lsx_vaddi_du:
7379 case Intrinsic::loongarch_lasx_xvaddi_bu:
7380 case Intrinsic::loongarch_lasx_xvaddi_hu:
7381 case Intrinsic::loongarch_lasx_xvaddi_wu:
7382 case Intrinsic::loongarch_lasx_xvaddi_du:
7385 case Intrinsic::loongarch_lsx_vsub_b:
7386 case Intrinsic::loongarch_lsx_vsub_h:
7387 case Intrinsic::loongarch_lsx_vsub_w:
7388 case Intrinsic::loongarch_lsx_vsub_d:
7389 case Intrinsic::loongarch_lasx_xvsub_b:
7390 case Intrinsic::loongarch_lasx_xvsub_h:
7391 case Intrinsic::loongarch_lasx_xvsub_w:
7392 case Intrinsic::loongarch_lasx_xvsub_d:
7395 case Intrinsic::loongarch_lsx_vsubi_bu:
7396 case Intrinsic::loongarch_lsx_vsubi_hu:
7397 case Intrinsic::loongarch_lsx_vsubi_wu:
7398 case Intrinsic::loongarch_lsx_vsubi_du:
7399 case Intrinsic::loongarch_lasx_xvsubi_bu:
7400 case Intrinsic::loongarch_lasx_xvsubi_hu:
7401 case Intrinsic::loongarch_lasx_xvsubi_wu:
7402 case Intrinsic::loongarch_lasx_xvsubi_du:
7405 case Intrinsic::loongarch_lsx_vneg_b:
7406 case Intrinsic::loongarch_lsx_vneg_h:
7407 case Intrinsic::loongarch_lsx_vneg_w:
7408 case Intrinsic::loongarch_lsx_vneg_d:
7409 case Intrinsic::loongarch_lasx_xvneg_b:
7410 case Intrinsic::loongarch_lasx_xvneg_h:
7411 case Intrinsic::loongarch_lasx_xvneg_w:
7412 case Intrinsic::loongarch_lasx_xvneg_d:
7416 APInt(
N->getValueType(0).getScalarType().getSizeInBits(), 0,
7418 SDLoc(
N),
N->getValueType(0)),
7420 case Intrinsic::loongarch_lsx_vmax_b:
7421 case Intrinsic::loongarch_lsx_vmax_h:
7422 case Intrinsic::loongarch_lsx_vmax_w:
7423 case Intrinsic::loongarch_lsx_vmax_d:
7424 case Intrinsic::loongarch_lasx_xvmax_b:
7425 case Intrinsic::loongarch_lasx_xvmax_h:
7426 case Intrinsic::loongarch_lasx_xvmax_w:
7427 case Intrinsic::loongarch_lasx_xvmax_d:
7430 case Intrinsic::loongarch_lsx_vmax_bu:
7431 case Intrinsic::loongarch_lsx_vmax_hu:
7432 case Intrinsic::loongarch_lsx_vmax_wu:
7433 case Intrinsic::loongarch_lsx_vmax_du:
7434 case Intrinsic::loongarch_lasx_xvmax_bu:
7435 case Intrinsic::loongarch_lasx_xvmax_hu:
7436 case Intrinsic::loongarch_lasx_xvmax_wu:
7437 case Intrinsic::loongarch_lasx_xvmax_du:
7440 case Intrinsic::loongarch_lsx_vmaxi_b:
7441 case Intrinsic::loongarch_lsx_vmaxi_h:
7442 case Intrinsic::loongarch_lsx_vmaxi_w:
7443 case Intrinsic::loongarch_lsx_vmaxi_d:
7444 case Intrinsic::loongarch_lasx_xvmaxi_b:
7445 case Intrinsic::loongarch_lasx_xvmaxi_h:
7446 case Intrinsic::loongarch_lasx_xvmaxi_w:
7447 case Intrinsic::loongarch_lasx_xvmaxi_d:
7450 case Intrinsic::loongarch_lsx_vmaxi_bu:
7451 case Intrinsic::loongarch_lsx_vmaxi_hu:
7452 case Intrinsic::loongarch_lsx_vmaxi_wu:
7453 case Intrinsic::loongarch_lsx_vmaxi_du:
7454 case Intrinsic::loongarch_lasx_xvmaxi_bu:
7455 case Intrinsic::loongarch_lasx_xvmaxi_hu:
7456 case Intrinsic::loongarch_lasx_xvmaxi_wu:
7457 case Intrinsic::loongarch_lasx_xvmaxi_du:
7460 case Intrinsic::loongarch_lsx_vmin_b:
7461 case Intrinsic::loongarch_lsx_vmin_h:
7462 case Intrinsic::loongarch_lsx_vmin_w:
7463 case Intrinsic::loongarch_lsx_vmin_d:
7464 case Intrinsic::loongarch_lasx_xvmin_b:
7465 case Intrinsic::loongarch_lasx_xvmin_h:
7466 case Intrinsic::loongarch_lasx_xvmin_w:
7467 case Intrinsic::loongarch_lasx_xvmin_d:
7470 case Intrinsic::loongarch_lsx_vmin_bu:
7471 case Intrinsic::loongarch_lsx_vmin_hu:
7472 case Intrinsic::loongarch_lsx_vmin_wu:
7473 case Intrinsic::loongarch_lsx_vmin_du:
7474 case Intrinsic::loongarch_lasx_xvmin_bu:
7475 case Intrinsic::loongarch_lasx_xvmin_hu:
7476 case Intrinsic::loongarch_lasx_xvmin_wu:
7477 case Intrinsic::loongarch_lasx_xvmin_du:
7480 case Intrinsic::loongarch_lsx_vmini_b:
7481 case Intrinsic::loongarch_lsx_vmini_h:
7482 case Intrinsic::loongarch_lsx_vmini_w:
7483 case Intrinsic::loongarch_lsx_vmini_d:
7484 case Intrinsic::loongarch_lasx_xvmini_b:
7485 case Intrinsic::loongarch_lasx_xvmini_h:
7486 case Intrinsic::loongarch_lasx_xvmini_w:
7487 case Intrinsic::loongarch_lasx_xvmini_d:
7490 case Intrinsic::loongarch_lsx_vmini_bu:
7491 case Intrinsic::loongarch_lsx_vmini_hu:
7492 case Intrinsic::loongarch_lsx_vmini_wu:
7493 case Intrinsic::loongarch_lsx_vmini_du:
7494 case Intrinsic::loongarch_lasx_xvmini_bu:
7495 case Intrinsic::loongarch_lasx_xvmini_hu:
7496 case Intrinsic::loongarch_lasx_xvmini_wu:
7497 case Intrinsic::loongarch_lasx_xvmini_du:
7500 case Intrinsic::loongarch_lsx_vmul_b:
7501 case Intrinsic::loongarch_lsx_vmul_h:
7502 case Intrinsic::loongarch_lsx_vmul_w:
7503 case Intrinsic::loongarch_lsx_vmul_d:
7504 case Intrinsic::loongarch_lasx_xvmul_b:
7505 case Intrinsic::loongarch_lasx_xvmul_h:
7506 case Intrinsic::loongarch_lasx_xvmul_w:
7507 case Intrinsic::loongarch_lasx_xvmul_d:
7510 case Intrinsic::loongarch_lsx_vmadd_b:
7511 case Intrinsic::loongarch_lsx_vmadd_h:
7512 case Intrinsic::loongarch_lsx_vmadd_w:
7513 case Intrinsic::loongarch_lsx_vmadd_d:
7514 case Intrinsic::loongarch_lasx_xvmadd_b:
7515 case Intrinsic::loongarch_lasx_xvmadd_h:
7516 case Intrinsic::loongarch_lasx_xvmadd_w:
7517 case Intrinsic::loongarch_lasx_xvmadd_d: {
7518 EVT ResTy =
N->getValueType(0);
7523 case Intrinsic::loongarch_lsx_vmsub_b:
7524 case Intrinsic::loongarch_lsx_vmsub_h:
7525 case Intrinsic::loongarch_lsx_vmsub_w:
7526 case Intrinsic::loongarch_lsx_vmsub_d:
7527 case Intrinsic::loongarch_lasx_xvmsub_b:
7528 case Intrinsic::loongarch_lasx_xvmsub_h:
7529 case Intrinsic::loongarch_lasx_xvmsub_w:
7530 case Intrinsic::loongarch_lasx_xvmsub_d: {
7531 EVT ResTy =
N->getValueType(0);
7536 case Intrinsic::loongarch_lsx_vdiv_b:
7537 case Intrinsic::loongarch_lsx_vdiv_h:
7538 case Intrinsic::loongarch_lsx_vdiv_w:
7539 case Intrinsic::loongarch_lsx_vdiv_d:
7540 case Intrinsic::loongarch_lasx_xvdiv_b:
7541 case Intrinsic::loongarch_lasx_xvdiv_h:
7542 case Intrinsic::loongarch_lasx_xvdiv_w:
7543 case Intrinsic::loongarch_lasx_xvdiv_d:
7546 case Intrinsic::loongarch_lsx_vdiv_bu:
7547 case Intrinsic::loongarch_lsx_vdiv_hu:
7548 case Intrinsic::loongarch_lsx_vdiv_wu:
7549 case Intrinsic::loongarch_lsx_vdiv_du:
7550 case Intrinsic::loongarch_lasx_xvdiv_bu:
7551 case Intrinsic::loongarch_lasx_xvdiv_hu:
7552 case Intrinsic::loongarch_lasx_xvdiv_wu:
7553 case Intrinsic::loongarch_lasx_xvdiv_du:
7556 case Intrinsic::loongarch_lsx_vmod_b:
7557 case Intrinsic::loongarch_lsx_vmod_h:
7558 case Intrinsic::loongarch_lsx_vmod_w:
7559 case Intrinsic::loongarch_lsx_vmod_d:
7560 case Intrinsic::loongarch_lasx_xvmod_b:
7561 case Intrinsic::loongarch_lasx_xvmod_h:
7562 case Intrinsic::loongarch_lasx_xvmod_w:
7563 case Intrinsic::loongarch_lasx_xvmod_d:
7566 case Intrinsic::loongarch_lsx_vmod_bu:
7567 case Intrinsic::loongarch_lsx_vmod_hu:
7568 case Intrinsic::loongarch_lsx_vmod_wu:
7569 case Intrinsic::loongarch_lsx_vmod_du:
7570 case Intrinsic::loongarch_lasx_xvmod_bu:
7571 case Intrinsic::loongarch_lasx_xvmod_hu:
7572 case Intrinsic::loongarch_lasx_xvmod_wu:
7573 case Intrinsic::loongarch_lasx_xvmod_du:
7576 case Intrinsic::loongarch_lsx_vand_v:
7577 case Intrinsic::loongarch_lasx_xvand_v:
7580 case Intrinsic::loongarch_lsx_vor_v:
7581 case Intrinsic::loongarch_lasx_xvor_v:
7584 case Intrinsic::loongarch_lsx_vxor_v:
7585 case Intrinsic::loongarch_lasx_xvxor_v:
7588 case Intrinsic::loongarch_lsx_vnor_v:
7589 case Intrinsic::loongarch_lasx_xvnor_v: {
7594 case Intrinsic::loongarch_lsx_vandi_b:
7595 case Intrinsic::loongarch_lasx_xvandi_b:
7598 case Intrinsic::loongarch_lsx_vori_b:
7599 case Intrinsic::loongarch_lasx_xvori_b:
7602 case Intrinsic::loongarch_lsx_vxori_b:
7603 case Intrinsic::loongarch_lasx_xvxori_b:
7606 case Intrinsic::loongarch_lsx_vsll_b:
7607 case Intrinsic::loongarch_lsx_vsll_h:
7608 case Intrinsic::loongarch_lsx_vsll_w:
7609 case Intrinsic::loongarch_lsx_vsll_d:
7610 case Intrinsic::loongarch_lasx_xvsll_b:
7611 case Intrinsic::loongarch_lasx_xvsll_h:
7612 case Intrinsic::loongarch_lasx_xvsll_w:
7613 case Intrinsic::loongarch_lasx_xvsll_d:
7616 case Intrinsic::loongarch_lsx_vslli_b:
7617 case Intrinsic::loongarch_lasx_xvslli_b:
7620 case Intrinsic::loongarch_lsx_vslli_h:
7621 case Intrinsic::loongarch_lasx_xvslli_h:
7624 case Intrinsic::loongarch_lsx_vslli_w:
7625 case Intrinsic::loongarch_lasx_xvslli_w:
7628 case Intrinsic::loongarch_lsx_vslli_d:
7629 case Intrinsic::loongarch_lasx_xvslli_d:
7632 case Intrinsic::loongarch_lsx_vsrl_b:
7633 case Intrinsic::loongarch_lsx_vsrl_h:
7634 case Intrinsic::loongarch_lsx_vsrl_w:
7635 case Intrinsic::loongarch_lsx_vsrl_d:
7636 case Intrinsic::loongarch_lasx_xvsrl_b:
7637 case Intrinsic::loongarch_lasx_xvsrl_h:
7638 case Intrinsic::loongarch_lasx_xvsrl_w:
7639 case Intrinsic::loongarch_lasx_xvsrl_d:
7642 case Intrinsic::loongarch_lsx_vsrli_b:
7643 case Intrinsic::loongarch_lasx_xvsrli_b:
7646 case Intrinsic::loongarch_lsx_vsrli_h:
7647 case Intrinsic::loongarch_lasx_xvsrli_h:
7650 case Intrinsic::loongarch_lsx_vsrli_w:
7651 case Intrinsic::loongarch_lasx_xvsrli_w:
7654 case Intrinsic::loongarch_lsx_vsrli_d:
7655 case Intrinsic::loongarch_lasx_xvsrli_d:
7658 case Intrinsic::loongarch_lsx_vsra_b:
7659 case Intrinsic::loongarch_lsx_vsra_h:
7660 case Intrinsic::loongarch_lsx_vsra_w:
7661 case Intrinsic::loongarch_lsx_vsra_d:
7662 case Intrinsic::loongarch_lasx_xvsra_b:
7663 case Intrinsic::loongarch_lasx_xvsra_h:
7664 case Intrinsic::loongarch_lasx_xvsra_w:
7665 case Intrinsic::loongarch_lasx_xvsra_d:
7668 case Intrinsic::loongarch_lsx_vsrai_b:
7669 case Intrinsic::loongarch_lasx_xvsrai_b:
7672 case Intrinsic::loongarch_lsx_vsrai_h:
7673 case Intrinsic::loongarch_lasx_xvsrai_h:
7676 case Intrinsic::loongarch_lsx_vsrai_w:
7677 case Intrinsic::loongarch_lasx_xvsrai_w:
7680 case Intrinsic::loongarch_lsx_vsrai_d:
7681 case Intrinsic::loongarch_lasx_xvsrai_d:
7684 case Intrinsic::loongarch_lsx_vclz_b:
7685 case Intrinsic::loongarch_lsx_vclz_h:
7686 case Intrinsic::loongarch_lsx_vclz_w:
7687 case Intrinsic::loongarch_lsx_vclz_d:
7688 case Intrinsic::loongarch_lasx_xvclz_b:
7689 case Intrinsic::loongarch_lasx_xvclz_h:
7690 case Intrinsic::loongarch_lasx_xvclz_w:
7691 case Intrinsic::loongarch_lasx_xvclz_d:
7693 case Intrinsic::loongarch_lsx_vpcnt_b:
7694 case Intrinsic::loongarch_lsx_vpcnt_h:
7695 case Intrinsic::loongarch_lsx_vpcnt_w:
7696 case Intrinsic::loongarch_lsx_vpcnt_d:
7697 case Intrinsic::loongarch_lasx_xvpcnt_b:
7698 case Intrinsic::loongarch_lasx_xvpcnt_h:
7699 case Intrinsic::loongarch_lasx_xvpcnt_w:
7700 case Intrinsic::loongarch_lasx_xvpcnt_d:
7702 case Intrinsic::loongarch_lsx_vbitclr_b:
7703 case Intrinsic::loongarch_lsx_vbitclr_h:
7704 case Intrinsic::loongarch_lsx_vbitclr_w:
7705 case Intrinsic::loongarch_lsx_vbitclr_d:
7706 case Intrinsic::loongarch_lasx_xvbitclr_b:
7707 case Intrinsic::loongarch_lasx_xvbitclr_h:
7708 case Intrinsic::loongarch_lasx_xvbitclr_w:
7709 case Intrinsic::loongarch_lasx_xvbitclr_d:
7711 case Intrinsic::loongarch_lsx_vbitclri_b:
7712 case Intrinsic::loongarch_lasx_xvbitclri_b:
7714 case Intrinsic::loongarch_lsx_vbitclri_h:
7715 case Intrinsic::loongarch_lasx_xvbitclri_h:
7717 case Intrinsic::loongarch_lsx_vbitclri_w:
7718 case Intrinsic::loongarch_lasx_xvbitclri_w:
7720 case Intrinsic::loongarch_lsx_vbitclri_d:
7721 case Intrinsic::loongarch_lasx_xvbitclri_d:
7723 case Intrinsic::loongarch_lsx_vbitset_b:
7724 case Intrinsic::loongarch_lsx_vbitset_h:
7725 case Intrinsic::loongarch_lsx_vbitset_w:
7726 case Intrinsic::loongarch_lsx_vbitset_d:
7727 case Intrinsic::loongarch_lasx_xvbitset_b:
7728 case Intrinsic::loongarch_lasx_xvbitset_h:
7729 case Intrinsic::loongarch_lasx_xvbitset_w:
7730 case Intrinsic::loongarch_lasx_xvbitset_d: {
7731 EVT VecTy =
N->getValueType(0);
7737 case Intrinsic::loongarch_lsx_vbitseti_b:
7738 case Intrinsic::loongarch_lasx_xvbitseti_b:
7740 case Intrinsic::loongarch_lsx_vbitseti_h:
7741 case Intrinsic::loongarch_lasx_xvbitseti_h:
7743 case Intrinsic::loongarch_lsx_vbitseti_w:
7744 case Intrinsic::loongarch_lasx_xvbitseti_w:
7746 case Intrinsic::loongarch_lsx_vbitseti_d:
7747 case Intrinsic::loongarch_lasx_xvbitseti_d:
7749 case Intrinsic::loongarch_lsx_vbitrev_b:
7750 case Intrinsic::loongarch_lsx_vbitrev_h:
7751 case Intrinsic::loongarch_lsx_vbitrev_w:
7752 case Intrinsic::loongarch_lsx_vbitrev_d:
7753 case Intrinsic::loongarch_lasx_xvbitrev_b:
7754 case Intrinsic::loongarch_lasx_xvbitrev_h:
7755 case Intrinsic::loongarch_lasx_xvbitrev_w:
7756 case Intrinsic::loongarch_lasx_xvbitrev_d: {
7757 EVT VecTy =
N->getValueType(0);
7763 case Intrinsic::loongarch_lsx_vbitrevi_b:
7764 case Intrinsic::loongarch_lasx_xvbitrevi_b:
7766 case Intrinsic::loongarch_lsx_vbitrevi_h:
7767 case Intrinsic::loongarch_lasx_xvbitrevi_h:
7769 case Intrinsic::loongarch_lsx_vbitrevi_w:
7770 case Intrinsic::loongarch_lasx_xvbitrevi_w:
7772 case Intrinsic::loongarch_lsx_vbitrevi_d:
7773 case Intrinsic::loongarch_lasx_xvbitrevi_d:
7775 case Intrinsic::loongarch_lsx_vfadd_s:
7776 case Intrinsic::loongarch_lsx_vfadd_d:
7777 case Intrinsic::loongarch_lasx_xvfadd_s:
7778 case Intrinsic::loongarch_lasx_xvfadd_d:
7781 case Intrinsic::loongarch_lsx_vfsub_s:
7782 case Intrinsic::loongarch_lsx_vfsub_d:
7783 case Intrinsic::loongarch_lasx_xvfsub_s:
7784 case Intrinsic::loongarch_lasx_xvfsub_d:
7787 case Intrinsic::loongarch_lsx_vfmul_s:
7788 case Intrinsic::loongarch_lsx_vfmul_d:
7789 case Intrinsic::loongarch_lasx_xvfmul_s:
7790 case Intrinsic::loongarch_lasx_xvfmul_d:
7793 case Intrinsic::loongarch_lsx_vfdiv_s:
7794 case Intrinsic::loongarch_lsx_vfdiv_d:
7795 case Intrinsic::loongarch_lasx_xvfdiv_s:
7796 case Intrinsic::loongarch_lasx_xvfdiv_d:
7799 case Intrinsic::loongarch_lsx_vfmadd_s:
7800 case Intrinsic::loongarch_lsx_vfmadd_d:
7801 case Intrinsic::loongarch_lasx_xvfmadd_s:
7802 case Intrinsic::loongarch_lasx_xvfmadd_d:
7804 N->getOperand(2),
N->getOperand(3));
7805 case Intrinsic::loongarch_lsx_vinsgr2vr_b:
7807 N->getOperand(1),
N->getOperand(2),
7809 case Intrinsic::loongarch_lsx_vinsgr2vr_h:
7810 case Intrinsic::loongarch_lasx_xvinsgr2vr_w:
7812 N->getOperand(1),
N->getOperand(2),
7814 case Intrinsic::loongarch_lsx_vinsgr2vr_w:
7815 case Intrinsic::loongarch_lasx_xvinsgr2vr_d:
7817 N->getOperand(1),
N->getOperand(2),
7819 case Intrinsic::loongarch_lsx_vinsgr2vr_d:
7821 N->getOperand(1),
N->getOperand(2),
7823 case Intrinsic::loongarch_lsx_vreplgr2vr_b:
7824 case Intrinsic::loongarch_lsx_vreplgr2vr_h:
7825 case Intrinsic::loongarch_lsx_vreplgr2vr_w:
7826 case Intrinsic::loongarch_lsx_vreplgr2vr_d:
7827 case Intrinsic::loongarch_lasx_xvreplgr2vr_b:
7828 case Intrinsic::loongarch_lasx_xvreplgr2vr_h:
7829 case Intrinsic::loongarch_lasx_xvreplgr2vr_w:
7830 case Intrinsic::loongarch_lasx_xvreplgr2vr_d:
7831 return DAG.
getNode(LoongArchISD::VREPLGR2VR,
DL,
N->getValueType(0),
7834 case Intrinsic::loongarch_lsx_vreplve_b:
7835 case Intrinsic::loongarch_lsx_vreplve_h:
7836 case Intrinsic::loongarch_lsx_vreplve_w:
7837 case Intrinsic::loongarch_lsx_vreplve_d:
7838 case Intrinsic::loongarch_lasx_xvreplve_b:
7839 case Intrinsic::loongarch_lasx_xvreplve_h:
7840 case Intrinsic::loongarch_lasx_xvreplve_w:
7841 case Intrinsic::loongarch_lasx_xvreplve_d:
7842 return DAG.
getNode(LoongArchISD::VREPLVE,
DL,
N->getValueType(0),
7846 case Intrinsic::loongarch_lsx_vpickve2gr_b:
7850 case Intrinsic::loongarch_lsx_vpickve2gr_h:
7851 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
7855 case Intrinsic::loongarch_lsx_vpickve2gr_w:
7859 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
7863 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
7864 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
7868 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
7872 case Intrinsic::loongarch_lsx_bz_b:
7873 case Intrinsic::loongarch_lsx_bz_h:
7874 case Intrinsic::loongarch_lsx_bz_w:
7875 case Intrinsic::loongarch_lsx_bz_d:
7876 case Intrinsic::loongarch_lasx_xbz_b:
7877 case Intrinsic::loongarch_lasx_xbz_h:
7878 case Intrinsic::loongarch_lasx_xbz_w:
7879 case Intrinsic::loongarch_lasx_xbz_d:
7881 return DAG.
getNode(LoongArchISD::VALL_ZERO,
DL,
N->getValueType(0),
7884 case Intrinsic::loongarch_lsx_bz_v:
7885 case Intrinsic::loongarch_lasx_xbz_v:
7887 return DAG.
getNode(LoongArchISD::VANY_ZERO,
DL,
N->getValueType(0),
7890 case Intrinsic::loongarch_lsx_bnz_b:
7891 case Intrinsic::loongarch_lsx_bnz_h:
7892 case Intrinsic::loongarch_lsx_bnz_w:
7893 case Intrinsic::loongarch_lsx_bnz_d:
7894 case Intrinsic::loongarch_lasx_xbnz_b:
7895 case Intrinsic::loongarch_lasx_xbnz_h:
7896 case Intrinsic::loongarch_lasx_xbnz_w:
7897 case Intrinsic::loongarch_lasx_xbnz_d:
7899 return DAG.
getNode(LoongArchISD::VALL_NONZERO,
DL,
N->getValueType(0),
7902 case Intrinsic::loongarch_lsx_bnz_v:
7903 case Intrinsic::loongarch_lasx_xbnz_v:
7905 return DAG.
getNode(LoongArchISD::VANY_NONZERO,
DL,
N->getValueType(0),
7908 case Intrinsic::loongarch_lasx_concat_128_s:
7909 case Intrinsic::loongarch_lasx_concat_128_d:
7910 case Intrinsic::loongarch_lasx_concat_128:
7912 N->getOperand(1),
N->getOperand(2));
7924 if (Op0.
getOpcode() == LoongArchISD::MOVFR2GR_S_LA64)
7936 if (Op0->
getOpcode() == LoongArchISD::MOVGR2FR_W_LA64) {
7938 "Unexpected value type!");
7947 MVT VT =
N->getSimpleValueType(0);
7968 if (Op0->
getOpcode() == LoongArchISD::BUILD_PAIR_F64)
7981 APInt V =
C->getValueAPF().bitcastToAPInt();
7996 MVT VT =
N->getSimpleValueType(0);
8057 EVT VT =
N->getValueType(0);
8059 if (VT != MVT::f32 && VT != MVT::f64)
8061 if (VT == MVT::f32 && !Subtarget.hasBasicF())
8063 if (VT == MVT::f64 && !Subtarget.hasBasicD())
8086 return DAG.
getNode(LoongArchISD::SITOF,
SDLoc(
N), VT, Load);
8100 if (!Subtarget.hasExtLSX())
8104 EVT DstVT =
N->getValueType(0);
8106 EVT SrcVT = Src.getValueType();
8116 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
8121 if (SrcBits % BlockBits != 0 && SrcBits != 128)
8124 if (DstEltBits < 32) {
8130 if (SrcEltBits != 64 || DstEltBits != 32)
8135 if (Subtarget.hasExtLASX())
8143 unsigned BlockNumElts = BlockBits / 64;
8146 Src.getOperand(0).getValueType() == BlockVT) {
8147 for (
unsigned i = 0; i < Src.getNumOperands(); i++)
8149 }
else if (SrcBits > BlockBits) {
8151 for (
unsigned i = 0; i < SrcBits / BlockBits; i++)
8156 BlockBits = SrcBits;
8160 MVT NativeVT = BlockBits == 256 ? MVT::v8i32 : MVT::v4i32;
8162 for (
unsigned i = 0; i < Blocks.
size(); i += 2) {
8167 if (BlockBits == 256) {
8177 if (Blocks.
size() == 1)
8234 Subtarget.hasExtLASX() && N1.
hasOneUse())
8246 return DAG.
getNode(
N.getOpcode(),
DL, VT, N0, N1);
8256 EVT VT =
N.getValueType();
8273 switch (
N.getOpcode()) {
8289 EVT VT =
N->getValueType(0);
8304 EVT VT =
N->getValueType(0);
8306 if (VT.
isVector() &&
N->getNumOperands() == 2)
8319 EVT VT =
N->getValueType(0);
8331 SDValue TrueVal =
N->getOperand(1);
8332 SDValue FalseVal =
N->getOperand(2);
8366 if (FalseVal.getOpcode() !=
ISD::ADD)
8369 SDValue Add0 = FalseVal.getOperand(0);
8370 SDValue Add1 = FalseVal.getOperand(1);
8440 : LoongArchISD::VSRAR,
8447 switch (
N->getOpcode()) {
8475 case LoongArchISD::BITREV_W:
8477 case LoongArchISD::BR_CC:
8479 case LoongArchISD::SELECT_CC:
8483 case LoongArchISD::MOVGR2FR_W_LA64:
8485 case LoongArchISD::MOVFR2GR_S_LA64:
8487 case LoongArchISD::CRC_W_B_W:
8488 case LoongArchISD::CRC_W_H_W:
8489 case LoongArchISD::CRCC_W_B_W:
8490 case LoongArchISD::CRCC_W_H_W:
8491 case LoongArchISD::VMSKLTZ:
8492 case LoongArchISD::XVMSKLTZ:
8494 case LoongArchISD::SPLIT_PAIR_F64:
8496 case LoongArchISD::VANDN:
8502 case LoongArchISD::VPACKEV:
8503 case LoongArchISD::VPERMI:
8529 MF->
insert(It, BreakMBB);
8533 SinkMBB->splice(SinkMBB->end(),
MBB, std::next(
MI.getIterator()),
MBB->end());
8534 SinkMBB->transferSuccessorsAndUpdatePHIs(
MBB);
8546 MBB->addSuccessor(BreakMBB);
8547 MBB->addSuccessor(SinkMBB);
8553 BreakMBB->addSuccessor(SinkMBB);
8565 switch (
MI.getOpcode()) {
8568 case LoongArch::PseudoVBZ:
8569 CondOpc = LoongArch::VSETEQZ_V;
8571 case LoongArch::PseudoVBZ_B:
8572 CondOpc = LoongArch::VSETANYEQZ_B;
8574 case LoongArch::PseudoVBZ_H:
8575 CondOpc = LoongArch::VSETANYEQZ_H;
8577 case LoongArch::PseudoVBZ_W:
8578 CondOpc = LoongArch::VSETANYEQZ_W;
8580 case LoongArch::PseudoVBZ_D:
8581 CondOpc = LoongArch::VSETANYEQZ_D;
8583 case LoongArch::PseudoVBNZ:
8584 CondOpc = LoongArch::VSETNEZ_V;
8586 case LoongArch::PseudoVBNZ_B:
8587 CondOpc = LoongArch::VSETALLNEZ_B;
8589 case LoongArch::PseudoVBNZ_H:
8590 CondOpc = LoongArch::VSETALLNEZ_H;
8592 case LoongArch::PseudoVBNZ_W:
8593 CondOpc = LoongArch::VSETALLNEZ_W;
8595 case LoongArch::PseudoVBNZ_D:
8596 CondOpc = LoongArch::VSETALLNEZ_D;
8598 case LoongArch::PseudoXVBZ:
8599 CondOpc = LoongArch::XVSETEQZ_V;
8601 case LoongArch::PseudoXVBZ_B:
8602 CondOpc = LoongArch::XVSETANYEQZ_B;
8604 case LoongArch::PseudoXVBZ_H:
8605 CondOpc = LoongArch::XVSETANYEQZ_H;
8607 case LoongArch::PseudoXVBZ_W:
8608 CondOpc = LoongArch::XVSETANYEQZ_W;
8610 case LoongArch::PseudoXVBZ_D:
8611 CondOpc = LoongArch::XVSETANYEQZ_D;
8613 case LoongArch::PseudoXVBNZ:
8614 CondOpc = LoongArch::XVSETNEZ_V;
8616 case LoongArch::PseudoXVBNZ_B:
8617 CondOpc = LoongArch::XVSETALLNEZ_B;
8619 case LoongArch::PseudoXVBNZ_H:
8620 CondOpc = LoongArch::XVSETALLNEZ_H;
8622 case LoongArch::PseudoXVBNZ_W:
8623 CondOpc = LoongArch::XVSETALLNEZ_W;
8625 case LoongArch::PseudoXVBNZ_D:
8626 CondOpc = LoongArch::XVSETALLNEZ_D;
8641 F->insert(It, FalseBB);
8642 F->insert(It, TrueBB);
8643 F->insert(It, SinkBB);
8646 SinkBB->
splice(SinkBB->
end(), BB, std::next(
MI.getIterator()), BB->
end());
8675 MI.getOperand(0).getReg())
8682 MI.eraseFromParent();
8690 unsigned BroadcastOp;
8692 switch (
MI.getOpcode()) {
8695 case LoongArch::PseudoXVINSGR2VR_B:
8697 BroadcastOp = LoongArch::XVREPLGR2VR_B;
8698 InsOp = LoongArch::XVEXTRINS_B;
8700 case LoongArch::PseudoXVINSGR2VR_H:
8702 BroadcastOp = LoongArch::XVREPLGR2VR_H;
8703 InsOp = LoongArch::XVEXTRINS_H;
8715 unsigned Idx =
MI.getOperand(3).getImm();
8723 .
addReg(XSrc, {}, LoongArch::sub_128);
8725 TII->get(HalfSize == 8 ? LoongArch::VINSGR2VR_H
8726 : LoongArch::VINSGR2VR_B),
8734 .
addImm(LoongArch::sub_128);
8741 BuildMI(*BB,
MI,
DL,
TII->get(LoongArch::XVPERMI_Q), ScratchReg2)
8744 .
addImm(Idx >= HalfSize ? 48 : 18);
8749 .
addImm((Idx >= HalfSize ? Idx - HalfSize : Idx) * 17);
8752 MI.eraseFromParent();
8759 assert(Subtarget.hasExtLSX());
8767 unsigned BroadcastOp, CTOp, PickOp;
8768 switch (
MI.getOpcode()) {
8771 case LoongArch::PseudoCTPOP_B:
8772 BroadcastOp = LoongArch::VREPLGR2VR_B;
8773 CTOp = LoongArch::VPCNT_B;
8774 PickOp = LoongArch::VPICKVE2GR_B;
8776 case LoongArch::PseudoCTPOP_H:
8777 case LoongArch::PseudoCTPOP_H_LA32:
8778 BroadcastOp = LoongArch::VREPLGR2VR_H;
8779 CTOp = LoongArch::VPCNT_H;
8780 PickOp = LoongArch::VPICKVE2GR_H;
8782 case LoongArch::PseudoCTPOP_W:
8783 case LoongArch::PseudoCTPOP_W_LA32:
8784 BroadcastOp = LoongArch::VREPLGR2VR_W;
8785 CTOp = LoongArch::VPCNT_W;
8786 PickOp = LoongArch::VPICKVE2GR_W;
8788 case LoongArch::PseudoCTPOP_D:
8789 BroadcastOp = LoongArch::VREPLGR2VR_D;
8790 CTOp = LoongArch::VPCNT_D;
8791 PickOp = LoongArch::VPICKVE2GR_D;
8801 MI.eraseFromParent();
8815 unsigned EleBits = 8;
8816 unsigned NotOpc = 0;
8819 switch (
MI.getOpcode()) {
8822 case LoongArch::PseudoVMSKLTZ_B:
8823 MskOpc = LoongArch::VMSKLTZ_B;
8825 case LoongArch::PseudoVMSKLTZ_H:
8826 MskOpc = LoongArch::VMSKLTZ_H;
8829 case LoongArch::PseudoVMSKLTZ_W:
8830 MskOpc = LoongArch::VMSKLTZ_W;
8833 case LoongArch::PseudoVMSKLTZ_D:
8834 MskOpc = LoongArch::VMSKLTZ_D;
8837 case LoongArch::PseudoVMSKGEZ_B:
8838 MskOpc = LoongArch::VMSKGEZ_B;
8840 case LoongArch::PseudoVMSKEQZ_B:
8841 MskOpc = LoongArch::VMSKNZ_B;
8842 NotOpc = LoongArch::VNOR_V;
8844 case LoongArch::PseudoVMSKNEZ_B:
8845 MskOpc = LoongArch::VMSKNZ_B;
8847 case LoongArch::PseudoXVMSKLTZ_B:
8848 MskOpc = LoongArch::XVMSKLTZ_B;
8849 RC = &LoongArch::LASX256RegClass;
8851 case LoongArch::PseudoXVMSKLTZ_H:
8852 MskOpc = LoongArch::XVMSKLTZ_H;
8853 RC = &LoongArch::LASX256RegClass;
8856 case LoongArch::PseudoXVMSKLTZ_W:
8857 MskOpc = LoongArch::XVMSKLTZ_W;
8858 RC = &LoongArch::LASX256RegClass;
8861 case LoongArch::PseudoXVMSKLTZ_D:
8862 MskOpc = LoongArch::XVMSKLTZ_D;
8863 RC = &LoongArch::LASX256RegClass;
8866 case LoongArch::PseudoXVMSKGEZ_B:
8867 MskOpc = LoongArch::XVMSKGEZ_B;
8868 RC = &LoongArch::LASX256RegClass;
8870 case LoongArch::PseudoXVMSKEQZ_B:
8871 MskOpc = LoongArch::XVMSKNZ_B;
8872 NotOpc = LoongArch::XVNOR_V;
8873 RC = &LoongArch::LASX256RegClass;
8875 case LoongArch::PseudoXVMSKNEZ_B:
8876 MskOpc = LoongArch::XVMSKNZ_B;
8877 RC = &LoongArch::LASX256RegClass;
8892 if (
TRI->getRegSizeInBits(*RC) > 128) {
8902 TII->get(Subtarget.
is64Bit() ? LoongArch::BSTRINS_D
8903 : LoongArch::BSTRINS_W),
8907 .
addImm(256 / EleBits - 1)
8915 MI.eraseFromParent();
8922 assert(
MI.getOpcode() == LoongArch::SplitPairF64Pseudo &&
8923 "Unexpected instruction");
8935 MI.eraseFromParent();
8942 assert(
MI.getOpcode() == LoongArch::BuildPairF64Pseudo &&
8943 "Unexpected instruction");
8959 MI.eraseFromParent();
8964 switch (
MI.getOpcode()) {
8967 case LoongArch::Select_GPR_Using_CC_GPR:
9003 if (
MI.getOperand(2).isReg())
9004 RHS =
MI.getOperand(2).getReg();
9005 auto CC =
static_cast<unsigned>(
MI.getOperand(3).
getImm());
9009 SelectDests.
insert(
MI.getOperand(0).getReg());
9013 SequenceMBBI !=
E; ++SequenceMBBI) {
9014 if (SequenceMBBI->isDebugInstr())
9017 if (SequenceMBBI->getOperand(1).getReg() !=
LHS ||
9018 !SequenceMBBI->getOperand(2).isReg() ||
9019 SequenceMBBI->getOperand(2).getReg() !=
RHS ||
9020 SequenceMBBI->getOperand(3).getImm() != CC ||
9021 SelectDests.
count(SequenceMBBI->getOperand(4).getReg()) ||
9022 SelectDests.
count(SequenceMBBI->getOperand(5).getReg()))
9024 LastSelectPseudo = &*SequenceMBBI;
9026 SelectDests.
insert(SequenceMBBI->getOperand(0).getReg());
9029 if (SequenceMBBI->hasUnmodeledSideEffects() ||
9030 SequenceMBBI->mayLoadOrStore() ||
9031 SequenceMBBI->usesCustomInsertionHook())
9034 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
9049 F->insert(
I, IfFalseMBB);
9050 F->insert(
I, TailMBB);
9053 unsigned CallFrameSize =
TII.getCallFrameSizeAt(*LastSelectPseudo);
9059 TailMBB->
push_back(DebugInstr->removeFromParent());
9063 TailMBB->
splice(TailMBB->
end(), HeadMBB,
9073 if (
MI.getOperand(2).isImm())
9085 auto SelectMBBI =
MI.getIterator();
9086 auto SelectEnd = std::next(LastSelectPseudo->
getIterator());
9088 while (SelectMBBI != SelectEnd) {
9089 auto Next = std::next(SelectMBBI);
9093 TII.get(LoongArch::PHI), SelectMBBI->getOperand(0).getReg())
9094 .
addReg(SelectMBBI->getOperand(4).getReg())
9096 .
addReg(SelectMBBI->getOperand(5).getReg())
9103 F->getProperties().resetNoPHIs();
9109 const TargetInstrInfo *
TII = Subtarget.getInstrInfo();
9112 switch (
MI.getOpcode()) {
9115 case LoongArch::DIV_W:
9116 case LoongArch::DIV_WU:
9117 case LoongArch::MOD_W:
9118 case LoongArch::MOD_WU:
9119 case LoongArch::DIV_D:
9120 case LoongArch::DIV_DU:
9121 case LoongArch::MOD_D:
9122 case LoongArch::MOD_DU:
9125 case LoongArch::WRFCSR: {
9127 LoongArch::FCSR0 +
MI.getOperand(0).getImm())
9128 .
addReg(
MI.getOperand(1).getReg());
9129 MI.eraseFromParent();
9132 case LoongArch::RDFCSR: {
9133 MachineInstr *ReadFCSR =
9135 MI.getOperand(0).getReg())
9136 .
addReg(LoongArch::FCSR0 +
MI.getOperand(1).getImm());
9138 MI.eraseFromParent();
9141 case LoongArch::Select_GPR_Using_CC_GPR:
9143 case LoongArch::BuildPairF64Pseudo:
9145 case LoongArch::SplitPairF64Pseudo:
9147 case LoongArch::PseudoVBZ:
9148 case LoongArch::PseudoVBZ_B:
9149 case LoongArch::PseudoVBZ_H:
9150 case LoongArch::PseudoVBZ_W:
9151 case LoongArch::PseudoVBZ_D:
9152 case LoongArch::PseudoVBNZ:
9153 case LoongArch::PseudoVBNZ_B:
9154 case LoongArch::PseudoVBNZ_H:
9155 case LoongArch::PseudoVBNZ_W:
9156 case LoongArch::PseudoVBNZ_D:
9157 case LoongArch::PseudoXVBZ:
9158 case LoongArch::PseudoXVBZ_B:
9159 case LoongArch::PseudoXVBZ_H:
9160 case LoongArch::PseudoXVBZ_W:
9161 case LoongArch::PseudoXVBZ_D:
9162 case LoongArch::PseudoXVBNZ:
9163 case LoongArch::PseudoXVBNZ_B:
9164 case LoongArch::PseudoXVBNZ_H:
9165 case LoongArch::PseudoXVBNZ_W:
9166 case LoongArch::PseudoXVBNZ_D:
9168 case LoongArch::PseudoXVINSGR2VR_B:
9169 case LoongArch::PseudoXVINSGR2VR_H:
9171 case LoongArch::PseudoCTPOP_B:
9172 case LoongArch::PseudoCTPOP_H:
9173 case LoongArch::PseudoCTPOP_W:
9174 case LoongArch::PseudoCTPOP_D:
9175 case LoongArch::PseudoCTPOP_H_LA32:
9176 case LoongArch::PseudoCTPOP_W_LA32:
9178 case LoongArch::PseudoVMSKLTZ_B:
9179 case LoongArch::PseudoVMSKLTZ_H:
9180 case LoongArch::PseudoVMSKLTZ_W:
9181 case LoongArch::PseudoVMSKLTZ_D:
9182 case LoongArch::PseudoVMSKGEZ_B:
9183 case LoongArch::PseudoVMSKEQZ_B:
9184 case LoongArch::PseudoVMSKNEZ_B:
9185 case LoongArch::PseudoXVMSKLTZ_B:
9186 case LoongArch::PseudoXVMSKLTZ_H:
9187 case LoongArch::PseudoXVMSKLTZ_W:
9188 case LoongArch::PseudoXVMSKLTZ_D:
9189 case LoongArch::PseudoXVMSKGEZ_B:
9190 case LoongArch::PseudoXVMSKEQZ_B:
9191 case LoongArch::PseudoXVMSKNEZ_B:
9193 case TargetOpcode::STATEPOINT:
9199 MI.addOperand(*
MI.getMF(),
9201 LoongArch::R1,
true,
9204 if (!Subtarget.is64Bit())
9207 case LoongArch::PROBED_STACKALLOC_DYN:
9214 unsigned *
Fast)
const {
9215 if (!Subtarget.hasUAL())
9233 LoongArch::R7, LoongArch::R8, LoongArch::R9,
9234 LoongArch::R10, LoongArch::R11};
9249 LoongArch::R23, LoongArch::R24, LoongArch::R25, LoongArch::R26,
9250 LoongArch::R27, LoongArch::R28, LoongArch::R29, LoongArch::R30,
9251 LoongArch::R4, LoongArch::R5, LoongArch::R6, LoongArch::R7,
9252 LoongArch::R8, LoongArch::R9, LoongArch::R10, LoongArch::R11,
9253 LoongArch::R12, LoongArch::R13, LoongArch::R14, LoongArch::R15,
9254 LoongArch::R16, LoongArch::R17, LoongArch::R18, LoongArch::R19,
9260 LoongArch::F3, LoongArch::F4, LoongArch::F5,
9261 LoongArch::F6, LoongArch::F7};
9264 LoongArch::F0_64, LoongArch::F1_64, LoongArch::F2_64, LoongArch::F3_64,
9265 LoongArch::F4_64, LoongArch::F5_64, LoongArch::F6_64, LoongArch::F7_64};
9268 LoongArch::VR3, LoongArch::VR4, LoongArch::VR5,
9269 LoongArch::VR6, LoongArch::VR7};
9272 LoongArch::XR3, LoongArch::XR4, LoongArch::XR5,
9273 LoongArch::XR6, LoongArch::XR7};
9276 switch (State.getCallingConv()) {
9278 if (!State.isVarArg())
9282 return State.AllocateReg(
ArgGPRs);
9290 unsigned ValNo2,
MVT ValVT2,
MVT LocVT2,
9292 unsigned GRLenInBytes = GRLen / 8;
9303 State.AllocateStack(GRLenInBytes, StackAlign),
9306 ValNo2, ValVT2, State.AllocateStack(GRLenInBytes,
Align(GRLenInBytes)),
9317 ValNo2, ValVT2, State.AllocateStack(GRLenInBytes,
Align(GRLenInBytes)),
9325 unsigned ValNo,
MVT ValVT,
9328 unsigned GRLen =
DL.getLargestLegalIntTypeSizeInBits();
9329 assert((GRLen == 32 || GRLen == 64) &&
"Unspport GRLen");
9330 MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64;
9335 if (IsRet && ValNo > 1)
9339 bool UseGPRForFloat =
true;
9349 UseGPRForFloat = ArgFlags.
isVarArg();
9362 unsigned TwoGRLenInBytes = (2 * GRLen) / 8;
9365 DL.getTypeAllocSize(OrigTy) == TwoGRLenInBytes) {
9366 unsigned RegIdx = State.getFirstUnallocated(
ArgGPRs);
9368 if (RegIdx != std::size(
ArgGPRs) && RegIdx % 2 == 1)
9374 State.getPendingArgFlags();
9377 "PendingLocs and PendingArgFlags out of sync");
9381 UseGPRForFloat =
true;
9383 if (UseGPRForFloat && ValVT == MVT::f32) {
9386 }
else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
9389 }
else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
9392 assert(PendingLocs.
empty() &&
"Can't lower f64 if it is split");
9434 PendingLocs.
size() <= 2) {
9435 assert(PendingLocs.
size() == 2 &&
"Unexpected PendingLocs.size()");
9440 PendingLocs.
clear();
9441 PendingArgFlags.
clear();
9448 unsigned StoreSizeBytes = GRLen / 8;
9451 if (ValVT == MVT::f32 && !UseGPRForFloat) {
9453 }
else if (ValVT == MVT::f64 && !UseGPRForFloat) {
9457 UseGPRForFloat =
false;
9458 StoreSizeBytes = 16;
9459 StackAlign =
Align(16);
9462 UseGPRForFloat =
false;
9463 StoreSizeBytes = 32;
9464 StackAlign =
Align(32);
9470 Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
9474 if (!PendingLocs.
empty()) {
9476 assert(PendingLocs.
size() > 2 &&
"Unexpected PendingLocs.size()");
9477 for (
auto &It : PendingLocs) {
9479 It.convertToReg(
Reg);
9484 PendingLocs.clear();
9485 PendingArgFlags.
clear();
9488 assert((!UseGPRForFloat || LocVT == GRLenVT) &&
9489 "Expected an GRLenVT at this stage");
9506void LoongArchTargetLowering::analyzeInputArgs(
9509 LoongArchCCAssignFn Fn)
const {
9511 for (
unsigned i = 0, e = Ins.
size(); i != e; ++i) {
9512 MVT ArgVT = Ins[i].VT;
9513 Type *ArgTy =
nullptr;
9515 ArgTy = FType->getReturnType();
9516 else if (Ins[i].isOrigArg())
9517 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
9521 CCInfo, IsRet, ArgTy)) {
9522 LLVM_DEBUG(
dbgs() <<
"InputArg #" << i <<
" has unhandled type " << ArgVT
9529void LoongArchTargetLowering::analyzeOutputArgs(
9532 CallLoweringInfo *CLI, LoongArchCCAssignFn Fn)
const {
9533 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
9534 MVT ArgVT = Outs[i].VT;
9535 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty :
nullptr;
9539 CCInfo, IsRet, OrigTy)) {
9540 LLVM_DEBUG(
dbgs() <<
"OutputArg #" << i <<
" has unhandled type " << ArgVT
9559 Val = DAG.
getNode(LoongArchISD::MOVGR2FR_W_LA64,
DL, MVT::f32, Val);
9581 if (In.isOrigArg()) {
9586 if ((
BitWidth <= 32 && In.Flags.isSExt()) ||
9587 (
BitWidth < 32 && In.Flags.isZExt())) {
9637 Register LoVReg =
RegInfo.createVirtualRegister(&LoongArch::GPRRegClass);
9650 Register HiVReg =
RegInfo.createVirtualRegister(&LoongArch::GPRRegClass);
9654 return DAG.
getNode(LoongArchISD::BUILD_PAIR_F64,
DL, MVT::f64,
Lo,
Hi);
9668 Val = DAG.
getNode(LoongArchISD::MOVFR2GR_S_LA64,
DL, MVT::i64, Val);
9680 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
9684 LoongArch::R23, LoongArch::R24, LoongArch::R25,
9685 LoongArch::R26, LoongArch::R27, LoongArch::R28,
9686 LoongArch::R29, LoongArch::R30, LoongArch::R31};
9693 if (LocVT == MVT::f32) {
9696 static const MCPhysReg FPR32List[] = {LoongArch::F24, LoongArch::F25,
9697 LoongArch::F26, LoongArch::F27};
9704 if (LocVT == MVT::f64) {
9707 static const MCPhysReg FPR64List[] = {LoongArch::F28_64, LoongArch::F29_64,
9708 LoongArch::F30_64, LoongArch::F31_64};
9739 "GHC calling convention requires the F and D extensions");
9744 MVT GRLenVT = Subtarget.getGRLenVT();
9745 unsigned GRLenInBytes = Subtarget.getGRLen() / 8;
9754 return CI->isMustTailCall();
9759 std::vector<SDValue> OutChains;
9768 analyzeInputArgs(MF, CCInfo, Ins,
false,
CC_LoongArch);
9770 for (
unsigned i = 0, e = ArgLocs.
size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
9787 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
9796 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
9797 assert(ArgPartOffset == 0);
9798 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
9800 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
9824 int VaArgOffset, VarArgsSaveSize;
9828 if (ArgRegs.
size() == Idx) {
9830 VarArgsSaveSize = 0;
9832 VarArgsSaveSize = GRLenInBytes * (ArgRegs.
size() - Idx);
9833 VaArgOffset = -VarArgsSaveSize;
9839 LoongArchFI->setVarArgsFrameIndex(FI);
9847 VarArgsSaveSize += GRLenInBytes;
9852 for (
unsigned I = Idx;
I < ArgRegs.
size();
9853 ++
I, VaArgOffset += GRLenInBytes) {
9854 const Register Reg = RegInfo.createVirtualRegister(RC);
9855 RegInfo.addLiveIn(ArgRegs[
I], Reg);
9863 ->setValue((
Value *)
nullptr);
9864 OutChains.push_back(Store);
9866 LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize);
9871 if (!OutChains.empty()) {
9872 OutChains.push_back(Chain);
9887 if (
N->getNumValues() != 1)
9889 if (!
N->hasNUsesOfValue(1, 0))
9892 SDNode *Copy = *
N->user_begin();
9898 if (Copy->getGluedNode())
9902 bool HasRet =
false;
9904 if (
Node->getOpcode() != LoongArchISD::RET)
9912 Chain = Copy->getOperand(0);
9917bool LoongArchTargetLowering::isEligibleForTailCallOptimization(
9921 auto CalleeCC = CLI.CallConv;
9922 auto &Outs = CLI.Outs;
9924 auto CallerCC = Caller.getCallingConv();
9926 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
9935 for (
auto &Arg : Outs)
9936 if (Arg.Flags.isByVal())
9952 for (
auto &VA : ArgLocs)
9958 auto IsCallerStructRet = Caller.hasStructRetAttr();
9959 auto IsCalleeStructRet = Outs.empty() ?
false : Outs[0].Flags.isSRet();
9960 if (IsCallerStructRet || IsCalleeStructRet)
9965 const uint32_t *CallerPreserved =
TRI->getCallPreservedMask(MF, CallerCC);
9966 if (CalleeCC != CallerCC) {
9967 const uint32_t *CalleePreserved =
TRI->getCallPreservedMask(MF, CalleeCC);
9968 if (!
TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
9994 MVT GRLenVT = Subtarget.getGRLenVT();
10006 analyzeOutputArgs(MF, ArgCCInfo, Outs,
false, &CLI,
CC_LoongArch);
10010 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
10016 "site marked musttail");
10023 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
10025 if (!Flags.isByVal())
10029 unsigned Size = Flags.getByValSize();
10030 Align Alignment = Flags.getNonZeroByValAlign();
10037 Chain = DAG.
getMemcpy(Chain,
DL, FIPtr, Arg, SizeNode, Alignment, Alignment,
10039 false,
nullptr, std::nullopt,
10051 for (
unsigned i = 0, j = 0, e = ArgLocs.
size(), OutIdx = 0; i != e;
10054 SDValue ArgValue = OutVals[OutIdx];
10062 DAG.
getNode(LoongArchISD::SPLIT_PAIR_F64,
DL,
10063 DAG.
getVTList(MVT::i32, MVT::i32), ArgValue);
10075 if (!StackPtr.getNode())
10087 RegsToPass.
push_back(std::make_pair(RegHigh,
Hi));
10101 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
10121 const Argument *FormalArg =
nullptr;
10122 unsigned FilteredIdx = 0;
10123 for (
const auto &CallArg : CLI.
CB->
args()) {
10124 if (CallArg->getType()->isEmptyTy())
10126 if (FilteredIdx == CallArgIdx) {
10135 unsigned FormalArgIdx = CallArgIdx;
10137 FormalArgIdx = FormalArg->
getArgNo();
10141 if (Arg.getType()->isEmptyTy())
10143 if (FilteredIdx == CallArgIdx) {
10144 FormalArgIdx = Arg.getArgNo();
10157 SDValue IncomingPtr = CopyOp;
10174 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
10175 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
10176 SDValue PartValue = OutVals[OutIdx + 1];
10177 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
10188 ArgValue = IncomingPtr;
10192 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
10204 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
10205 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
10206 assert(ArgPartOffset == 0);
10211 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
10212 SDValue PartValue = OutVals[OutIdx + 1];
10213 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
10225 DAG.
getStore(Chain,
DL, ArgValue, SpillSlot,
10227 for (
const auto &Part : Parts) {
10228 SDValue PartValue = Part.first;
10229 SDValue PartOffset = Part.second;
10236 ArgValue = SpillSlot;
10243 if (Flags.isByVal())
10244 ArgValue = ByValArgs[j++];
10252 "Tail call not allowed if stack is used for passing parameters");
10255 if (!StackPtr.getNode())
10268 if (!MemOpChains.
empty())
10274 for (
auto &Reg : RegsToPass) {
10275 Chain = DAG.
getCopyToReg(Chain,
DL, Reg.first, Reg.second, Glue);
10297 Ops.push_back(Chain);
10298 Ops.push_back(Callee);
10302 for (
auto &Reg : RegsToPass)
10303 Ops.push_back(DAG.
getRegister(Reg.first, Reg.second.getValueType()));
10308 const uint32_t *Mask =
TRI->getCallPreservedMask(MF, CallConv);
10309 assert(Mask &&
"Missing call preserved mask for calling convention");
10315 Ops.push_back(Glue);
10324 Op = IsTailCall ? LoongArchISD::TAIL : LoongArchISD::CALL;
10327 Op = IsTailCall ? LoongArchISD::TAIL_MEDIUM : LoongArchISD::CALL_MEDIUM;
10330 assert(Subtarget.is64Bit() &&
"Large code model requires LA64");
10331 Op = IsTailCall ? LoongArchISD::TAIL_LARGE : LoongArchISD::CALL_LARGE;
10353 analyzeInputArgs(MF, RetCCInfo, Ins,
true,
CC_LoongArch);
10356 for (
unsigned i = 0, e = RVLocs.
size(); i != e; ++i) {
10357 auto &VA = RVLocs[i];
10365 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10366 assert(VA.needsCustom());
10371 RetValue = DAG.
getNode(LoongArchISD::BUILD_PAIR_F64,
DL, MVT::f64,
10372 RetValue, RetValue2);
10385 const Type *RetTy)
const {
10387 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
10389 for (
unsigned i = 0, e = Outs.
size(); i != e; ++i) {
10393 Outs[i].Flags, CCInfo,
true,
nullptr))
10419 for (
unsigned i = 0, e = RVLocs.
size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
10420 SDValue Val = OutVals[OutIdx];
10429 DAG.
getVTList(MVT::i32, MVT::i32), Val);
10433 Register RegHi = RVLocs[++i].getLocReg();
10458 return DAG.
getNode(LoongArchISD::RET,
DL, MVT::Other, RetOps);
10466 const APInt &SplatValue,
const unsigned SplatBitSize)
const {
10469 if (SplatBitSize == 16 && !(V & 0x00FF)) {
10471 RequiredImm = (0b10101 << 8) | (V >> 8);
10472 return {
true, RequiredImm};
10473 }
else if (SplatBitSize == 32) {
10475 if (!(V & 0xFFFF00FF)) {
10476 RequiredImm = (0b10001 << 8) | (V >> 8);
10477 return {
true, RequiredImm};
10480 if (!(V & 0xFF00FFFF)) {
10481 RequiredImm = (0b10010 << 8) | (V >> 16);
10482 return {
true, RequiredImm};
10485 if (!(V & 0x00FFFFFF)) {
10486 RequiredImm = (0b10011 << 8) | (V >> 24);
10487 return {
true, RequiredImm};
10490 if ((V & 0xFFFF00FF) == 0xFF) {
10491 RequiredImm = (0b10110 << 8) | (V >> 8);
10492 return {
true, RequiredImm};
10495 if ((V & 0xFF00FFFF) == 0xFFFF) {
10496 RequiredImm = (0b10111 << 8) | (V >> 16);
10497 return {
true, RequiredImm};
10500 if ((V & 0x7E07FFFF) == 0x3E000000 || (V & 0x7E07FFFF) == 0x40000000) {
10502 (0b11010 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
10503 return {
true, RequiredImm};
10505 }
else if (SplatBitSize == 64) {
10507 if ((V & 0xFFFFFFFF7E07FFFFULL) == 0x3E000000ULL ||
10508 (V & 0xFFFFFFFF7E07FFFFULL) == 0x40000000ULL) {
10510 (0b11011 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
10511 return {
true, RequiredImm};
10514 if ((V & 0x7FC0FFFFFFFFFFFFULL) == 0x4000000000000000ULL ||
10515 (V & 0x7FC0FFFFFFFFFFFFULL) == 0x3FC0000000000000ULL) {
10517 (0b11100 << 8) | (((V >> 56) & 0xC0) ^ 0x40) | ((V >> 48) & 0x3F);
10518 return {
true, RequiredImm};
10521 auto sameBitsPreByte = [](
uint64_t x) -> std::pair<bool, uint8_t> {
10523 for (
int i = 0; i < 8; ++i) {
10525 if (
byte == 0 ||
byte == 0xFF)
10526 res |= ((
byte & 1) << i);
10531 return {
true, res};
10533 auto [IsSame, Suffix] = sameBitsPreByte(V);
10535 RequiredImm = (0b11001 << 8) | Suffix;
10536 return {
true, RequiredImm};
10539 return {
false, RequiredImm};
10544 if (!Subtarget.hasExtLSX())
10547 if (VT == MVT::f32) {
10548 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7e07ffff;
10549 return (masked == 0x3e000000 || masked == 0x40000000);
10552 if (VT == MVT::f64) {
10553 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7fc0ffffffffffff;
10554 return (masked == 0x3fc0000000000000 || masked == 0x4000000000000000);
10560bool LoongArchTargetLowering::isFPImmLegal(
const APFloat &Imm,
EVT VT,
10561 bool ForCodeSize)
const {
10563 if (VT == MVT::f32 && !Subtarget.hasBasicF())
10565 if (VT == MVT::f64 && !Subtarget.hasBasicD())
10567 return (Imm.isZero() || Imm.isExactlyValue(1.0) ||
isFPImmVLDILegal(Imm, VT));
10578bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
10588 Type *Ty =
I->getOperand(0)->getType();
10590 unsigned Size = Ty->getIntegerBitWidth();
10610 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
10615 if (Subtarget.hasExtLASX())
10617 else if (Subtarget.hasExtLSX())
10624 EVT VT =
Y.getValueType();
10627 return Subtarget.hasExtLSX() && VT.
isInteger();
10638 case Intrinsic::loongarch_masked_atomicrmw_xchg_i32:
10639 case Intrinsic::loongarch_masked_atomicrmw_add_i32:
10640 case Intrinsic::loongarch_masked_atomicrmw_sub_i32:
10641 case Intrinsic::loongarch_masked_atomicrmw_nand_i32: {
10644 Info.memVT = MVT::i32;
10645 Info.ptrVal =
I.getArgOperand(0);
10647 Info.align =
Align(4);
10666 "Unable to expand");
10667 unsigned MinWordSize = 4;
10679 Value *AlignedAddr = Builder.CreateIntrinsic(
10680 Intrinsic::ptrmask, {PtrTy, IntTy},
10681 {Addr, ConstantInt::get(IntTy, ~(
uint64_t)(MinWordSize - 1))},
nullptr,
10684 Value *AddrInt = Builder.CreatePtrToInt(Addr, IntTy);
10685 Value *PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1,
"PtrLSB");
10686 Value *ShiftAmt = Builder.CreateShl(PtrLSB, 3);
10687 ShiftAmt = Builder.CreateTrunc(ShiftAmt, WordType,
"ShiftAmt");
10688 Value *Mask = Builder.CreateShl(
10689 ConstantInt::get(WordType,
10690 (1 << (
DL.getTypeStoreSize(
ValueType) * 8)) - 1),
10692 Value *Inv_Mask = Builder.CreateNot(Mask,
"Inv_Mask");
10693 Value *ValOperand_Shifted =
10694 Builder.CreateShl(Builder.CreateZExt(AI->
getValOperand(), WordType),
10695 ShiftAmt,
"ValOperand_Shifted");
10698 NewOperand = Builder.CreateOr(ValOperand_Shifted, Inv_Mask,
"AndOperand");
10700 NewOperand = ValOperand_Shifted;
10703 Builder.CreateAtomicRMW(
Op, AlignedAddr, NewOperand,
Align(MinWordSize),
10706 Value *Shift = Builder.CreateLShr(NewAI, ShiftAmt,
"shifted");
10707 Value *Trunc = Builder.CreateTrunc(Shift,
ValueType,
"extracted");
10708 Value *FinalOldResult = Builder.CreateBitCast(Trunc,
ValueType);
10727 if (Subtarget.hasLAM_BH() && Subtarget.is64Bit() &&
10735 if (Subtarget.hasLAMCAS()) {
10757 return Intrinsic::loongarch_masked_atomicrmw_xchg_i64;
10759 return Intrinsic::loongarch_masked_atomicrmw_add_i64;
10761 return Intrinsic::loongarch_masked_atomicrmw_sub_i64;
10763 return Intrinsic::loongarch_masked_atomicrmw_nand_i64;
10765 return Intrinsic::loongarch_masked_atomicrmw_umax_i64;
10767 return Intrinsic::loongarch_masked_atomicrmw_umin_i64;
10769 return Intrinsic::loongarch_masked_atomicrmw_max_i64;
10771 return Intrinsic::loongarch_masked_atomicrmw_min_i64;
10781 return Intrinsic::loongarch_masked_atomicrmw_xchg_i32;
10783 return Intrinsic::loongarch_masked_atomicrmw_add_i32;
10785 return Intrinsic::loongarch_masked_atomicrmw_sub_i32;
10787 return Intrinsic::loongarch_masked_atomicrmw_nand_i32;
10789 return Intrinsic::loongarch_masked_atomicrmw_umax_i32;
10791 return Intrinsic::loongarch_masked_atomicrmw_umin_i32;
10793 return Intrinsic::loongarch_masked_atomicrmw_max_i32;
10795 return Intrinsic::loongarch_masked_atomicrmw_min_i32;
10807 if (Subtarget.hasLAMCAS())
10819 unsigned GRLen = Subtarget.getGRLen();
10821 Value *FailureOrdering =
10822 Builder.getIntN(Subtarget.getGRLen(),
static_cast<uint64_t>(FailOrd));
10823 Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i32;
10825 CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
10826 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
10827 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
10828 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10831 Value *Result = Builder.CreateIntrinsic(
10832 CmpXchgIntrID, Tys, {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
10834 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10850 Builder.CreateNot(Mask,
"Inv_Mask"),
10857 unsigned GRLen = Subtarget.getGRLen();
10866 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
10867 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
10868 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
10881 unsigned ValWidth =
10884 Builder.CreateSub(Builder.getIntN(GRLen, GRLen - ValWidth), ShiftAmt);
10885 Result = Builder.CreateCall(LlwOpScwLoop,
10886 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
10889 Builder.CreateCall(LlwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
10893 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
10916 const Constant *PersonalityFn)
const {
10917 return LoongArch::R4;
10921 const Constant *PersonalityFn)
const {
10922 return LoongArch::R5;
10933 int RefinementSteps = VT.
getScalarType() == MVT::f64 ? 2 : 1;
10934 return RefinementSteps;
10939 assert(Subtarget.hasFrecipe() &&
10940 "Reciprocal estimate queried on unsupported target");
10951 return Subtarget.hasBasicD();
10955 return Subtarget.hasExtLSX();
10959 return Subtarget.hasExtLASX();
10968 int &RefinementSteps,
10969 bool &UseOneConstNR,
10970 bool Reciprocal)
const {
10972 "Enabled should never be Disabled here");
10974 if (!Subtarget.hasFrecipe())
10989 UseOneConstNR =
false;
10995 if (Reciprocal || RefinementSteps > 0)
11005 int &RefinementSteps)
const {
11007 "Enabled should never be Disabled here");
11009 if (!Subtarget.hasFrecipe())
11023 return DAG.
getNode(LoongArchISD::FRECIPE,
DL, VT, Operand);
11031LoongArchTargetLowering::getConstraintType(
StringRef Constraint)
const {
11051 if (Constraint.
size() == 1) {
11052 switch (Constraint[0]) {
11068 if (Constraint ==
"ZC" || Constraint ==
"ZB")
11077 return StringSwitch<InlineAsm::ConstraintCode>(ConstraintCode)
11084std::pair<unsigned, const TargetRegisterClass *>
11085LoongArchTargetLowering::getRegForInlineAsmConstraint(
11089 if (Constraint.
size() == 1) {
11090 switch (Constraint[0]) {
11095 return std::make_pair(0U, &LoongArch::GPRRegClass);
11097 return std::make_pair(0U, &LoongArch::GPRNoR0R1RegClass);
11099 if (Subtarget.hasBasicF() && VT == MVT::f32)
11100 return std::make_pair(0U, &LoongArch::FPR32RegClass);
11101 if (Subtarget.hasBasicD() && VT == MVT::f64)
11102 return std::make_pair(0U, &LoongArch::FPR64RegClass);
11103 if (Subtarget.hasExtLSX() &&
11104 TRI->isTypeLegalForClass(LoongArch::LSX128RegClass, VT))
11105 return std::make_pair(0U, &LoongArch::LSX128RegClass);
11106 if (Subtarget.hasExtLASX() &&
11107 TRI->isTypeLegalForClass(LoongArch::LASX256RegClass, VT))
11108 return std::make_pair(0U, &LoongArch::LASX256RegClass);
11128 bool IsFP = Constraint[2] ==
'f';
11129 std::pair<StringRef, StringRef> Temp = Constraint.
split(
'$');
11130 std::pair<unsigned, const TargetRegisterClass *>
R;
11135 unsigned RegNo =
R.first;
11136 if (LoongArch::F0 <= RegNo && RegNo <= LoongArch::F31) {
11137 if (Subtarget.hasBasicD() && (VT == MVT::f64 || VT == MVT::Other)) {
11138 unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64;
11139 return std::make_pair(DReg, &LoongArch::FPR64RegClass);
11149void LoongArchTargetLowering::LowerAsmOperandForConstraint(
11153 if (Constraint.
size() == 1) {
11154 switch (Constraint[0]) {
11158 uint64_t CVal =
C->getSExtValue();
11161 Subtarget.getGRLenVT()));
11167 uint64_t CVal =
C->getSExtValue();
11170 Subtarget.getGRLenVT()));
11176 if (
C->getZExtValue() == 0)
11183 uint64_t CVal =
C->getZExtValue();
11196#define GET_REGISTER_MATCHER
11197#include "LoongArchGenAsmMatcher.inc"
11203 std::string NewRegName = Name.second.str();
11209 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
11210 if (!ReservedRegs.
test(Reg))
11227 const APInt &Imm = ConstNode->getAPIntValue();
11229 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
11230 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
11233 if (ConstNode->hasOneUse() &&
11234 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
11235 (Imm - 8).isPowerOf2() || (Imm - 16).isPowerOf2()))
11241 if (ConstNode->hasOneUse() && !(Imm.sge(-2048) && Imm.sle(4095))) {
11242 unsigned Shifts = Imm.countr_zero();
11248 APInt ImmPop = Imm.ashr(Shifts);
11249 if (ImmPop == 3 || ImmPop == 5 || ImmPop == 9 || ImmPop == 17)
11253 APInt ImmSmall =
APInt(Imm.getBitWidth(), 1ULL << Shifts,
true);
11254 if ((Imm - ImmSmall).isPowerOf2() || (Imm + ImmSmall).isPowerOf2() ||
11255 (ImmSmall - Imm).isPowerOf2())
11265 Type *Ty,
unsigned AS,
11284 switch (AM.
Scale) {
11320 EVT MemVT = LD->getMemoryVT();
11321 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
11332 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
11341 if (
Y.getValueType().isVector())
11353 Type *Ty,
bool IsSigned)
const {
11354 if (Subtarget.is64Bit() && Ty->isIntegerTy(32))
11363 if (Subtarget.isSoftFPABI() && (
Type.isFloatingPoint() && !
Type.isVector() &&
11364 Type.getSizeInBits() < Subtarget.getGRLen()))
11374 Align &PrefAlign)
const {
11378 if (Subtarget.is64Bit()) {
11380 PrefAlign =
Align(8);
11383 PrefAlign =
Align(4);
11398bool LoongArchTargetLowering::splitValueIntoRegisterParts(
11400 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID> CC)
const {
11401 bool IsABIRegCopy = CC.has_value();
11404 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
11405 PartVT == MVT::f32) {
11420SDValue LoongArchTargetLowering::joinRegisterPartsIntoValue(
11422 MVT PartVT,
EVT ValueVT, std::optional<CallingConv::ID> CC)
const {
11423 bool IsABIRegCopy = CC.has_value();
11425 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
11426 PartVT == MVT::f32) {
11443 if (VT == MVT::f16 && Subtarget.hasBasicF())
11449unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
11452 if (VT == MVT::f16 && Subtarget.hasBasicF())
11461 unsigned Opc =
Op.getOpcode();
11466 case LoongArchISD::VPICK_ZEXT_ELT: {
11480 unsigned Depth)
const {
11481 EVT VT =
Op.getValueType();
11483 unsigned Opc =
Op.getOpcode();
11487 case LoongArchISD::CRC_W_B_W:
11488 case LoongArchISD::CRC_W_H_W:
11489 case LoongArchISD::CRCC_W_B_W:
11490 case LoongArchISD::CRCC_W_H_W: {
11492 APInt DemandedSrcBits =
11494 Opc == LoongArchISD::CRCC_W_B_W)
11498 OriginalDemandedElts, KnownSrc, TLO,
Depth + 1);
11500 case LoongArchISD::VMSKLTZ:
11501 case LoongArchISD::XVMSKLTZ: {
11503 MVT SrcVT = Src.getSimpleValueType();
11508 if (OriginalDemandedBits.
countr_zero() >= NumElts)
11512 APInt KnownUndef, KnownZero;
11528 if (KnownSrc.
One[SrcBits - 1])
11530 else if (KnownSrc.
Zero[SrcBits - 1])
11535 Src, DemandedSrcBits, DemandedElts, TLO.
DAG,
Depth + 1))
11542 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO,
Depth);
11565 unsigned Index)
const {
11574 unsigned Index)
const {
11578 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
11593 Align StackAlign)
const {
11597 unsigned StackProbeSize =
11601 return StackProbeSize ? StackProbeSize : StackAlign.
value();
11605LoongArchTargetLowering::lowerDYNAMIC_STACKALLOC(
SDValue Op,
11619 const EVT VT =
Op.getValueType();
11630 Chain = DAG.
getNode(LoongArchISD::PROBED_ALLOCA, dl, MVT::Other, Chain, SP);
11640 const Register TargetReg =
MI.getOperand(0).getReg();
11643 const bool IsLA64 = Subtarget.is64Bit();
11644 const Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
11651 MF.
insert(MBBInsertPoint, LoopTestMBB);
11654 MF.
insert(MBBInsertPoint, ExitMBB);
11665 TII->get(IsLA64 ? LoongArch::SUB_D : LoongArch::SUB_W),
SPReg)
11671 TII->get(IsLA64 ? LoongArch::ST_D : LoongArch::ST_W))
11677 BuildMI(*LoopTestMBB, LoopTestMBB->
end(),
DL,
TII->get(LoongArch::BLTU))
11692 MBB->addSuccessor(LoopTestMBB);
11694 MI.eraseFromParent();
11696 return ExitMBB->
begin()->getParent();
static MCRegister MatchRegisterName(StringRef Name)
static SDValue performSHLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
If the operand is a bitwise AND with a constant RHS, and the shift has a constant RHS and is the only...
static bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSELECT_CCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static uint64_t getConstant(const Value *IndexValue)
static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
static bool isSigned(unsigned Opcode)
const HexagonInstrInfo * TII
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static SDValue performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
const MCPhysReg ArgFPR32s[]
static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Dispatching routine to lower various 128-bit LoongArch vector shuffles.
static SDValue lowerVECTOR_SHUFFLE_XVSHUF4I(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).
static SDValue lowerVECTOR_SHUFFLE_VPICKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPICKEV (if possible).
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVPICKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPICKOD (if possible).
static SDValue unpackF64OnLA32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static bool fitsRegularPattern(typename SmallVectorImpl< ValType >::const_iterator Begin, unsigned CheckStride, typename SmallVectorImpl< ValType >::const_iterator End, ValType ExpectedIndex, unsigned ExpectedIndexStride)
Determine whether a range fits a regular pattern of values.
static SDValue lowerVECTOR_SHUFFLE_IsReverse(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE whose result is the reversed source vector.
static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned Depth)
static SDValue performHorizWideningCombine(SDNode *N, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg, SelectionDAG &DAG)
static cl::opt< bool > ZeroDivCheck("loongarch-check-zero-division", cl::Hidden, cl::desc("Trap on integer division by zero."), cl::init(false))
static SDValue lowerVECTOR_SHUFFLE_XVPERMI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVPERMI (if possible).
static SDValue lowerVECTOR_SHUFFLE_VSHUF(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VSHUF.
static int getEstimateRefinementSteps(EVT VT, const LoongArchSubtarget &Subtarget)
static bool isSupportedReciprocalEstimateType(EVT VT, const LoongArchSubtarget &Subtarget)
static void emitErrorAndReplaceIntrinsicResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, StringRef ErrorMsg, bool WithChain=true)
static SDValue lowerVECTOR_SHUFFLEAsByteRotate(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE as byte rotate (if possible).
static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp, SelectionDAG &DAG, bool IsSigned=false)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVINSVE0(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVINSVE0 (if possible).
static SDValue performMOVFR2GR_SCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VILVH(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VILVH (if possible).
static SDValue performDemandedBitsCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI, unsigned ValNo, MVT ValVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsRet, Type *OrigTy)
static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue performSPLIT_PAIR_F64Combine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static MachineBasicBlock * emitSplitPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue lowerVectorBitSetImm(SDNode *Node, SelectionDAG &DAG)
static SDValue performSETCC_BITCASTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performEXTENDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVPACKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPACKOD (if possible).
static bool buildVPERMIInfo(ArrayRef< int > Mask, SDValue V1, SDValue V2, SmallVectorImpl< SDValue > &SrcVec, unsigned &MaskImm)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue combineAndNotIntoVANDN(SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
Try to fold: (and (xor X, -1), Y) -> (vandn X, Y).
static SDValue matchLowHalfOf128BitLanes(SDValue N)
static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp, const SDLoc &DL, SelectionDAG &DAG)
#define CRC_CASE_EXT_BINARYOP(NAME, NODE)
static SDValue lowerVectorBitRevImm(SDNode *Node, SelectionDAG &DAG)
static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size, unsigned Depth)
static bool isConstantSplatVector(SDValue N, APInt &SplatValue, unsigned MinSizeInBits)
static SDValue lowerVECTOR_SHUFFLEAsShift(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, const APInt &Zeroable)
Lower VECTOR_SHUFFLE as shift (if possible).
static SDValue lowerVECTOR_SHUFFLE_VSHUF4I(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VSHUF4I (if possible).
static SDValue truncateVecElts(SDNode *Node, SelectionDAG &DAG)
static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
static MachineBasicBlock * insertDivByZeroTrap(MachineInstr &MI, MachineBasicBlock *MBB)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE_VEXTRINS(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VEXTRINS (if possible).
static SDValue lowerVectorBitClear(SDNode *Node, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE_VPACKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPACKEV (if possible).
static MachineBasicBlock * emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue performSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performVANDNCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
Do target-specific dag combines on LoongArchISD::VANDN nodes.
static void replaceVPICKVE2GRResults(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned ResOp)
static SDValue lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const APInt &Zeroable)
Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
static SDValue legalizeIntrinsicImmArg(SDNode *Node, unsigned ImmOp, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, bool IsSigned=false)
static cl::opt< MaterializeFPImm > MaterializeFPImmInsNum("loongarch-materialize-float-imm", cl::Hidden, cl::desc("Maximum number of instructions used (including code sequence " "to generate the value and moving the value to FPR) when " "materializing floating-point immediates (default = 3)"), cl::init(MaterializeFPImm3Ins), cl::values(clEnumValN(NoMaterializeFPImm, "0", "Use constant pool"), clEnumValN(MaterializeFPImm2Ins, "2", "Materialize FP immediate within 2 instructions"), clEnumValN(MaterializeFPImm3Ins, "3", "Materialize FP immediate within 3 instructions"), clEnumValN(MaterializeFPImm4Ins, "4", "Materialize FP immediate within 4 instructions"), clEnumValN(MaterializeFPImm5Ins, "5", "Materialize FP immediate within 5 instructions"), clEnumValN(MaterializeFPImm6Ins, "6", "Materialize FP immediate within 6 instructions " "(behaves same as 5 on loongarch64)")))
static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op, StringRef ErrorMsg, SelectionDAG &DAG)
static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2)
static unsigned getLoongArchWOpcode(unsigned Opcode)
const MCPhysReg ArgFPR64s[]
static MachineBasicBlock * emitPseudoCTPOP(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue performMOVGR2FR_WCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
#define IOCSRWR_CASE(NAME, NODE)
#define CRC_CASE_EXT_UNARYOP(NAME, NODE)
static SDValue lowerVECTOR_SHUFFLE_VPACKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPACKOD (if possible).
static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL)
static SDValue isNOT(SDValue V, SelectionDAG &DAG)
static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Dispatching routine to lower various 256-bit LoongArch vector shuffles.
static SDValue lowerVECTOR_SHUFFLE_VREPLVEI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VREPLVEI (if possible).
static MachineBasicBlock * emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
const MCPhysReg PreserveNoneArgGPRs[]
static void fillVector(ArrayRef< SDValue > Ops, SelectionDAG &DAG, SDLoc DL, const LoongArchSubtarget &Subtarget, SDValue &Vector, EVT ResTy)
static SDValue fillSubVectorFromBuildVector(BuildVectorSDNode *Node, SelectionDAG &DAG, SDLoc DL, const LoongArchSubtarget &Subtarget, EVT ResTy, unsigned first)
static bool isSelectPseudo(MachineInstr &MI)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp, SelectionDAG &DAG, bool IsSigned=false)
const MCPhysReg ArgGPRs[]
static SDValue lowerVECTOR_SHUFFLE_XVPERM(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVPERM (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVILVL(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVILVL (if possible).
static SDValue performFP_TO_INTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VPERMI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VPERMI (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVEXTRINS(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVEXTRINS (if possible).
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static void replaceVecCondBranchResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned ResOp)
#define ASRT_LE_GT_CASE(NAME)
static SDValue lowerVECTOR_SHUFFLE_XVPACKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPACKEV (if possible).
static SDValue matchDeinterleaveBuildVector(SDValue N, unsigned &StartIndex)
static SDValue performBR_CCCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static void computeZeroableShuffleElements(ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero)
Compute whether each element of a shuffle is zeroable.
static SDValue combineFP_ROUND(SDValue N, const SDLoc &DL, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue widenShuffleMask(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
static MachineBasicBlock * emitVecCondBranchPseudo(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static bool canonicalizeShuffleVectorByLane(const SDLoc &DL, MutableArrayRef< int > Mask, MVT VT, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Shuffle vectors by lane to generate more optimized instructions.
static SDValue lowerVECTOR_SHUFFLE_XVILVH(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVILVH (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVSHUF (if possible).
static void replaceCMP_XCHG_128Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static SDValue lowerVectorPickVE2GR(SDNode *N, SelectionDAG &DAG, unsigned ResOp)
static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
#define IOCSRRD_CASE(NAME, NODE)
static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
Attempts to match vector shuffle as byte rotation.
static SDValue lowerVECTOR_SHUFFLE_XVPICKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPICKEV (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVREPLVEI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).
static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI instruction.
static SDValue lowerVECTOR_SHUFFLE_VILVL(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VILVL (if possible).
static SDValue lowerVectorBitClearImm(SDNode *Node, SelectionDAG &DAG)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
static void replaceINTRINSIC_WO_CHAINResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VPICKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPICKOD (if possible).
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static Register allocateArgGPR(CCState &State)
static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
Test whether a shuffle mask is equivalent within each sub-lane.
Register const TargetRegisterInfo * TRI
Promote Memory to Register
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static constexpr MCPhysReg SPReg
const SmallVectorImpl< MachineOperand > & Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static bool inRange(const MCExpr *Expr, int64_t MinValue, int64_t MaxValue, bool AllowSymbol=false)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static bool isSequentialOrUndefInRange(ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
Return true if every element in Mask, beginning from position Pos and ending in Pos + Size,...
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
uint64_t getZExtValue() const
Get zero extended value.
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
unsigned countr_zero() const
Count the number of trailing zero bits.
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
This class represents an incoming formal argument to a Function.
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
Get the array size.
An instruction that atomically checks whether a specified value is in a memory location,...
Value * getCompareOperand()
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
Value * getPointerOperand()
bool isFloatingPointOperation() const
BinOp getOperation() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
LLVM_ABI StringRef getValueAsString() const
Return the attribute's value as a string.
LLVM Basic Block Representation.
bool test(unsigned Idx) const
Returns true if bit Idx is set.
size_type count() const
Returns the number of bits which are set.
A "pseudo-class" with methods for operating on BUILD_VECTORs.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
LLVM_ABI void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getReg(unsigned ValNo, MVT ValVT, MCRegister Reg, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, MCRegister Reg, MVT LocVT, LocInfo HTP)
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
This class represents a function call, abstracting a target machine's calling convention.
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
uint64_t getZExtValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getPointerSizeInBits(unsigned AS=0) const
The size in bits of the pointer representation in a given address space.
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
iterator_range< arg_iterator > args()
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Argument * getArg(unsigned i) const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Common base class shared among various IRBuilders.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
This is an important class for using LLVM in a threaded context.
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
LoongArchMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private Lo...
void setIncomingIndirectArg(unsigned ArgIndex, Register Reg)
void addSExt32Register(Register Reg)
Register getIncomingIndirectArg(unsigned ArgIndex) const
const LoongArchRegisterInfo * getRegisterInfo() const override
const LoongArchInstrInfo * getInstrInfo() const override
unsigned getGRLen() const
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const override
Hooks for building estimates in place of slower divisions and square roots.
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
std::pair< bool, uint64_t > isImmVLDILegalForMode1(const APInt &SplatValue, const unsigned SplatBitSize) const
Check if a constant splat can be generated using [x]vldi, where imm[12] is 1.
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this function.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Determine if the target supports unaligned memory accesses.
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
MachineBasicBlock * emitDynamicProbedAlloc(MachineInstr &MI, MachineBasicBlock *MBB) const
bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isFPImmVLDILegal(const APFloat &Imm, EVT VT) const
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
unsigned getStackProbeSize(const MachineFunction &MF, Align StackAlign) const
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
void emitExpandAtomicRMW(AtomicRMWInst *AI) const override
Perform a atomicrmw expansion using a target-specific way.
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
LoongArchTargetLowering(const TargetMachine &TM, const LoongArchSubtarget &STI)
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const override
Return a reciprocal estimate value for the input operand.
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context, const Type *RetTy) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
Wrapper class representing physical registers. Should be passed by value.
bool hasFeature(unsigned Feature) const
static MVT getFloatingPointVT(unsigned BitWidth)
bool is128BitVector() const
Return true if this is a 128-bit vector type.
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
bool is256BitVector() const
Return true if this is a 256-bit vector type.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
void push_back(MachineInstr *MI)
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
bool isImplicitDef() const
LLVM_ABI void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
void setIsKill(bool Val=true)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Class to represent pointers.
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
LLVM_ABI bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
size_t use_size() const
Return the number of uses of this node.
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getExtractSubvector(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Return the VT typed sub-vector of Vec at Idx.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getInsertSubvector(const SDLoc &DL, SDValue Vec, SDValue SubVec, unsigned Idx)
Insert SubVec at the Idx element of Vec.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
LLVM_ABI SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static LLVM_ABI bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
ArrayRef< int > getMask() const
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
void reserve(size_type N)
typename SuperClass::const_iterator const_iterator
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
constexpr size_t size() const
Get the string size.
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setMaxBytesForAlignment(unsigned MaxBytes)
bool isOperationLegalOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal using promotion.
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
@ ZeroOrOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
TargetLowering(const TargetLowering &)=delete
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Primary interface to the complete machine description for the target machine.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
bool shouldAssumeDSOLocal(const GlobalValue *GV) const
CodeModel::Model getCodeModel() const
Returns the code model.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
LLVM_ABI unsigned getIntegerBitWidth() const
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ PreserveMost
Used for runtime calls that preserves most registers.
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
@ Fast
Attempts to make calls as fast as possible (e.g.
@ PreserveNone
Used for runtime calls that preserves none general registers.
@ C
The default llvm calling convention, compatible with C.
LLVM_ABI bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum maximum on two values, following IEEE-754 definition...
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isExtVecInRegOpcode(unsigned Opcode)
LLVM_ABI bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
LLVM_ABI bool isFreezeUndef(const SDNode *N)
Return true if the specified node is FREEZE(UNDEF).
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
ABI getTargetABI(StringRef ABIName)
InstSeq generateInstSeq(int64_t Val)
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
NodeAddr< NodeBase * > Node
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
FunctionAddr VTableAddr Value
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
constexpr RegState getKillRegState(bool B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
LLVM_ABI bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
AtomicOrdering
Atomic ordering for LLVM's memory model.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
FunctionAddr VTableAddr Next
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr bool isShiftedInt(int64_t x)
Checks if a signed integer is an N bit number shifted left by S.
constexpr unsigned BitWidth
std::string join_items(Sep Separator, Args &&... Items)
Joins the strings in the parameter pack Items, adding Separator between the elements....
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool is256BitVector() const
Return true if this is a 256-bit vector type.
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
bool isInteger() const
Return true if this is an integer or a vector integer type.
Align getNonZeroOrigAlign() const
unsigned getBitWidth() const
Get the bit width of this value.
void resetAll()
Resets the known state of all bits.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals
bool isBeforeLegalizeOps() const
LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
bool CombineTo(SDValue O, SDValue N)