51#define DEBUG_TYPE "legalizevectorops"
55class VectorLegalizer {
67 LegalizedNodes.
insert(std::make_pair(
From, To));
70 LegalizedNodes.
insert(std::make_pair(To, To));
139 std::pair<SDValue, SDValue> ExpandLoad(
SDNode *
N);
197bool VectorLegalizer::Run() {
199 bool HasVectors =
false;
201 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I) {
221 DAG.AssignTopologicalOrder();
223 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I)
227 SDValue OldRoot = DAG.getRoot();
228 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
229 DAG.setRoot(LegalizedNodes[OldRoot]);
231 LegalizedNodes.clear();
234 DAG.RemoveDeadNodes();
241 "Unexpected number of results");
243 for (
unsigned i = 0, e =
Op->getNumValues(); i != e; ++i)
244 AddLegalizedOperand(
Op.getValue(i),
SDValue(Result, i));
249VectorLegalizer::RecursivelyLegalizeResults(
SDValue Op,
252 "Unexpected number of results");
254 for (
unsigned i = 0, e =
Results.
size(); i != e; ++i) {
256 AddLegalizedOperand(
Op.getValue(i),
Results[i]);
266 if (
I != LegalizedNodes.end())
return I->second;
270 for (
const SDValue &Oper :
Op->op_values())
273 SDNode *
Node = DAG.UpdateNodeOperands(
Op.getNode(), Ops);
275 bool HasVectorValueOrOp =
278 [](
SDValue O) { return O.getValueType().isVector(); });
279 if (!HasVectorValueOrOp)
280 return TranslateLegalizeResults(
Op,
Node);
284 switch (
Op.getOpcode()) {
286 return TranslateLegalizeResults(
Op,
Node);
290 EVT LoadedVT =
LD->getMemoryVT();
292 Action = TLI.getLoadExtAction(ExtType,
LD->getValueType(0), LoadedVT);
297 EVT StVT =
ST->getMemoryVT();
298 MVT ValVT =
ST->getValue().getSimpleValueType();
299 if (StVT.
isVector() &&
ST->isTruncatingStore())
300 Action = TLI.getTruncStoreAction(ValVT, StVT);
304 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
307 if (Action == TargetLowering::Legal)
308 Action = TargetLowering::Expand;
310#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
311 case ISD::STRICT_##DAGN:
312#include "llvm/IR/ConstrainedOps.def"
313 ValVT =
Node->getValueType(0);
316 ValVT =
Node->getOperand(1).getValueType();
319 MVT OpVT =
Node->getOperand(1).getSimpleValueType();
321 Action = TLI.getCondCodeAction(CCCode, OpVT);
322 if (Action == TargetLowering::Legal)
323 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
325 Action = TLI.getOperationAction(
Node->getOpcode(), ValVT);
332 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
333 TLI.getStrictFPOperationAction(
Node->getOpcode(), ValVT) ==
334 TargetLowering::Legal) {
336 if (TLI.getOperationAction(
Node->getOpcode(), EltVT)
337 == TargetLowering::Expand &&
338 TLI.getStrictFPOperationAction(
Node->getOpcode(), EltVT)
339 == TargetLowering::Legal)
340 Action = TargetLowering::Legal;
447 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
457 unsigned Scale =
Node->getConstantOperandVal(2);
458 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
459 Node->getValueType(0), Scale);
481 Action = TLI.getOperationAction(
Node->getOpcode(),
482 Node->getOperand(0).getValueType());
486 Action = TLI.getOperationAction(
Node->getOpcode(),
487 Node->getOperand(1).getValueType());
490 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
492 Action = TLI.getCondCodeAction(CCCode, OpVT);
493 if (Action == TargetLowering::Legal)
494 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
498#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
500 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
501 : Node->getOperand(LEGALPOS).getValueType(); \
502 if (ISD::VPID == ISD::VP_SETCC) { \
503 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
504 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
505 if (Action != TargetLowering::Legal) \
509 if (!Node->getValueType(0).isVector()) { \
510 Action = TargetLowering::Legal; \
513 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
515#include "llvm/IR/VPIntrinsics.def"
523 case TargetLowering::Promote:
525 "This action is not supported yet!");
527 Promote(
Node, ResultVals);
528 assert(!ResultVals.
empty() &&
"No results for promotion?");
530 case TargetLowering::Legal:
533 case TargetLowering::Custom:
535 if (LowerOperationWrapper(
Node, ResultVals))
539 case TargetLowering::Expand:
541 Expand(
Node, ResultVals);
545 if (ResultVals.
empty())
546 return TranslateLegalizeResults(
Op,
Node);
549 return RecursivelyLegalizeResults(
Op, ResultVals);
554bool VectorLegalizer::LowerOperationWrapper(
SDNode *
Node,
566 if (
Node->getNumValues() == 1) {
574 "Lowering returned the wrong number of results!");
577 for (
unsigned I = 0, E =
Node->getNumValues();
I != E; ++
I)
583void VectorLegalizer::PromoteSETCC(
SDNode *
Node,
585 MVT VecVT =
Node->getOperand(0).getSimpleValueType();
586 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
593 Operands[0] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(0));
594 Operands[1] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(1));
597 if (
Node->getOpcode() == ISD::VP_SETCC) {
608void VectorLegalizer::PromoteSTRICT(
SDNode *
Node,
610 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
611 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
619 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
620 if (
Node->getOperand(j).getValueType().isVector() &&
627 {
Node->getOperand(0),
Node->getOperand(j)});
633 SDVTList VTs = DAG.getVTList(NewVecVT,
Node->getValueType(1));
643 DAG.getIntPtrConstant(0,
DL,
true)});
652 switch (
Node->getOpcode()) {
693 "Can't promote a vector with multiple results!");
694 MVT VT =
Node->getSimpleValueType(0);
695 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
699 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
703 if (
Node->getOperand(j).getValueType().isVector() && !SkipPromote)
704 if (
Node->getOperand(j)
706 .getVectorElementType()
707 .isFloatingPoint() &&
723 DAG.getIntPtrConstant(0, dl,
true));
730void VectorLegalizer::PromoteINT_TO_FP(
SDNode *
Node,
734 bool IsStrict =
Node->isStrictFPOpcode();
735 MVT VT =
Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
736 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
738 "Vectors have different number of elements!");
747 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
748 if (
Node->getOperand(j).getValueType().isVector())
749 Operands[j] = DAG.getNode(Opc, dl, NVT,
Node->getOperand(j));
771void VectorLegalizer::PromoteFP_TO_INT(
SDNode *
Node,
773 MVT VT =
Node->getSimpleValueType(0);
774 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
775 bool IsStrict =
Node->isStrictFPOpcode();
777 "Vectors have different number of elements!");
779 unsigned NewOpc =
Node->getOpcode();
793 Promoted = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
794 {
Node->getOperand(0),
Node->getOperand(1)});
797 Promoted = DAG.
getNode(NewOpc, dl, NVT,
Node->getOperand(0));
808 Promoted = DAG.
getNode(NewOpc, dl, NVT, Promoted,
816std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(
SDNode *
N) {
818 return TLI.scalarizeVectorLoad(LD, DAG);
823 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
828 switch (
Node->getOpcode()) {
830 std::pair<SDValue, SDValue> Tmp = ExpandLoad(
Node);
839 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
846 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(
Node));
849 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(
Node));
852 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(
Node));
877 if (
Node->getValueType(0).isScalableVector()) {
878 EVT CondVT = TLI.getSetCCResultType(
879 DAG.getDataLayout(), *DAG.getContext(),
Node->getValueType(0));
882 Node->getOperand(1),
Node->getOperand(4));
885 Node->getOperand(3)));
922 case ISD::VP_BITREVERSE:
923 if (
SDValue Expanded = TLI.expandVPBITREVERSE(
Node, DAG)) {
929 if (
SDValue Expanded = TLI.expandCTPOP(
Node, DAG)) {
935 if (
SDValue Expanded = TLI.expandVPCTPOP(
Node, DAG)) {
942 if (
SDValue Expanded = TLI.expandCTLZ(
Node, DAG)) {
948 case ISD::VP_CTLZ_ZERO_UNDEF:
949 if (
SDValue Expanded = TLI.expandVPCTLZ(
Node, DAG)) {
956 if (
SDValue Expanded = TLI.expandCTTZ(
Node, DAG)) {
962 case ISD::VP_CTTZ_ZERO_UNDEF:
963 if (
SDValue Expanded = TLI.expandVPCTTZ(
Node, DAG)) {
972 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG)) {
979 if (
SDValue Expanded = TLI.expandROT(
Node,
false , DAG)) {
986 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG)) {
993 if (
SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(
Node, DAG)) {
1002 if (
SDValue Expanded = TLI.expandIntMINMAX(
Node, DAG)) {
1023 if (
SDValue Expanded = TLI.expandAddSubSat(
Node, DAG)) {
1030 if (
SDValue Expanded = TLI.expandShlSat(
Node, DAG)) {
1038 if (
Node->getValueType(0).isScalableVector()) {
1039 if (
SDValue Expanded = TLI.expandFP_TO_INT_SAT(
Node, DAG)) {
1047 if (
SDValue Expanded = TLI.expandFixedPointMul(
Node, DAG)) {
1066#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1067 case ISD::STRICT_##DAGN:
1068#include "llvm/IR/ConstrainedOps.def"
1086 Results.push_back(TLI.expandVecReduce(
Node, DAG));
1090 Results.push_back(TLI.expandVecReduceSeq(
Node, DAG));
1100 if (tryExpandVecMathCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
1101 RTLIB::REM_F80, RTLIB::REM_F128,
1109 if (
Node->getNumValues() == 1) {
1113 "VectorLegalizer Expand returned wrong number of results!");
1123 EVT VT =
Node->getValueType(0);
1141 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1142 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1143 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand ||
1146 VT) == TargetLowering::Expand)
1147 return DAG.UnrollVectorOp(
Node);
1155 Mask = DAG.getSelect(
DL, BitTy, Mask, DAG.getAllOnesConstant(
DL, BitTy),
1156 DAG.getConstant(0,
DL, BitTy));
1159 Mask = DAG.getSplat(MaskTy,
DL, Mask);
1167 SDValue NotMask = DAG.getNOT(
DL, Mask, MaskTy);
1176 EVT VT =
Node->getValueType(0);
1179 if (TLI.getOperationAction(
ISD::SRA, VT) == TargetLowering::Expand ||
1180 TLI.getOperationAction(
ISD::SHL, VT) == TargetLowering::Expand)
1181 return DAG.UnrollVectorOp(
Node);
1184 EVT OrigTy = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1188 SDValue ShiftSz = DAG.getConstant(BW - OrigBW,
DL, VT);
1198 EVT VT =
Node->getValueType(0);
1201 EVT SrcVT = Src.getValueType();
1208 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1213 Src, DAG.getVectorIdxConstant(0,
DL));
1218 ShuffleMask.
resize(NumSrcElements, -1);
1221 int ExtLaneScale = NumSrcElements / NumElements;
1222 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1223 for (
int i = 0; i < NumElements; ++i)
1224 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1228 DAG.getVectorShuffle(SrcVT,
DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1233 EVT VT =
Node->getValueType(0);
1235 EVT SrcVT = Src.getValueType();
1246 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth,
DL, VT);
1257 EVT VT =
Node->getValueType(0);
1260 EVT SrcVT = Src.getValueType();
1267 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1272 Src, DAG.getVectorIdxConstant(0,
DL));
1280 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1282 int ExtLaneScale = NumSrcElements / NumElements;
1283 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1284 for (
int i = 0; i < NumElements; ++i)
1285 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1288 DAG.getVectorShuffle(SrcVT,
DL, Zero, Src, ShuffleMask));
1294 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
1295 ShuffleMask.
push_back((
I * ScalarSizeInBytes) + J);
1299 EVT VT =
Node->getValueType(0);
1303 return TLI.expandBSWAP(
Node, DAG);
1311 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1314 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1320 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1321 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1322 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1323 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
1324 return TLI.expandBSWAP(
Node, DAG);
1327 return DAG.UnrollVectorOp(
Node);
1330void VectorLegalizer::ExpandBITREVERSE(
SDNode *
Node,
1332 EVT VT =
Node->getValueType(0);
1336 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1351 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1356 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1358 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
1359 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
1360 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
1361 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
1364 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT),
1375 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1376 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1377 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1378 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT)) {
1379 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1403 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1404 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1405 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand)
1406 return DAG.UnrollVectorOp(
Node);
1412 auto BoolContents = TLI.getBooleanContents(Op1.
getValueType());
1413 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1414 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1416 return DAG.UnrollVectorOp(
Node);
1422 return DAG.UnrollVectorOp(
Node);
1430 SDValue NotMask = DAG.getNOT(
DL, Mask, VT);
1452 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1453 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1454 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1455 return DAG.UnrollVectorOp(
Node);
1459 return DAG.UnrollVectorOp(
Node);
1461 SDValue Ones = DAG.getAllOnesConstant(
DL, VT);
1464 Op1 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op1, Mask, Ones, EVL);
1465 Op2 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op2, NotMask, Ones, EVL);
1466 return DAG.getNode(ISD::VP_OR,
DL, VT, Op1, Op2, Ones, EVL);
1481 EVT MaskVT =
Mask.getValueType();
1493 return DAG.UnrollVectorOp(
Node);
1497 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1498 EVLVecVT) != MaskVT)
1499 return DAG.UnrollVectorOp(
Node);
1501 SDValue StepVec = DAG.getStepVector(
DL, EVLVecVT);
1502 SDValue SplatEVL = DAG.getSplat(EVLVecVT,
DL, EVL);
1504 DAG.getSetCC(
DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1507 return DAG.getSelect(
DL,
Node->getValueType(0), FullMask, Op1, Op2);
1512 EVT VT =
Node->getValueType(0);
1514 unsigned DivOpc =
Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1516 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1517 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1518 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1530 SDValue Mul = DAG.getNode(ISD::VP_MUL,
DL, VT, Divisor, Div, Mask, EVL);
1531 return DAG.getNode(ISD::VP_SUB,
DL, VT, Dividend,
Mul, Mask, EVL);
1534void VectorLegalizer::ExpandFP_TO_UINT(
SDNode *
Node,
1538 if (TLI.expandFP_TO_UINT(
Node, Result, Chain, DAG)) {
1540 if (
Node->isStrictFPOpcode())
1546 if (
Node->isStrictFPOpcode()) {
1554void VectorLegalizer::ExpandUINT_TO_FLOAT(
SDNode *
Node,
1556 bool IsStrict =
Node->isStrictFPOpcode();
1557 unsigned OpNo = IsStrict ? 1 : 0;
1559 EVT VT = Src.getValueType();
1565 if (TLI.expandUINT_TO_FP(
Node, Result, Chain, DAG)) {
1574 TargetLowering::Expand) ||
1576 TargetLowering::Expand)) ||
1577 TLI.getOperationAction(
ISD::SRL, VT) == TargetLowering::Expand) {
1588 assert((BW == 64 || BW == 32) &&
1589 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1591 SDValue HalfWord = DAG.getConstant(BW / 2,
DL, VT);
1596 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1597 SDValue HalfWordMask = DAG.getConstant(HWMask,
DL, VT);
1601 DAG.getConstantFP(1ULL << (BW / 2),
DL,
Node->getValueType(0));
1612 {
Node->getValueType(0), MVT::Other},
1613 {
Node->getOperand(0),
HI});
1617 {
Node->getValueType(0), MVT::Other},
1618 {
Node->getOperand(0),
LO});
1646 if (TLI.isOperationLegalOrCustom(
ISD::FSUB,
Node->getValueType(0))) {
1651 Node->getOperand(0));
1653 return DAG.UnrollVectorOp(
Node);
1656void VectorLegalizer::ExpandFSUB(
SDNode *
Node,
1661 EVT VT =
Node->getValueType(0);
1662 if (TLI.isOperationLegalOrCustom(
ISD::FNEG, VT) &&
1663 TLI.isOperationLegalOrCustom(
ISD::FADD, VT))
1670void VectorLegalizer::ExpandSETCC(
SDNode *
Node,
1672 bool NeedInvert =
false;
1673 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
1677 unsigned Offset = IsStrict ? 1 : 0;
1684 MVT OpVT =
LHS.getSimpleValueType();
1687 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1704 TLI.LegalizeSetCCCondCode(DAG,
Node->getValueType(0), LHS, RHS,
CC, Mask,
1705 EVL, NeedInvert, dl, Chain, IsSignaling);
1712 LHS = DAG.getNode(
Node->getOpcode(), dl,
Node->getVTList(),
1713 {Chain, LHS, RHS, CC},
Node->getFlags());
1714 Chain =
LHS.getValue(1);
1716 LHS = DAG.getNode(ISD::VP_SETCC, dl,
Node->getValueType(0),
1717 {LHS, RHS, CC, Mask, EVL},
Node->getFlags());
1728 LHS = DAG.getLogicalNOT(dl, LHS,
LHS->getValueType(0));
1730 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL,
LHS->getValueType(0));
1733 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
1737 EVT VT =
Node->getValueType(0);
1740 DAG.getBoolConstant(
true, dl, VT,
LHS.getValueType()),
1741 DAG.getBoolConstant(
false, dl, VT,
LHS.getValueType()),
CC);
1742 LHS->setFlags(
Node->getFlags());
1750void VectorLegalizer::ExpandUADDSUBO(
SDNode *
Node,
1753 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
1758void VectorLegalizer::ExpandSADDSUBO(
SDNode *
Node,
1761 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
1766void VectorLegalizer::ExpandMULO(
SDNode *
Node,
1769 if (!TLI.expandMULO(
Node, Result, Overflow, DAG))
1770 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(
Node);
1776void VectorLegalizer::ExpandFixedPointDiv(
SDNode *
Node,
1779 if (
SDValue Expanded = TLI.expandFixedPointDiv(
N->getOpcode(),
SDLoc(
N),
1780 N->getOperand(0),
N->getOperand(1),
N->getConstantOperandVal(2), DAG))
1784void VectorLegalizer::ExpandStrictFPOp(
SDNode *
Node,
1807 "Expected REM node");
1810 if (!TLI.expandREM(
Node, Result, DAG))
1825 assert(!
Node->isStrictFPOpcode() &&
"Unexpected strict fp operation!");
1827 const char *LCName = TLI.getLibcallName(LC);
1830 LLVM_DEBUG(
dbgs() <<
"Looking for vector variant of " << LCName <<
"\n");
1832 EVT VT =
Node->getValueType(0);
1850 for (
unsigned i = 0; i <
Node->getNumOperands(); ++i) {
1851 assert(
Node->getOperand(i).getValueType() == VT &&
1852 "Expected matching vector types!");
1855 FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys,
false);
1867 if (OptVFInfo->Shape.Parameters.size() !=
1876 Entry.IsSExt =
false;
1877 Entry.IsZExt =
false;
1880 for (
auto &VFParam : OptVFInfo->Shape.Parameters) {
1881 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
1882 EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
1883 Entry.Node = DAG.getBoolConstant(
true,
DL, MaskVT, VT);
1885 Args.push_back(Entry);
1890 if (VFParam.ParamKind != VFParamKind::Vector)
1893 Entry.Node =
Node->getOperand(OpNum++);
1895 Args.push_back(Entry);
1900 TLI.getPointerTy(DAG.getDataLayout()));
1903 .setChain(DAG.getEntryNode())
1906 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
1907 Results.push_back(CallResult.first);
1912bool VectorLegalizer::tryExpandVecMathCall(
1917 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,
1918 Call_F80, Call_F128, Call_PPCF128);
1920 if (LC == RTLIB::UNKNOWN_LIBCALL)
1926void VectorLegalizer::UnrollStrictFPOp(
SDNode *
Node,
1928 EVT VT =
Node->getValueType(0);
1931 unsigned NumOpers =
Node->getNumOperands();
1934 EVT TmpEltVT = EltVT;
1938 *DAG.getContext(), TmpEltVT);
1940 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1946 for (
unsigned i = 0; i < NumElems; ++i) {
1948 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1954 for (
unsigned j = 1;
j < NumOpers; ++
j) {
1971 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1972 DAG.getAllOnesConstant(dl, EltVT),
1973 DAG.getConstant(0, dl, EltVT));
1987 EVT VT =
Node->getValueType(0);
1993 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
1996 for (
unsigned i = 0; i < NumElems; ++i) {
1998 DAG.getVectorIdxConstant(i, dl));
2000 DAG.getVectorIdxConstant(i, dl));
2003 *DAG.getContext(), TmpEltVT),
2004 LHSElem, RHSElem,
CC);
2005 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
2006 DAG.getConstant(0, dl, EltVT));
2008 return DAG.getBuildVector(VT, dl, Ops);
2012 return VectorLegalizer(*this).Run();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Provides info so a possible vectorization of a function can be computed.
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.