51#define DEBUG_TYPE "legalizevectorops"
55class VectorLegalizer {
67 LegalizedNodes.
insert(std::make_pair(
From, To));
70 LegalizedNodes.
insert(std::make_pair(To, To));
139 std::pair<SDValue, SDValue> ExpandLoad(
SDNode *
N);
197bool VectorLegalizer::Run() {
199 bool HasVectors =
false;
201 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I) {
221 DAG.AssignTopologicalOrder();
223 E = std::prev(DAG.allnodes_end());
I != std::next(E); ++
I)
227 SDValue OldRoot = DAG.getRoot();
228 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
229 DAG.setRoot(LegalizedNodes[OldRoot]);
231 LegalizedNodes.clear();
234 DAG.RemoveDeadNodes();
241 "Unexpected number of results");
243 for (
unsigned i = 0, e =
Op->getNumValues(); i != e; ++i)
244 AddLegalizedOperand(
Op.getValue(i),
SDValue(Result, i));
249VectorLegalizer::RecursivelyLegalizeResults(
SDValue Op,
252 "Unexpected number of results");
254 for (
unsigned i = 0, e =
Results.
size(); i != e; ++i) {
256 AddLegalizedOperand(
Op.getValue(i),
Results[i]);
266 if (
I != LegalizedNodes.end())
return I->second;
270 for (
const SDValue &Oper :
Op->op_values())
273 SDNode *
Node = DAG.UpdateNodeOperands(
Op.getNode(), Ops);
275 bool HasVectorValueOrOp =
278 [](
SDValue O) { return O.getValueType().isVector(); });
279 if (!HasVectorValueOrOp)
280 return TranslateLegalizeResults(
Op,
Node);
284 switch (
Op.getOpcode()) {
286 return TranslateLegalizeResults(
Op,
Node);
290 EVT LoadedVT =
LD->getMemoryVT();
292 Action = TLI.getLoadExtAction(ExtType,
LD->getValueType(0), LoadedVT);
297 EVT StVT =
ST->getMemoryVT();
298 MVT ValVT =
ST->getValue().getSimpleValueType();
299 if (StVT.
isVector() &&
ST->isTruncatingStore())
300 Action = TLI.getTruncStoreAction(ValVT, StVT);
304 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
307 if (Action == TargetLowering::Legal)
308 Action = TargetLowering::Expand;
310#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
311 case ISD::STRICT_##DAGN:
312#include "llvm/IR/ConstrainedOps.def"
313 ValVT =
Node->getValueType(0);
316 ValVT =
Node->getOperand(1).getValueType();
319 MVT OpVT =
Node->getOperand(1).getSimpleValueType();
321 Action = TLI.getCondCodeAction(CCCode, OpVT);
322 if (Action == TargetLowering::Legal)
323 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
325 Action = TLI.getOperationAction(
Node->getOpcode(), ValVT);
332 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
333 TLI.getStrictFPOperationAction(
Node->getOpcode(), ValVT) ==
334 TargetLowering::Legal) {
336 if (TLI.getOperationAction(
Node->getOpcode(), EltVT)
337 == TargetLowering::Expand &&
338 TLI.getStrictFPOperationAction(
Node->getOpcode(), EltVT)
339 == TargetLowering::Legal)
340 Action = TargetLowering::Legal;
445 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
455 unsigned Scale =
Node->getConstantOperandVal(2);
456 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
457 Node->getValueType(0), Scale);
479 Action = TLI.getOperationAction(
Node->getOpcode(),
480 Node->getOperand(0).getValueType());
484 Action = TLI.getOperationAction(
Node->getOpcode(),
485 Node->getOperand(1).getValueType());
488 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
490 Action = TLI.getCondCodeAction(CCCode, OpVT);
491 if (Action == TargetLowering::Legal)
492 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
496#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
498 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
499 : Node->getOperand(LEGALPOS).getValueType(); \
500 if (ISD::VPID == ISD::VP_SETCC) { \
501 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
502 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
503 if (Action != TargetLowering::Legal) \
507 if (!Node->getValueType(0).isVector()) { \
508 Action = TargetLowering::Legal; \
511 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
513#include "llvm/IR/VPIntrinsics.def"
521 case TargetLowering::Promote:
523 "This action is not supported yet!");
525 Promote(
Node, ResultVals);
526 assert(!ResultVals.
empty() &&
"No results for promotion?");
528 case TargetLowering::Legal:
531 case TargetLowering::Custom:
533 if (LowerOperationWrapper(
Node, ResultVals))
537 case TargetLowering::Expand:
539 Expand(
Node, ResultVals);
543 if (ResultVals.
empty())
544 return TranslateLegalizeResults(
Op,
Node);
547 return RecursivelyLegalizeResults(
Op, ResultVals);
552bool VectorLegalizer::LowerOperationWrapper(
SDNode *
Node,
564 if (
Node->getNumValues() == 1) {
572 "Lowering returned the wrong number of results!");
575 for (
unsigned I = 0, E =
Node->getNumValues();
I != E; ++
I)
581void VectorLegalizer::PromoteSETCC(
SDNode *
Node,
583 MVT VecVT =
Node->getOperand(0).getSimpleValueType();
584 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
591 Operands[0] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(0));
592 Operands[1] = DAG.getNode(ExtOp,
DL, NewVecVT,
Node->getOperand(1));
595 if (
Node->getOpcode() == ISD::VP_SETCC) {
606void VectorLegalizer::PromoteSTRICT(
SDNode *
Node,
608 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
609 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
617 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
618 if (
Node->getOperand(j).getValueType().isVector() &&
625 {
Node->getOperand(0),
Node->getOperand(j)});
631 SDVTList VTs = DAG.getVTList(NewVecVT,
Node->getValueType(1));
641 DAG.getIntPtrConstant(0,
DL,
true)});
650 switch (
Node->getOpcode()) {
691 "Can't promote a vector with multiple results!");
692 MVT VT =
Node->getSimpleValueType(0);
693 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
697 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
701 if (
Node->getOperand(j).getValueType().isVector() && !SkipPromote)
702 if (
Node->getOperand(j)
704 .getVectorElementType()
705 .isFloatingPoint() &&
721 DAG.getIntPtrConstant(0, dl,
true));
728void VectorLegalizer::PromoteINT_TO_FP(
SDNode *
Node,
732 bool IsStrict =
Node->isStrictFPOpcode();
733 MVT VT =
Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
734 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
736 "Vectors have different number of elements!");
745 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
746 if (
Node->getOperand(j).getValueType().isVector())
747 Operands[j] = DAG.getNode(Opc, dl, NVT,
Node->getOperand(j));
769void VectorLegalizer::PromoteFP_TO_INT(
SDNode *
Node,
771 MVT VT =
Node->getSimpleValueType(0);
772 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
773 bool IsStrict =
Node->isStrictFPOpcode();
775 "Vectors have different number of elements!");
777 unsigned NewOpc =
Node->getOpcode();
791 Promoted = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
792 {
Node->getOperand(0),
Node->getOperand(1)});
795 Promoted = DAG.
getNode(NewOpc, dl, NVT,
Node->getOperand(0));
806 Promoted = DAG.
getNode(NewOpc, dl, NVT, Promoted,
814std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(
SDNode *
N) {
816 return TLI.scalarizeVectorLoad(LD, DAG);
821 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
826 switch (
Node->getOpcode()) {
828 std::pair<SDValue, SDValue> Tmp = ExpandLoad(
Node);
837 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
844 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(
Node));
847 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(
Node));
850 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(
Node));
875 if (
Node->getValueType(0).isScalableVector()) {
876 EVT CondVT = TLI.getSetCCResultType(
877 DAG.getDataLayout(), *DAG.getContext(),
Node->getValueType(0));
880 Node->getOperand(1),
Node->getOperand(4));
883 Node->getOperand(3)));
920 case ISD::VP_BITREVERSE:
921 if (
SDValue Expanded = TLI.expandVPBITREVERSE(
Node, DAG)) {
927 if (
SDValue Expanded = TLI.expandCTPOP(
Node, DAG)) {
933 if (
SDValue Expanded = TLI.expandVPCTPOP(
Node, DAG)) {
940 if (
SDValue Expanded = TLI.expandCTLZ(
Node, DAG)) {
946 case ISD::VP_CTLZ_ZERO_UNDEF:
947 if (
SDValue Expanded = TLI.expandVPCTLZ(
Node, DAG)) {
954 if (
SDValue Expanded = TLI.expandCTTZ(
Node, DAG)) {
960 case ISD::VP_CTTZ_ZERO_UNDEF:
961 if (
SDValue Expanded = TLI.expandVPCTTZ(
Node, DAG)) {
970 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG)) {
977 if (
SDValue Expanded = TLI.expandROT(
Node,
false , DAG)) {
984 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG)) {
991 if (
SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(
Node, DAG)) {
1000 if (
SDValue Expanded = TLI.expandIntMINMAX(
Node, DAG)) {
1021 if (
SDValue Expanded = TLI.expandAddSubSat(
Node, DAG)) {
1028 if (
SDValue Expanded = TLI.expandShlSat(
Node, DAG)) {
1036 if (
Node->getValueType(0).isScalableVector()) {
1037 if (
SDValue Expanded = TLI.expandFP_TO_INT_SAT(
Node, DAG)) {
1045 if (
SDValue Expanded = TLI.expandFixedPointMul(
Node, DAG)) {
1064#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1065 case ISD::STRICT_##DAGN:
1066#include "llvm/IR/ConstrainedOps.def"
1084 Results.push_back(TLI.expandVecReduce(
Node, DAG));
1088 Results.push_back(TLI.expandVecReduceSeq(
Node, DAG));
1098 if (tryExpandVecMathCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
1099 RTLIB::REM_F80, RTLIB::REM_F128,
1107 if (
Node->getNumValues() == 1) {
1111 "VectorLegalizer Expand returned wrong number of results!");
1121 EVT VT =
Node->getValueType(0);
1139 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1140 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1141 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand ||
1144 VT) == TargetLowering::Expand)
1145 return DAG.UnrollVectorOp(
Node);
1153 Mask = DAG.getSelect(
DL, BitTy, Mask, DAG.getAllOnesConstant(
DL, BitTy),
1154 DAG.getConstant(0,
DL, BitTy));
1157 Mask = DAG.getSplat(MaskTy,
DL, Mask);
1165 SDValue NotMask = DAG.getNOT(
DL, Mask, MaskTy);
1174 EVT VT =
Node->getValueType(0);
1177 if (TLI.getOperationAction(
ISD::SRA, VT) == TargetLowering::Expand ||
1178 TLI.getOperationAction(
ISD::SHL, VT) == TargetLowering::Expand)
1179 return DAG.UnrollVectorOp(
Node);
1182 EVT OrigTy = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1186 SDValue ShiftSz = DAG.getConstant(BW - OrigBW,
DL, VT);
1196 EVT VT =
Node->getValueType(0);
1199 EVT SrcVT = Src.getValueType();
1206 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1211 Src, DAG.getVectorIdxConstant(0,
DL));
1216 ShuffleMask.
resize(NumSrcElements, -1);
1219 int ExtLaneScale = NumSrcElements / NumElements;
1220 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1221 for (
int i = 0; i < NumElements; ++i)
1222 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1226 DAG.getVectorShuffle(SrcVT,
DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1231 EVT VT =
Node->getValueType(0);
1233 EVT SrcVT = Src.getValueType();
1244 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth,
DL, VT);
1255 EVT VT =
Node->getValueType(0);
1258 EVT SrcVT = Src.getValueType();
1265 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1270 Src, DAG.getVectorIdxConstant(0,
DL));
1278 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1280 int ExtLaneScale = NumSrcElements / NumElements;
1281 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1282 for (
int i = 0; i < NumElements; ++i)
1283 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1286 DAG.getVectorShuffle(SrcVT,
DL, Zero, Src, ShuffleMask));
1292 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
1293 ShuffleMask.
push_back((
I * ScalarSizeInBytes) + J);
1297 EVT VT =
Node->getValueType(0);
1301 return TLI.expandBSWAP(
Node, DAG);
1309 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1312 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1318 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1319 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1320 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1321 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
1322 return TLI.expandBSWAP(
Node, DAG);
1325 return DAG.UnrollVectorOp(
Node);
1328void VectorLegalizer::ExpandBITREVERSE(
SDNode *
Node,
1330 EVT VT =
Node->getValueType(0);
1334 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1349 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1354 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1356 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
1357 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
1358 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
1359 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
1362 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT),
1373 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1374 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1375 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1376 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT)) {
1377 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1401 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1402 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1403 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand)
1404 return DAG.UnrollVectorOp(
Node);
1410 auto BoolContents = TLI.getBooleanContents(Op1.
getValueType());
1411 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1412 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1414 return DAG.UnrollVectorOp(
Node);
1420 return DAG.UnrollVectorOp(
Node);
1428 SDValue NotMask = DAG.getNOT(
DL, Mask, VT);
1450 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1451 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1452 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1453 return DAG.UnrollVectorOp(
Node);
1457 return DAG.UnrollVectorOp(
Node);
1459 SDValue Ones = DAG.getAllOnesConstant(
DL, VT);
1462 Op1 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op1, Mask, Ones, EVL);
1463 Op2 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op2, NotMask, Ones, EVL);
1464 return DAG.getNode(ISD::VP_OR,
DL, VT, Op1, Op2, Ones, EVL);
1479 EVT MaskVT =
Mask.getValueType();
1491 return DAG.UnrollVectorOp(
Node);
1495 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1496 EVLVecVT) != MaskVT)
1497 return DAG.UnrollVectorOp(
Node);
1499 SDValue StepVec = DAG.getStepVector(
DL, EVLVecVT);
1500 SDValue SplatEVL = DAG.getSplat(EVLVecVT,
DL, EVL);
1502 DAG.getSetCC(
DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1505 return DAG.getSelect(
DL,
Node->getValueType(0), FullMask, Op1, Op2);
1510 EVT VT =
Node->getValueType(0);
1512 unsigned DivOpc =
Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1514 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1515 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1516 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1528 SDValue Mul = DAG.getNode(ISD::VP_MUL,
DL, VT, Divisor, Div, Mask, EVL);
1529 return DAG.getNode(ISD::VP_SUB,
DL, VT, Dividend,
Mul, Mask, EVL);
1532void VectorLegalizer::ExpandFP_TO_UINT(
SDNode *
Node,
1536 if (TLI.expandFP_TO_UINT(
Node, Result, Chain, DAG)) {
1538 if (
Node->isStrictFPOpcode())
1544 if (
Node->isStrictFPOpcode()) {
1552void VectorLegalizer::ExpandUINT_TO_FLOAT(
SDNode *
Node,
1554 bool IsStrict =
Node->isStrictFPOpcode();
1555 unsigned OpNo = IsStrict ? 1 : 0;
1557 EVT VT = Src.getValueType();
1563 if (TLI.expandUINT_TO_FP(
Node, Result, Chain, DAG)) {
1572 TargetLowering::Expand) ||
1574 TargetLowering::Expand)) ||
1575 TLI.getOperationAction(
ISD::SRL, VT) == TargetLowering::Expand) {
1586 assert((BW == 64 || BW == 32) &&
1587 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1589 SDValue HalfWord = DAG.getConstant(BW / 2,
DL, VT);
1594 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1595 SDValue HalfWordMask = DAG.getConstant(HWMask,
DL, VT);
1599 DAG.getConstantFP(1ULL << (BW / 2),
DL,
Node->getValueType(0));
1610 {
Node->getValueType(0), MVT::Other},
1611 {
Node->getOperand(0),
HI});
1615 {
Node->getValueType(0), MVT::Other},
1616 {
Node->getOperand(0),
LO});
1644 if (TLI.isOperationLegalOrCustom(
ISD::FSUB,
Node->getValueType(0))) {
1649 Node->getOperand(0));
1651 return DAG.UnrollVectorOp(
Node);
1654void VectorLegalizer::ExpandFSUB(
SDNode *
Node,
1659 EVT VT =
Node->getValueType(0);
1660 if (TLI.isOperationLegalOrCustom(
ISD::FNEG, VT) &&
1661 TLI.isOperationLegalOrCustom(
ISD::FADD, VT))
1668void VectorLegalizer::ExpandSETCC(
SDNode *
Node,
1670 bool NeedInvert =
false;
1671 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
1675 unsigned Offset = IsStrict ? 1 : 0;
1682 MVT OpVT =
LHS.getSimpleValueType();
1685 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1702 TLI.LegalizeSetCCCondCode(DAG,
Node->getValueType(0), LHS, RHS,
CC, Mask,
1703 EVL, NeedInvert, dl, Chain, IsSignaling);
1710 LHS = DAG.getNode(
Node->getOpcode(), dl,
Node->getVTList(),
1711 {Chain, LHS, RHS, CC},
Node->getFlags());
1712 Chain =
LHS.getValue(1);
1714 LHS = DAG.getNode(ISD::VP_SETCC, dl,
Node->getValueType(0),
1715 {LHS, RHS, CC, Mask, EVL},
Node->getFlags());
1726 LHS = DAG.getLogicalNOT(dl, LHS,
LHS->getValueType(0));
1728 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL,
LHS->getValueType(0));
1731 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
1735 EVT VT =
Node->getValueType(0);
1738 DAG.getBoolConstant(
true, dl, VT,
LHS.getValueType()),
1739 DAG.getBoolConstant(
false, dl, VT,
LHS.getValueType()),
CC);
1740 LHS->setFlags(
Node->getFlags());
1748void VectorLegalizer::ExpandUADDSUBO(
SDNode *
Node,
1751 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
1756void VectorLegalizer::ExpandSADDSUBO(
SDNode *
Node,
1759 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
1764void VectorLegalizer::ExpandMULO(
SDNode *
Node,
1767 if (!TLI.expandMULO(
Node, Result, Overflow, DAG))
1768 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(
Node);
1774void VectorLegalizer::ExpandFixedPointDiv(
SDNode *
Node,
1777 if (
SDValue Expanded = TLI.expandFixedPointDiv(
N->getOpcode(),
SDLoc(
N),
1778 N->getOperand(0),
N->getOperand(1),
N->getConstantOperandVal(2), DAG))
1782void VectorLegalizer::ExpandStrictFPOp(
SDNode *
Node,
1805 "Expected REM node");
1808 if (!TLI.expandREM(
Node, Result, DAG))
1823 assert(!
Node->isStrictFPOpcode() &&
"Unexpected strict fp operation!");
1825 const char *LCName = TLI.getLibcallName(LC);
1828 LLVM_DEBUG(
dbgs() <<
"Looking for vector variant of " << LCName <<
"\n");
1830 EVT VT =
Node->getValueType(0);
1848 for (
unsigned i = 0; i <
Node->getNumOperands(); ++i) {
1849 assert(
Node->getOperand(i).getValueType() == VT &&
1850 "Expected matching vector types!");
1853 FunctionType *ScalarFTy = FunctionType::get(ScalarTy, ArgTys,
false);
1865 if (OptVFInfo->Shape.Parameters.size() !=
1874 Entry.IsSExt =
false;
1875 Entry.IsZExt =
false;
1878 for (
auto &VFParam : OptVFInfo->Shape.Parameters) {
1879 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
1880 EVT MaskVT = TLI.getSetCCResultType(DAG.getDataLayout(), *Ctx, VT);
1881 Entry.Node = DAG.getBoolConstant(
true,
DL, MaskVT, VT);
1883 Args.push_back(Entry);
1888 if (VFParam.ParamKind != VFParamKind::Vector)
1891 Entry.Node =
Node->getOperand(OpNum++);
1893 Args.push_back(Entry);
1898 TLI.getPointerTy(DAG.getDataLayout()));
1901 .setChain(DAG.getEntryNode())
1904 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
1905 Results.push_back(CallResult.first);
1910bool VectorLegalizer::tryExpandVecMathCall(
1915 Node->getValueType(0).getVectorElementType(), Call_F32, Call_F64,
1916 Call_F80, Call_F128, Call_PPCF128);
1918 if (LC == RTLIB::UNKNOWN_LIBCALL)
1924void VectorLegalizer::UnrollStrictFPOp(
SDNode *
Node,
1926 EVT VT =
Node->getValueType(0);
1929 unsigned NumOpers =
Node->getNumOperands();
1932 EVT TmpEltVT = EltVT;
1936 *DAG.getContext(), TmpEltVT);
1938 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1944 for (
unsigned i = 0; i < NumElems; ++i) {
1946 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1952 for (
unsigned j = 1;
j < NumOpers; ++
j) {
1969 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1970 DAG.getAllOnesConstant(dl, EltVT),
1971 DAG.getConstant(0, dl, EltVT));
1985 EVT VT =
Node->getValueType(0);
1991 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
1994 for (
unsigned i = 0; i < NumElems; ++i) {
1996 DAG.getVectorIdxConstant(i, dl));
1998 DAG.getVectorIdxConstant(i, dl));
2001 *DAG.getContext(), TmpEltVT),
2002 LHSElem, RHSElem,
CC);
2003 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
2004 DAG.getConstant(0, dl, EltVT));
2006 return DAG.getBuildVector(VT, dl, Ops);
2010 return VectorLegalizer(*this).Run();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This is an important class for using LLVM in a threaded context.
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Provides information about what library functions are available for the current target.
const VecDesc * getVectorMappingInfo(StringRef F, const ElementCount &VF, bool Masked) const
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
The instances of the Type class are immutable: once they are created, they are never changed.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Provides info so a possible vectorization of a function can be computed.
std::string getVectorFunctionABIVariantString() const
Returns a vector function ABI variant string on the form: ZGV<isa><mask><vlen><vparams><scalarname>(<...
StringRef getVectorFnName() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
std::optional< VFInfo > tryDemangleForVFABI(StringRef MangledName, const FunctionType *FTy)
Function to construct a VFInfo out of a mangled names in the following format:
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.