49#define DEBUG_TYPE "legalizevectorops"
53class VectorLegalizer {
65 LegalizedNodes.
insert(std::make_pair(
From, To));
68 LegalizedNodes.
insert(std::make_pair(To, To));
137 std::pair<SDValue, SDValue> ExpandLoad(
SDNode *
N);
186bool VectorLegalizer::Run() {
188 bool HasVectors =
false;
190 E = std::prev(DAG.allnodes_end());
I != std::next(
E); ++
I) {
210 DAG.AssignTopologicalOrder();
212 E = std::prev(DAG.allnodes_end());
I != std::next(
E); ++
I)
216 SDValue OldRoot = DAG.getRoot();
217 assert(LegalizedNodes.count(OldRoot) &&
"Root didn't get legalized?");
218 DAG.setRoot(LegalizedNodes[OldRoot]);
220 LegalizedNodes.clear();
223 DAG.RemoveDeadNodes();
230 "Unexpected number of results");
232 for (
unsigned i = 0, e =
Op->getNumValues(); i != e; ++i)
233 AddLegalizedOperand(
Op.getValue(i),
SDValue(Result, i));
238VectorLegalizer::RecursivelyLegalizeResults(
SDValue Op,
241 "Unexpected number of results");
243 for (
unsigned i = 0, e =
Results.
size(); i != e; ++i) {
245 AddLegalizedOperand(
Op.getValue(i),
Results[i]);
255 if (
I != LegalizedNodes.end())
return I->second;
259 for (
const SDValue &Oper :
Op->op_values())
262 SDNode *
Node = DAG.UpdateNodeOperands(
Op.getNode(), Ops);
264 bool HasVectorValueOrOp =
267 [](
SDValue O) { return O.getValueType().isVector(); });
268 if (!HasVectorValueOrOp)
269 return TranslateLegalizeResults(
Op,
Node);
273 switch (
Op.getOpcode()) {
275 return TranslateLegalizeResults(
Op,
Node);
279 EVT LoadedVT =
LD->getMemoryVT();
281 Action = TLI.getLoadExtAction(ExtType,
LD->getValueType(0), LoadedVT);
286 EVT StVT =
ST->getMemoryVT();
287 MVT ValVT =
ST->getValue().getSimpleValueType();
288 if (StVT.
isVector() &&
ST->isTruncatingStore())
289 Action = TLI.getTruncStoreAction(ValVT, StVT);
293 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
296 if (Action == TargetLowering::Legal)
297 Action = TargetLowering::Expand;
299#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
300 case ISD::STRICT_##DAGN:
301#include "llvm/IR/ConstrainedOps.def"
302 ValVT =
Node->getValueType(0);
305 ValVT =
Node->getOperand(1).getValueType();
308 MVT OpVT =
Node->getOperand(1).getSimpleValueType();
310 Action = TLI.getCondCodeAction(CCCode, OpVT);
311 if (Action == TargetLowering::Legal)
312 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
314 Action = TLI.getOperationAction(
Node->getOpcode(), ValVT);
321 if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
322 TLI.getStrictFPOperationAction(
Node->getOpcode(), ValVT) ==
323 TargetLowering::Legal) {
325 if (TLI.getOperationAction(
Node->getOpcode(), EltVT)
326 == TargetLowering::Expand &&
327 TLI.getStrictFPOperationAction(
Node->getOpcode(), EltVT)
328 == TargetLowering::Legal)
329 Action = TargetLowering::Legal;
433 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
443 unsigned Scale =
Node->getConstantOperandVal(2);
444 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
445 Node->getValueType(0), Scale);
465 Action = TLI.getOperationAction(
Node->getOpcode(),
466 Node->getOperand(0).getValueType());
470 Action = TLI.getOperationAction(
Node->getOpcode(),
471 Node->getOperand(1).getValueType());
474 MVT OpVT =
Node->getOperand(0).getSimpleValueType();
476 Action = TLI.getCondCodeAction(CCCode, OpVT);
477 if (Action == TargetLowering::Legal)
478 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
482#define BEGIN_REGISTER_VP_SDNODE(VPID, LEGALPOS, ...) \
484 EVT LegalizeVT = LEGALPOS < 0 ? Node->getValueType(-(1 + LEGALPOS)) \
485 : Node->getOperand(LEGALPOS).getValueType(); \
486 if (ISD::VPID == ISD::VP_SETCC) { \
487 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get(); \
488 Action = TLI.getCondCodeAction(CCCode, LegalizeVT.getSimpleVT()); \
489 if (Action != TargetLowering::Legal) \
492 Action = TLI.getOperationAction(Node->getOpcode(), LegalizeVT); \
494#include "llvm/IR/VPIntrinsics.def"
502 case TargetLowering::Promote:
504 "This action is not supported yet!");
506 Promote(
Node, ResultVals);
507 assert(!ResultVals.
empty() &&
"No results for promotion?");
509 case TargetLowering::Legal:
512 case TargetLowering::Custom:
514 if (LowerOperationWrapper(
Node, ResultVals))
518 case TargetLowering::Expand:
520 Expand(
Node, ResultVals);
524 if (ResultVals.
empty())
525 return TranslateLegalizeResults(
Op,
Node);
528 return RecursivelyLegalizeResults(
Op, ResultVals);
533bool VectorLegalizer::LowerOperationWrapper(
SDNode *
Node,
545 if (
Node->getNumValues() == 1) {
553 "Lowering returned the wrong number of results!");
556 for (
unsigned I = 0,
E =
Node->getNumValues();
I !=
E; ++
I)
562void VectorLegalizer::PromoteReduction(
SDNode *
Node,
564 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
565 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
566 MVT ScalarVT =
Node->getSimpleValueType(0);
573 if (
Node->getOperand(0).getValueType().isFloatingPoint())
580 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
581 if (
Node->getOperand(j).getValueType().isVector() &&
585 if (
Node->getOperand(j).getValueType().isFloatingPoint())
599 DAG.getIntPtrConstant(0,
DL,
true));
609 switch (
Node->getOpcode()) {
624 case ISD::VP_REDUCE_ADD:
625 case ISD::VP_REDUCE_MUL:
626 case ISD::VP_REDUCE_AND:
627 case ISD::VP_REDUCE_OR:
628 case ISD::VP_REDUCE_XOR:
629 case ISD::VP_REDUCE_SMAX:
630 case ISD::VP_REDUCE_SMIN:
631 case ISD::VP_REDUCE_UMAX:
632 case ISD::VP_REDUCE_UMIN:
633 case ISD::VP_REDUCE_FADD:
634 case ISD::VP_REDUCE_FMUL:
635 case ISD::VP_REDUCE_FMAX:
636 case ISD::VP_REDUCE_FMIN:
637 case ISD::VP_REDUCE_SEQ_FADD:
654 "Can't promote a vector with multiple results!");
655 MVT VT =
Node->getSimpleValueType(0);
656 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
660 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
664 if (
Node->getOperand(j).getValueType().isVector() && !SkipPromote)
665 if (
Node->getOperand(j)
667 .getVectorElementType()
668 .isFloatingPoint() &&
684 DAG.getIntPtrConstant(0, dl,
true));
691void VectorLegalizer::PromoteINT_TO_FP(
SDNode *
Node,
695 bool IsStrict =
Node->isStrictFPOpcode();
696 MVT VT =
Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
697 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
699 "Vectors have different number of elements!");
708 for (
unsigned j = 0;
j !=
Node->getNumOperands(); ++
j) {
709 if (
Node->getOperand(j).getValueType().isVector())
710 Operands[j] = DAG.getNode(Opc, dl, NVT,
Node->getOperand(j));
732void VectorLegalizer::PromoteFP_TO_INT(
SDNode *
Node,
734 MVT VT =
Node->getSimpleValueType(0);
735 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
736 bool IsStrict =
Node->isStrictFPOpcode();
738 "Vectors have different number of elements!");
740 unsigned NewOpc =
Node->getOpcode();
754 Promoted = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
755 {
Node->getOperand(0),
Node->getOperand(1)});
758 Promoted = DAG.
getNode(NewOpc, dl, NVT,
Node->getOperand(0));
769 Promoted = DAG.
getNode(NewOpc, dl, NVT, Promoted,
777std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(
SDNode *
N) {
779 return TLI.scalarizeVectorLoad(LD, DAG);
784 SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
789 switch (
Node->getOpcode()) {
791 std::pair<SDValue, SDValue> Tmp = ExpandLoad(
Node);
800 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
807 Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(
Node));
810 Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(
Node));
813 Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(
Node));
838 if (
Node->getValueType(0).isScalableVector()) {
839 EVT CondVT = TLI.getSetCCResultType(
840 DAG.getDataLayout(), *DAG.getContext(),
Node->getValueType(0));
843 Node->getOperand(1),
Node->getOperand(4));
846 Node->getOperand(3)));
883 case ISD::VP_BITREVERSE:
884 if (
SDValue Expanded = TLI.expandVPBITREVERSE(
Node, DAG)) {
890 if (
SDValue Expanded = TLI.expandCTPOP(
Node, DAG)) {
896 if (
SDValue Expanded = TLI.expandVPCTPOP(
Node, DAG)) {
903 if (
SDValue Expanded = TLI.expandCTLZ(
Node, DAG)) {
909 case ISD::VP_CTLZ_ZERO_UNDEF:
910 if (
SDValue Expanded = TLI.expandVPCTLZ(
Node, DAG)) {
917 if (
SDValue Expanded = TLI.expandCTTZ(
Node, DAG)) {
923 case ISD::VP_CTTZ_ZERO_UNDEF:
924 if (
SDValue Expanded = TLI.expandVPCTTZ(
Node, DAG)) {
933 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG)) {
940 if (
SDValue Expanded = TLI.expandROT(
Node,
false , DAG)) {
947 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG)) {
956 if (
SDValue Expanded = TLI.expandIntMINMAX(
Node, DAG)) {
977 if (
SDValue Expanded = TLI.expandAddSubSat(
Node, DAG)) {
984 if (
SDValue Expanded = TLI.expandShlSat(
Node, DAG)) {
992 if (
Node->getValueType(0).isScalableVector()) {
993 if (
SDValue Expanded = TLI.expandFP_TO_INT_SAT(
Node, DAG)) {
1001 if (
SDValue Expanded = TLI.expandFixedPointMul(
Node, DAG)) {
1020#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1021 case ISD::STRICT_##DAGN:
1022#include "llvm/IR/ConstrainedOps.def"
1040 Results.push_back(TLI.expandVecReduce(
Node, DAG));
1044 Results.push_back(TLI.expandVecReduceSeq(
Node, DAG));
1064 EVT VT =
Node->getValueType(0);
1082 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1083 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1084 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand ||
1087 VT) == TargetLowering::Expand)
1088 return DAG.UnrollVectorOp(
Node);
1096 Mask = DAG.getSelect(
DL, BitTy, Mask, DAG.getAllOnesConstant(
DL, BitTy),
1097 DAG.getConstant(0,
DL, BitTy));
1100 Mask = DAG.getSplat(MaskTy,
DL, Mask);
1108 SDValue NotMask = DAG.getNOT(
DL, Mask, MaskTy);
1117 EVT VT =
Node->getValueType(0);
1120 if (TLI.getOperationAction(
ISD::SRA, VT) == TargetLowering::Expand ||
1121 TLI.getOperationAction(
ISD::SHL, VT) == TargetLowering::Expand)
1122 return DAG.UnrollVectorOp(
Node);
1125 EVT OrigTy = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1129 SDValue ShiftSz = DAG.getConstant(BW - OrigBW,
DL, VT);
1139 EVT VT =
Node->getValueType(0);
1142 EVT SrcVT = Src.getValueType();
1149 "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1154 Src, DAG.getVectorIdxConstant(0,
DL));
1159 ShuffleMask.
resize(NumSrcElements, -1);
1162 int ExtLaneScale = NumSrcElements / NumElements;
1163 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1164 for (
int i = 0; i < NumElements; ++i)
1165 ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1169 DAG.getVectorShuffle(SrcVT,
DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1174 EVT VT =
Node->getValueType(0);
1176 EVT SrcVT = Src.getValueType();
1187 SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth,
DL, VT);
1198 EVT VT =
Node->getValueType(0);
1201 EVT SrcVT = Src.getValueType();
1208 "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1213 Src, DAG.getVectorIdxConstant(0,
DL));
1221 auto ShuffleMask = llvm::to_vector<16>(llvm::seq<int>(0, NumSrcElements));
1223 int ExtLaneScale = NumSrcElements / NumElements;
1224 int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1225 for (
int i = 0; i < NumElements; ++i)
1226 ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1229 DAG.getVectorShuffle(SrcVT,
DL, Zero, Src, ShuffleMask));
1235 for (
int J = ScalarSizeInBytes - 1; J >= 0; --J)
1236 ShuffleMask.
push_back((
I * ScalarSizeInBytes) + J);
1240 EVT VT =
Node->getValueType(0);
1244 return TLI.expandBSWAP(
Node, DAG);
1252 if (TLI.isShuffleMaskLegal(ShuffleMask, ByteVT)) {
1255 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1261 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1262 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1263 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1264 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT))
1265 return TLI.expandBSWAP(
Node, DAG);
1268 return DAG.UnrollVectorOp(
Node);
1271void VectorLegalizer::ExpandBITREVERSE(
SDNode *
Node,
1273 EVT VT =
Node->getValueType(0);
1277 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1292 if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1297 if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1299 (TLI.isOperationLegalOrCustom(
ISD::SHL, ByteVT) &&
1300 TLI.isOperationLegalOrCustom(
ISD::SRL, ByteVT) &&
1301 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, ByteVT) &&
1302 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, ByteVT)))) {
1305 Op = DAG.getVectorShuffle(ByteVT,
DL,
Op, DAG.getUNDEF(ByteVT),
1316 if (TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
1317 TLI.isOperationLegalOrCustom(
ISD::SRL, VT) &&
1318 TLI.isOperationLegalOrCustomOrPromote(
ISD::AND, VT) &&
1319 TLI.isOperationLegalOrCustomOrPromote(
ISD::OR, VT)) {
1320 Results.push_back(TLI.expandBITREVERSE(
Node, DAG));
1344 if (TLI.getOperationAction(
ISD::AND, VT) == TargetLowering::Expand ||
1345 TLI.getOperationAction(
ISD::XOR, VT) == TargetLowering::Expand ||
1346 TLI.getOperationAction(
ISD::OR, VT) == TargetLowering::Expand)
1347 return DAG.UnrollVectorOp(
Node);
1353 auto BoolContents = TLI.getBooleanContents(Op1.
getValueType());
1354 if (BoolContents != TargetLowering::ZeroOrNegativeOneBooleanContent &&
1355 !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1357 return DAG.UnrollVectorOp(
Node);
1363 return DAG.UnrollVectorOp(
Node);
1371 SDValue NotMask = DAG.getNOT(
DL, Mask, VT);
1393 if (TLI.getOperationAction(ISD::VP_AND, VT) == TargetLowering::Expand ||
1394 TLI.getOperationAction(ISD::VP_XOR, VT) == TargetLowering::Expand ||
1395 TLI.getOperationAction(ISD::VP_OR, VT) == TargetLowering::Expand)
1396 return DAG.UnrollVectorOp(
Node);
1400 return DAG.UnrollVectorOp(
Node);
1402 SDValue Ones = DAG.getAllOnesConstant(
DL, VT);
1405 Op1 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op1, Mask, Ones, EVL);
1406 Op2 = DAG.
getNode(ISD::VP_AND,
DL, VT, Op2, NotMask, Ones, EVL);
1407 return DAG.getNode(ISD::VP_OR,
DL, VT, Op1, Op2, Ones, EVL);
1422 EVT MaskVT =
Mask.getValueType();
1434 return DAG.UnrollVectorOp(
Node);
1438 if (TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
1439 EVLVecVT) != MaskVT)
1440 return DAG.UnrollVectorOp(
Node);
1442 SDValue StepVec = DAG.getStepVector(
DL, EVLVecVT);
1443 SDValue SplatEVL = DAG.getSplat(EVLVecVT,
DL, EVL);
1445 DAG.getSetCC(
DL, MaskVT, StepVec, SplatEVL, ISD::CondCode::SETULT);
1448 return DAG.getSelect(
DL,
Node->getValueType(0), FullMask, Op1, Op2);
1453 EVT VT =
Node->getValueType(0);
1455 unsigned DivOpc =
Node->getOpcode() == ISD::VP_SREM ? ISD::VP_SDIV : ISD::VP_UDIV;
1457 if (!TLI.isOperationLegalOrCustom(DivOpc, VT) ||
1458 !TLI.isOperationLegalOrCustom(ISD::VP_MUL, VT) ||
1459 !TLI.isOperationLegalOrCustom(ISD::VP_SUB, VT))
1471 SDValue Mul = DAG.getNode(ISD::VP_MUL,
DL, VT, Divisor, Div, Mask, EVL);
1472 return DAG.getNode(ISD::VP_SUB,
DL, VT, Dividend,
Mul, Mask, EVL);
1475void VectorLegalizer::ExpandFP_TO_UINT(
SDNode *
Node,
1479 if (TLI.expandFP_TO_UINT(
Node, Result, Chain, DAG)) {
1481 if (
Node->isStrictFPOpcode())
1487 if (
Node->isStrictFPOpcode()) {
1495void VectorLegalizer::ExpandUINT_TO_FLOAT(
SDNode *
Node,
1497 bool IsStrict =
Node->isStrictFPOpcode();
1498 unsigned OpNo = IsStrict ? 1 : 0;
1500 EVT VT = Src.getValueType();
1506 if (TLI.expandUINT_TO_FP(
Node, Result, Chain, DAG)) {
1515 TargetLowering::Expand) ||
1517 TargetLowering::Expand)) ||
1518 TLI.getOperationAction(
ISD::SRL, VT) == TargetLowering::Expand) {
1529 assert((BW == 64 || BW == 32) &&
1530 "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1532 SDValue HalfWord = DAG.getConstant(BW / 2,
DL, VT);
1537 uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1538 SDValue HalfWordMask = DAG.getConstant(HWMask,
DL, VT);
1542 DAG.getConstantFP(1ULL << (BW / 2),
DL,
Node->getValueType(0));
1553 {
Node->getValueType(0), MVT::Other},
1554 {
Node->getOperand(0),
HI});
1558 {
Node->getValueType(0), MVT::Other},
1559 {
Node->getOperand(0),
LO});
1587 if (TLI.isOperationLegalOrCustom(
ISD::FSUB,
Node->getValueType(0))) {
1592 Node->getOperand(0));
1594 return DAG.UnrollVectorOp(
Node);
1597void VectorLegalizer::ExpandFSUB(
SDNode *
Node,
1602 EVT VT =
Node->getValueType(0);
1603 if (TLI.isOperationLegalOrCustom(
ISD::FNEG, VT) &&
1604 TLI.isOperationLegalOrCustom(
ISD::FADD, VT))
1611void VectorLegalizer::ExpandSETCC(
SDNode *
Node,
1613 bool NeedInvert =
false;
1614 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
1618 unsigned Offset = IsStrict ? 1 : 0;
1625 MVT OpVT =
LHS.getSimpleValueType();
1628 if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1645 TLI.LegalizeSetCCCondCode(DAG,
Node->getValueType(0), LHS, RHS,
CC, Mask,
1646 EVL, NeedInvert, dl, Chain, IsSignaling);
1653 LHS = DAG.getNode(
Node->getOpcode(), dl,
Node->getVTList(),
1654 {Chain, LHS, RHS, CC},
Node->getFlags());
1655 Chain =
LHS.getValue(1);
1657 LHS = DAG.getNode(ISD::VP_SETCC, dl,
Node->getValueType(0),
1658 {LHS, RHS, CC, Mask, EVL},
Node->getFlags());
1669 LHS = DAG.getLogicalNOT(dl, LHS,
LHS->getValueType(0));
1671 LHS = DAG.getVPLogicalNOT(dl, LHS, Mask, EVL,
LHS->getValueType(0));
1674 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
1678 EVT VT =
Node->getValueType(0);
1681 DAG.getBoolConstant(
true, dl, VT,
LHS.getValueType()),
1682 DAG.getBoolConstant(
false, dl, VT,
LHS.getValueType()),
CC);
1683 LHS->setFlags(
Node->getFlags());
1691void VectorLegalizer::ExpandUADDSUBO(
SDNode *
Node,
1694 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
1699void VectorLegalizer::ExpandSADDSUBO(
SDNode *
Node,
1702 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
1707void VectorLegalizer::ExpandMULO(
SDNode *
Node,
1710 if (!TLI.expandMULO(
Node, Result, Overflow, DAG))
1711 std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(
Node);
1717void VectorLegalizer::ExpandFixedPointDiv(
SDNode *
Node,
1720 if (
SDValue Expanded = TLI.expandFixedPointDiv(
N->getOpcode(),
SDLoc(
N),
1721 N->getOperand(0),
N->getOperand(1),
N->getConstantOperandVal(2), DAG))
1725void VectorLegalizer::ExpandStrictFPOp(
SDNode *
Node,
1748 "Expected REM node");
1751 if (!TLI.expandREM(
Node, Result, DAG))
1756void VectorLegalizer::UnrollStrictFPOp(
SDNode *
Node,
1758 EVT VT =
Node->getValueType(0);
1761 unsigned NumOpers =
Node->getNumOperands();
1764 EVT TmpEltVT = EltVT;
1768 *DAG.getContext(), TmpEltVT);
1770 EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1776 for (
unsigned i = 0; i < NumElems; ++i) {
1778 SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1784 for (
unsigned j = 1;
j < NumOpers; ++
j) {
1801 ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1802 DAG.getAllOnesConstant(dl, EltVT),
1803 DAG.getConstant(0, dl, EltVT));
1817 EVT VT =
Node->getValueType(0);
1823 EVT TmpEltVT =
LHS.getValueType().getVectorElementType();
1826 for (
unsigned i = 0; i < NumElems; ++i) {
1828 DAG.getVectorIdxConstant(i, dl));
1830 DAG.getVectorIdxConstant(i, dl));
1833 *DAG.getContext(), TmpEltVT),
1834 LHSElem, RHSElem,
CC);
1835 Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
1836 DAG.getConstant(0, dl, EltVT));
1838 return DAG.getBuildVector(VT, dl, Ops);
1842 return VectorLegalizer(*this).Run();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
BlockVerifier::State From
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
mir Rename Register Operands
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
DEMANGLE_DUMP_METHOD void dump() const
This class represents an Operation in the Expression.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This class is used to represent ISD::LOAD nodes.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
const TargetLowering & getTargetLoweringInfo() const
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Iterator for intrusive lists based on ilist_node.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ SSUBO
Same for subtraction.
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
uint64_t getScalarSizeInBits() const
bool isFixedLengthVector() const
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.