30#define DEBUG_TYPE "legalize-types"
40void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
51 switch (
N->getOpcode()) {
54 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
55 N->dump(&DAG);
dbgs() <<
"\n";
63 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
67 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
69 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
71 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
73 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
74 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
75 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
77 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
83 Res = PromoteIntRes_Select(
N);
88 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
90 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
92 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
95 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
97 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
99 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(
N);
break;
101 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(
N);
break;
102 case ISD::VP_TRUNCATE:
104 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
105 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
109 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
111 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
113 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
115 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
117 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
119 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
121 Res = PromoteIntRes_BUILD_VECTOR(
N);
125 Res = PromoteIntRes_ScalarOp(
N);
129 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
134 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
140 case ISD::VP_FP_TO_SINT:
141 case ISD::VP_FP_TO_UINT:
149 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
153 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
169 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
176 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
183 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
186 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
188 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
190 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
195 case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(
N, ResNo);
break;
205 case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(
N);
break;
217 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
220 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
234 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
238 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
250 Res = PromoteIntRes_VECREDUCE(
N);
253 case ISD::VP_REDUCE_ADD:
254 case ISD::VP_REDUCE_MUL:
255 case ISD::VP_REDUCE_AND:
256 case ISD::VP_REDUCE_OR:
257 case ISD::VP_REDUCE_XOR:
258 case ISD::VP_REDUCE_SMAX:
259 case ISD::VP_REDUCE_SMIN:
260 case ISD::VP_REDUCE_UMAX:
261 case ISD::VP_REDUCE_UMIN:
262 Res = PromoteIntRes_VP_REDUCE(
N);
266 Res = PromoteIntRes_FREEZE(
N);
271 Res = PromoteIntRes_Rotate(
N);
276 Res = PromoteIntRes_FunnelShift(
N);
280 Res = PromoteIntRes_IS_FPCLASS(
N);
286 SetPromotedInteger(
SDValue(
N, ResNo), Res);
291 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
292 return GetPromotedInteger(Op);
297 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
299 Op.getValueType(), Op,
N->getOperand(1));
304 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
306 Op.getValueType(), Op,
N->getOperand(1));
312 N->getMemoryVT(), ResVT,
313 N->getChain(),
N->getBasePtr(),
322 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
325 N->getChain(),
N->getBasePtr(),
326 Op2,
N->getMemOperand());
337 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
348 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
358 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
361 Op2 = SExtPromotedInteger(Op2);
364 Op2 = ZExtPromotedInteger(Op2);
367 Op2 = GetPromotedInteger(Op2);
376 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
377 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
379 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
388 EVT OutVT =
N->getValueType(0);
392 switch (getTypeAction(InVT)) {
419 BitConvertToInteger(GetScalarizedVector(InOp)));
428 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
429 Lo = BitConvertToInteger(
Lo);
430 Hi = BitConvertToInteger(
Hi);
438 JoinIntegers(
Lo,
Hi));
472 if (isTypeLegal(WideOutVT)) {
473 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
483 CreateStackStoreLoad(InOp, OutVT));
487 SDValue V = GetPromotedInteger(
N->getOperand(0));
493 SDValue Op = GetPromotedInteger(
N->getOperand(0));
494 EVT OVT =
N->getValueType(0);
495 EVT NVT =
Op.getValueType();
514 SDValue Op = GetPromotedInteger(
N->getOperand(0));
515 EVT OVT =
N->getValueType(0);
516 EVT NVT =
Op.getValueType();
540 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
545 EVT VT =
N->getValueType(0);
554 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
559 EVT OVT =
N->getValueType(0);
576 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
577 Op = DAG.
getNode(
N->getOpcode(), dl, NVT, Op);
586 EVT OVT =
N->getValueType(0);
603 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
608 SDValue Op = GetPromotedInteger(
N->getOperand(0));
609 EVT OVT =
N->getValueType(0);
610 EVT NVT =
Op.getValueType();
636 return DAG.
getNode(
N->getOpcode(), dl, NVT, Op);
639SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
654 EVT SVT =
In.getValueType().getScalarType();
666 unsigned NewOpc =
N->getOpcode();
683 if (
N->getOpcode() == ISD::VP_FP_TO_UINT &&
686 NewOpc = ISD::VP_FP_TO_SINT;
689 if (
N->isStrictFPOpcode()) {
691 {
N->getOperand(0),
N->getOperand(1)});
695 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
696 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
699 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
711 N->getOpcode() == ISD::VP_FP_TO_UINT)
718SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
722 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
726SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
730 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
738 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
750 if (getTypeAction(
N->getOperand(0).getValueType())
752 SDValue Res = GetPromotedInteger(
N->getOperand(0));
770 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
780 N->getMemoryVT(),
N->getMemOperand());
790 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
798 N->getOffset(),
N->getMask(), ExtPassThru,
799 N->getMemoryVT(),
N->getMemOperand(),
800 N->getAddressingMode(), ExtType,
801 N->isExpandingLoad());
810 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
812 "Gather result type and the passThru argument type should be the same");
819 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
820 N->getIndex(),
N->getScale() };
822 N->getMemoryVT(), dl, Ops,
823 N->getMemOperand(),
N->getIndexType(),
836 EVT VT =
N->getValueType(0);
837 EVT SVT = getSetCCResultType(VT);
838 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
839 unsigned NumOps =
N->getNumOperands();
840 assert(NumOps <= 3 &&
"Too many operands");
842 Ops[2] =
N->getOperand(2);
850 ReplaceValueWith(
SDValue(
N, 0), Res);
869 unsigned Opcode =
N->getOpcode();
872 SDValue Op1Promoted, Op2Promoted;
874 Op1Promoted = GetPromotedInteger(Op1);
875 Op2Promoted = ZExtPromotedInteger(Op2);
877 Op1Promoted = ZExtPromotedInteger(Op1);
878 Op2Promoted = ZExtPromotedInteger(Op2);
880 Op1Promoted = SExtPromotedInteger(Op1);
881 Op2Promoted = SExtPromotedInteger(Op2);
914 "addition, subtraction or left shift");
917 unsigned SHLAmount = NewBits - OldBits;
927 DAG.
getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
928 return DAG.
getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
937 DAG.
getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
946 SDValue Op1Promoted, Op2Promoted;
952 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
953 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
955 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
956 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
958 EVT OldType =
N->getOperand(0).getValueType();
972 Op2Promoted,
N->getOperand(2));
974 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
977 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
982 unsigned SatW,
bool Signed,
1012 EVT VT =
LHS.getValueType();
1036 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1042 "Tried to saturate to more than the original type?");
1051 SDValue Op1Promoted, Op2Promoted;
1057 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1058 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1060 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1061 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1064 unsigned Scale =
N->getConstantOperandVal(2);
1073 N->getValueType(0).getScalarSizeInBits();
1078 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1079 Op2Promoted,
N->getOperand(2));
1089 Op2Promoted, Scale, DAG)) {
1092 N->getValueType(0).getScalarSizeInBits(),
1100 N->getValueType(0).getScalarSizeInBits());
1103SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1105 return PromoteIntRes_Overflow(
N);
1109 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1110 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1111 EVT OVT =
N->getOperand(0).getValueType();
1112 EVT NVT =
LHS.getValueType();
1127 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1135 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1136 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1138 unsigned Opcode =
N->getOpcode();
1139 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1146 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1147 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1149 LHS.getValueType(),
N->getOperand(0),
1150 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1154 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1155 EVT InVT =
N->getOperand(OpNo).getValueType();
1158 EVT SVT = getSetCCResultType(InVT);
1166 SVT = getSetCCResultType(InVT);
1174 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1175 "Vector compare must return a vector result!");
1179 if (
N->isStrictFPOpcode()) {
1181 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1182 N->getOperand(2),
N->getOperand(3)};
1183 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers);
1188 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1189 N->getOperand(1),
N->getOperand(2));
1204 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1207 RHS = ZExtPromotedInteger(RHS);
1208 if (
N->getOpcode() != ISD::VP_SHL)
1211 N->getOperand(2),
N->getOperand(3));
1214SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1215 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1217 Op.getValueType(), Op,
N->getOperand(1));
1220SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1224 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1225 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1226 if (
N->getNumOperands() == 2)
1228 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1229 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1231 N->getOperand(2),
N->getOperand(3));
1236 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1237 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1238 if (
N->getNumOperands() == 2)
1240 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1241 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1243 N->getOperand(2),
N->getOperand(3));
1248 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1249 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1250 if (
N->getNumOperands() == 2)
1252 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1253 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1255 N->getOperand(2),
N->getOperand(3));
1261 SDValue LHS = SExtOrZExtPromotedInteger(
N->getOperand(0));
1262 SDValue RHS = SExtOrZExtPromotedInteger(
N->getOperand(1));
1264 LHS.getValueType(), LHS, RHS);
1269 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1272 RHS = ZExtPromotedInteger(RHS);
1273 if (
N->getOpcode() != ISD::VP_ASHR)
1276 N->getOperand(2),
N->getOperand(3));
1281 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1284 RHS = ZExtPromotedInteger(RHS);
1285 if (
N->getOpcode() != ISD::VP_LSHR)
1288 N->getOperand(2),
N->getOperand(3));
1294 ReplaceValueWith(
SDValue(
N, 0), Res);
1299 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1300 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1303 Amt = ZExtPromotedInteger(Amt);
1307 EVT OldVT =
N->getOperand(0).getValueType();
1308 EVT VT =
Lo.getValueType();
1309 unsigned Opcode =
N->getOpcode();
1323 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1360 Res = GetPromotedInteger(InOp);
1367 "Dst and Src must have the same number of elements");
1369 "Promoted vector type must be a power of two");
1372 GetSplitVector(InOp, EOp1, EOp2);
1380 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1381 "Expected VP_TRUNCATE opcode");
1382 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1383 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1384 std::tie(EVLLo, EVLHi) =
1385 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1386 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1387 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1394 SDValue WideInOp = GetWidenedVector(InOp);
1399 N->getValueType(0).getScalarType(), NumElem);
1414 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1415 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1420SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1422 return PromoteIntRes_Overflow(
N);
1426 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1427 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1428 EVT OVT =
N->getOperand(0).getValueType();
1429 EVT NVT =
LHS.getValueType();
1443 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1451SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(
SDNode *
N,
unsigned ResNo) {
1453 return PromoteIntRes_Overflow(
N);
1465 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1466 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1468 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1472 LHS, RHS,
N->getOperand(2));
1482 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1483 return PromoteIntRes_Overflow(
N);
1487 EVT OVT =
N->getValueType(0);
1500 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1504SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1507 return PromoteIntRes_Overflow(
N);
1511 EVT SmallVT =
LHS.getValueType();
1518 LHS = SExtPromotedInteger(LHS);
1519 RHS = SExtPromotedInteger(RHS);
1521 LHS = ZExtPromotedInteger(LHS);
1522 RHS = ZExtPromotedInteger(RHS);
1553 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1559 N->getValueType(0)));
1565 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1572 EVT VT =
N->getValueType(0);
1580 for (
unsigned i = 0; i < NumRegs; ++i) {
1581 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1582 N->getConstantOperandVal(3));
1588 std::reverse(Parts.begin(), Parts.end());
1593 for (
unsigned i = 1; i < NumRegs; ++i) {
1604 ReplaceValueWith(
SDValue(
N, 1), Chain);
1617bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1621 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1626 switch (
N->getOpcode()) {
1629 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1630 N->dump(&DAG);
dbgs() <<
"\n";
1636 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1639 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1640 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1646 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1650 Res = PromoteIntOp_ScalarOp(
N);
1653 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1656 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1658 case ISD::VP_SINT_TO_FP:
1661 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1663 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1665 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1667 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1669 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1671 case ISD::VP_TRUNCATE:
1675 case ISD::VP_UINT_TO_FP:
1686 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1689 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
1694 case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(
N, OpNo);
break;
1722 case ISD::VP_REDUCE_ADD:
1723 case ISD::VP_REDUCE_MUL:
1724 case ISD::VP_REDUCE_AND:
1725 case ISD::VP_REDUCE_OR:
1726 case ISD::VP_REDUCE_XOR:
1727 case ISD::VP_REDUCE_SMAX:
1728 case ISD::VP_REDUCE_SMIN:
1729 case ISD::VP_REDUCE_UMAX:
1730 case ISD::VP_REDUCE_UMIN:
1731 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
1736 Res = PromoteIntOp_STACKMAP(
N, OpNo);
1739 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
1741 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1742 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1743 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
1748 if (!Res.getNode())
return false;
1752 if (Res.getNode() ==
N)
1755 const bool IsStrictFp =
N->isStrictFPOpcode();
1756 assert(Res.getValueType() ==
N->getValueType(0) &&
1757 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1758 "Invalid operand expansion");
1762 ReplaceValueWith(
SDValue(
N, 0), Res);
1771void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
1780 LHS = SExtPromotedInteger(LHS);
1781 RHS = SExtPromotedInteger(RHS);
1786 "Unknown integer comparison!");
1788 SDValue OpL = GetPromotedInteger(LHS);
1789 SDValue OpR = GetPromotedInteger(RHS);
1795 unsigned OpLEffectiveBits =
1797 unsigned OpREffectiveBits =
1799 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1800 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1807 LHS = SExtPromotedInteger(LHS);
1808 RHS = SExtPromotedInteger(RHS);
1819 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1820 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1827 LHS = ZExtPromotedInteger(LHS);
1828 RHS = ZExtPromotedInteger(RHS);
1832 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1837 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
1839 N->getChain(),
N->getBasePtr(), Op2,
N->getMemOperand());
1845 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
1848SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
1849 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
1853 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
1858 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
1862SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
1863 assert(OpNo == 1 &&
"only know how to promote condition");
1870 N->getOperand(2)), 0);
1875 EVT OVT =
N->getOperand(0).getValueType();
1876 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
1877 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
1878 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
1891 EVT VecVT =
N->getValueType(0);
1894 "Legal vector of one illegal element?");
1899 assert(
N->getOperand(0).getValueSizeInBits() >=
1900 N->getValueType(0).getScalarSizeInBits() &&
1901 "Type of inserted value narrower than vector element type!");
1904 for (
unsigned i = 0; i < NumElts; ++i)
1905 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
1910SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
1917 assert(
N->getOperand(1).getValueSizeInBits() >=
1918 N->getValueType(0).getScalarSizeInBits() &&
1919 "Type of inserted value narrower than vector element type!");
1921 GetPromotedInteger(
N->getOperand(1)),
1926 assert(OpNo == 2 &&
"Different operand and result vector types?");
1932 N->getOperand(1),
Idx), 0);
1939 GetPromotedInteger(
N->getOperand(0))), 0);
1942SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
1943 assert(OpNo == 0 &&
"Only know how to promote the condition!");
1945 EVT OpTy =
N->getOperand(1).getValueType();
1948 if (
SDValue Res = WidenVSELECTMask(
N))
1950 Res,
N->getOperand(1),
N->getOperand(2));
1954 Cond = PromoteTargetBoolean(
Cond, OpVT);
1957 N->getOperand(2)), 0);
1960SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
1961 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
1965 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
1969 N->getOperand(3),
N->getOperand(4)), 0);
1972SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
1973 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
1977 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
1983 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
1986 N->getOperand(3),
N->getOperand(4)),
1992 ZExtPromotedInteger(
N->getOperand(1))), 0);
1997 ZExtPromotedInteger(
N->getOperand(2))), 0);
2001 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2009 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2011 SExtPromotedInteger(
N->getOperand(0)),
2012 N->getOperand(1),
N->getOperand(2)),
2015 SExtPromotedInteger(
N->getOperand(0))), 0);
2018SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2020 SExtPromotedInteger(
N->getOperand(1))), 0);
2028 SDValue Val = GetPromotedInteger(
N->getValue());
2032 N->getMemoryVT(),
N->getMemOperand());
2043 Mask = PromoteTargetBoolean(Mask, DataVT);
2049 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2050 DataOp = GetPromotedInteger(DataOp);
2053 N->getOffset(), Mask,
N->getMemoryVT(),
2054 N->getMemOperand(),
N->getAddressingMode(),
2055 true,
N->isCompressingStore());
2060 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2061 EVT DataVT =
N->getValueType(0);
2062 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2064 NewOps[OpNo] =
Mask;
2081 EVT DataVT =
N->getValueType(0);
2082 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2083 }
else if (OpNo == 4) {
2085 if (
N->isIndexSigned())
2087 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2089 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2091 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2105 bool TruncateStore =
N->isTruncatingStore();
2110 EVT DataVT =
N->getValue().getValueType();
2111 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2112 }
else if (OpNo == 4) {
2114 if (
N->isIndexSigned())
2116 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2118 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2120 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2121 TruncateStore =
true;
2125 SDLoc(
N), NewOps,
N->getMemOperand(),
2126 N->getIndexType(), TruncateStore);
2130 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2131 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2132 return DAG.
getNode(ISD::VP_TRUNCATE,
SDLoc(
N),
N->getValueType(0), Op,
2133 N->getOperand(1),
N->getOperand(2));
2138 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2140 ZExtPromotedInteger(
N->getOperand(0)),
2141 N->getOperand(1),
N->getOperand(2)),
2144 ZExtPromotedInteger(
N->getOperand(0))), 0);
2147SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2149 ZExtPromotedInteger(
N->getOperand(1))), 0);
2154 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2159SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(
SDNode *
N,
unsigned OpNo) {
2160 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2167 Carry = PromoteTargetBoolean(Carry,
LHS.getValueType());
2173 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2178SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2180 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2184SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(
SDNode *
N,
unsigned OpNo) {
2185 assert(OpNo > 1 &&
"Don't know how to promote this operand!");
2188 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2189 SDValue Op3 = ZExtPromotedInteger(
N->getOperand(3));
2190 SDValue Op4 = ZExtPromotedInteger(
N->getOperand(4));
2197 bool IsStrict =
N->isStrictFPOpcode();
2209 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
2214 return DAG.
getUNDEF(
N->getValueType(0));
2216 unsigned OpOffset = IsStrict ? 1 : 0;
2219 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2220 "POWI exponent should match with sizeof(int) when doing the libcall.");
2223 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2224 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2225 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2226 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2228 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2233 switch (
N->getOpcode()) {
2241 case ISD::VP_REDUCE_ADD:
2242 case ISD::VP_REDUCE_MUL:
2243 case ISD::VP_REDUCE_AND:
2244 case ISD::VP_REDUCE_OR:
2245 case ISD::VP_REDUCE_XOR:
2249 case ISD::VP_REDUCE_SMAX:
2250 case ISD::VP_REDUCE_SMIN:
2254 case ISD::VP_REDUCE_UMAX:
2255 case ISD::VP_REDUCE_UMIN:
2265 return GetPromotedInteger(V);
2267 return SExtPromotedInteger(V);
2269 return ZExtPromotedInteger(V);
2275 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2277 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2278 EVT InVT =
Op.getValueType();
2280 EVT ResVT =
N->getValueType(0);
2281 unsigned Opcode =
N->getOpcode();
2313SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2320 NewOps[2] = PromoteTargetBoolean(Op,
N->getOperand(1).getValueType());
2324 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2326 Op = PromoteIntOpVectorReduction(
N, Op);
2330 EVT VT =
N->getValueType(0);
2331 EVT EltVT =
Op.getValueType().getScalarType();
2346 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2350SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2353 SDValue Operand =
N->getOperand(OpNo);
2359SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2362 SDValue Operand =
N->getOperand(OpNo);
2368SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2369 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2370 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2373 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2386void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2393 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2396 switch (
N->getOpcode()) {
2399 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2400 N->dump(&DAG);
dbgs() <<
"\n";
2446 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
2472 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2473 SplitInteger(Tmp.first,
Lo,
Hi);
2474 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2482 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2491 SplitInteger(Tmp,
Lo,
Hi);
2562 ExpandIntRes_Rotate(
N,
Lo,
Hi);
2567 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
2571 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
2581std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
2582 unsigned Opc =
Node->getOpcode();
2588 EVT RetVT =
Node->getValueType(0);
2596 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2597 "Unexpected atomic op or value type!");
2601 Node->getOperand(0));
2606void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2611 GetExpandedInteger(
N->getOperand(0), InL, InH);
2622 unsigned VTBits =
N->getValueType(0).getSizeInBits();
2624 EVT ShTy =
N->getOperand(1).getValueType();
2627 if (Amt.
uge(VTBits)) {
2629 }
else if (Amt.
ugt(NVTBits)) {
2633 }
else if (Amt == NVTBits) {
2648 if (Amt.
uge(VTBits)) {
2650 }
else if (Amt.
ugt(NVTBits)) {
2654 }
else if (Amt == NVTBits) {
2669 if (Amt.
uge(VTBits)) {
2672 }
else if (Amt.
ugt(NVTBits)) {
2677 }
else if (Amt == NVTBits) {
2695bool DAGTypeLegalizer::
2703 "Expanded integer type size not a power of two!");
2710 if (((Known.
Zero|Known.
One) & HighBitMask) == 0)
2715 GetExpandedInteger(
N->getOperand(0), InL, InH);
2724 switch (
N->getOpcode()) {
2752 switch (
N->getOpcode()) {
2769 Lo = DAG.
getNode(
N->getOpcode(), dl, NVT, InL, Amt);
2782bool DAGTypeLegalizer::
2789 "Expanded integer type size not a power of two!");
2794 GetExpandedInteger(
N->getOperand(0), InL, InH);
2806 switch (
N->getOpcode()) {
2821 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
2837 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
2853 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
2874void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
2885 SDValue LHSL, LHSH, RHSL, RHSH;
2886 GetExpandedInteger(LHS, LHSL, LHSH);
2887 GetExpandedInteger(RHS, RHSL, RHSH);
2891 EVT CCT = getSetCCResultType(NVT);
2905 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
2921void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
2925 SDValue LHSL, LHSH, RHSL, RHSH;
2926 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
2927 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
2930 SDValue LoOps[2] = { LHSL, RHSL };
2931 SDValue HiOps[3] = { LHSH, RHSH };
2940 HiOps[2] =
Lo.getValue(1);
2946 HiOps[2] =
Lo.getValue(1);
2968 HiOps[2] =
Lo.getValue(1);
2972 HiOps[2] =
Lo.getValue(1);
2985 EVT OvfVT = getSetCCResultType(NVT);
3032 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3046void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3049 SDValue LHSL, LHSH, RHSL, RHSH;
3051 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3052 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3054 SDValue LoOps[2] = { LHSL, RHSL };
3055 SDValue HiOps[3] = { LHSH, RHSH };
3059 HiOps[2] =
Lo.getValue(1);
3063 HiOps[2] =
Lo.getValue(1);
3069 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3072void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3075 SDValue LHSL, LHSH, RHSL, RHSH;
3077 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3078 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3080 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3081 SDValue HiOps[3] = { LHSH, RHSH };
3083 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3084 HiOps[2] =
Lo.getValue(1);
3085 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3089 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3092void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3100 unsigned CarryOp, NoCarryOp;
3102 switch(
N->getOpcode()) {
3122 SDValue LHSL, LHSH, RHSL, RHSH;
3123 GetExpandedInteger(LHS, LHSL, LHSH);
3124 GetExpandedInteger(RHS, RHSL, RHSH);
3126 SDValue LoOps[2] = { LHSL, RHSL };
3127 SDValue HiOps[3] = { LHSH, RHSH };
3129 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3130 HiOps[2] =
Lo.getValue(1);
3131 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3133 Ovf =
Hi.getValue(1);
3138 SplitInteger(Sum,
Lo,
Hi);
3147 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3150void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(
SDNode *
N,
3153 SDValue LHSL, LHSH, RHSL, RHSH;
3155 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3156 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3158 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3161 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3162 HiOps[2] =
Lo.getValue(1);
3163 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3167 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3170void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3173 SDValue LHSL, LHSH, RHSL, RHSH;
3175 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3176 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3182 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3183 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3187 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3190void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3195 if (
Op.getValueType().bitsLE(NVT)) {
3202 assert(getTypeAction(
Op.getValueType()) ==
3204 "Only know how to promote this result!");
3205 SDValue Res = GetPromotedInteger(Op);
3207 "Operand over promoted?");
3209 SplitInteger(Res,
Lo,
Hi);
3213void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3216 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3217 EVT NVT =
Lo.getValueType();
3218 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3222 if (NVTBits < EVTBits) {
3225 EVTBits - NVTBits)));
3235void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3238 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3239 EVT NVT =
Lo.getValueType();
3240 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3244 if (NVTBits < EVTBits) {
3247 EVTBits - NVTBits)));
3255void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3258 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3263void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3266 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3275 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3276 EVT NVT =
Lo.getValueType();
3282void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3286 auto Constant = cast<ConstantSDNode>(
N);
3288 bool IsTarget =
Constant->isTargetOpcode();
3289 bool IsOpaque =
Constant->isOpaque();
3300 GetExpandedInteger(N0,
Lo,
Hi);
3301 EVT NVT =
Lo.getValueType();
3332 EVT VT =
N->getValueType(0);
3336 SplitInteger(Neg, NegLo, NegHi);
3344void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3348 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3349 EVT NVT =
Lo.getValueType();
3364void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3368 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3369 EVT NVT =
Lo.getValueType();
3375void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3379 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3380 EVT NVT =
Lo.getValueType();
3409 ReplaceValueWith(
SDValue(
N, 1), Chain);
3415 EVT VT =
N->getValueType(0);
3417 bool IsStrict =
N->isStrictFPOpcode();
3419 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3421 Op = GetPromotedFloat(Op);
3425 Op = GetSoftPromotedHalf(Op);
3428 SplitInteger(Op,
Lo,
Hi);
3433 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-sint conversion!");
3436 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT, Op,
3437 CallOptions, dl, Chain);
3438 SplitInteger(Tmp.first,
Lo,
Hi);
3441 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3447 EVT VT =
N->getValueType(0);
3449 bool IsStrict =
N->isStrictFPOpcode();
3451 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3453 Op = GetPromotedFloat(Op);
3457 Op = GetSoftPromotedHalf(Op);
3460 SplitInteger(Op,
Lo,
Hi);
3465 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-uint conversion!");
3467 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT, Op,
3468 CallOptions, dl, Chain);
3469 SplitInteger(Tmp.first,
Lo,
Hi);
3472 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3478 SplitInteger(Res,
Lo,
Hi);
3484 bool IsStrict =
N->isStrictFPOpcode();
3485 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3489 "Input type needs to be promoted!");
3491 EVT VT =
Op.getValueType();
3498 Chain =
Op.getValue(1);
3508 LC = RTLIB::LROUND_F32;
3510 LC = RTLIB::LROUND_F64;
3512 LC = RTLIB::LROUND_F80;
3514 LC = RTLIB::LROUND_F128;
3516 LC = RTLIB::LROUND_PPCF128;
3517 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
3521 LC = RTLIB::LRINT_F32;
3523 LC = RTLIB::LRINT_F64;
3525 LC = RTLIB::LRINT_F80;
3527 LC = RTLIB::LRINT_F128;
3529 LC = RTLIB::LRINT_PPCF128;
3530 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
3534 LC = RTLIB::LLROUND_F32;
3536 LC = RTLIB::LLROUND_F64;
3538 LC = RTLIB::LLROUND_F80;
3540 LC = RTLIB::LLROUND_F128;
3542 LC = RTLIB::LLROUND_PPCF128;
3543 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
3547 LC = RTLIB::LLRINT_F32;
3549 LC = RTLIB::LLRINT_F64;
3551 LC = RTLIB::LLRINT_F80;
3553 LC = RTLIB::LLRINT_F128;
3555 LC = RTLIB::LLRINT_PPCF128;
3556 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
3560 EVT RetVT =
N->getValueType(0);
3564 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
3565 Op, CallOptions, dl,
3567 SplitInteger(Tmp.first,
Lo,
Hi);
3569 if (
N->isStrictFPOpcode())
3570 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3573void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
3575 if (
N->isAtomic()) {
3578 EVT VT =
N->getMemoryVT();
3583 VT, VTs,
N->getOperand(0),
3584 N->getOperand(1), Zero, Zero,
N->getMemOperand());
3591 ExpandRes_NormalLoad(
N,
Lo,
Hi);
3597 EVT VT =
N->getValueType(0);
3608 if (
N->getMemoryVT().bitsLE(NVT)) {
3609 EVT MemVT =
N->getMemoryVT();
3612 N->getOriginalAlign(), MMOFlags, AAInfo);
3615 Ch =
Lo.getValue(1);
3620 unsigned LoSize =
Lo.getValueSizeInBits();
3635 N->getOriginalAlign(), MMOFlags, AAInfo);
3637 unsigned ExcessBits =
3645 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3646 N->getOriginalAlign(), MMOFlags, AAInfo);
3655 EVT MemVT =
N->getMemoryVT();
3658 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3664 N->getOriginalAlign(), MMOFlags, AAInfo);
3670 N->getPointerInfo().getWithOffset(IncrementSize),
3672 N->getOriginalAlign(), MMOFlags, AAInfo);
3696 ReplaceValueWith(
SDValue(
N, 1), Ch);
3699void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
3703 GetExpandedInteger(
N->getOperand(0), LL, LH);
3704 GetExpandedInteger(
N->getOperand(1), RL, RH);
3709void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
3711 EVT VT =
N->getValueType(0);
3716 GetExpandedInteger(
N->getOperand(0), LL, LH);
3717 GetExpandedInteger(
N->getOperand(1), RL, RH);
3727 LC = RTLIB::MUL_I16;
3729 LC = RTLIB::MUL_I32;
3731 LC = RTLIB::MUL_I64;
3733 LC = RTLIB::MUL_I128;
3741 unsigned HalfBits =
Bits >> 1;
3777 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
3780 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3784void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(
SDNode *
N,
SDValue &
Lo,
3792 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
3798 SplitInteger(Result,
Lo,
Hi);
3804 SplitInteger(Result,
Lo,
Hi);
3815 EVT VT =
N->getValueType(0);
3819 uint64_t Scale =
N->getConstantOperandVal(2);
3831 EVT BoolVT = getSetCCResultType(VT);