27#define DEBUG_TYPE "legalize-types"
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
51void DAGTypeLegalizer::SoftenFloatResult(
SDNode *
N,
unsigned ResNo) {
52 LLVM_DEBUG(
dbgs() <<
"Soften float result " << ResNo <<
": ";
N->dump(&DAG));
55 switch (
N->getOpcode()) {
58 dbgs() <<
"SoftenFloatResult #" << ResNo <<
": ";
59 N->dump(&DAG);
dbgs() <<
"\n";
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(
N, ResNo);
break;
100 case ISD::FMA:
R = SoftenFloatRes_FMA(
N);
break;
119 R = SoftenFloatRes_FFREXP(
N);
154 R = SoftenFloatRes_VECREDUCE(
N);
158 R = SoftenFloatRes_VECREDUCE_SEQ(
N);
165 SetSoftenedFloat(
SDValue(
N, ResNo), R);
170 bool IsStrict =
N->isStrictFPOpcode();
172 unsigned Offset = IsStrict ? 1 : 0;
174 "Unexpected number of operands!");
178 EVT OpVT =
N->getOperand(0 +
Offset).getValueType();
180 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
184 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
189 bool IsStrict =
N->isStrictFPOpcode();
191 unsigned Offset = IsStrict ? 1 : 0;
193 "Unexpected number of operands!");
195 GetSoftenedFloat(
N->getOperand(1 +
Offset)) };
198 EVT OpsVT[2] = {
N->getOperand(0 +
Offset).getValueType(),
199 N->getOperand(1 +
Offset).getValueType() };
201 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Ops,
205 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
210 return BitConvertToInteger(
N->getOperand(0));
216 GetSoftenedFloat(
N->getOperand(0)));
222 GetSoftenedFloat(
N->getOperand(0)));
228 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
229 return BitConvertToInteger(
Op);
237 BitConvertToInteger(
N->getOperand(0)),
238 BitConvertToInteger(
N->getOperand(1)));
254 APInt Val(128, words);
265SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(
SDNode *
N,
unsigned ResNo) {
266 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
269 NewOp,
N->getOperand(1));
280 SDValue Op = GetSoftenedFloat(
N->getOperand(0));
286 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
292 RTLIB::FMIN_PPCF128));
297 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
303 RTLIB::FMAX_PPCF128));
312 RTLIB::ADD_PPCF128));
321 RTLIB::CBRT_PPCF128));
330 RTLIB::CEIL_PPCF128));
335 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
338 EVT LVT =
LHS.getValueType();
339 EVT RVT =
RHS.getValueType();
360 }
else if (SizeDiff < 0) {
387 RTLIB::COS_PPCF128));
396 RTLIB::DIV_PPCF128));
405 RTLIB::EXP_PPCF128));
414 RTLIB::EXP2_PPCF128));
418 return SoftenFloatRes_Unary(
420 GetFPLibCall(
N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
421 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
430 RTLIB::FLOOR_PPCF128));
439 RTLIB::LOG_PPCF128));
448 RTLIB::LOG2_PPCF128));
457 RTLIB::LOG10_PPCF128));
461 bool IsStrict =
N->isStrictFPOpcode();
463 unsigned Offset = IsStrict ? 1 : 0;
465 GetSoftenedFloat(
N->getOperand(1 +
Offset)),
466 GetSoftenedFloat(
N->getOperand(2 +
Offset)) };
469 EVT OpsVT[3] = {
N->getOperand(0 +
Offset).getValueType(),
470 N->getOperand(1 +
Offset).getValueType(),
471 N->getOperand(2 +
Offset).getValueType() };
473 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG,
480 NVT, Ops, CallOptions,
SDLoc(
N), Chain);
482 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
492 RTLIB::MUL_PPCF128));
497 RTLIB::NEARBYINT_F32,
498 RTLIB::NEARBYINT_F64,
499 RTLIB::NEARBYINT_F80,
500 RTLIB::NEARBYINT_F128,
501 RTLIB::NEARBYINT_PPCF128));
515 bool IsStrict =
N->isStrictFPOpcode();
522 Op = GetPromotedFloat(
Op);
525 if (
Op.getValueType() ==
N->getValueType(0))
526 return BitConvertToInteger(
Op);
533 if ((
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16) &&
534 N->getValueType(0) != MVT::f32) {
537 { MVT::f32, MVT::Other }, { Chain,
Op });
538 Chain =
Op.getValue(1);
544 if (
Op.getValueType() == MVT::bf16)
545 return SoftenFloatRes_BF16_TO_FP(
N);
548 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_EXTEND!");
550 EVT OpVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
552 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
556 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
566 EVT OpsVT[1] = {
N->getOperand(0).getValueType() };
569 CallOptions,
SDLoc(
N)).first;
570 if (
N->getValueType(0) == MVT::f32)
575 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_EXTEND!");
582 assert(
N->getValueType(0) == MVT::f32 &&
583 "Can only soften BF16_TO_FP with f32 result");
595 bool IsStrict =
N->isStrictFPOpcode();
600 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND!");
602 EVT OpVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
604 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
608 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
618 RTLIB::POW_PPCF128));
622 bool IsStrict =
N->isStrictFPOpcode();
623 unsigned Offset = IsStrict ? 1 : 0;
624 assert((
N->getOperand(1 +
Offset).getValueType() == MVT::i16 ||
625 N->getOperand(1 +
Offset).getValueType() == MVT::i32) &&
626 "Unsupported power type!");
632 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
641 N->getOperand(1 +
Offset).getValueType().getSizeInBits()) {
653 EVT OpsVT[2] = {
N->getOperand(0 +
Offset).getValueType(),
654 N->getOperand(1 +
Offset).getValueType() };
656 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Ops,
660 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
665 assert(!
N->isStrictFPOpcode() &&
"strictfp not implemented for frexp");
666 EVT VT0 =
N->getValueType(0);
667 EVT VT1 =
N->getValueType(1);
684 SDValue Ops[2] = {GetSoftenedFloat(
N->getOperand(0)), StackSlot};
691 auto [ReturnVal, Chain] = TLI.
makeLibCall(DAG, LC, NVT0, Ops, CallOptions,
DL,
693 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
699 ReplaceValueWith(
SDValue(
N, 1), LoadExp);
709 RTLIB::REM_PPCF128));
718 RTLIB::RINT_PPCF128));
727 RTLIB::ROUND_PPCF128));
732 RTLIB::ROUNDEVEN_F32,
733 RTLIB::ROUNDEVEN_F64,
734 RTLIB::ROUNDEVEN_F80,
735 RTLIB::ROUNDEVEN_F128,
736 RTLIB::ROUNDEVEN_PPCF128));
745 RTLIB::SIN_PPCF128));
754 RTLIB::SQRT_PPCF128));
763 RTLIB::SUB_PPCF128));
772 RTLIB::TRUNC_PPCF128));
777 EVT VT =
N->getValueType(0);
782 L->getMemOperand()->getFlags() &
786 NewL = DAG.
getLoad(
L->getAddressingMode(),
L->getExtensionType(), NVT, dl,
787 L->getChain(),
L->getBasePtr(),
L->getOffset(),
788 L->getPointerInfo(), NVT,
L->getOriginalAlign(),
789 MMOFlags,
L->getAAInfo());
798 dl,
L->getChain(),
L->getBasePtr(),
L->getOffset(),
799 L->getPointerInfo(),
L->getMemoryVT(),
800 L->getOriginalAlign(), MMOFlags,
L->getAAInfo());
805 return BitConvertToInteger(ExtendNode);
812 LHS.getValueType(),
N->getOperand(0), LHS, RHS);
819 LHS.getValueType(),
N->getOperand(0),
820 N->getOperand(1), LHS, RHS,
N->getOperand(4));
825 N->getValueType(0)));
831 EVT VT =
N->getValueType(0);
836 NewVAARG = DAG.
getVAArg(NVT, dl, Chain,
Ptr,
N->getOperand(2),
837 N->getConstantOperandVal(3));
847 bool IsStrict =
N->isStrictFPOpcode();
850 EVT SVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
851 EVT RVT =
N->getValueType(0);
859 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
860 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
866 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported XINT_TO_FP!");
871 NVT,
N->getOperand(IsStrict ? 1 : 0));
875 std::pair<SDValue, SDValue> Tmp =
877 Op, CallOptions, dl, Chain);
880 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
890SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(
SDNode *
N) {
899bool DAGTypeLegalizer::SoftenFloatOperand(
SDNode *
N,
unsigned OpNo) {
900 LLVM_DEBUG(
dbgs() <<
"Soften float operand " << OpNo <<
": ";
N->dump(&DAG));
903 switch (
N->getOpcode()) {
906 dbgs() <<
"SoftenFloatOperand Op #" << OpNo <<
": ";
907 N->dump(&DAG);
dbgs() <<
"\n";
912 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(
N);
break;
924 Res = SoftenFloatOp_FP_TO_XINT_SAT(
N);
break;
930 case ISD::LRINT: Res = SoftenFloatOp_LRINT(
N);
break;
936 case ISD::SETCC: Res = SoftenFloatOp_SETCC(
N);
break;
937 case ISD::STORE: Res = SoftenFloatOp_STORE(
N, OpNo);
break;
942 if (!Res.
getNode())
return false;
950 "Invalid operand softening");
952 ReplaceValueWith(
SDValue(
N, 0), Res);
957 SDValue Op0 = GetSoftenedFloat(
N->getOperand(0));
970 bool IsStrict =
N->isStrictFPOpcode();
972 EVT SVT =
Op.getValueType();
973 EVT RVT =
N->getValueType(0);
979 FloatRVT = MVT::bf16;
982 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND libcall");
985 Op = GetSoftenedFloat(
Op);
988 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RVT,
Op,
992 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
993 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
1000 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
1001 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
1004 NewLHS = GetSoftenedFloat(NewLHS);
1005 NewRHS = GetSoftenedFloat(NewRHS);
1007 N->getOperand(2),
N->getOperand(3));
1011 if (!NewRHS.getNode()) {
1029 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1030 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1034 if (Promoted.
bitsGE(RetVT))
1042 bool IsStrict =
N->isStrictFPOpcode();
1046 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
1047 EVT SVT =
Op.getValueType();
1048 EVT RVT =
N->getValueType(0);
1058 "Unsupported FP_TO_XINT!");
1060 Op = GetSoftenedFloat(
Op);
1064 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
1065 CallOptions, dl, Chain);
1073 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1074 ReplaceValueWith(
SDValue(
N, 0), Res);
1078SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(
SDNode *
N) {
1084 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
1085 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
1088 NewLHS = GetSoftenedFloat(NewLHS);
1089 NewRHS = GetSoftenedFloat(NewRHS);
1091 N->getOperand(0),
N->getOperand(1));
1095 if (!NewRHS.getNode()) {
1102 N->getOperand(2),
N->getOperand(3),
1108 bool IsStrict =
N->isStrictFPOpcode();
1109 SDValue Op0 =
N->getOperand(IsStrict ? 1 : 0);
1110 SDValue Op1 =
N->getOperand(IsStrict ? 2 : 1);
1113 cast<CondCodeSDNode>(
N->getOperand(IsStrict ? 3 : 2))->get();
1116 SDValue NewLHS = GetSoftenedFloat(Op0);
1117 SDValue NewRHS = GetSoftenedFloat(Op1);
1133 "Unexpected setcc expansion!");
1136 ReplaceValueWith(
SDValue(
N, 0), NewLHS);
1137 ReplaceValueWith(
SDValue(
N, 1), Chain);
1143SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
1145 assert(OpNo == 1 &&
"Can only soften the stored value!");
1150 if (
ST->isTruncatingStore())
1152 Val = BitConvertToInteger(
1156 Val = GetSoftenedFloat(Val);
1158 return DAG.
getStore(
ST->getChain(), dl, Val,
ST->getBasePtr(),
1159 ST->getMemOperand());
1164 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
1167 EVT LVT =
LHS.getValueType();
1169 EVT RVT =
RHS.getValueType();
1175 int SizeDiff = RSize - LSize;
1183 }
else if (SizeDiff < 0) {
1198 bool IsStrict =
N->isStrictFPOpcode();
1199 unsigned Offset = IsStrict ? 1 : 0;
1203 EVT OpVT =
N->getOperand(0 +
Offset).getValueType();
1205 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT,
Op,
1209 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1210 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
1218 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1224 RTLIB::LROUND_PPCF128));
1228 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1233 RTLIB::LLROUND_F128,
1234 RTLIB::LLROUND_PPCF128));
1238 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1244 RTLIB::LRINT_PPCF128));
1248 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1254 RTLIB::LLRINT_PPCF128));
1265void DAGTypeLegalizer::ExpandFloatResult(
SDNode *
N,
unsigned ResNo) {
1271 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
1274 switch (
N->getOpcode()) {
1277 dbgs() <<
"ExpandFloatResult #" << ResNo <<
": ";
1278 N->dump(&DAG);
dbgs() <<
"\n";
1371 "Do not know how to expand this float constant!");
1372 APInt C = cast<ConstantFPSDNode>(
N)->getValueAPF().bitcastToAPInt();
1375 APInt(64,
C.getRawData()[1])),
1378 APInt(64,
C.getRawData()[0])),
1384 bool IsStrict =
N->isStrictFPOpcode();
1385 unsigned Offset = IsStrict ? 1 : 0;
1389 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC,
N->getValueType(0),
1393 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1394 GetPairElements(Tmp.first,
Lo,
Hi);
1399 bool IsStrict =
N->isStrictFPOpcode();
1400 unsigned Offset = IsStrict ? 1 : 0;
1404 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC,
N->getValueType(0),
1405 Ops, CallOptions,
SDLoc(
N),
1408 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1409 GetPairElements(Tmp.first,
Lo,
Hi);
1414 assert(
N->getValueType(0) == MVT::ppcf128 &&
1415 "Logic only correct for ppcf128!");
1418 GetExpandedFloat(
N->getOperand(0),
Lo, Tmp);
1429 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1430 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1431 RTLIB::FMIN_PPCF128),
Lo,
Hi);
1437 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1438 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1439 RTLIB::FMAX_PPCF128),
Lo,
Hi);
1445 RTLIB::ADD_F32, RTLIB::ADD_F64,
1446 RTLIB::ADD_F80, RTLIB::ADD_F128,
1447 RTLIB::ADD_PPCF128),
Lo,
Hi);
1452 ExpandFloatRes_Unary(
N,
GetFPLibCall(
N->getValueType(0), RTLIB::CBRT_F32,
1453 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1455 RTLIB::CBRT_PPCF128),
Lo,
Hi);
1458void DAGTypeLegalizer::ExpandFloatRes_FCEIL(
SDNode *
N,
1461 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1462 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1463 RTLIB::CEIL_PPCF128),
Lo,
Hi);
1466void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(
SDNode *
N,
1469 RTLIB::COPYSIGN_F32,
1470 RTLIB::COPYSIGN_F64,
1471 RTLIB::COPYSIGN_F80,
1472 RTLIB::COPYSIGN_F128,
1473 RTLIB::COPYSIGN_PPCF128),
Lo,
Hi);
1476void DAGTypeLegalizer::ExpandFloatRes_FCOS(
SDNode *
N,
1479 RTLIB::COS_F32, RTLIB::COS_F64,
1480 RTLIB::COS_F80, RTLIB::COS_F128,
1481 RTLIB::COS_PPCF128),
Lo,
Hi);
1491 RTLIB::DIV_PPCF128),
Lo,
Hi);
1494void DAGTypeLegalizer::ExpandFloatRes_FEXP(
SDNode *
N,
1497 RTLIB::EXP_F32, RTLIB::EXP_F64,
1498 RTLIB::EXP_F80, RTLIB::EXP_F128,
1499 RTLIB::EXP_PPCF128),
Lo,
Hi);
1502void DAGTypeLegalizer::ExpandFloatRes_FEXP2(
SDNode *
N,
1505 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1506 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1507 RTLIB::EXP2_PPCF128),
Lo,
Hi);
1512 ExpandFloatRes_Unary(
N,
1514 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1515 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1519void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(
SDNode *
N,
1522 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1523 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1524 RTLIB::FLOOR_PPCF128),
Lo,
Hi);
1527void DAGTypeLegalizer::ExpandFloatRes_FLOG(
SDNode *
N,
1530 RTLIB::LOG_F32, RTLIB::LOG_F64,
1531 RTLIB::LOG_F80, RTLIB::LOG_F128,
1532 RTLIB::LOG_PPCF128),
Lo,
Hi);
1535void DAGTypeLegalizer::ExpandFloatRes_FLOG2(
SDNode *
N,
1538 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1539 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1540 RTLIB::LOG2_PPCF128),
Lo,
Hi);
1543void DAGTypeLegalizer::ExpandFloatRes_FLOG10(
SDNode *
N,
1546 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1547 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1548 RTLIB::LOG10_PPCF128),
Lo,
Hi);
1553 bool IsStrict =
N->isStrictFPOpcode();
1554 unsigned Offset = IsStrict ? 1 : 0;
1564 RTLIB::FMA_PPCF128),
1565 N->getValueType(0), Ops, CallOptions,
1568 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1569 GetPairElements(Tmp.first,
Lo,
Hi);
1579 RTLIB::MUL_PPCF128),
Lo,
Hi);
1582void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(
SDNode *
N,
1585 RTLIB::NEARBYINT_F32,
1586 RTLIB::NEARBYINT_F64,
1587 RTLIB::NEARBYINT_F80,
1588 RTLIB::NEARBYINT_F128,
1589 RTLIB::NEARBYINT_PPCF128),
Lo,
Hi);
1595 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
1604 bool IsStrict =
N->isStrictFPOpcode();
1609 if (NVT ==
N->getOperand(1).getValueType()) {
1610 Hi =
N->getOperand(1);
1611 Chain =
N->getOperand(0);
1615 {
N->getOperand(0),
N->getOperand(1) });
1616 Chain =
Hi.getValue(1);
1626 ReplaceValueWith(
SDValue(
N, 1), Chain);
1629void DAGTypeLegalizer::ExpandFloatRes_FPOW(
SDNode *
N,
1632 RTLIB::POW_F32, RTLIB::POW_F64,
1633 RTLIB::POW_F80, RTLIB::POW_F128,
1634 RTLIB::POW_PPCF128),
Lo,
Hi);
1637void DAGTypeLegalizer::ExpandFloatRes_FPOWI(
SDNode *
N,
1647void DAGTypeLegalizer::ExpandFloatRes_FREEZE(
SDNode *
N,
1649 assert(
N->getValueType(0) == MVT::ppcf128 &&
1650 "Logic only correct for ppcf128!");
1653 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
1658void DAGTypeLegalizer::ExpandFloatRes_FREM(
SDNode *
N,
1661 RTLIB::REM_F32, RTLIB::REM_F64,
1662 RTLIB::REM_F80, RTLIB::REM_F128,
1663 RTLIB::REM_PPCF128),
Lo,
Hi);
1666void DAGTypeLegalizer::ExpandFloatRes_FRINT(
SDNode *
N,
1669 RTLIB::RINT_F32, RTLIB::RINT_F64,
1670 RTLIB::RINT_F80, RTLIB::RINT_F128,
1671 RTLIB::RINT_PPCF128),
Lo,
Hi);
1674void DAGTypeLegalizer::ExpandFloatRes_FROUND(
SDNode *
N,
1681 RTLIB::ROUND_PPCF128),
Lo,
Hi);
1684void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(
SDNode *
N,
1687 RTLIB::ROUNDEVEN_F32,
1688 RTLIB::ROUNDEVEN_F64,
1689 RTLIB::ROUNDEVEN_F80,
1690 RTLIB::ROUNDEVEN_F128,
1691 RTLIB::ROUNDEVEN_PPCF128),
Lo,
Hi);
1694void DAGTypeLegalizer::ExpandFloatRes_FSIN(
SDNode *
N,
1697 RTLIB::SIN_F32, RTLIB::SIN_F64,
1698 RTLIB::SIN_F80, RTLIB::SIN_F128,
1699 RTLIB::SIN_PPCF128),
Lo,
Hi);
1702void DAGTypeLegalizer::ExpandFloatRes_FSQRT(
SDNode *
N,
1705 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1706 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1707 RTLIB::SQRT_PPCF128),
Lo,
Hi);
1717 RTLIB::SUB_PPCF128),
Lo,
Hi);
1720void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(
SDNode *
N,
1723 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1724 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1725 RTLIB::TRUNC_PPCF128),
Lo,
Hi);
1731 ExpandRes_NormalLoad(
N,
Lo,
Hi);
1743 assert(
LD->getMemoryVT().bitsLE(NVT) &&
"Float type not round?");
1746 LD->getMemoryVT(),
LD->getMemOperand());
1749 Chain =
Hi.getValue(1);
1757 ReplaceValueWith(
SDValue(LD, 1), Chain);
1762 assert(
N->getValueType(0) == MVT::ppcf128 &&
"Unsupported XINT_TO_FP!");
1763 EVT VT =
N->getValueType(0);
1765 bool Strict =
N->isStrictFPOpcode();
1766 SDValue Src =
N->getOperand(Strict ? 1 : 0);
1767 EVT SrcVT = Src.getValueType();
1775 Flags.setNoFPExcept(
N->getFlags().hasNoFPExcept());
1780 if (SrcVT.
bitsLE(MVT::i32)) {
1786 {Chain, Src}, Flags);
1787 Chain =
Hi.getValue(1);
1789 Hi = DAG.
getNode(
N->getOpcode(), dl, NVT, Src);
1792 if (SrcVT.
bitsLE(MVT::i64)) {
1795 LC = RTLIB::SINTTOFP_I64_PPCF128;
1796 }
else if (SrcVT.
bitsLE(MVT::i128)) {
1798 LC = RTLIB::SINTTOFP_I128_PPCF128;
1800 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported XINT_TO_FP!");
1804 std::pair<SDValue, SDValue> Tmp =
1805 TLI.
makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1808 GetPairElements(Tmp.first,
Lo,
Hi);
1814 ReplaceValueWith(
SDValue(
N, 1), Chain);
1824 SrcVT = Src.getValueType();
1827 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1828 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1829 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1851 {Chain, Hi, NewLo}, Flags);
1852 Chain =
Lo.getValue(1);
1853 ReplaceValueWith(
SDValue(
N, 1), Chain);
1858 GetPairElements(
Lo,
Lo,
Hi);
1870bool DAGTypeLegalizer::ExpandFloatOperand(
SDNode *
N,
unsigned OpNo) {
1875 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
1878 switch (
N->getOpcode()) {
1881 dbgs() <<
"ExpandFloatOperand Op #" << OpNo <<
": ";
1882 N->dump(&DAG);
dbgs() <<
"\n";
1890 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(
N);
break;
1898 case ISD::LROUND: Res = ExpandFloatOp_LROUND(
N);
break;
1900 case ISD::LRINT: Res = ExpandFloatOp_LRINT(
N);
break;
1901 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(
N);
break;
1905 case ISD::SETCC: Res = ExpandFloatOp_SETCC(
N);
break;
1906 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(
N),
1911 if (!Res.
getNode())
return false;
1919 "Invalid operand expansion");
1921 ReplaceValueWith(
SDValue(
N, 0), Res);
1927void DAGTypeLegalizer::FloatExpandSetCCOperands(
SDValue &NewLHS,
1932 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1933 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1934 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1943 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1948 RHSLo, CCCode, OutputChain, IsSignaling);
1956 RHSHi, CCCode, OutputChain, IsSignaling);
1961 Chain = OutputChain;
1965 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
1966 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
1968 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain);
1980 N->getOperand(4)), 0);
1984 assert(
N->getOperand(1).getValueType() == MVT::ppcf128 &&
1985 "Logic only correct for ppcf128!");
1987 GetExpandedFloat(
N->getOperand(1),
Lo,
Hi);
1991 N->getValueType(0),
N->getOperand(0),
Hi);
1995 bool IsStrict =
N->isStrictFPOpcode();
1996 assert(
N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1997 "Logic only correct for ppcf128!");
1999 GetExpandedFloat(
N->getOperand(IsStrict ? 1 : 0),
Lo,
Hi);
2004 N->getValueType(0),
Hi,
N->getOperand(1));
2008 if (
Hi.getValueType() ==
N->getValueType(0)) {
2010 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
2016 {
N->getValueType(0), MVT::Other},
2017 {
N->getOperand(0),
Hi,
N->getOperand(2)});
2019 ReplaceValueWith(
SDValue(
N, 0), Expansion);
2024 EVT RVT =
N->getValueType(0);
2027 bool IsStrict =
N->isStrictFPOpcode();
2030 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
2036 "Unsupported FP_TO_XINT!");
2038 std::pair<SDValue, SDValue> Tmp =
2043 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2044 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2049 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
2050 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
2052 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain);
2063 N->getOperand(2),
N->getOperand(3),
2068 bool IsStrict =
N->isStrictFPOpcode();
2069 SDValue NewLHS =
N->getOperand(IsStrict ? 1 : 0);
2070 SDValue NewRHS =
N->getOperand(IsStrict ? 2 : 1);
2073 cast<CondCodeSDNode>(
N->getOperand(IsStrict ? 3 : 2))->get();
2074 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain,
2080 "Unexpected setcc expansion!");
2082 ReplaceValueWith(
SDValue(
N, 0), NewLHS);
2083 ReplaceValueWith(
SDValue(
N, 1), Chain);
2089SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
2091 return ExpandOp_NormalStore(
N, OpNo);
2094 assert(OpNo == 1 &&
"Can only expand the stored value so far");
2101 ST->getValue().getValueType());
2103 assert(
ST->getMemoryVT().bitsLE(NVT) &&
"Float type not round?");
2107 GetExpandedOp(
ST->getValue(),
Lo,
Hi);
2110 ST->getMemoryVT(),
ST->getMemOperand());
2114 EVT RVT =
N->getValueType(0);
2115 EVT RetVT =
N->getOperand(0).getValueType();
2122 RTLIB::LROUND_PPCF128),
2123 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2127 EVT RVT =
N->getValueType(0);
2128 EVT RetVT =
N->getOperand(0).getValueType();
2134 RTLIB::LLROUND_F128,
2135 RTLIB::LLROUND_PPCF128),
2136 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2140 EVT RVT =
N->getValueType(0);
2141 EVT RetVT =
N->getOperand(0).getValueType();
2148 RTLIB::LRINT_PPCF128),
2149 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2153 EVT RVT =
N->getValueType(0);
2154 EVT RetVT =
N->getOperand(0).getValueType();
2161 RTLIB::LLRINT_PPCF128),
2162 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2171 if (OpVT == MVT::f16) {
2173 }
else if (RetVT == MVT::f16) {
2175 }
else if (OpVT == MVT::bf16) {
2177 }
else if (RetVT == MVT::bf16) {
2184bool DAGTypeLegalizer::PromoteFloatOperand(
SDNode *
N,
unsigned OpNo) {
2185 LLVM_DEBUG(
dbgs() <<
"Promote float operand " << OpNo <<
": ";
N->dump(&DAG));
2188 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
2199 switch (
N->getOpcode()) {
2202 dbgs() <<
"PromoteFloatOperand Op #" << OpNo <<
": ";
2203 N->dump(&DAG);
dbgs() <<
"\n";
2212 case ISD::LLRINT:
R = PromoteFloatOp_UnaryOp(
N, OpNo);
break;
2215 R = PromoteFloatOp_FP_TO_XINT_SAT(
N, OpNo);
break;
2218 case ISD::SETCC:
R = PromoteFloatOp_SETCC(
N, OpNo);
break;
2219 case ISD::STORE:
R = PromoteFloatOp_STORE(
N, OpNo);
break;
2224 ReplaceValueWith(
SDValue(
N, 0), R);
2228SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(
SDNode *
N,
unsigned OpNo) {
2230 EVT OpVT =
Op->getValueType(0);
2232 SDValue Promoted = GetPromotedFloat(
N->getOperand(0));
2241 return DAG.
getBitcast(
N->getValueType(0), Convert);
2246SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(
SDNode *
N,
unsigned OpNo) {
2247 assert (OpNo == 1 &&
"Only Operand 1 must need promotion here");
2248 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2251 N->getOperand(0), Op1);
2255SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(
SDNode *
N,
unsigned OpNo) {
2256 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2260SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(
SDNode *
N,
2262 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2267SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(
SDNode *
N,
unsigned OpNo) {
2268 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2269 EVT VT =
N->getValueType(0);
2272 if (VT ==
Op->getValueType(0))
2282SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2287 LHS, RHS,
N->getOperand(2),
N->getOperand(3),
2293SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2294 EVT VT =
N->getValueType(0);
2295 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2296 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2297 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
2305SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
2310 SDValue Promoted = GetPromotedFloat(Val);
2311 EVT VT =
ST->getOperand(1).getValueType();
2318 return DAG.
getStore(
ST->getChain(),
DL, NewVal,
ST->getBasePtr(),
2319 ST->getMemOperand());
2326void DAGTypeLegalizer::PromoteFloatResult(
SDNode *
N,
unsigned ResNo) {
2327 LLVM_DEBUG(
dbgs() <<
"Promote float result " << ResNo <<
": ";
N->dump(&DAG));
2331 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
2336 switch (
N->getOpcode()) {
2343 dbgs() <<
"PromoteFloatResult #" << ResNo <<
": ";
2344 N->dump(&DAG);
dbgs() <<
"\n";
2351 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(
N);
break;
2386 case ISD::FSUB:
R = PromoteFloatRes_BinOp(
N);
break;
2389 case ISD::FMAD:
R = PromoteFloatRes_FMAD(
N);
break;
2396 case ISD::LOAD:
R = PromoteFloatRes_LOAD(
N);
break;
2410 R = PromoteFloatRes_VECREDUCE(
N);
2414 R = PromoteFloatRes_VECREDUCE_SEQ(
N);
2419 SetPromotedFloat(
SDValue(
N, ResNo), R);
2428 EVT VT =
N->getValueType(0);
2433 N->getOperand(0).getValueType().getSizeInBits());
2440 EVT VT =
N->getValueType(0);
2459SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
2464 if (isa<ConstantSDNode>(
N->getOperand(1))) {
2470 uint64_t IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
2472 switch (getTypeAction(VecVT)) {
2475 SDValue Res = GetScalarizedVector(
N->getOperand(0));
2476 ReplaceValueWith(
SDValue(
N, 0), Res);
2480 Vec = GetWidenedVector(Vec);
2482 ReplaceValueWith(
SDValue(
N, 0), Res);
2487 GetSplitVector(Vec,
Lo,
Hi);
2489 uint64_t LoElts =
Lo.getValueType().getVectorNumElements();
2491 if (IdxVal < LoElts)
2496 Idx.getValueType()));
2497 ReplaceValueWith(
SDValue(
N, 0), Res);
2505 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
2510 NewOp,
N->getOperand(1));
2513 EVT VT =
N->getValueType(0);
2522 EVT VT =
N->getValueType(0);
2524 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2535 EVT VT =
N->getValueType(0);
2537 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2546 EVT VT =
N->getValueType(0);
2548 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2549 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2550 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), NVT, Op0, Op1,
N->getFlags());
2554 EVT VT =
N->getValueType(0);
2556 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2557 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2558 SDValue Op2 = GetPromotedFloat(
N->getOperand(2));
2560 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), NVT, Op0, Op1, Op2);
2565 EVT VT =
N->getValueType(0);
2567 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2574 EVT VT =
N->getValueType(0);
2576 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2590 EVT VT =
N->getValueType(0);
2591 EVT OpVT =
Op->getValueType(0);
2603 EVT VT =
N->getValueType(0);
2608 L->getAddressingMode(),
L->getExtensionType(), IVT,
SDLoc(
N),
2609 L->getChain(),
L->getBasePtr(),
L->getOffset(),
L->getPointerInfo(), IVT,
2610 L->getOriginalAlign(),
L->getMemOperand()->getFlags(),
L->getAAInfo());
2626 N->getOperand(0), TrueVal, FalseVal);
2636 TrueVal.getNode()->getValueType(0),
N->getOperand(0),
2637 N->getOperand(1), TrueVal, FalseVal,
N->getOperand(4));
2644 EVT VT =
N->getValueType(0);
2656 N->getValueType(0)));
2668SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(
SDNode *
N) {
2674 EVT VT =
N->getValueType(0);
2685 { AM->getChain(), AM->getBasePtr(), CastVal },
2708void DAGTypeLegalizer::SoftPromoteHalfResult(
SDNode *
N,
unsigned ResNo) {
2709 LLVM_DEBUG(
dbgs() <<
"Soft promote half result " << ResNo <<
": ";
2714 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
2719 switch (
N->getOpcode()) {
2722 dbgs() <<
"SoftPromoteHalfResult #" << ResNo <<
": ";
2723 N->dump(&DAG);
dbgs() <<
"\n";
2731 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(
N);
break;
2769 case ISD::FSUB:
R = SoftPromoteHalfRes_BinOp(
N);
break;
2772 case ISD::FMAD:
R = SoftPromoteHalfRes_FMAD(
N);
break;
2777 case ISD::LOAD:
R = SoftPromoteHalfRes_LOAD(
N);
break;
2782 case ISD::UNDEF:
R = SoftPromoteHalfRes_UNDEF(
N);
break;
2790 R = SoftPromoteHalfRes_VECREDUCE(
N);
2794 R = SoftPromoteHalfRes_VECREDUCE_SEQ(
N);
2799 SetSoftPromotedHalf(
SDValue(
N, ResNo), R);
2803 return BitConvertToInteger(
N->getOperand(0));
2806SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(
SDNode *
N) {
2814SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
2815 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
2821SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(
SDNode *
N) {
2822 SDValue LHS = GetSoftPromotedHalf(
N->getOperand(0));
2823 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
2826 EVT LVT =
LHS.getValueType();
2827 EVT RVT =
RHS.getValueType();
2848 }
else if (SizeDiff < 0) {
2870 EVT OVT =
N->getValueType(0);
2872 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
2873 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
2874 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
2879 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
2880 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
2881 Op2 = DAG.
getNode(PromotionOpcode, dl, NVT, Op2);
2890 EVT OVT =
N->getValueType(0);
2892 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
2905SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(
SDNode *
N) {
2906 EVT RVT =
N->getValueType(0);
2907 EVT SVT =
N->getOperand(0).getValueType();
2909 if (
N->isStrictFPOpcode()) {
2913 {
N->getOperand(0),
N->getOperand(1)});
2928 DAG.
getLoad(
L->getAddressingMode(),
L->getExtensionType(), MVT::i16,
2929 SDLoc(
N),
L->getChain(),
L->getBasePtr(),
L->getOffset(),
2930 L->getPointerInfo(), MVT::i16,
L->getOriginalAlign(),
2931 L->getMemOperand()->getFlags(),
L->getAAInfo());
2939 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
2940 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
2945SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(
SDNode *
N) {
2946 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
2947 SDValue Op3 = GetSoftPromotedHalf(
N->getOperand(3));
2949 N->getOperand(0),
N->getOperand(1), Op2, Op3,
2953SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(
SDNode *
N) {
2954 EVT OVT =
N->getValueType(0);
2969 EVT OVT =
N->getValueType(0);
2971 SDValue Op = GetSoftPromotedHalf(
N->getOperand(0));
2984 EVT OVT =
N->getValueType(0);
2986 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
2987 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
2992 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
2993 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3001SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(
SDNode *
N) {
3007SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(
SDNode *
N) {
3017bool DAGTypeLegalizer::SoftPromoteHalfOperand(
SDNode *
N,
unsigned OpNo) {
3018 LLVM_DEBUG(
dbgs() <<
"Soft promote half operand " << OpNo <<
": ";
3022 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
3032 switch (
N->getOpcode()) {
3035 dbgs() <<
"SoftPromoteHalfOperand Op #" << OpNo <<
": ";
3036 N->dump(&DAG);
dbgs() <<
"\n";
3041 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(
N);
break;
3042 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(
N, OpNo);
break;
3047 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(
N);
break;
3050 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(
N, OpNo);
break;
3051 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(
N);
break;
3052 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(
N, OpNo);
break;
3054 Res = SoftPromoteHalfOp_STACKMAP(
N, OpNo);
3057 Res = SoftPromoteHalfOp_PATCHPOINT(
N, OpNo);
3067 "Invalid operand expansion");
3069 ReplaceValueWith(
SDValue(
N, 0), Res);
3074 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
3081 assert(OpNo == 1 &&
"Only Operand 1 must need promotion here");
3088 Op1 = GetSoftPromotedHalf(Op1);
3091 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0),
N->getOperand(0),
3095SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(
SDNode *
N) {
3096 EVT RVT =
N->getValueType(0);
3097 bool IsStrict =
N->isStrictFPOpcode();
3098 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3099 EVT SVT =
Op.getValueType();
3100 Op = GetSoftPromotedHalf(
N->getOperand(IsStrict ? 1 : 0));
3106 {
N->getValueType(0), MVT::Other}, {
N->getOperand(0),
Op});
3108 ReplaceValueWith(
SDValue(
N, 0), Res);
3115SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(
SDNode *
N) {
3116 EVT RVT =
N->getValueType(0);
3118 EVT SVT =
Op.getValueType();
3123 Op = GetSoftPromotedHalf(
Op);
3127 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0), Res);
3130SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(
SDNode *
N) {
3131 EVT RVT =
N->getValueType(0);
3133 EVT SVT =
Op.getValueType();
3138 Op = GetSoftPromotedHalf(
Op);
3142 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0), Res,
3148 assert(OpNo == 0 &&
"Can only soften the comparison values");
3156 Op0 = GetSoftPromotedHalf(Op0);
3157 Op1 = GetSoftPromotedHalf(Op1);
3161 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3162 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3165 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
3171 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
3177 Op0 = GetSoftPromotedHalf(Op0);
3178 Op1 = GetSoftPromotedHalf(Op1);
3182 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3183 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3188SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(
SDNode *
N,
unsigned OpNo) {
3189 assert(OpNo == 1 &&
"Can only soften the stored value!");
3194 assert(!
ST->isTruncatingStore() &&
"Unexpected truncating store.");
3195 SDValue Promoted = GetSoftPromotedHalf(Val);
3196 return DAG.
getStore(
ST->getChain(), dl, Promoted,
ST->getBasePtr(),
3197 ST->getMemOperand());
3200SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
3204 NewOps[OpNo] = GetSoftPromotedHalf(
Op);
3208 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
3219 NewOps[OpNo] = GetSoftPromotedHalf(
Op);
3223 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
MachineFunction & getMachineFunction() const
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
DWARFExpression::Operation Op
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)