27#define DEBUG_TYPE "legalize-types"
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
51void DAGTypeLegalizer::SoftenFloatResult(
SDNode *
N,
unsigned ResNo) {
52 LLVM_DEBUG(
dbgs() <<
"Soften float result " << ResNo <<
": ";
N->dump(&DAG);
56 switch (
N->getOpcode()) {
59 dbgs() <<
"SoftenFloatResult #" << ResNo <<
": ";
60 N->dump(&DAG);
dbgs() <<
"\n";
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(
N, ResNo);
break;
99 case ISD::FMA:
R = SoftenFloatRes_FMA(
N);
break;
146 R = SoftenFloatRes_VECREDUCE(
N);
150 R = SoftenFloatRes_VECREDUCE_SEQ(
N);
157 SetSoftenedFloat(
SDValue(
N, ResNo), R);
162 bool IsStrict =
N->isStrictFPOpcode();
164 unsigned Offset = IsStrict ? 1 : 0;
166 "Unexpected number of operands!");
170 EVT OpVT =
N->getOperand(0 +
Offset).getValueType();
172 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Op,
176 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
181 bool IsStrict =
N->isStrictFPOpcode();
183 unsigned Offset = IsStrict ? 1 : 0;
185 "Unexpected number of operands!");
187 GetSoftenedFloat(
N->getOperand(1 +
Offset)) };
190 EVT OpsVT[2] = {
N->getOperand(0 +
Offset).getValueType(),
191 N->getOperand(1 +
Offset).getValueType() };
193 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Ops,
197 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
202 return BitConvertToInteger(
N->getOperand(0));
208 GetSoftenedFloat(
N->getOperand(0)));
214 GetSoftenedFloat(
N->getOperand(0)));
220 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
221 return BitConvertToInteger(Op);
229 BitConvertToInteger(
N->getOperand(0)),
230 BitConvertToInteger(
N->getOperand(1)));
246 APInt Val(128, words);
257SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(
SDNode *
N,
unsigned ResNo) {
258 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
261 NewOp,
N->getOperand(1));
272 SDValue Op = GetSoftenedFloat(
N->getOperand(0));
278 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
284 RTLIB::FMIN_PPCF128));
289 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
295 RTLIB::FMAX_PPCF128));
304 RTLIB::ADD_PPCF128));
313 RTLIB::CBRT_PPCF128));
322 RTLIB::CEIL_PPCF128));
327 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
330 EVT LVT =
LHS.getValueType();
331 EVT RVT =
RHS.getValueType();
352 }
else if (SizeDiff < 0) {
379 RTLIB::COS_PPCF128));
388 RTLIB::DIV_PPCF128));
397 RTLIB::EXP_PPCF128));
406 RTLIB::EXP2_PPCF128));
415 RTLIB::FLOOR_PPCF128));
424 RTLIB::LOG_PPCF128));
433 RTLIB::LOG2_PPCF128));
442 RTLIB::LOG10_PPCF128));
446 bool IsStrict =
N->isStrictFPOpcode();
448 unsigned Offset = IsStrict ? 1 : 0;
450 GetSoftenedFloat(
N->getOperand(1 +
Offset)),
451 GetSoftenedFloat(
N->getOperand(2 +
Offset)) };
454 EVT OpsVT[3] = {
N->getOperand(0 +
Offset).getValueType(),
455 N->getOperand(1 +
Offset).getValueType(),
456 N->getOperand(2 +
Offset).getValueType() };
458 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG,
465 NVT, Ops, CallOptions,
SDLoc(
N), Chain);
467 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
477 RTLIB::MUL_PPCF128));
482 RTLIB::NEARBYINT_F32,
483 RTLIB::NEARBYINT_F64,
484 RTLIB::NEARBYINT_F80,
485 RTLIB::NEARBYINT_F128,
486 RTLIB::NEARBYINT_PPCF128));
500 bool IsStrict =
N->isStrictFPOpcode();
507 Op = GetPromotedFloat(Op);
510 if (
Op.getValueType() ==
N->getValueType(0))
511 return BitConvertToInteger(Op);
518 if ((
Op.getValueType() == MVT::f16 ||
Op.getValueType() == MVT::bf16) &&
519 N->getValueType(0) != MVT::f32) {
522 { MVT::f32, MVT::Other }, { Chain,
Op });
523 Chain =
Op.getValue(1);
529 if (
Op.getValueType() == MVT::bf16)
530 return SoftenFloatRes_BF16_TO_FP(
N);
533 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_EXTEND!");
535 EVT OpVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
537 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Op,
541 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
551 EVT OpsVT[1] = {
N->getOperand(0).getValueType() };
554 CallOptions,
SDLoc(
N)).first;
555 if (
N->getValueType(0) == MVT::f32)
560 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_EXTEND!");
567 assert(
N->getValueType(0) == MVT::f32 &&
568 "Can only soften BF16_TO_FP with f32 result");
580 bool IsStrict =
N->isStrictFPOpcode();
585 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND!");
587 EVT OpVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
589 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Op,
593 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
603 RTLIB::POW_PPCF128));
607 bool IsStrict =
N->isStrictFPOpcode();
608 unsigned Offset = IsStrict ? 1 : 0;
609 assert((
N->getOperand(1 +
Offset).getValueType() == MVT::i16 ||
610 N->getOperand(1 +
Offset).getValueType() == MVT::i32) &&
611 "Unsupported power type!");
613 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
622 N->getOperand(1 +
Offset).getValueType().getSizeInBits()) {
634 EVT OpsVT[2] = {
N->getOperand(0 +
Offset).getValueType(),
635 N->getOperand(1 +
Offset).getValueType() };
637 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Ops,
641 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
651 RTLIB::REM_PPCF128));
660 RTLIB::RINT_PPCF128));
669 RTLIB::ROUND_PPCF128));
674 RTLIB::ROUNDEVEN_F32,
675 RTLIB::ROUNDEVEN_F64,
676 RTLIB::ROUNDEVEN_F80,
677 RTLIB::ROUNDEVEN_F128,
678 RTLIB::ROUNDEVEN_PPCF128));
687 RTLIB::SIN_PPCF128));
696 RTLIB::SQRT_PPCF128));
705 RTLIB::SUB_PPCF128));
714 RTLIB::TRUNC_PPCF128));
719 EVT VT =
N->getValueType(0);
724 L->getMemOperand()->getFlags() &
728 NewL = DAG.
getLoad(
L->getAddressingMode(),
L->getExtensionType(), NVT, dl,
729 L->getChain(),
L->getBasePtr(),
L->getOffset(),
730 L->getPointerInfo(), NVT,
L->getOriginalAlign(),
731 MMOFlags,
L->getAAInfo());
740 dl,
L->getChain(),
L->getBasePtr(),
L->getOffset(),
741 L->getPointerInfo(),
L->getMemoryVT(),
742 L->getOriginalAlign(), MMOFlags,
L->getAAInfo());
747 return BitConvertToInteger(ExtendNode);
754 LHS.getValueType(),
N->getOperand(0), LHS, RHS);
761 LHS.getValueType(),
N->getOperand(0),
762 N->getOperand(1), LHS, RHS,
N->getOperand(4));
767 N->getValueType(0)));
773 EVT VT =
N->getValueType(0);
778 NewVAARG = DAG.
getVAArg(NVT, dl, Chain,
Ptr,
N->getOperand(2),
779 N->getConstantOperandVal(3));
789 bool IsStrict =
N->isStrictFPOpcode();
792 EVT SVT =
N->getOperand(IsStrict ? 1 : 0).getValueType();
793 EVT RVT =
N->getValueType(0);
801 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
802 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
808 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported XINT_TO_FP!");
813 NVT,
N->getOperand(IsStrict ? 1 : 0));
817 std::pair<SDValue, SDValue> Tmp =
819 Op, CallOptions, dl, Chain);
822 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
832SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(
SDNode *
N) {
841bool DAGTypeLegalizer::SoftenFloatOperand(
SDNode *
N,
unsigned OpNo) {
842 LLVM_DEBUG(
dbgs() <<
"Soften float operand " << OpNo <<
": ";
N->dump(&DAG);
846 switch (
N->getOpcode()) {
849 dbgs() <<
"SoftenFloatOperand Op #" << OpNo <<
": ";
850 N->dump(&DAG);
dbgs() <<
"\n";
855 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(
N);
break;
867 Res = SoftenFloatOp_FP_TO_XINT_SAT(
N);
break;
873 case ISD::LRINT: Res = SoftenFloatOp_LRINT(
N);
break;
879 case ISD::SETCC: Res = SoftenFloatOp_SETCC(
N);
break;
880 case ISD::STORE: Res = SoftenFloatOp_STORE(
N, OpNo);
break;
885 if (!Res.getNode())
return false;
889 if (Res.getNode() ==
N)
892 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 1 &&
893 "Invalid operand softening");
895 ReplaceValueWith(
SDValue(
N, 0), Res);
900 SDValue Op0 = GetSoftenedFloat(
N->getOperand(0));
913 bool IsStrict =
N->isStrictFPOpcode();
915 EVT SVT =
Op.getValueType();
916 EVT RVT =
N->getValueType(0);
922 FloatRVT = MVT::bf16;
925 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported FP_ROUND libcall");
928 Op = GetSoftenedFloat(Op);
931 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RVT, Op,
935 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
936 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
943 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
944 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
947 NewLHS = GetSoftenedFloat(NewLHS);
948 NewRHS = GetSoftenedFloat(NewRHS);
950 N->getOperand(2),
N->getOperand(3));
954 if (!NewRHS.getNode()) {
972 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
973 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
977 if (Promoted.
bitsGE(RetVT))
985 bool IsStrict =
N->isStrictFPOpcode();
990 EVT SVT =
Op.getValueType();
991 EVT RVT =
N->getValueType(0);
1001 "Unsupported FP_TO_XINT!");
1003 Op = GetSoftenedFloat(Op);
1007 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Op,
1008 CallOptions, dl, Chain);
1016 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1017 ReplaceValueWith(
SDValue(
N, 0), Res);
1021SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(
SDNode *
N) {
1027 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
1028 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
1031 NewLHS = GetSoftenedFloat(NewLHS);
1032 NewRHS = GetSoftenedFloat(NewRHS);
1034 N->getOperand(0),
N->getOperand(1));
1038 if (!NewRHS.getNode()) {
1045 N->getOperand(2),
N->getOperand(3),
1051 bool IsStrict =
N->isStrictFPOpcode();
1052 SDValue Op0 =
N->getOperand(IsStrict ? 1 : 0);
1053 SDValue Op1 =
N->getOperand(IsStrict ? 2 : 1);
1056 cast<CondCodeSDNode>(
N->getOperand(IsStrict ? 3 : 2))->get();
1059 SDValue NewLHS = GetSoftenedFloat(Op0);
1060 SDValue NewRHS = GetSoftenedFloat(Op1);
1076 "Unexpected setcc expansion!");
1079 ReplaceValueWith(
SDValue(
N, 0), NewLHS);
1080 ReplaceValueWith(
SDValue(
N, 1), Chain);
1086SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
1088 assert(OpNo == 1 &&
"Can only soften the stored value!");
1093 if (
ST->isTruncatingStore())
1095 Val = BitConvertToInteger(
1099 Val = GetSoftenedFloat(Val);
1101 return DAG.
getStore(
ST->getChain(), dl, Val,
ST->getBasePtr(),
1102 ST->getMemOperand());
1107 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
1110 EVT LVT =
LHS.getValueType();
1112 EVT RVT =
RHS.getValueType();
1118 int SizeDiff = RSize - LSize;
1126 }
else if (SizeDiff < 0) {
1141 bool IsStrict =
N->isStrictFPOpcode();
1142 unsigned Offset = IsStrict ? 1 : 0;
1146 EVT OpVT =
N->getOperand(0 +
Offset).getValueType();
1148 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, NVT, Op,
1152 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1153 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
1161 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1167 RTLIB::LROUND_PPCF128));
1171 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1176 RTLIB::LLROUND_F128,
1177 RTLIB::LLROUND_PPCF128));
1181 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1187 RTLIB::LRINT_PPCF128));
1191 EVT OpVT =
N->getOperand(
N->isStrictFPOpcode() ? 1 : 0).getValueType();
1197 RTLIB::LLRINT_PPCF128));
1208void DAGTypeLegalizer::ExpandFloatResult(
SDNode *
N,
unsigned ResNo) {
1214 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
1217 switch (
N->getOpcode()) {
1220 dbgs() <<
"ExpandFloatResult #" << ResNo <<
": ";
1221 N->dump(&DAG);
dbgs() <<
"\n";
1223 llvm_unreachable(
"Do not know how to expand the result of this operator!");
1310 "Do not know how to expand this float constant!");
1311 APInt C = cast<ConstantFPSDNode>(
N)->getValueAPF().bitcastToAPInt();
1314 APInt(64,
C.getRawData()[1])),
1317 APInt(64,
C.getRawData()[0])),
1323 bool IsStrict =
N->isStrictFPOpcode();
1324 unsigned Offset = IsStrict ? 1 : 0;
1328 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC,
N->getValueType(0),
1329 Op, CallOptions,
SDLoc(
N),
1332 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1333 GetPairElements(Tmp.first,
Lo,
Hi);
1338 bool IsStrict =
N->isStrictFPOpcode();
1339 unsigned Offset = IsStrict ? 1 : 0;
1343 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC,
N->getValueType(0),
1344 Ops, CallOptions,
SDLoc(
N),
1347 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1348 GetPairElements(Tmp.first,
Lo,
Hi);
1353 assert(
N->getValueType(0) == MVT::ppcf128 &&
1354 "Logic only correct for ppcf128!");
1357 GetExpandedFloat(
N->getOperand(0),
Lo, Tmp);
1368 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1369 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1370 RTLIB::FMIN_PPCF128),
Lo,
Hi);
1376 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1377 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1378 RTLIB::FMAX_PPCF128),
Lo,
Hi);
1384 RTLIB::ADD_F32, RTLIB::ADD_F64,
1385 RTLIB::ADD_F80, RTLIB::ADD_F128,
1386 RTLIB::ADD_PPCF128),
Lo,
Hi);
1391 ExpandFloatRes_Unary(
N,
GetFPLibCall(
N->getValueType(0), RTLIB::CBRT_F32,
1392 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1394 RTLIB::CBRT_PPCF128),
Lo,
Hi);
1397void DAGTypeLegalizer::ExpandFloatRes_FCEIL(
SDNode *
N,
1400 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1401 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1402 RTLIB::CEIL_PPCF128),
Lo,
Hi);
1405void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(
SDNode *
N,
1408 RTLIB::COPYSIGN_F32,
1409 RTLIB::COPYSIGN_F64,
1410 RTLIB::COPYSIGN_F80,
1411 RTLIB::COPYSIGN_F128,
1412 RTLIB::COPYSIGN_PPCF128),
Lo,
Hi);
1415void DAGTypeLegalizer::ExpandFloatRes_FCOS(
SDNode *
N,
1418 RTLIB::COS_F32, RTLIB::COS_F64,
1419 RTLIB::COS_F80, RTLIB::COS_F128,
1420 RTLIB::COS_PPCF128),
Lo,
Hi);
1430 RTLIB::DIV_PPCF128),
Lo,
Hi);
1433void DAGTypeLegalizer::ExpandFloatRes_FEXP(
SDNode *
N,
1436 RTLIB::EXP_F32, RTLIB::EXP_F64,
1437 RTLIB::EXP_F80, RTLIB::EXP_F128,
1438 RTLIB::EXP_PPCF128),
Lo,
Hi);
1441void DAGTypeLegalizer::ExpandFloatRes_FEXP2(
SDNode *
N,
1444 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1445 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1446 RTLIB::EXP2_PPCF128),
Lo,
Hi);
1449void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(
SDNode *
N,
1452 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1453 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1454 RTLIB::FLOOR_PPCF128),
Lo,
Hi);
1457void DAGTypeLegalizer::ExpandFloatRes_FLOG(
SDNode *
N,
1460 RTLIB::LOG_F32, RTLIB::LOG_F64,
1461 RTLIB::LOG_F80, RTLIB::LOG_F128,
1462 RTLIB::LOG_PPCF128),
Lo,
Hi);
1465void DAGTypeLegalizer::ExpandFloatRes_FLOG2(
SDNode *
N,
1468 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1469 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1470 RTLIB::LOG2_PPCF128),
Lo,
Hi);
1473void DAGTypeLegalizer::ExpandFloatRes_FLOG10(
SDNode *
N,
1476 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1477 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1478 RTLIB::LOG10_PPCF128),
Lo,
Hi);
1483 bool IsStrict =
N->isStrictFPOpcode();
1484 unsigned Offset = IsStrict ? 1 : 0;
1494 RTLIB::FMA_PPCF128),
1495 N->getValueType(0), Ops, CallOptions,
1498 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1499 GetPairElements(Tmp.first,
Lo,
Hi);
1509 RTLIB::MUL_PPCF128),
Lo,
Hi);
1512void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(
SDNode *
N,
1515 RTLIB::NEARBYINT_F32,
1516 RTLIB::NEARBYINT_F64,
1517 RTLIB::NEARBYINT_F80,
1518 RTLIB::NEARBYINT_F128,
1519 RTLIB::NEARBYINT_PPCF128),
Lo,
Hi);
1525 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
1534 bool IsStrict =
N->isStrictFPOpcode();
1539 if (NVT ==
N->getOperand(1).getValueType()) {
1540 Hi =
N->getOperand(1);
1541 Chain =
N->getOperand(0);
1545 {
N->getOperand(0),
N->getOperand(1) });
1546 Chain =
Hi.getValue(1);
1556 ReplaceValueWith(
SDValue(
N, 1), Chain);
1559void DAGTypeLegalizer::ExpandFloatRes_FPOW(
SDNode *
N,
1562 RTLIB::POW_F32, RTLIB::POW_F64,
1563 RTLIB::POW_F80, RTLIB::POW_F128,
1564 RTLIB::POW_PPCF128),
Lo,
Hi);
1567void DAGTypeLegalizer::ExpandFloatRes_FPOWI(
SDNode *
N,
1572void DAGTypeLegalizer::ExpandFloatRes_FREEZE(
SDNode *
N,
1574 assert(
N->getValueType(0) == MVT::ppcf128 &&
1575 "Logic only correct for ppcf128!");
1578 GetExpandedFloat(
N->getOperand(0),
Lo,
Hi);
1583void DAGTypeLegalizer::ExpandFloatRes_FREM(
SDNode *
N,
1586 RTLIB::REM_F32, RTLIB::REM_F64,
1587 RTLIB::REM_F80, RTLIB::REM_F128,
1588 RTLIB::REM_PPCF128),
Lo,
Hi);
1591void DAGTypeLegalizer::ExpandFloatRes_FRINT(
SDNode *
N,
1594 RTLIB::RINT_F32, RTLIB::RINT_F64,
1595 RTLIB::RINT_F80, RTLIB::RINT_F128,
1596 RTLIB::RINT_PPCF128),
Lo,
Hi);
1599void DAGTypeLegalizer::ExpandFloatRes_FROUND(
SDNode *
N,
1606 RTLIB::ROUND_PPCF128),
Lo,
Hi);
1609void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(
SDNode *
N,
1612 RTLIB::ROUNDEVEN_F32,
1613 RTLIB::ROUNDEVEN_F64,
1614 RTLIB::ROUNDEVEN_F80,
1615 RTLIB::ROUNDEVEN_F128,
1616 RTLIB::ROUNDEVEN_PPCF128),
Lo,
Hi);
1619void DAGTypeLegalizer::ExpandFloatRes_FSIN(
SDNode *
N,
1622 RTLIB::SIN_F32, RTLIB::SIN_F64,
1623 RTLIB::SIN_F80, RTLIB::SIN_F128,
1624 RTLIB::SIN_PPCF128),
Lo,
Hi);
1627void DAGTypeLegalizer::ExpandFloatRes_FSQRT(
SDNode *
N,
1630 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1631 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1632 RTLIB::SQRT_PPCF128),
Lo,
Hi);
1642 RTLIB::SUB_PPCF128),
Lo,
Hi);
1645void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(
SDNode *
N,
1648 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1649 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1650 RTLIB::TRUNC_PPCF128),
Lo,
Hi);
1656 ExpandRes_NormalLoad(
N,
Lo,
Hi);
1668 assert(
LD->getMemoryVT().bitsLE(NVT) &&
"Float type not round?");
1671 LD->getMemoryVT(),
LD->getMemOperand());
1674 Chain =
Hi.getValue(1);
1682 ReplaceValueWith(
SDValue(LD, 1), Chain);
1687 assert(
N->getValueType(0) == MVT::ppcf128 &&
"Unsupported XINT_TO_FP!");
1688 EVT VT =
N->getValueType(0);
1690 bool Strict =
N->isStrictFPOpcode();
1691 SDValue Src =
N->getOperand(Strict ? 1 : 0);
1692 EVT SrcVT = Src.getValueType();
1700 Flags.setNoFPExcept(
N->getFlags().hasNoFPExcept());
1705 if (SrcVT.
bitsLE(MVT::i32)) {
1711 {Chain, Src}, Flags);
1712 Chain =
Hi.getValue(1);
1714 Hi = DAG.
getNode(
N->getOpcode(), dl, NVT, Src);
1717 if (SrcVT.
bitsLE(MVT::i64)) {
1720 LC = RTLIB::SINTTOFP_I64_PPCF128;
1721 }
else if (SrcVT.
bitsLE(MVT::i128)) {
1723 LC = RTLIB::SINTTOFP_I128_PPCF128;
1725 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported XINT_TO_FP!");
1729 std::pair<SDValue, SDValue> Tmp =
1730 TLI.
makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1733 GetPairElements(Tmp.first,
Lo,
Hi);
1739 ReplaceValueWith(
SDValue(
N, 1), Chain);
1749 SrcVT = Src.getValueType();
1752 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1753 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1754 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1776 {Chain, Hi, NewLo}, Flags);
1777 Chain =
Lo.getValue(1);
1778 ReplaceValueWith(
SDValue(
N, 1), Chain);
1783 GetPairElements(
Lo,
Lo,
Hi);
1795bool DAGTypeLegalizer::ExpandFloatOperand(
SDNode *
N,
unsigned OpNo) {
1800 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
1803 switch (
N->getOpcode()) {
1806 dbgs() <<
"ExpandFloatOperand Op #" << OpNo <<
": ";
1807 N->dump(&DAG);
dbgs() <<
"\n";
1815 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(
N);
break;
1823 case ISD::LROUND: Res = ExpandFloatOp_LROUND(
N);
break;
1825 case ISD::LRINT: Res = ExpandFloatOp_LRINT(
N);
break;
1826 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(
N);
break;
1830 case ISD::SETCC: Res = ExpandFloatOp_SETCC(
N);
break;
1831 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(
N),
1836 if (!Res.getNode())
return false;
1840 if (Res.getNode() ==
N)
1843 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 1 &&
1844 "Invalid operand expansion");
1846 ReplaceValueWith(
SDValue(
N, 0), Res);
1852void DAGTypeLegalizer::FloatExpandSetCCOperands(
SDValue &NewLHS,
1857 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1858 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1859 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1868 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1873 RHSLo, CCCode, OutputChain, IsSignaling);
1881 RHSHi, CCCode, OutputChain, IsSignaling);
1886 Chain = OutputChain;
1890 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
1891 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
1893 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain);
1905 N->getOperand(4)), 0);
1909 assert(
N->getOperand(1).getValueType() == MVT::ppcf128 &&
1910 "Logic only correct for ppcf128!");
1912 GetExpandedFloat(
N->getOperand(1),
Lo,
Hi);
1916 N->getValueType(0),
N->getOperand(0),
Hi);
1920 bool IsStrict =
N->isStrictFPOpcode();
1921 assert(
N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1922 "Logic only correct for ppcf128!");
1924 GetExpandedFloat(
N->getOperand(IsStrict ? 1 : 0),
Lo,
Hi);
1929 N->getValueType(0),
Hi,
N->getOperand(1));
1933 if (
Hi.getValueType() ==
N->getValueType(0)) {
1935 ReplaceValueWith(
SDValue(
N, 1),
N->getOperand(0));
1941 {
N->getValueType(0), MVT::Other},
1942 {
N->getOperand(0),
Hi,
N->getOperand(2)});
1944 ReplaceValueWith(
SDValue(
N, 0), Expansion);
1949 EVT RVT =
N->getValueType(0);
1952 bool IsStrict =
N->isStrictFPOpcode();
1955 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
1961 "Unsupported FP_TO_XINT!");
1963 std::pair<SDValue, SDValue> Tmp =
1964 TLI.
makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
1968 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
1969 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
1974 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
1975 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
1977 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain);
1988 N->getOperand(2),
N->getOperand(3),
1993 bool IsStrict =
N->isStrictFPOpcode();
1994 SDValue NewLHS =
N->getOperand(IsStrict ? 1 : 0);
1995 SDValue NewRHS =
N->getOperand(IsStrict ? 2 : 1);
1998 cast<CondCodeSDNode>(
N->getOperand(IsStrict ? 3 : 2))->get();
1999 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N), Chain,
2005 "Unexpected setcc expansion!");
2007 ReplaceValueWith(
SDValue(
N, 0), NewLHS);
2008 ReplaceValueWith(
SDValue(
N, 1), Chain);
2014SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
2016 return ExpandOp_NormalStore(
N, OpNo);
2019 assert(OpNo == 1 &&
"Can only expand the stored value so far");
2026 ST->getValue().getValueType());
2028 assert(
ST->getMemoryVT().bitsLE(NVT) &&
"Float type not round?");
2032 GetExpandedOp(
ST->getValue(),
Lo,
Hi);
2035 ST->getMemoryVT(),
ST->getMemOperand());
2039 EVT RVT =
N->getValueType(0);
2040 EVT RetVT =
N->getOperand(0).getValueType();
2047 RTLIB::LROUND_PPCF128),
2048 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2052 EVT RVT =
N->getValueType(0);
2053 EVT RetVT =
N->getOperand(0).getValueType();
2059 RTLIB::LLROUND_F128,
2060 RTLIB::LLROUND_PPCF128),
2061 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2065 EVT RVT =
N->getValueType(0);
2066 EVT RetVT =
N->getOperand(0).getValueType();
2073 RTLIB::LRINT_PPCF128),
2074 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2078 EVT RVT =
N->getValueType(0);
2079 EVT RetVT =
N->getOperand(0).getValueType();
2086 RTLIB::LLRINT_PPCF128),
2087 RVT,
N->getOperand(0), CallOptions,
SDLoc(
N)).first;
2096 if (OpVT == MVT::f16) {
2098 }
else if (RetVT == MVT::f16) {
2100 }
else if (OpVT == MVT::bf16) {
2102 }
else if (RetVT == MVT::bf16) {
2109bool DAGTypeLegalizer::PromoteFloatOperand(
SDNode *
N,
unsigned OpNo) {
2110 LLVM_DEBUG(
dbgs() <<
"Promote float operand " << OpNo <<
": ";
N->dump(&DAG);
2114 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
2124 switch (
N->getOpcode()) {
2127 dbgs() <<
"PromoteFloatOperand Op #" << OpNo <<
": ";
2128 N->dump(&DAG);
dbgs() <<
"\n";
2138 R = PromoteFloatOp_FP_TO_XINT_SAT(
N, OpNo);
break;
2141 case ISD::SETCC:
R = PromoteFloatOp_SETCC(
N, OpNo);
break;
2142 case ISD::STORE:
R = PromoteFloatOp_STORE(
N, OpNo);
break;
2146 ReplaceValueWith(
SDValue(
N, 0), R);
2150SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(
SDNode *
N,
unsigned OpNo) {
2152 EVT OpVT =
Op->getValueType(0);
2154 SDValue Promoted = GetPromotedFloat(
N->getOperand(0));
2163 return DAG.
getBitcast(
N->getValueType(0), Convert);
2168SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(
SDNode *
N,
unsigned OpNo) {
2169 assert (OpNo == 1 &&
"Only Operand 1 must need promotion here");
2170 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2173 N->getOperand(0), Op1);
2177SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(
SDNode *
N,
unsigned OpNo) {
2178 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2182SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(
SDNode *
N,
2184 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2189SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(
SDNode *
N,
unsigned OpNo) {
2190 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2191 EVT VT =
N->getValueType(0);
2194 if (VT ==
Op->getValueType(0))
2204SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2209 LHS, RHS,
N->getOperand(2),
N->getOperand(3),
2215SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2216 EVT VT =
N->getValueType(0);
2217 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2218 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2219 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
2227SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(
SDNode *
N,
unsigned OpNo) {
2232 SDValue Promoted = GetPromotedFloat(Val);
2233 EVT VT =
ST->getOperand(1).getValueType();
2240 return DAG.
getStore(
ST->getChain(),
DL, NewVal,
ST->getBasePtr(),
2241 ST->getMemOperand());
2248void DAGTypeLegalizer::PromoteFloatResult(
SDNode *
N,
unsigned ResNo) {
2249 LLVM_DEBUG(
dbgs() <<
"Promote float result " << ResNo <<
": ";
N->dump(&DAG);
2254 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
2259 switch (
N->getOpcode()) {
2266 dbgs() <<
"PromoteFloatResult #" << ResNo <<
": ";
2267 N->dump(&DAG);
dbgs() <<
"\n";
2274 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(
N);
break;
2308 case ISD::FSUB:
R = PromoteFloatRes_BinOp(
N);
break;
2311 case ISD::FMAD:
R = PromoteFloatRes_FMAD(
N);
break;
2316 case ISD::LOAD:
R = PromoteFloatRes_LOAD(
N);
break;
2328 R = PromoteFloatRes_VECREDUCE(
N);
2332 R = PromoteFloatRes_VECREDUCE_SEQ(
N);
2337 SetPromotedFloat(
SDValue(
N, ResNo), R);
2346 EVT VT =
N->getValueType(0);
2351 N->getOperand(0).getValueType().getSizeInBits());
2358 EVT VT =
N->getValueType(0);
2377SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
2382 if (isa<ConstantSDNode>(
N->getOperand(1))) {
2388 uint64_t IdxVal = cast<ConstantSDNode>(
Idx)->getZExtValue();
2390 switch (getTypeAction(VecVT)) {
2393 SDValue Res = GetScalarizedVector(
N->getOperand(0));
2394 ReplaceValueWith(
SDValue(
N, 0), Res);
2398 Vec = GetWidenedVector(Vec);
2400 ReplaceValueWith(
SDValue(
N, 0), Res);
2405 GetSplitVector(Vec,
Lo,
Hi);
2407 uint64_t LoElts =
Lo.getValueType().getVectorNumElements();
2409 if (IdxVal < LoElts)
2414 Idx.getValueType()));
2415 ReplaceValueWith(
SDValue(
N, 0), Res);
2423 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
2428 NewOp,
N->getOperand(1));
2431 EVT VT =
N->getValueType(0);
2440 EVT VT =
N->getValueType(0);
2442 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2453 EVT VT =
N->getValueType(0);
2455 SDValue Op = GetPromotedFloat(
N->getOperand(0));
2464 EVT VT =
N->getValueType(0);
2466 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2467 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2468 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), NVT, Op0, Op1,
N->getFlags());
2472 EVT VT =
N->getValueType(0);
2474 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2475 SDValue Op1 = GetPromotedFloat(
N->getOperand(1));
2476 SDValue Op2 = GetPromotedFloat(
N->getOperand(2));
2478 return DAG.
getNode(
N->getOpcode(),
SDLoc(
N), NVT, Op0, Op1, Op2);
2483 EVT VT =
N->getValueType(0);
2485 SDValue Op0 = GetPromotedFloat(
N->getOperand(0));
2497 EVT VT =
N->getValueType(0);
2498 EVT OpVT =
Op->getValueType(0);
2510 EVT VT =
N->getValueType(0);
2515 L->getAddressingMode(),
L->getExtensionType(), IVT,
SDLoc(
N),
2516 L->getChain(),
L->getBasePtr(),
L->getOffset(),
L->getPointerInfo(), IVT,
2517 L->getOriginalAlign(),
L->getMemOperand()->getFlags(),
L->getAAInfo());
2533 N->getOperand(0), TrueVal, FalseVal);
2543 TrueVal.getNode()->getValueType(0),
N->getOperand(0),
2544 N->getOperand(1), TrueVal, FalseVal,
N->getOperand(4));
2551 EVT VT =
N->getValueType(0);
2563 N->getValueType(0)));
2575SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(
SDNode *
N) {
2581 EVT VT =
N->getValueType(0);
2592 { AM->getChain(), AM->getBasePtr(), CastVal },
2615void DAGTypeLegalizer::SoftPromoteHalfResult(
SDNode *
N,
unsigned ResNo) {
2616 LLVM_DEBUG(
dbgs() <<
"Soft promote half result " << ResNo <<
": ";
2617 N->dump(&DAG);
dbgs() <<
"\n");
2621 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
2626 switch (
N->getOpcode()) {
2629 dbgs() <<
"SoftPromoteHalfResult #" << ResNo <<
": ";
2630 N->dump(&DAG);
dbgs() <<
"\n";
2632 llvm_unreachable(
"Do not know how to soft promote this operator's result!");
2637 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(
N);
break;
2674 case ISD::FSUB:
R = SoftPromoteHalfRes_BinOp(
N);
break;
2677 case ISD::FMAD:
R = SoftPromoteHalfRes_FMAD(
N);
break;
2679 case ISD::FPOWI:
R = SoftPromoteHalfRes_FPOWI(
N);
break;
2681 case ISD::LOAD:
R = SoftPromoteHalfRes_LOAD(
N);
break;
2686 case ISD::UNDEF:
R = SoftPromoteHalfRes_UNDEF(
N);
break;
2692 R = SoftPromoteHalfRes_VECREDUCE(
N);
2696 R = SoftPromoteHalfRes_VECREDUCE_SEQ(
N);
2701 SetSoftPromotedHalf(
SDValue(
N, ResNo), R);
2705 return BitConvertToInteger(
N->getOperand(0));
2708SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(
SDNode *
N) {
2716SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
2717 SDValue NewOp = BitConvertVectorToIntegerVector(
N->getOperand(0));
2723SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(
SDNode *
N) {
2724 SDValue LHS = GetSoftPromotedHalf(
N->getOperand(0));
2725 SDValue RHS = BitConvertToInteger(
N->getOperand(1));
2728 EVT LVT =
LHS.getValueType();
2729 EVT RVT =
RHS.getValueType();
2750 }
else if (SizeDiff < 0) {
2772 EVT OVT =
N->getValueType(0);
2774 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
2775 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
2776 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
2781 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
2782 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
2783 Op2 = DAG.
getNode(PromotionOpcode, dl, NVT, Op2);
2792 EVT OVT =
N->getValueType(0);
2794 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
2807SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(
SDNode *
N) {
2808 EVT RVT =
N->getValueType(0);
2809 EVT SVT =
N->getOperand(0).getValueType();
2811 if (
N->isStrictFPOpcode()) {
2815 {
N->getOperand(0),
N->getOperand(1)});
2830 DAG.
getLoad(
L->getAddressingMode(),
L->getExtensionType(), MVT::i16,
2831 SDLoc(
N),
L->getChain(),
L->getBasePtr(),
L->getOffset(),
2832 L->getPointerInfo(), MVT::i16,
L->getOriginalAlign(),
2833 L->getMemOperand()->getFlags(),
L->getAAInfo());
2841 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
2842 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
2847SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(
SDNode *
N) {
2848 SDValue Op2 = GetSoftPromotedHalf(
N->getOperand(2));
2849 SDValue Op3 = GetSoftPromotedHalf(
N->getOperand(3));
2851 N->getOperand(0),
N->getOperand(1), Op2, Op3,
2855SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(
SDNode *
N) {
2856 EVT OVT =
N->getValueType(0);
2871 EVT OVT =
N->getValueType(0);
2873 SDValue Op = GetSoftPromotedHalf(
N->getOperand(0));
2886 EVT OVT =
N->getValueType(0);
2888 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
2889 SDValue Op1 = GetSoftPromotedHalf(
N->getOperand(1));
2894 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
2895 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
2903SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(
SDNode *
N) {
2909SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(
SDNode *
N) {
2919bool DAGTypeLegalizer::SoftPromoteHalfOperand(
SDNode *
N,
unsigned OpNo) {
2920 LLVM_DEBUG(
dbgs() <<
"Soft promote half operand " << OpNo <<
": ";
2921 N->dump(&DAG);
dbgs() <<
"\n");
2924 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
2934 switch (
N->getOpcode()) {
2937 dbgs() <<
"SoftPromoteHalfOperand Op #" << OpNo <<
": ";
2938 N->dump(&DAG);
dbgs() <<
"\n";
2940 llvm_unreachable(
"Do not know how to soft promote this operator's operand!");
2942 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(
N);
break;
2943 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(
N, OpNo);
break;
2948 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(
N);
break;
2951 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(
N, OpNo);
break;
2952 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(
N);
break;
2953 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(
N, OpNo);
break;
2955 Res = SoftPromoteHalfOp_STACKMAP(
N, OpNo);
2958 Res = SoftPromoteHalfOp_PATCHPOINT(
N, OpNo);
2965 assert(Res.getNode() !=
N &&
"Expected a new node!");
2967 assert(Res.getValueType() ==
N->getValueType(0) &&
N->getNumValues() == 1 &&
2968 "Invalid operand expansion");
2970 ReplaceValueWith(
SDValue(
N, 0), Res);
2975 SDValue Op0 = GetSoftPromotedHalf(
N->getOperand(0));
2982 assert(OpNo == 1 &&
"Only Operand 1 must need promotion here");
2989 Op1 = GetSoftPromotedHalf(Op1);
2992 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0),
N->getOperand(0),
2996SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(
SDNode *
N) {
2997 EVT RVT =
N->getValueType(0);
2998 bool IsStrict =
N->isStrictFPOpcode();
2999 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3000 EVT SVT =
Op.getValueType();
3001 Op = GetSoftPromotedHalf(
N->getOperand(IsStrict ? 1 : 0));
3007 {
N->getValueType(0), MVT::Other}, {
N->getOperand(0), Op});
3009 ReplaceValueWith(
SDValue(
N, 0), Res);
3016SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(
SDNode *
N) {
3017 EVT RVT =
N->getValueType(0);
3019 EVT SVT =
Op.getValueType();
3024 Op = GetSoftPromotedHalf(Op);
3028 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0), Res);
3031SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(
SDNode *
N) {
3032 EVT RVT =
N->getValueType(0);
3034 EVT SVT =
Op.getValueType();
3039 Op = GetSoftPromotedHalf(Op);
3043 return DAG.
getNode(
N->getOpcode(), dl,
N->getValueType(0), Res,
3049 assert(OpNo == 0 &&
"Can only soften the comparison values");
3057 Op0 = GetSoftPromotedHalf(Op0);
3058 Op1 = GetSoftPromotedHalf(Op1);
3062 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3063 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3066 N->getOperand(2),
N->getOperand(3),
N->getOperand(4));
3072 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
3078 Op0 = GetSoftPromotedHalf(Op0);
3079 Op1 = GetSoftPromotedHalf(Op1);
3083 Op0 = DAG.
getNode(PromotionOpcode, dl, NVT, Op0);
3084 Op1 = DAG.
getNode(PromotionOpcode, dl, NVT, Op1);
3089SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(
SDNode *
N,
unsigned OpNo) {
3090 assert(OpNo == 1 &&
"Can only soften the stored value!");
3095 assert(!
ST->isTruncatingStore() &&
"Unexpected truncating store.");
3096 SDValue Promoted = GetSoftPromotedHalf(Val);
3097 return DAG.
getStore(
ST->getChain(), dl, Promoted,
ST->getBasePtr(),
3098 ST->getMemOperand());
3101SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
3105 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3109 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
3120 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3124 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ SIGN_EXTEND
Conversion operators.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ SHL
Shift and rotation operations.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
This is an optimization pass for GlobalISel generic memory operations.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)