31#define DEBUG_TYPE "legalize-types"
41void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
51 switch (
N->getOpcode()) {
54 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
55 N->dump(&DAG);
dbgs() <<
"\n";
62 case ISD::VP_BITREVERSE:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
80 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
81 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
82 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
84 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
90 Res = PromoteIntRes_Select(
N);
95 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
97 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
99 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
102 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
104 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
106 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(
N);
break;
108 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(
N);
break;
109 case ISD::VP_TRUNCATE:
111 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
112 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
116 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
118 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
120 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
122 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
124 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
127 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
130 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
132 Res = PromoteIntRes_BUILD_VECTOR(
N);
136 Res = PromoteIntRes_ScalarOp(
N);
140 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
145 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
148 case ISD::VP_SIGN_EXTEND:
150 case ISD::VP_ZERO_EXTEND:
153 case ISD::VP_FP_TO_SINT:
154 case ISD::VP_FP_TO_UINT:
162 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
166 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
170 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
185 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
192 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
199 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
202 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
204 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
206 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
222 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(
N);
224 case ISD::VP_SADDSAT:
225 case ISD::VP_UADDSAT:
226 case ISD::VP_SSUBSAT:
227 case ISD::VP_USUBSAT:
228 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(
N);
241 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
244 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
258 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
262 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
274 Res = PromoteIntRes_VECREDUCE(
N);
277 case ISD::VP_REDUCE_ADD:
278 case ISD::VP_REDUCE_MUL:
279 case ISD::VP_REDUCE_AND:
280 case ISD::VP_REDUCE_OR:
281 case ISD::VP_REDUCE_XOR:
282 case ISD::VP_REDUCE_SMAX:
283 case ISD::VP_REDUCE_SMIN:
284 case ISD::VP_REDUCE_UMAX:
285 case ISD::VP_REDUCE_UMIN:
286 Res = PromoteIntRes_VP_REDUCE(
N);
290 Res = PromoteIntRes_FREEZE(
N);
295 Res = PromoteIntRes_Rotate(
N);
300 Res = PromoteIntRes_FunnelShift(
N);
305 Res = PromoteIntRes_VPFunnelShift(
N);
309 Res = PromoteIntRes_IS_FPCLASS(
N);
312 Res = PromoteIntRes_FFREXP(
N);
317 Res = PromoteIntRes_XRINT(
N);
323 SetPromotedInteger(
SDValue(
N, ResNo), Res);
328 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
329 return GetPromotedInteger(
Op);
334 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
336 Op.getValueType(),
Op,
N->getOperand(1));
341 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
343 Op.getValueType(),
Op,
N->getOperand(1));
349 N->getMemoryVT(), ResVT,
350 N->getChain(),
N->getBasePtr(),
369 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
379 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
382 N->getChain(),
N->getBasePtr(),
383 Op2,
N->getMemOperand());
394 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
405 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
415 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
418 Op2 = SExtPromotedInteger(Op2);
421 Op2 = ZExtPromotedInteger(Op2);
424 Op2 = GetPromotedInteger(Op2);
433 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
434 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
436 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
445 EVT OutVT =
N->getValueType(0);
449 switch (getTypeAction(InVT)) {
476 BitConvertToInteger(GetScalarizedVector(InOp)));
485 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
486 Lo = BitConvertToInteger(
Lo);
487 Hi = BitConvertToInteger(
Hi);
495 JoinIntegers(
Lo,
Hi));
529 if (isTypeLegal(WideOutVT)) {
530 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
540 CreateStackStoreLoad(InOp, OutVT));
544 SDValue V = GetPromotedInteger(
N->getOperand(0));
546 V.getValueType(), V);
550 SDValue Op = GetPromotedInteger(
N->getOperand(0));
551 EVT OVT =
N->getValueType(0);
552 EVT NVT =
Op.getValueType();
572 return DAG.
getNode(ISD::VP_LSHR, dl, NVT,
573 DAG.
getNode(ISD::VP_BSWAP, dl, NVT,
Op, Mask, EVL), ShAmt,
578 SDValue Op = GetPromotedInteger(
N->getOperand(0));
579 EVT OVT =
N->getValueType(0);
580 EVT NVT =
Op.getValueType();
600 return DAG.
getNode(ISD::VP_LSHR, dl, NVT,
601 DAG.
getNode(ISD::VP_BITREVERSE, dl, NVT,
Op, Mask, EVL),
610 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
615 EVT VT =
N->getValueType(0);
624 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
629 EVT OVT =
N->getValueType(0);
646 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
651 if (!
N->isVPOpcode())
657 return DAG.
getNode(ISD::VP_SUB, dl, NVT,
658 DAG.
getNode(
N->getOpcode(), dl, NVT,
Op, Mask, EVL),
659 ExtractLeadingBits, Mask, EVL);
663 EVT OVT =
N->getValueType(0);
680 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
681 if (!
N->isVPOpcode())
684 N->getOperand(1),
N->getOperand(2));
688 SDValue Op = GetPromotedInteger(
N->getOperand(0));
689 EVT OVT =
N->getValueType(0);
690 EVT NVT =
Op.getValueType();
708 if (
N->getOpcode() ==
ISD::CTTZ ||
N->getOpcode() == ISD::VP_CTTZ) {
719 N->getOperand(1),
N->getOperand(2));
721 if (!
N->isVPOpcode())
722 return DAG.
getNode(
N->getOpcode(), dl, NVT,
Op);
723 return DAG.
getNode(
N->getOpcode(), dl, NVT,
Op,
N->getOperand(1),
727SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
742 EVT SVT =
In.getValueType().getScalarType();
754 unsigned NewOpc =
N->getOpcode();
771 if (
N->getOpcode() == ISD::VP_FP_TO_UINT &&
774 NewOpc = ISD::VP_FP_TO_SINT;
777 if (
N->isStrictFPOpcode()) {
778 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
779 {
N->getOperand(0),
N->getOperand(1)});
783 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
784 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
787 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
799 N->getOpcode() == ISD::VP_FP_TO_UINT)
806SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
810 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
814SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
818 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
821SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
826 N->getOperand(0),
N->getOperand(1));
834 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
842 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
854 if (getTypeAction(
N->getOperand(0).getValueType())
856 SDValue Res = GetPromotedInteger(
N->getOperand(0));
874 if (
N->getNumOperands() != 1) {
875 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
876 assert(
N->isVPOpcode() &&
"Expected VP opcode");
877 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
878 N->getOperand(1),
N->getOperand(2));
880 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
890 N->getMemoryVT(),
N->getMemOperand());
900 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
908 N->getOffset(),
N->getMask(), ExtPassThru,
909 N->getMemoryVT(),
N->getMemOperand(),
910 N->getAddressingMode(), ExtType,
911 N->isExpandingLoad());
920 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
922 "Gather result type and the passThru argument type should be the same");
929 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
930 N->getIndex(),
N->getScale() };
932 N->getMemoryVT(), dl, Ops,
933 N->getMemOperand(),
N->getIndexType(),
946 EVT VT =
N->getValueType(0);
947 EVT SVT = getSetCCResultType(VT);
948 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
949 unsigned NumOps =
N->getNumOperands();
950 assert(NumOps <= 3 &&
"Too many operands");
952 Ops[2] = PromoteTargetBoolean(
N->getOperand(2), VT);
960 ReplaceValueWith(
SDValue(
N, 0), Res);
966template <
class MatchContextClass>
978 MatchContextClass matcher(DAG, TLI,
N);
981 unsigned Opcode = matcher.getRootBaseOpcode();
985 SDValue Op1Promoted, Op2Promoted;
987 Op1Promoted = GetPromotedInteger(Op1);
988 Op2Promoted = ZExtPromotedInteger(Op2);
990 Op1Promoted = ZExtPromotedInteger(Op1);
991 Op2Promoted = ZExtPromotedInteger(Op2);
993 Op1Promoted = SExtPromotedInteger(Op1);
994 Op2Promoted = SExtPromotedInteger(Op2);
1003 matcher.getNode(
ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1004 return matcher.getNode(
ISD::UMIN, dl, PromotedType,
Add, SatMax);
1009 return matcher.getNode(
ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1014 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1027 "addition, subtraction or left shift");
1030 unsigned SHLAmount = NewBits - OldBits;
1037 matcher.
getNode(
ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1040 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1041 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1050 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1051 Result = matcher.getNode(
ISD::SMIN, dl, PromotedType, Result, SatMax);
1052 Result = matcher.getNode(
ISD::SMAX, dl, PromotedType, Result, SatMin);
1059 SDValue Op1Promoted, Op2Promoted;
1065 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1066 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1068 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1069 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1071 EVT OldType =
N->getOperand(0).getValueType();
1085 Op2Promoted,
N->getOperand(2));
1087 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
1090 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1095 unsigned SatW,
bool Signed,
1098 EVT VT = V.getValueType();
1125 EVT VT =
LHS.getValueType();
1143 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1149 "Tried to saturate to more than the original type?");
1158 SDValue Op1Promoted, Op2Promoted;
1164 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1165 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1167 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1168 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1171 unsigned Scale =
N->getConstantOperandVal(2);
1180 N->getValueType(0).getScalarSizeInBits();
1185 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1186 Op2Promoted,
N->getOperand(2));
1196 Op2Promoted, Scale, DAG)) {
1199 N->getValueType(0).getScalarSizeInBits(),
1207 N->getValueType(0).getScalarSizeInBits());
1210SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1212 return PromoteIntRes_Overflow(
N);
1216 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1217 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1218 EVT OVT =
N->getOperand(0).getValueType();
1219 EVT NVT =
LHS.getValueType();
1234 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1242 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1243 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1245 unsigned Opcode =
N->getOpcode();
1246 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1253 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1254 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1256 LHS.getValueType(),
N->getOperand(0),
1257 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1261 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1262 EVT InVT =
N->getOperand(OpNo).getValueType();
1265 EVT SVT = getSetCCResultType(InVT);
1273 SVT = getSetCCResultType(InVT);
1281 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1282 "Vector compare must return a vector result!");
1286 if (
N->isStrictFPOpcode()) {
1288 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1289 N->getOperand(2),
N->getOperand(3)};
1290 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1295 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1296 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1312 EVT VT =
N->getValueType(0);
1318 ReplaceValueWith(
SDValue(
N, 0), Res);
1323 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1326 RHS = ZExtPromotedInteger(RHS);
1327 if (
N->getOpcode() != ISD::VP_SHL)
1330 N->getOperand(2),
N->getOperand(3));
1333SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1334 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1336 Op.getValueType(),
Op,
N->getOperand(1));
1339SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1343 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1344 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1345 if (
N->getNumOperands() == 2)
1347 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1348 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1350 N->getOperand(2),
N->getOperand(3));
1355 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1356 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1357 if (
N->getNumOperands() == 2)
1359 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1360 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1362 N->getOperand(2),
N->getOperand(3));
1367 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1368 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1369 if (
N->getNumOperands() == 2)
1371 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1372 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1374 N->getOperand(2),
N->getOperand(3));
1383 SExtOrZExtPromotedOperands(LHS, RHS);
1386 LHS.getValueType(), LHS, RHS);
1391 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1394 RHS = ZExtPromotedInteger(RHS);
1395 if (
N->getOpcode() != ISD::VP_ASHR)
1398 N->getOperand(2),
N->getOperand(3));
1403 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1406 RHS = ZExtPromotedInteger(RHS);
1407 if (
N->getOpcode() != ISD::VP_LSHR)
1410 N->getOperand(2),
N->getOperand(3));
1416 ReplaceValueWith(
SDValue(
N, 0), Res);
1421 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1422 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1425 Amt = ZExtPromotedInteger(Amt);
1429 EVT OldVT =
N->getOperand(0).getValueType();
1430 EVT VT =
Lo.getValueType();
1431 unsigned Opcode =
N->getOpcode();
1445 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1470SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(
SDNode *
N) {
1471 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1472 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1477 Amt = ZExtPromotedInteger(Amt);
1481 EVT OldVT =
N->getOperand(0).getValueType();
1482 EVT VT =
Lo.getValueType();
1483 unsigned Opcode =
N->getOpcode();
1484 bool IsFSHR = Opcode == ISD::VP_FSHR;
1489 Amt = DAG.
getNode(ISD::VP_UREM,
DL, AmtVT, Amt,
1497 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1500 Hi = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Hi, HiShift, Mask, EVL);
1504 Res = DAG.
getNode(IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL,
DL, VT, Res, Amt,
1507 Res = DAG.
getNode(ISD::VP_LSHR,
DL, VT, Res, HiShift, Mask, EVL);
1513 Lo = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Lo, ShiftOffset, Mask, EVL);
1518 Amt = DAG.
getNode(ISD::VP_ADD,
DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1536 Res = GetPromotedInteger(InOp);
1543 "Dst and Src must have the same number of elements");
1545 "Promoted vector type must be a power of two");
1548 GetSplitVector(InOp, EOp1, EOp2);
1556 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1557 "Expected VP_TRUNCATE opcode");
1558 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1559 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1560 std::tie(EVLLo, EVLHi) =
1561 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1562 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1563 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1570 SDValue WideInOp = GetWidenedVector(InOp);
1575 N->getValueType(0).getScalarType(), NumElem);
1590 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1591 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1596SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1598 return PromoteIntRes_Overflow(
N);
1602 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1603 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1604 EVT OVT =
N->getOperand(0).getValueType();
1605 EVT NVT =
LHS.getValueType();
1619 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1630 return PromoteIntRes_Overflow(
N);
1642 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1643 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1645 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1649 LHS, RHS,
N->getOperand(2));
1659 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1660 return PromoteIntRes_Overflow(
N);
1664 EVT OVT =
N->getValueType(0);
1677 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1681SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1684 return PromoteIntRes_Overflow(
N);
1688 EVT SmallVT =
LHS.getValueType();
1695 LHS = SExtPromotedInteger(LHS);
1696 RHS = SExtPromotedInteger(RHS);
1698 LHS = ZExtPromotedInteger(LHS);
1699 RHS = ZExtPromotedInteger(RHS);
1730 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1736 N->getValueType(0)));
1742 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1749 EVT VT =
N->getValueType(0);
1757 for (
unsigned i = 0; i < NumRegs; ++i) {
1758 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1759 N->getConstantOperandVal(3));
1765 std::reverse(Parts.begin(), Parts.end());
1770 for (
unsigned i = 1; i < NumRegs; ++i) {
1781 ReplaceValueWith(
SDValue(
N, 1), Chain);
1794bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1797 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1802 switch (
N->getOpcode()) {
1805 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1806 N->dump(&DAG);
dbgs() <<
"\n";
1812 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1815 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1816 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1822 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1826 Res = PromoteIntOp_ScalarOp(
N);
1829 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1832 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1834 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(
N);
break;
1835 case ISD::VP_SINT_TO_FP:
1838 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1840 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1842 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1844 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1846 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1848 case ISD::VP_TRUNCATE:
1852 case ISD::VP_UINT_TO_FP:
1857 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(
N);
break;
1865 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1868 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
1894 case ISD::VP_REDUCE_ADD:
1895 case ISD::VP_REDUCE_MUL:
1896 case ISD::VP_REDUCE_AND:
1897 case ISD::VP_REDUCE_OR:
1898 case ISD::VP_REDUCE_XOR:
1899 case ISD::VP_REDUCE_SMAX:
1900 case ISD::VP_REDUCE_SMIN:
1901 case ISD::VP_REDUCE_UMAX:
1902 case ISD::VP_REDUCE_UMIN:
1903 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
1908 Res = PromoteIntOp_STACKMAP(
N, OpNo);
1911 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
1913 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1914 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1915 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
1917 case ISD::EXPERIMENTAL_VP_SPLICE:
1918 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
1923 if (!Res.
getNode())
return false;
1930 const bool IsStrictFp =
N->isStrictFPOpcode();
1932 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1933 "Invalid operand expansion");
1937 ReplaceValueWith(
SDValue(
N, 0), Res);
1947void DAGTypeLegalizer::SExtOrZExtPromotedOperands(
SDValue &LHS,
SDValue &RHS) {
1948 SDValue OpL = GetPromotedInteger(LHS);
1949 SDValue OpR = GetPromotedInteger(RHS);
1955 unsigned OpLEffectiveBits =
1957 unsigned OpREffectiveBits =
1959 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1960 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1967 LHS = SExtPromotedInteger(LHS);
1968 RHS = SExtPromotedInteger(RHS);
1979 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1980 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1987 LHS = ZExtPromotedInteger(LHS);
1988 RHS = ZExtPromotedInteger(RHS);
1993void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
2002 LHS = SExtPromotedInteger(LHS);
2003 RHS = SExtPromotedInteger(RHS);
2008 "Unknown integer comparison!");
2010 SExtOrZExtPromotedOperands(LHS, RHS);
2014 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2019 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2021 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2027 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
2030SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2031 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2035 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
2040 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
2044SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2045 assert(OpNo == 1 &&
"only know how to promote condition");
2048 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2052 N->getOperand(2)), 0);
2057 EVT OVT =
N->getOperand(0).getValueType();
2058 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2059 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2060 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2073 EVT VecVT =
N->getValueType(0);
2076 "Legal vector of one illegal element?");
2081 assert(
N->getOperand(0).getValueSizeInBits() >=
2082 N->getValueType(0).getScalarSizeInBits() &&
2083 "Type of inserted value narrower than vector element type!");
2086 for (
unsigned i = 0; i < NumElts; ++i)
2087 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2092SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2099 assert(
N->getOperand(1).getValueSizeInBits() >=
2100 N->getValueType(0).getScalarSizeInBits() &&
2101 "Type of inserted value narrower than vector element type!");
2103 GetPromotedInteger(
N->getOperand(1)),
2108 assert(OpNo == 2 &&
"Different operand and result vector types?");
2114 N->getOperand(1),
Idx), 0);
2121 GetPromotedInteger(
N->getOperand(0))), 0);
2124SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2125 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2127 EVT OpTy =
N->getOperand(1).getValueType();
2130 if (
SDValue Res = WidenVSELECTMask(
N))
2132 Res,
N->getOperand(1),
N->getOperand(2));
2136 Cond = PromoteTargetBoolean(
Cond, OpVT);
2139 N->getOperand(2)), 0);
2142SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2143 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2147 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
2151 N->getOperand(3),
N->getOperand(4)), 0);
2154SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2155 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2159 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
2165 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2168 N->getOperand(3),
N->getOperand(4)),
2174 ZExtPromotedInteger(
N->getOperand(1))), 0);
2179 ZExtPromotedInteger(
N->getOperand(2))), 0);
2183 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2190SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(
SDNode *
N) {
2192 EVT VT =
N->getValueType(0);
2193 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2195 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2203 return DAG.
getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt,
N->getOperand(1),
2208 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2210 SExtPromotedInteger(
N->getOperand(0)),
2211 N->getOperand(1),
N->getOperand(2)),
2214 SExtPromotedInteger(
N->getOperand(0))), 0);
2217SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2219 SExtPromotedInteger(
N->getOperand(1))), 0);
2227 SDValue Val = GetPromotedInteger(
N->getValue());
2231 N->getMemoryVT(),
N->getMemOperand());
2242 Mask = PromoteTargetBoolean(Mask, DataVT);
2248 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2249 DataOp = GetPromotedInteger(DataOp);
2252 N->getOffset(), Mask,
N->getMemoryVT(),
2253 N->getMemOperand(),
N->getAddressingMode(),
2254 true,
N->isCompressingStore());
2259 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2260 EVT DataVT =
N->getValueType(0);
2261 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2263 NewOps[OpNo] =
Mask;
2280 EVT DataVT =
N->getValueType(0);
2281 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2282 }
else if (OpNo == 4) {
2284 if (
N->isIndexSigned())
2286 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2288 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2290 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2304 bool TruncateStore =
N->isTruncatingStore();
2309 EVT DataVT =
N->getValue().getValueType();
2310 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2311 }
else if (OpNo == 4) {
2313 if (
N->isIndexSigned())
2315 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2317 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2319 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2320 TruncateStore =
true;
2324 SDLoc(
N), NewOps,
N->getMemOperand(),
2325 N->getIndexType(), TruncateStore);
2329 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2330 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2332 N->getOperand(1),
N->getOperand(2));
2337 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2339 ZExtPromotedInteger(
N->getOperand(0)),
2340 N->getOperand(1),
N->getOperand(2)),
2343 ZExtPromotedInteger(
N->getOperand(0))), 0);
2346SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2348 ZExtPromotedInteger(
N->getOperand(1))), 0);
2353 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2358SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(
SDNode *
N) {
2360 EVT VT =
N->getValueType(0);
2361 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2363 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2366 N->getOperand(0).getScalarValueSizeInBits());
2368 N->getOperand(1),
N->getOperand(2));
2372 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2377SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2379 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2384 bool IsStrict =
N->isStrictFPOpcode();
2396 SDValue Op = SExtPromotedInteger(
N->getOperand(1));
2406 unsigned OpOffset = IsStrict ? 1 : 0;
2409 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2410 "POWI exponent should match with sizeof(int) when doing the libcall.");
2413 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2414 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2415 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2416 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2418 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2423 switch (
N->getOpcode()) {
2431 case ISD::VP_REDUCE_ADD:
2432 case ISD::VP_REDUCE_MUL:
2433 case ISD::VP_REDUCE_AND:
2434 case ISD::VP_REDUCE_OR:
2435 case ISD::VP_REDUCE_XOR:
2439 case ISD::VP_REDUCE_SMAX:
2440 case ISD::VP_REDUCE_SMIN:
2444 case ISD::VP_REDUCE_UMAX:
2445 case ISD::VP_REDUCE_UMIN:
2455 return GetPromotedInteger(V);
2457 return SExtPromotedInteger(V);
2459 return ZExtPromotedInteger(V);
2465 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2467 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2468 EVT InVT =
Op.getValueType();
2470 EVT ResVT =
N->getValueType(0);
2471 unsigned Opcode =
N->getOpcode();
2491 Op = ZExtPromotedInteger(
N->getOperand(0));
2494 Op = SExtPromotedInteger(
N->getOperand(0));
2510 Op = ZExtPromotedInteger(
N->getOperand(0));
2513 Op = SExtPromotedInteger(
N->getOperand(0));
2527SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2534 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2538 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2540 Op = PromoteIntOpVectorReduction(
N,
Op);
2544 EVT VT =
N->getValueType(0);
2545 EVT EltVT =
Op.getValueType().getScalarType();
2560 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2564SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2567 SDValue Operand =
N->getOperand(OpNo);
2573SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2576 SDValue Operand =
N->getOperand(OpNo);
2582SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2583 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2584 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2587 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2592SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
2596 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2600 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
2602 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2614void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2620 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2623 switch (
N->getOpcode()) {
2626 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2627 N->dump(&DAG);
dbgs() <<
"\n";
2673 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
2700 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2701 SplitInteger(Tmp.first,
Lo,
Hi);
2702 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2710 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2719 SplitInteger(Tmp,
Lo,
Hi);
2790 ExpandIntRes_Rotate(
N,
Lo,
Hi);
2795 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
2799 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
2809std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
2810 unsigned Opc =
Node->getOpcode();
2816 EVT RetVT =
Node->getValueType(0);
2824 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2825 "Unexpected atomic op or value type!");
2829 Node->getOperand(0));
2834void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2839 GetExpandedInteger(
N->getOperand(0), InL, InH);
2850 unsigned VTBits =
N->getValueType(0).getSizeInBits();
2854 if (Amt.
uge(VTBits)) {
2856 }
else if (Amt.
ugt(NVTBits)) {
2860 }
else if (Amt == NVTBits) {
2877 if (Amt.
uge(VTBits)) {
2879 }
else if (Amt.
ugt(NVTBits)) {
2883 }
else if (Amt == NVTBits) {
2900 if (Amt.
uge(VTBits)) {
2903 }
else if (Amt.
ugt(NVTBits)) {
2908 }
else if (Amt == NVTBits) {
2928bool DAGTypeLegalizer::
2930 unsigned Opc =
N->getOpcode();
2938 "Expanded integer type size not a power of two!");
2945 if (((Known.
Zero | Known.
One) & HighBitMask) == 0)
2950 GetExpandedInteger(In, InL, InH);
3004 Lo = DAG.
getNode(Opc, dl, NVT, InL, Amt);
3017bool DAGTypeLegalizer::
3024 "Expanded integer type size not a power of two!");
3029 GetExpandedInteger(
N->getOperand(0), InL, InH);
3041 switch (
N->getOpcode()) {
3056 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
3072 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3088 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3109void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3118 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3119 unsigned NumHalfBits = NumBits / 2;
3122 SDValue LHSL, LHSH, RHSL, RHSH;
3123 GetExpandedInteger(LHS, LHSL, LHSH);
3124 GetExpandedInteger(RHS, RHSL, RHSH);
3137 SDValue LHSL, LHSH, RHSL, RHSH;
3138 GetExpandedInteger(LHS, LHSL, LHSH);
3139 GetExpandedInteger(RHS, RHSL, RHSH);
3141 EVT CCT = getSetCCResultType(NVT);
3150 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3154 const APInt *RHSVal =
nullptr;
3155 if (
auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3156 RHSVal = &RHSConst->getAPIntValue();
3163 SDValue LHSL, LHSH, RHSL, RHSH;
3164 GetExpandedInteger(LHS, LHSL, LHSH);
3165 GetExpandedInteger(RHS, RHSL, RHSH);
3167 EVT CCT = getSetCCResultType(NVT);
3173 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3192 switch (
N->getOpcode()) {
3219 EVT VT =
N->getValueType(0);
3220 EVT CCT = getSetCCResultType(VT);
3223 SplitInteger(Result,
Lo,
Hi);
3226void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3230 SDValue LHSL, LHSH, RHSL, RHSH;
3231 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3232 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3235 SDValue LoOps[2] = { LHSL, RHSL };
3236 SDValue HiOps[3] = { LHSH, RHSH };
3245 HiOps[2] =
Lo.getValue(1);
3251 HiOps[2] =
Lo.getValue(1);
3273 HiOps[2] =
Lo.getValue(1);
3277 HiOps[2] =
Lo.getValue(1);
3290 EVT OvfVT = getSetCCResultType(NVT);
3330 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3333 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3336 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3354 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3368void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3371 SDValue LHSL, LHSH, RHSL, RHSH;
3373 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3374 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3376 SDValue LoOps[2] = { LHSL, RHSL };
3377 SDValue HiOps[3] = { LHSH, RHSH };
3381 HiOps[2] =
Lo.getValue(1);
3385 HiOps[2] =
Lo.getValue(1);
3391 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3394void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3397 SDValue LHSL, LHSH, RHSL, RHSH;
3399 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3400 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3402 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3403 SDValue HiOps[3] = { LHSH, RHSH };
3405 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3406 HiOps[2] =
Lo.getValue(1);
3407 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3411 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3414void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3422 unsigned CarryOp, NoCarryOp;
3424 switch(
N->getOpcode()) {
3444 SDValue LHSL, LHSH, RHSL, RHSH;
3445 GetExpandedInteger(LHS, LHSL, LHSH);
3446 GetExpandedInteger(RHS, RHSL, RHSH);
3448 SDValue LoOps[2] = { LHSL, RHSL };
3449 SDValue HiOps[3] = { LHSH, RHSH };
3451 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3452 HiOps[2] =
Lo.getValue(1);
3453 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3455 Ovf =
Hi.getValue(1);
3460 SplitInteger(Sum,
Lo,
Hi);
3471 DAG.
getSetCC(dl,
N->getValueType(1), LHS,
3482 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3488 SDValue LHSL, LHSH, RHSL, RHSH;
3490 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3491 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3493 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3496 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3497 HiOps[2] =
Lo.getValue(1);
3498 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3502 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3505void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3508 SDValue LHSL, LHSH, RHSL, RHSH;
3510 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3511 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3517 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3518 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3522 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3525void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3530 if (
Op.getValueType().bitsLE(NVT)) {
3537 assert(getTypeAction(
Op.getValueType()) ==
3539 "Only know how to promote this result!");
3542 "Operand over promoted?");
3544 SplitInteger(Res,
Lo,
Hi);
3548void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3551 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3552 EVT NVT =
Lo.getValueType();
3553 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3557 if (NVTBits < EVTBits) {
3560 EVTBits - NVTBits)));
3570void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3573 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3574 EVT NVT =
Lo.getValueType();
3575 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3579 if (NVTBits < EVTBits) {
3582 EVTBits - NVTBits)));
3590void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3593 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3598void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3601 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3610 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3611 EVT NVT =
Lo.getValueType();
3617void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3621 auto Constant = cast<ConstantSDNode>(
N);
3623 bool IsTarget =
Constant->isTargetOpcode();
3624 bool IsOpaque =
Constant->isOpaque();
3635 GetExpandedInteger(N0,
Lo,
Hi);
3636 EVT NVT =
Lo.getValueType();
3667 EVT VT =
N->getValueType(0);
3671 SplitInteger(Neg, NegLo, NegHi);
3679void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3683 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3684 EVT NVT =
Lo.getValueType();
3699void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3703 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3704 EVT NVT =
Lo.getValueType();
3710void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3714 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3715 EVT NVT =
Lo.getValueType();
3744 ReplaceValueWith(
SDValue(
N, 1), Chain);
3752 Chain =
Op.getValue(1);
3761 EVT VT =
N->getValueType(0);
3765 bool IsStrict =
N->isStrictFPOpcode();
3767 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3769 Op = GetPromotedFloat(
Op);
3772 EVT OFPVT =
Op.getValueType();
3774 Op = GetSoftPromotedHalf(
Op);
3778 SplitInteger(
Op,
Lo,
Hi);
3782 if (
Op.getValueType() == MVT::bf16) {
3789 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
3792 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3793 CallOptions, dl, Chain);
3794 SplitInteger(Tmp.first,
Lo,
Hi);
3797 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3803 SplitInteger(Res,
Lo,
Hi);
3809 bool IsStrict =
N->isStrictFPOpcode();
3810 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3814 "Input type needs to be promoted!");
3816 EVT VT =
Op.getValueType();
3818 if (VT == MVT::f16) {
3828 LC = RTLIB::LROUND_F32;
3829 else if (VT == MVT::f64)
3830 LC = RTLIB::LROUND_F64;
3831 else if (VT == MVT::f80)
3832 LC = RTLIB::LROUND_F80;
3833 else if (VT == MVT::f128)
3834 LC = RTLIB::LROUND_F128;
3835 else if (VT == MVT::ppcf128)
3836 LC = RTLIB::LROUND_PPCF128;
3837 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
3841 LC = RTLIB::LRINT_F32;
3842 else if (VT == MVT::f64)
3843 LC = RTLIB::LRINT_F64;
3844 else if (VT == MVT::f80)
3845 LC = RTLIB::LRINT_F80;
3846 else if (VT == MVT::f128)
3847 LC = RTLIB::LRINT_F128;
3848 else if (VT == MVT::ppcf128)
3849 LC = RTLIB::LRINT_PPCF128;
3850 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
3854 LC = RTLIB::LLROUND_F32;
3855 else if (VT == MVT::f64)
3856 LC = RTLIB::LLROUND_F64;
3857 else if (VT == MVT::f80)
3858 LC = RTLIB::LLROUND_F80;
3859 else if (VT == MVT::f128)
3860 LC = RTLIB::LLROUND_F128;
3861 else if (VT == MVT::ppcf128)
3862 LC = RTLIB::LLROUND_PPCF128;
3863 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
3867 LC = RTLIB::LLRINT_F32;
3868 else if (VT == MVT::f64)
3869 LC = RTLIB::LLRINT_F64;
3870 else if (VT == MVT::f80)
3871 LC = RTLIB::LLRINT_F80;
3872 else if (VT == MVT::f128)
3873 LC = RTLIB::LLRINT_F128;
3874 else if (VT == MVT::ppcf128)
3875 LC = RTLIB::LLRINT_PPCF128;
3876 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
3880 EVT RetVT =
N->getValueType(0);
3884 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
3885 Op, CallOptions, dl,
3887 SplitInteger(Tmp.first,
Lo,
Hi);
3889 if (
N->isStrictFPOpcode())
3890 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3893void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
3895 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
3898 ExpandRes_NormalLoad(
N,
Lo,
Hi);
3904 EVT VT =
N->getValueType(0);
3915 if (
N->getMemoryVT().bitsLE(NVT)) {
3916 EVT MemVT =
N->getMemoryVT();
3919 N->getOriginalAlign(), MMOFlags, AAInfo);
3922 Ch =
Lo.getValue(1);
3927 unsigned LoSize =
Lo.getValueSizeInBits();
3942 N->getOriginalAlign(), MMOFlags, AAInfo);
3944 unsigned ExcessBits =
3952 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3953 N->getOriginalAlign(), MMOFlags, AAInfo);
3962 EVT MemVT =
N->getMemoryVT();
3965 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3971 N->getOriginalAlign(), MMOFlags, AAInfo);
3977 N->getPointerInfo().getWithOffset(IncrementSize),
3979 N->getOriginalAlign(), MMOFlags, AAInfo);
4003 ReplaceValueWith(
SDValue(
N, 1), Ch);
4006void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4010 GetExpandedInteger(
N->getOperand(0), LL, LH);
4011 GetExpandedInteger(
N->getOperand(1), RL, RH);
4016void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4018 EVT VT =
N->getValueType(0);
4023 GetExpandedInteger(
N->getOperand(0), LL, LH);
4024 GetExpandedInteger(
N->getOperand(1), RL, RH);
4034 LC = RTLIB::MUL_I16;
4035 else if (VT == MVT::i32)
4036 LC = RTLIB::MUL_I32;
4037 else if (VT == MVT::i64)
4038 LC = RTLIB::MUL_I64;
4039 else if (VT == MVT::i128)
4040 LC = RTLIB::MUL_I128;
4052 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4055 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4067 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4073 SplitInteger(Result,
Lo,
Hi);
4079 SplitInteger(Result,
Lo,
Hi);
4090 EVT VT =
N->getValueType(0);
4094 uint64_t Scale =
N->getConstantOperandVal(2);
4106 EVT BoolVT = getSetCCResultType(VT);
4131 SplitInteger(Result,
Lo,
Hi);
4137 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4141 GetExpandedInteger(LHS, LL, LH);
4142 GetExpandedInteger(RHS, RL, RH);
4146 if (!TLI.
expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4154 SplitInteger(LoTmp, Result[0], Result[1]);
4155 SplitInteger(HiTmp, Result[2], Result[3]);
4157 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4160 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4161 "the size of the current value type");
4183 if (Scale % NVTSize) {
4199 if (Scale == VTSize)
4220 EVT BoolNVT = getSetCCResultType(NVT);
4223 if (Scale < NVTSize) {
4230 }
else if (Scale == NVTSize) {
4233 }
else if (Scale < VTSize) {
4241 "(and saturation can't happen with Scale==VTSize).");
4248 if (Scale < NVTSize) {
4253 unsigned OverflowBits = VTSize - Scale + 1;
4254 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4255 "Extent of overflow bits must start within HL");
4272 }
else if (Scale == NVTSize) {
4285 }
else if (Scale < VTSize) {
4288 unsigned OverflowBits = VTSize - Scale + 1;
4315 N->getConstantOperandVal(2), DAG);
4319 N->getConstantOperandVal(2), TLI, DAG);
4320 SplitInteger(Res,
Lo,
Hi);
4323void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *
Node,
4326 "Node has unexpected Opcode");
4341 SDValue LHSL, LHSH, RHSL, RHSH;
4342 GetExpandedInteger(LHS, LHSL, LHSH);
4343 GetExpandedInteger(RHS, RHSL, RHSH);
4347 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
4349 Ovf =
Hi.getValue(1);
4356 SplitInteger(Sum,
Lo,
Hi);
4381 EVT VT =
LHS.getValueType();
4384 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
4388 EVT OType =
Node->getValueType(1);
4396void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4398 EVT VT =
N->getValueType(0);
4400 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4410 LC = RTLIB::SDIV_I16;
4411 else if (VT == MVT::i32)
4412 LC = RTLIB::SDIV_I32;
4413 else if (VT == MVT::i64)
4414 LC = RTLIB::SDIV_I64;
4415 else if (VT == MVT::i128)
4416 LC = RTLIB::SDIV_I128;
4417 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4421 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4424void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
4427 SDValue Shiftee =
N->getOperand(0);
4434 bool ShiftByByteMultiple =
4439 if (!ShiftByByteMultiple)
4443 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
4444 unsigned VTByteWidth = VTBitWidth / 8;
4446 "Shiftee type size is not a power of two!");
4447 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4448 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4454 Align StackSlotAlignment(1);
4462 cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex());
4467 unsigned WideningOpc =
4469 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4476 Ch = DAG.
getStore(Ch, dl,
Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4482 if (ShiftByByteMultiple)
4483 Flags.setExact(
true);
4495 bool WillIndexUpwards =
N->getOpcode() !=
ISD::SHL;
4497 WillIndexUpwards = !WillIndexUpwards;
4500 if (WillIndexUpwards) {
4504 StackPtr, DAG.
getConstant(VTByteWidth, dl, PtrTy), dl);
4505 ByteOffset = DAG.
getNegative(ByteOffset, dl, ShAmtVT);
4514 VT, dl, Ch, AdjStackPtr,
4519 if (!ShiftByByteMultiple) {
4522 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
4526 SplitInteger(Res,
Lo,
Hi);
4529void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4531 EVT VT =
N->getValueType(0);
4532 unsigned Opc =
N->getOpcode();
4538 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
4542 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
4561 const bool LegalOrCustom =
4565 unsigned ExpansionFactor = 1;
4567 for (
EVT TmpVT = NVT;;) {
4569 if (NewTMPVT == TmpVT)
4579 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
4581 if (LegalOrCustom &&
4585 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4591 SDValue ShiftOp =
N->getOperand(1);
4596 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4598 Hi =
Lo.getValue(1);
4608 LC = RTLIB::SHL_I16;
4609 else if (VT == MVT::i32)
4610 LC = RTLIB::SHL_I32;
4611 else if (VT == MVT::i64)
4612 LC = RTLIB::SHL_I64;
4613 else if (VT == MVT::i128)
4614 LC = RTLIB::SHL_I128;
4618 LC = RTLIB::SRL_I16;
4619 else if (VT == MVT::i32)
4620 LC = RTLIB::SRL_I32;
4621 else if (VT == MVT::i64)
4622 LC = RTLIB::SRL_I64;
4623 else if (VT == MVT::i128)
4624 LC = RTLIB::SRL_I128;
4629 LC = RTLIB::SRA_I16;
4630 else if (VT == MVT::i32)
4631 LC = RTLIB::SRA_I32;
4632 else if (VT == MVT::i64)
4633 LC = RTLIB::SRA_I64;
4634 else if (VT == MVT::i128)
4635 LC = RTLIB::SRA_I128;
4642 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
4645 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4649 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
4653void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
4658 if (
Op.getValueType().bitsLE(NVT)) {
4669 assert(getTypeAction(
Op.getValueType()) ==
4671 "Only know how to promote this result!");
4674 "Operand over promoted?");
4676 SplitInteger(Res,
Lo,
Hi);
4684void DAGTypeLegalizer::
4687 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4688 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
4710void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
4712 EVT VT =
N->getValueType(0);
4714 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4724 LC = RTLIB::SREM_I16;
4725 else if (VT == MVT::i32)
4726 LC = RTLIB::SREM_I32;
4727 else if (VT == MVT::i64)
4728 LC = RTLIB::SREM_I64;
4729 else if (VT == MVT::i128)
4730 LC = RTLIB::SREM_I128;
4731 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
4735 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4738void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
4750void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
4752 EVT VT =
N->getValueType(0);
4771 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4772 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4773 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4775 EVT BitVT =
N->getValueType(1);
4800 SplitInteger(Three,
Lo,
Hi);
4804 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4815 LC = RTLIB::MULO_I32;
4816 else if (VT == MVT::i64)
4817 LC = RTLIB::MULO_I64;
4818 else if (VT == MVT::i128)
4819 LC = RTLIB::MULO_I128;
4832 SplitInteger(
Mul, MulLo, MulHi);
4838 SplitInteger(MulLo,
Lo,
Hi);
4839 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4852 EVT ArgVT =
Op.getValueType();
4856 Entry.IsSExt =
true;
4857 Entry.IsZExt =
false;
4858 Args.push_back(Entry);
4864 Entry.IsSExt =
true;
4865 Entry.IsZExt =
false;
4866 Args.push_back(Entry);
4885 ReplaceValueWith(
SDValue(
N, 1), Ofl);
4888void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
4890 EVT VT =
N->getValueType(0);
4892 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4901 if (isa<ConstantSDNode>(
N->getOperand(1))) {
4904 if (isTypeLegal(NVT)) {
4906 GetExpandedInteger(
N->getOperand(0), InL, InH);
4918 LC = RTLIB::UDIV_I16;
4919 else if (VT == MVT::i32)
4920 LC = RTLIB::UDIV_I32;
4921 else if (VT == MVT::i64)
4922 LC = RTLIB::UDIV_I64;
4923 else if (VT == MVT::i128)
4924 LC = RTLIB::UDIV_I128;
4925 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
4928 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4931void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
4933 EVT VT =
N->getValueType(0);
4935 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4944 if (isa<ConstantSDNode>(
N->getOperand(1))) {
4947 if (isTypeLegal(NVT)) {
4949 GetExpandedInteger(
N->getOperand(0), InL, InH);
4961 LC = RTLIB::UREM_I16;
4962 else if (VT == MVT::i32)
4963 LC = RTLIB::UREM_I32;
4964 else if (VT == MVT::i64)
4965 LC = RTLIB::UREM_I64;
4966 else if (VT == MVT::i128)
4967 LC = RTLIB::UREM_I128;
4968 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
4971 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4974void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
4979 if (
Op.getValueType().bitsLE(NVT)) {
4986 assert(getTypeAction(
Op.getValueType()) ==
4988 "Only know how to promote this result!");
4991 "Operand over promoted?");
4993 SplitInteger(Res,
Lo,
Hi);
5001void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5004 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
5009 cast<AtomicSDNode>(
N)->getMemoryVT(), VTs,
N->getOperand(0),
5010 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(
N)->getMemOperand());
5016void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5021 SplitInteger(Res,
Lo,
Hi);
5024void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5030 N->getOperand(0),
N->getOperand(1));
5031 SplitInteger(Res,
Lo,
Hi);
5038 GetExpandedInteger(
N->getOperand(0), In3, In4);
5039 GetExpandedInteger(
N->getOperand(1), In1, In2);
5043 unsigned Opc =
N->getOpcode();
5046 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5063 Lo = DAG.
getNode(Opc,
DL, HalfVT, Select2, Select1, NewShAmt);
5064 Hi = DAG.
getNode(Opc,
DL, HalfVT, Select3, Select2, NewShAmt);
5069 EVT VT =
N->getValueType(0);
5079 SplitInteger(Res,
Lo,
Hi);
5090bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5094 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5097 switch (
N->getOpcode()) {
5100 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5101 N->dump(&DAG);
dbgs() <<
"\n";
5106 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5113 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5119 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(
N), OpNo);
break;
5126 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5132 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5135 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5137 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5138 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5139 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5144 if (!Res.
getNode())
return false;
5152 "Invalid operand expansion");
5154 ReplaceValueWith(
SDValue(
N, 0), Res);
5160void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5164 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5165 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5166 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5186 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5187 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5219 RHSLo, LowCC,
false, DagCombineInfo, dl);
5226 RHSHi, CCCode,
false, DagCombineInfo, dl);
5239 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5241 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5250 if (LHSHi == RHSHi) {
5263 if (HasSETCCCARRY) {
5266 bool FlipOperands =
false;
5294 false, DagCombineInfo, dl);
5303 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
5304 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
5305 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5317 N->getOperand(4)), 0);
5321 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5322 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
5323 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5334 N->getOperand(2),
N->getOperand(3),
5339 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5340 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
5341 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5346 "Unexpected setcc expansion!");
5362 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5363 GetExpandedInteger(LHS, LHSLo, LHSHi);
5364 GetExpandedInteger(RHS, RHSLo, RHSHi);
5377 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5387 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
5396 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5401 bool IsStrict =
N->isStrictFPOpcode();
5405 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
5406 EVT DstVT =
N->getValueType(0);
5409 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5410 "Don't know how to expand this XINT_TO_FP!");
5413 std::pair<SDValue, SDValue> Tmp =
5419 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
5420 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
5425 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
5428 return ExpandOp_NormalStore(
N, OpNo);
5431 assert(OpNo == 1 &&
"Can only expand the stored value so far");
5433 EVT VT =
N->getOperand(1).getValueType();
5444 if (
N->getMemoryVT().bitsLE(NVT)) {
5445 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5447 N->getMemoryVT(),
N->getOriginalAlign(), MMOFlags,
5453 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5456 N->getOriginalAlign(), MMOFlags, AAInfo);
5458 unsigned ExcessBits =
5466 N->getPointerInfo().getWithOffset(IncrementSize),
5467 NEVT,
N->getOriginalAlign(), MMOFlags, AAInfo);
5473 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5475 EVT ExtVT =
N->getMemoryVT();
5478 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5496 N->getOriginalAlign(), MMOFlags, AAInfo);
5502 N->getPointerInfo().getWithOffset(IncrementSize),
5504 N->getOriginalAlign(), MMOFlags, AAInfo);
5510 GetExpandedInteger(
N->getOperand(0), InL, InH);
5519 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
5520 cast<AtomicSDNode>(
N)->getMemOperand());
5524SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
5525 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5526 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5530 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
5535SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
5538 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5539 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5545SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
5548 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5549 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5558SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
5560 EVT OutVT =
N->getValueType(0);
5562 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5566 SDValue BaseIdx =
N->getOperand(1);
5593 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5602 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5606 "Promoted operand has an element type greater than result");
5619 InOp0 = GetPromotedInteger(
N->getOperand(0));
5626 for (
unsigned i = 0; i != OutNumElems; ++i) {
5642SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
5643 EVT OutVT =
N->getValueType(0);
5645 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5657 Vec = GetPromotedInteger(Vec);
5663SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
5666 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5672SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
5674 EVT VT =
N->getValueType(0);
5679 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5680 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5687 EVT OutVT =
N->getValueType(0);
5689 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5690 unsigned NumElems =
N->getNumOperands();
5698 for (
unsigned i = 0; i != NumElems; ++i) {
5700 EVT OpVT =
Op.getValueType();
5705 if (OpVT.
bitsLT(NOutVTElem)) {
5711 ExtOpc = NOutExtOpc;
5724 assert(!
N->getOperand(0).getValueType().isVector() &&
5725 "Input must be a scalar");
5727 EVT OutVT =
N->getValueType(0);
5729 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5734 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op);
5739 EVT OutVT =
N->getValueType(0);
5742 "Type must be promoted to a scalable vector type");
5743 const APInt &StepVal =
N->getConstantOperandAPInt(0);
5748SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
5751 EVT OutVT =
N->getValueType(0);
5753 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5755 unsigned NumOperands =
N->getNumOperands();
5760 SDUse *MaxSizedValue = std::max_element(
5762 EVT AVT = A.getValueType().getVectorElementType();
5763 EVT BVT = B.getValueType().getVectorElementType();
5764 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5770 for (
unsigned I = 0;
I < NumOperands; ++
I) {
5772 EVT OpVT =
Op.getValueType();
5774 Op = GetPromotedInteger(
Op);
5777 "Unhandled legalization type");
5794 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
5795 assert(NumElem * NumOperands == NumOutElem &&
5796 "Unexpected number of elements");
5800 for (
unsigned i = 0; i < NumOperands; ++i) {
5803 Op = GetPromotedInteger(
Op);
5804 EVT SclrTy =
Op.getValueType().getVectorElementType();
5805 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
5806 "Unexpected number of elements");
5808 for (
unsigned j = 0;
j < NumElem; ++
j) {
5818SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
5819 EVT VT =
N->getValueType(0);
5821 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
5829 if (getTypeAction(
N->getOperand(0).getValueType())
5833 switch(
N->getOpcode()) {
5835 Promoted = SExtPromotedInteger(
N->getOperand(0));
5838 Promoted = ZExtPromotedInteger(
N->getOperand(0));
5841 Promoted = GetPromotedInteger(
N->getOperand(0));
5846 return DAG.
getNode(
N->getOpcode(), dl, NVT, Promoted);
5850 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
5853SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
5854 EVT OutVT =
N->getValueType(0);
5856 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5861 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5864 NOutVTElem,
N->getOperand(1));
5866 V0, ConvElem,
N->getOperand(2));
5874 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->ops());
5882 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
5883 return DAG.
getNode(
N->getOpcode(),
DL, Start.getValueType(), Start,
5884 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
5887SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
5889 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5901SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
5906 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5916SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
5918 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5921 N->getValueType(0).getVectorNumElements());
5926SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
5929 EVT ResVT =
N->getValueType(0);
5930 unsigned NumElems =
N->getNumOperands();
5935 for (
unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
5937 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
5945 EVT RetSclrTy =
N->getValueType(0).getVectorElementType();
5951 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
5953 EVT SclrTy =
Incoming->getValueType(0).getVectorElementType();
5954 unsigned NumElem =
Incoming->getValueType(0).getVectorNumElements();
5956 for (
unsigned i=0; i<NumElem; ++i) {
5968SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
5981 for (
unsigned I = 0;
I < OpNo;
I++)
5984 EVT Ty =
Op.getValueType();
5996 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6001 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6007SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6020 for (
unsigned I = 0;
I < OpNo;
I++)
6023 EVT Ty =
Op.getValueType();
6035 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6040 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
This class is used to represent ISD::LOAD nodes.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
@ TypeScalarizeScalableVector
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)