31#define DEBUG_TYPE "legalize-types"
41void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
51 switch (
N->getOpcode()) {
54 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
55 N->dump(&DAG);
dbgs() <<
"\n";
62 case ISD::VP_BITREVERSE:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
79 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
80 case ISD::VP_CTTZ_ELTS:
81 Res = PromoteIntRes_VP_CttzElements(
N);
84 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
85 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
86 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
88 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
94 Res = PromoteIntRes_Select(
N);
99 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
101 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
103 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
106 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
108 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
110 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(
N);
break;
112 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(
N);
break;
113 case ISD::VP_TRUNCATE:
115 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
116 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
120 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
122 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
124 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
126 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
128 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
131 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
134 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
136 Res = PromoteIntRes_BUILD_VECTOR(
N);
140 Res = PromoteIntRes_ScalarOp(
N);
144 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
149 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
152 case ISD::VP_SIGN_EXTEND:
154 case ISD::VP_ZERO_EXTEND:
157 case ISD::VP_FP_TO_SINT:
158 case ISD::VP_FP_TO_UINT:
166 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
170 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
174 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
189 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
196 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
203 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
206 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
208 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
210 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
226 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(
N);
228 case ISD::VP_SADDSAT:
229 case ISD::VP_UADDSAT:
230 case ISD::VP_SSUBSAT:
231 case ISD::VP_USUBSAT:
232 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(
N);
245 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
248 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
262 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
266 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
278 Res = PromoteIntRes_VECREDUCE(
N);
281 case ISD::VP_REDUCE_ADD:
282 case ISD::VP_REDUCE_MUL:
283 case ISD::VP_REDUCE_AND:
284 case ISD::VP_REDUCE_OR:
285 case ISD::VP_REDUCE_XOR:
286 case ISD::VP_REDUCE_SMAX:
287 case ISD::VP_REDUCE_SMIN:
288 case ISD::VP_REDUCE_UMAX:
289 case ISD::VP_REDUCE_UMIN:
290 Res = PromoteIntRes_VP_REDUCE(
N);
294 Res = PromoteIntRes_FREEZE(
N);
299 Res = PromoteIntRes_Rotate(
N);
304 Res = PromoteIntRes_FunnelShift(
N);
309 Res = PromoteIntRes_VPFunnelShift(
N);
313 Res = PromoteIntRes_IS_FPCLASS(
N);
316 Res = PromoteIntRes_FFREXP(
N);
321 Res = PromoteIntRes_XRINT(
N);
327 SetPromotedInteger(
SDValue(
N, ResNo), Res);
332 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
333 return GetPromotedInteger(
Op);
338 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
340 Op.getValueType(),
Op,
N->getOperand(1));
345 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
347 Op.getValueType(),
Op,
N->getOperand(1));
353 N->getMemoryVT(), ResVT,
354 N->getChain(),
N->getBasePtr(),
373 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
383 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
386 N->getChain(),
N->getBasePtr(),
387 Op2,
N->getMemOperand());
398 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
409 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
419 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
422 Op2 = SExtPromotedInteger(Op2);
425 Op2 = ZExtPromotedInteger(Op2);
428 Op2 = GetPromotedInteger(Op2);
437 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
438 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
440 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
449 EVT OutVT =
N->getValueType(0);
453 switch (getTypeAction(InVT)) {
480 BitConvertToInteger(GetScalarizedVector(InOp)));
489 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
490 Lo = BitConvertToInteger(
Lo);
491 Hi = BitConvertToInteger(
Hi);
499 JoinIntegers(
Lo,
Hi));
533 if (isTypeLegal(WideOutVT)) {
534 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
544 CreateStackStoreLoad(InOp, OutVT));
548 SDValue V = GetPromotedInteger(
N->getOperand(0));
550 V.getValueType(), V);
554 SDValue Op = GetPromotedInteger(
N->getOperand(0));
555 EVT OVT =
N->getValueType(0);
556 EVT NVT =
Op.getValueType();
576 return DAG.
getNode(ISD::VP_LSHR, dl, NVT,
577 DAG.
getNode(ISD::VP_BSWAP, dl, NVT,
Op, Mask, EVL), ShAmt,
582 SDValue Op = GetPromotedInteger(
N->getOperand(0));
583 EVT OVT =
N->getValueType(0);
584 EVT NVT =
Op.getValueType();
604 return DAG.
getNode(ISD::VP_LSHR, dl, NVT,
605 DAG.
getNode(ISD::VP_BITREVERSE, dl, NVT,
Op, Mask, EVL),
614 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
619 EVT VT =
N->getValueType(0);
628 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
633 EVT OVT =
N->getValueType(0);
650 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
655 if (!
N->isVPOpcode())
661 return DAG.
getNode(ISD::VP_SUB, dl, NVT,
662 DAG.
getNode(
N->getOpcode(), dl, NVT,
Op, Mask, EVL),
663 ExtractLeadingBits, Mask, EVL);
667 EVT OVT =
N->getValueType(0);
684 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
685 if (!
N->isVPOpcode())
688 N->getOperand(1),
N->getOperand(2));
692 SDValue Op = GetPromotedInteger(
N->getOperand(0));
693 EVT OVT =
N->getValueType(0);
694 EVT NVT =
Op.getValueType();
712 unsigned NewOpc =
N->getOpcode();
713 if (NewOpc ==
ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
725 N->getOperand(1),
N->getOperand(2));
726 NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
729 if (!
N->isVPOpcode())
731 return DAG.
getNode(NewOpc, dl, NVT,
Op,
N->getOperand(1),
N->getOperand(2));
734SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(
SDNode *
N) {
737 return DAG.
getNode(
N->getOpcode(),
DL, NewVT,
N->ops());
740SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
755 EVT SVT =
In.getValueType().getScalarType();
767 unsigned NewOpc =
N->getOpcode();
784 if (
N->getOpcode() == ISD::VP_FP_TO_UINT &&
787 NewOpc = ISD::VP_FP_TO_SINT;
790 if (
N->isStrictFPOpcode()) {
791 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
792 {
N->getOperand(0),
N->getOperand(1)});
796 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
797 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
800 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
812 N->getOpcode() == ISD::VP_FP_TO_UINT)
819SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
823 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
827SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
831 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
834SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
839 N->getOperand(0),
N->getOperand(1));
847 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
855 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
867 if (getTypeAction(
N->getOperand(0).getValueType())
869 SDValue Res = GetPromotedInteger(
N->getOperand(0));
887 if (
N->getNumOperands() != 1) {
888 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
889 assert(
N->isVPOpcode() &&
"Expected VP opcode");
890 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
891 N->getOperand(1),
N->getOperand(2));
893 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
903 N->getMemoryVT(),
N->getMemOperand());
913 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
921 N->getOffset(),
N->getMask(), ExtPassThru,
922 N->getMemoryVT(),
N->getMemOperand(),
923 N->getAddressingMode(), ExtType,
924 N->isExpandingLoad());
933 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
935 "Gather result type and the passThru argument type should be the same");
942 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
943 N->getIndex(),
N->getScale() };
945 N->getMemoryVT(), dl, Ops,
946 N->getMemOperand(),
N->getIndexType(),
959 EVT VT =
N->getValueType(0);
960 EVT SVT = getSetCCResultType(VT);
961 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
962 unsigned NumOps =
N->getNumOperands();
963 assert(NumOps <= 3 &&
"Too many operands");
965 Ops[2] = PromoteTargetBoolean(
N->getOperand(2), VT);
973 ReplaceValueWith(
SDValue(
N, 0), Res);
979template <
class MatchContextClass>
991 MatchContextClass matcher(DAG, TLI,
N);
994 unsigned Opcode = matcher.getRootBaseOpcode();
998 SDValue Op1Promoted, Op2Promoted;
1000 Op1Promoted = GetPromotedInteger(Op1);
1001 Op2Promoted = ZExtPromotedInteger(Op2);
1003 Op1Promoted = ZExtPromotedInteger(Op1);
1004 Op2Promoted = ZExtPromotedInteger(Op2);
1006 Op1Promoted = SExtPromotedInteger(Op1);
1007 Op2Promoted = SExtPromotedInteger(Op2);
1016 matcher.getNode(
ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1017 return matcher.getNode(
ISD::UMIN, dl, PromotedType,
Add, SatMax);
1022 return matcher.getNode(
ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1027 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1040 "addition, subtraction or left shift");
1043 unsigned SHLAmount = NewBits - OldBits;
1050 matcher.
getNode(
ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1053 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1054 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1063 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1064 Result = matcher.getNode(
ISD::SMIN, dl, PromotedType, Result, SatMax);
1065 Result = matcher.getNode(
ISD::SMAX, dl, PromotedType, Result, SatMin);
1072 SDValue Op1Promoted, Op2Promoted;
1078 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1079 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1081 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1082 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1084 EVT OldType =
N->getOperand(0).getValueType();
1098 Op2Promoted,
N->getOperand(2));
1100 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
1103 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1108 unsigned SatW,
bool Signed,
1111 EVT VT = V.getValueType();
1138 EVT VT =
LHS.getValueType();
1156 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1162 "Tried to saturate to more than the original type?");
1171 SDValue Op1Promoted, Op2Promoted;
1177 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1178 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1180 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1181 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1184 unsigned Scale =
N->getConstantOperandVal(2);
1193 N->getValueType(0).getScalarSizeInBits();
1198 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1199 Op2Promoted,
N->getOperand(2));
1209 Op2Promoted, Scale, DAG)) {
1212 N->getValueType(0).getScalarSizeInBits(),
1220 N->getValueType(0).getScalarSizeInBits());
1223SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1225 return PromoteIntRes_Overflow(
N);
1229 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1230 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1231 EVT OVT =
N->getOperand(0).getValueType();
1232 EVT NVT =
LHS.getValueType();
1247 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1255 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1256 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1258 unsigned Opcode =
N->getOpcode();
1259 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1266 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1267 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1269 LHS.getValueType(),
N->getOperand(0),
1270 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1274 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1275 EVT InVT =
N->getOperand(OpNo).getValueType();
1278 EVT SVT = getSetCCResultType(InVT);
1286 SVT = getSetCCResultType(InVT);
1294 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1295 "Vector compare must return a vector result!");
1299 if (
N->isStrictFPOpcode()) {
1301 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1302 N->getOperand(2),
N->getOperand(3)};
1303 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1308 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1309 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1325 EVT VT =
N->getValueType(0);
1331 ReplaceValueWith(
SDValue(
N, 0), Res);
1336 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1339 RHS = ZExtPromotedInteger(RHS);
1340 if (
N->getOpcode() != ISD::VP_SHL)
1343 N->getOperand(2),
N->getOperand(3));
1346SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1347 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1349 Op.getValueType(),
Op,
N->getOperand(1));
1352SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1356 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1357 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1358 if (
N->getNumOperands() == 2)
1360 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1361 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1363 N->getOperand(2),
N->getOperand(3));
1368 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1369 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1370 if (
N->getNumOperands() == 2)
1372 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1373 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1375 N->getOperand(2),
N->getOperand(3));
1380 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1381 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1382 if (
N->getNumOperands() == 2)
1384 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1385 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1387 N->getOperand(2),
N->getOperand(3));
1396 SExtOrZExtPromotedOperands(LHS, RHS);
1399 LHS.getValueType(), LHS, RHS);
1404 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1407 RHS = ZExtPromotedInteger(RHS);
1408 if (
N->getOpcode() != ISD::VP_ASHR)
1411 N->getOperand(2),
N->getOperand(3));
1416 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1419 RHS = ZExtPromotedInteger(RHS);
1420 if (
N->getOpcode() != ISD::VP_LSHR)
1423 N->getOperand(2),
N->getOperand(3));
1429 ReplaceValueWith(
SDValue(
N, 0), Res);
1434 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1435 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1438 Amt = ZExtPromotedInteger(Amt);
1442 EVT OldVT =
N->getOperand(0).getValueType();
1443 EVT VT =
Lo.getValueType();
1444 unsigned Opcode =
N->getOpcode();
1458 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1483SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(
SDNode *
N) {
1484 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1485 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1490 Amt = ZExtPromotedInteger(Amt);
1494 EVT OldVT =
N->getOperand(0).getValueType();
1495 EVT VT =
Lo.getValueType();
1496 unsigned Opcode =
N->getOpcode();
1497 bool IsFSHR = Opcode == ISD::VP_FSHR;
1502 Amt = DAG.
getNode(ISD::VP_UREM,
DL, AmtVT, Amt,
1510 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1513 Hi = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Hi, HiShift, Mask, EVL);
1517 Res = DAG.
getNode(IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL,
DL, VT, Res, Amt,
1520 Res = DAG.
getNode(ISD::VP_LSHR,
DL, VT, Res, HiShift, Mask, EVL);
1526 Lo = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Lo, ShiftOffset, Mask, EVL);
1531 Amt = DAG.
getNode(ISD::VP_ADD,
DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1549 Res = GetPromotedInteger(InOp);
1556 "Dst and Src must have the same number of elements");
1558 "Promoted vector type must be a power of two");
1561 GetSplitVector(InOp, EOp1, EOp2);
1569 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1570 "Expected VP_TRUNCATE opcode");
1571 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1572 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1573 std::tie(EVLLo, EVLHi) =
1574 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1575 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1576 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1583 SDValue WideInOp = GetWidenedVector(InOp);
1588 N->getValueType(0).getScalarType(), NumElem);
1603 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1604 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1609SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1611 return PromoteIntRes_Overflow(
N);
1615 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1616 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1617 EVT OVT =
N->getOperand(0).getValueType();
1618 EVT NVT =
LHS.getValueType();
1632 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1643 return PromoteIntRes_Overflow(
N);
1655 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1656 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1658 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1662 LHS, RHS,
N->getOperand(2));
1672 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1673 return PromoteIntRes_Overflow(
N);
1677 EVT OVT =
N->getValueType(0);
1690 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1694SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1697 return PromoteIntRes_Overflow(
N);
1701 EVT SmallVT =
LHS.getValueType();
1708 LHS = SExtPromotedInteger(LHS);
1709 RHS = SExtPromotedInteger(RHS);
1711 LHS = ZExtPromotedInteger(LHS);
1712 RHS = ZExtPromotedInteger(RHS);
1743 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1749 N->getValueType(0)));
1755 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1762 EVT VT =
N->getValueType(0);
1770 for (
unsigned i = 0; i < NumRegs; ++i) {
1771 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1772 N->getConstantOperandVal(3));
1778 std::reverse(Parts.begin(), Parts.end());
1783 for (
unsigned i = 1; i < NumRegs; ++i) {
1794 ReplaceValueWith(
SDValue(
N, 1), Chain);
1807bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1810 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1815 switch (
N->getOpcode()) {
1818 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1819 N->dump(&DAG);
dbgs() <<
"\n";
1825 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1828 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1829 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1835 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1839 Res = PromoteIntOp_ScalarOp(
N);
1842 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1845 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1847 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(
N);
break;
1848 case ISD::VP_SINT_TO_FP:
1851 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1853 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1855 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1857 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1859 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1861 case ISD::VP_TRUNCATE:
1865 case ISD::VP_UINT_TO_FP:
1870 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(
N);
break;
1878 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1881 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
1907 case ISD::VP_REDUCE_ADD:
1908 case ISD::VP_REDUCE_MUL:
1909 case ISD::VP_REDUCE_AND:
1910 case ISD::VP_REDUCE_OR:
1911 case ISD::VP_REDUCE_XOR:
1912 case ISD::VP_REDUCE_SMAX:
1913 case ISD::VP_REDUCE_SMIN:
1914 case ISD::VP_REDUCE_UMAX:
1915 case ISD::VP_REDUCE_UMIN:
1916 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
1921 Res = PromoteIntOp_STACKMAP(
N, OpNo);
1924 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
1926 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1927 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1928 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
1930 case ISD::EXPERIMENTAL_VP_SPLICE:
1931 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
1936 if (!Res.
getNode())
return false;
1943 const bool IsStrictFp =
N->isStrictFPOpcode();
1945 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1946 "Invalid operand expansion");
1950 ReplaceValueWith(
SDValue(
N, 0), Res);
1960void DAGTypeLegalizer::SExtOrZExtPromotedOperands(
SDValue &LHS,
SDValue &RHS) {
1961 SDValue OpL = GetPromotedInteger(LHS);
1962 SDValue OpR = GetPromotedInteger(RHS);
1968 unsigned OpLEffectiveBits =
1970 unsigned OpREffectiveBits =
1972 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1973 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
1980 LHS = SExtPromotedInteger(LHS);
1981 RHS = SExtPromotedInteger(RHS);
1992 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
1993 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2000 LHS = ZExtPromotedInteger(LHS);
2001 RHS = ZExtPromotedInteger(RHS);
2006void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
2015 LHS = SExtPromotedInteger(LHS);
2016 RHS = SExtPromotedInteger(RHS);
2021 "Unknown integer comparison!");
2023 SExtOrZExtPromotedOperands(LHS, RHS);
2027 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2032 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2034 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2040 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
2043SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2044 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2048 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
2053 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
2057SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2058 assert(OpNo == 1 &&
"only know how to promote condition");
2061 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2065 N->getOperand(2)), 0);
2070 EVT OVT =
N->getOperand(0).getValueType();
2071 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2072 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2073 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2086 EVT VecVT =
N->getValueType(0);
2089 "Legal vector of one illegal element?");
2094 assert(
N->getOperand(0).getValueSizeInBits() >=
2095 N->getValueType(0).getScalarSizeInBits() &&
2096 "Type of inserted value narrower than vector element type!");
2099 for (
unsigned i = 0; i < NumElts; ++i)
2100 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2105SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2112 assert(
N->getOperand(1).getValueSizeInBits() >=
2113 N->getValueType(0).getScalarSizeInBits() &&
2114 "Type of inserted value narrower than vector element type!");
2116 GetPromotedInteger(
N->getOperand(1)),
2121 assert(OpNo == 2 &&
"Different operand and result vector types?");
2127 N->getOperand(1),
Idx), 0);
2134 GetPromotedInteger(
N->getOperand(0))), 0);
2137SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2138 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2140 EVT OpTy =
N->getOperand(1).getValueType();
2143 if (
SDValue Res = WidenVSELECTMask(
N))
2145 Res,
N->getOperand(1),
N->getOperand(2));
2149 Cond = PromoteTargetBoolean(
Cond, OpVT);
2152 N->getOperand(2)), 0);
2155SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2156 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2160 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
2164 N->getOperand(3),
N->getOperand(4)), 0);
2167SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2168 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2172 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
2178 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2181 N->getOperand(3),
N->getOperand(4)),
2187 ZExtPromotedInteger(
N->getOperand(1))), 0);
2192 ZExtPromotedInteger(
N->getOperand(2))), 0);
2196 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2203SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(
SDNode *
N) {
2205 EVT VT =
N->getValueType(0);
2206 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2208 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2216 return DAG.
getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt,
N->getOperand(1),
2221 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2223 SExtPromotedInteger(
N->getOperand(0)),
2224 N->getOperand(1),
N->getOperand(2)),
2227 SExtPromotedInteger(
N->getOperand(0))), 0);
2230SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2232 SExtPromotedInteger(
N->getOperand(1))), 0);
2240 SDValue Val = GetPromotedInteger(
N->getValue());
2244 N->getMemoryVT(),
N->getMemOperand());
2255 Mask = PromoteTargetBoolean(Mask, DataVT);
2261 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2262 DataOp = GetPromotedInteger(DataOp);
2265 N->getOffset(), Mask,
N->getMemoryVT(),
2266 N->getMemOperand(),
N->getAddressingMode(),
2267 true,
N->isCompressingStore());
2272 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2273 EVT DataVT =
N->getValueType(0);
2274 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2276 NewOps[OpNo] =
Mask;
2293 EVT DataVT =
N->getValueType(0);
2294 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2295 }
else if (OpNo == 4) {
2297 if (
N->isIndexSigned())
2299 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2301 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2303 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2317 bool TruncateStore =
N->isTruncatingStore();
2322 EVT DataVT =
N->getValue().getValueType();
2323 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2324 }
else if (OpNo == 4) {
2326 if (
N->isIndexSigned())
2328 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2330 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2332 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2333 TruncateStore =
true;
2337 SDLoc(
N), NewOps,
N->getMemOperand(),
2338 N->getIndexType(), TruncateStore);
2342 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2343 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2345 N->getOperand(1),
N->getOperand(2));
2350 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2352 ZExtPromotedInteger(
N->getOperand(0)),
2353 N->getOperand(1),
N->getOperand(2)),
2356 ZExtPromotedInteger(
N->getOperand(0))), 0);
2359SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2361 ZExtPromotedInteger(
N->getOperand(1))), 0);
2366 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2371SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(
SDNode *
N) {
2373 EVT VT =
N->getValueType(0);
2374 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2376 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2379 N->getOperand(0).getScalarValueSizeInBits());
2381 N->getOperand(1),
N->getOperand(2));
2385 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2390SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2392 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2397 bool IsStrict =
N->isStrictFPOpcode();
2409 SDValue Op = SExtPromotedInteger(
N->getOperand(1));
2419 unsigned OpOffset = IsStrict ? 1 : 0;
2422 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2423 "POWI exponent should match with sizeof(int) when doing the libcall.");
2426 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2427 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2428 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2429 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2431 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2436 switch (
N->getOpcode()) {
2444 case ISD::VP_REDUCE_ADD:
2445 case ISD::VP_REDUCE_MUL:
2446 case ISD::VP_REDUCE_AND:
2447 case ISD::VP_REDUCE_OR:
2448 case ISD::VP_REDUCE_XOR:
2452 case ISD::VP_REDUCE_SMAX:
2453 case ISD::VP_REDUCE_SMIN:
2457 case ISD::VP_REDUCE_UMAX:
2458 case ISD::VP_REDUCE_UMIN:
2468 return GetPromotedInteger(V);
2470 return SExtPromotedInteger(V);
2472 return ZExtPromotedInteger(V);
2478 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2480 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2481 EVT InVT =
Op.getValueType();
2483 EVT ResVT =
N->getValueType(0);
2484 unsigned Opcode =
N->getOpcode();
2504 Op = ZExtPromotedInteger(
N->getOperand(0));
2507 Op = SExtPromotedInteger(
N->getOperand(0));
2523 Op = ZExtPromotedInteger(
N->getOperand(0));
2526 Op = SExtPromotedInteger(
N->getOperand(0));
2540SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2547 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2551 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2553 Op = PromoteIntOpVectorReduction(
N,
Op);
2557 EVT VT =
N->getValueType(0);
2558 EVT EltVT =
Op.getValueType().getScalarType();
2573 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2577SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2580 SDValue Operand =
N->getOperand(OpNo);
2586SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2589 SDValue Operand =
N->getOperand(OpNo);
2595SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2596 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2597 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2600 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2605SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
2609 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2613 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
2615 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2627void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2633 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2636 switch (
N->getOpcode()) {
2639 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2640 N->dump(&DAG);
dbgs() <<
"\n";
2686 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
2713 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2714 SplitInteger(Tmp.first,
Lo,
Hi);
2715 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2723 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2732 SplitInteger(Tmp,
Lo,
Hi);
2803 ExpandIntRes_Rotate(
N,
Lo,
Hi);
2808 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
2812 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
2822std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
2823 unsigned Opc =
Node->getOpcode();
2829 EVT RetVT =
Node->getValueType(0);
2837 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2838 "Unexpected atomic op or value type!");
2842 Node->getOperand(0));
2847void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2852 GetExpandedInteger(
N->getOperand(0), InL, InH);
2863 unsigned VTBits =
N->getValueType(0).getSizeInBits();
2867 if (Amt.
uge(VTBits)) {
2869 }
else if (Amt.
ugt(NVTBits)) {
2873 }
else if (Amt == NVTBits) {
2890 if (Amt.
uge(VTBits)) {
2892 }
else if (Amt.
ugt(NVTBits)) {
2896 }
else if (Amt == NVTBits) {
2913 if (Amt.
uge(VTBits)) {
2916 }
else if (Amt.
ugt(NVTBits)) {
2921 }
else if (Amt == NVTBits) {
2941bool DAGTypeLegalizer::
2943 unsigned Opc =
N->getOpcode();
2951 "Expanded integer type size not a power of two!");
2958 if (((Known.
Zero | Known.
One) & HighBitMask) == 0)
2963 GetExpandedInteger(In, InL, InH);
3017 Lo = DAG.
getNode(Opc, dl, NVT, InL, Amt);
3030bool DAGTypeLegalizer::
3037 "Expanded integer type size not a power of two!");
3042 GetExpandedInteger(
N->getOperand(0), InL, InH);
3054 switch (
N->getOpcode()) {
3069 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
3085 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3101 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3122void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3131 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3132 unsigned NumHalfBits = NumBits / 2;
3135 SDValue LHSL, LHSH, RHSL, RHSH;
3136 GetExpandedInteger(LHS, LHSL, LHSH);
3137 GetExpandedInteger(RHS, RHSL, RHSH);
3150 SDValue LHSL, LHSH, RHSL, RHSH;
3151 GetExpandedInteger(LHS, LHSL, LHSH);
3152 GetExpandedInteger(RHS, RHSL, RHSH);
3154 EVT CCT = getSetCCResultType(NVT);
3163 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3167 const APInt *RHSVal =
nullptr;
3168 if (
auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3169 RHSVal = &RHSConst->getAPIntValue();
3176 SDValue LHSL, LHSH, RHSL, RHSH;
3177 GetExpandedInteger(LHS, LHSL, LHSH);
3178 GetExpandedInteger(RHS, RHSL, RHSH);
3180 EVT CCT = getSetCCResultType(NVT);
3186 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3205 switch (
N->getOpcode()) {
3232 EVT VT =
N->getValueType(0);
3233 EVT CCT = getSetCCResultType(VT);
3236 SplitInteger(Result,
Lo,
Hi);
3239void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3243 SDValue LHSL, LHSH, RHSL, RHSH;
3244 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3245 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3248 SDValue LoOps[2] = { LHSL, RHSL };
3249 SDValue HiOps[3] = { LHSH, RHSH };
3258 HiOps[2] =
Lo.getValue(1);
3264 HiOps[2] =
Lo.getValue(1);
3286 HiOps[2] =
Lo.getValue(1);
3290 HiOps[2] =
Lo.getValue(1);
3303 EVT OvfVT = getSetCCResultType(NVT);
3343 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3346 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3349 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3367 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3381void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3384 SDValue LHSL, LHSH, RHSL, RHSH;
3386 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3387 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3389 SDValue LoOps[2] = { LHSL, RHSL };
3390 SDValue HiOps[3] = { LHSH, RHSH };
3394 HiOps[2] =
Lo.getValue(1);
3398 HiOps[2] =
Lo.getValue(1);
3404 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3407void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3410 SDValue LHSL, LHSH, RHSL, RHSH;
3412 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3413 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3415 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3416 SDValue HiOps[3] = { LHSH, RHSH };
3418 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3419 HiOps[2] =
Lo.getValue(1);
3420 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3424 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3427void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3435 unsigned CarryOp, NoCarryOp;
3437 switch(
N->getOpcode()) {
3457 SDValue LHSL, LHSH, RHSL, RHSH;
3458 GetExpandedInteger(LHS, LHSL, LHSH);
3459 GetExpandedInteger(RHS, RHSL, RHSH);
3461 SDValue LoOps[2] = { LHSL, RHSL };
3462 SDValue HiOps[3] = { LHSH, RHSH };
3464 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3465 HiOps[2] =
Lo.getValue(1);
3466 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3468 Ovf =
Hi.getValue(1);
3473 SplitInteger(Sum,
Lo,
Hi);
3484 DAG.
getSetCC(dl,
N->getValueType(1), LHS,
3495 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3501 SDValue LHSL, LHSH, RHSL, RHSH;
3503 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3504 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3506 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3509 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3510 HiOps[2] =
Lo.getValue(1);
3511 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3515 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3518void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3521 SDValue LHSL, LHSH, RHSL, RHSH;
3523 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3524 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3530 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3531 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3535 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3538void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3543 if (
Op.getValueType().bitsLE(NVT)) {
3550 assert(getTypeAction(
Op.getValueType()) ==
3552 "Only know how to promote this result!");
3555 "Operand over promoted?");
3557 SplitInteger(Res,
Lo,
Hi);
3561void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3564 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3565 EVT NVT =
Lo.getValueType();
3566 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3570 if (NVTBits < EVTBits) {
3573 EVTBits - NVTBits)));
3583void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3586 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3587 EVT NVT =
Lo.getValueType();
3588 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3592 if (NVTBits < EVTBits) {
3595 EVTBits - NVTBits)));
3603void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3606 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3611void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3614 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3623 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3624 EVT NVT =
Lo.getValueType();
3630void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3634 auto Constant = cast<ConstantSDNode>(
N);
3636 bool IsTarget =
Constant->isTargetOpcode();
3637 bool IsOpaque =
Constant->isOpaque();
3648 GetExpandedInteger(N0,
Lo,
Hi);
3649 EVT NVT =
Lo.getValueType();
3680 EVT VT =
N->getValueType(0);
3684 SplitInteger(Neg, NegLo, NegHi);
3692void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3696 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3697 EVT NVT =
Lo.getValueType();
3712void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3716 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3717 EVT NVT =
Lo.getValueType();
3723void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3727 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3728 EVT NVT =
Lo.getValueType();
3757 ReplaceValueWith(
SDValue(
N, 1), Chain);
3765 Chain =
Op.getValue(1);
3774 EVT VT =
N->getValueType(0);
3778 bool IsStrict =
N->isStrictFPOpcode();
3780 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3782 Op = GetPromotedFloat(
Op);
3785 EVT OFPVT =
Op.getValueType();
3787 Op = GetSoftPromotedHalf(
Op);
3791 SplitInteger(
Op,
Lo,
Hi);
3795 if (
Op.getValueType() == MVT::bf16) {
3802 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
3805 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3806 CallOptions, dl, Chain);
3807 SplitInteger(Tmp.first,
Lo,
Hi);
3810 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3816 SplitInteger(Res,
Lo,
Hi);
3822 bool IsStrict =
N->isStrictFPOpcode();
3823 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3827 "Input type needs to be promoted!");
3829 EVT VT =
Op.getValueType();
3831 if (VT == MVT::f16) {
3841 LC = RTLIB::LROUND_F32;
3842 else if (VT == MVT::f64)
3843 LC = RTLIB::LROUND_F64;
3844 else if (VT == MVT::f80)
3845 LC = RTLIB::LROUND_F80;
3846 else if (VT == MVT::f128)
3847 LC = RTLIB::LROUND_F128;
3848 else if (VT == MVT::ppcf128)
3849 LC = RTLIB::LROUND_PPCF128;
3850 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
3854 LC = RTLIB::LRINT_F32;
3855 else if (VT == MVT::f64)
3856 LC = RTLIB::LRINT_F64;
3857 else if (VT == MVT::f80)
3858 LC = RTLIB::LRINT_F80;
3859 else if (VT == MVT::f128)
3860 LC = RTLIB::LRINT_F128;
3861 else if (VT == MVT::ppcf128)
3862 LC = RTLIB::LRINT_PPCF128;
3863 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
3867 LC = RTLIB::LLROUND_F32;
3868 else if (VT == MVT::f64)
3869 LC = RTLIB::LLROUND_F64;
3870 else if (VT == MVT::f80)
3871 LC = RTLIB::LLROUND_F80;
3872 else if (VT == MVT::f128)
3873 LC = RTLIB::LLROUND_F128;
3874 else if (VT == MVT::ppcf128)
3875 LC = RTLIB::LLROUND_PPCF128;
3876 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
3880 LC = RTLIB::LLRINT_F32;
3881 else if (VT == MVT::f64)
3882 LC = RTLIB::LLRINT_F64;
3883 else if (VT == MVT::f80)
3884 LC = RTLIB::LLRINT_F80;
3885 else if (VT == MVT::f128)
3886 LC = RTLIB::LLRINT_F128;
3887 else if (VT == MVT::ppcf128)
3888 LC = RTLIB::LLRINT_PPCF128;
3889 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
3893 EVT RetVT =
N->getValueType(0);
3897 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
3898 Op, CallOptions, dl,
3900 SplitInteger(Tmp.first,
Lo,
Hi);
3902 if (
N->isStrictFPOpcode())
3903 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3906void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
3908 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
3911 ExpandRes_NormalLoad(
N,
Lo,
Hi);
3917 EVT VT =
N->getValueType(0);
3928 if (
N->getMemoryVT().bitsLE(NVT)) {
3929 EVT MemVT =
N->getMemoryVT();
3932 N->getOriginalAlign(), MMOFlags, AAInfo);
3935 Ch =
Lo.getValue(1);
3940 unsigned LoSize =
Lo.getValueSizeInBits();
3955 N->getOriginalAlign(), MMOFlags, AAInfo);
3957 unsigned ExcessBits =
3965 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3966 N->getOriginalAlign(), MMOFlags, AAInfo);
3975 EVT MemVT =
N->getMemoryVT();
3978 unsigned ExcessBits = (EBytes - IncrementSize)*8;
3984 N->getOriginalAlign(), MMOFlags, AAInfo);
3990 N->getPointerInfo().getWithOffset(IncrementSize),
3992 N->getOriginalAlign(), MMOFlags, AAInfo);
4016 ReplaceValueWith(
SDValue(
N, 1), Ch);
4019void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4023 GetExpandedInteger(
N->getOperand(0), LL, LH);
4024 GetExpandedInteger(
N->getOperand(1), RL, RH);
4029void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4031 EVT VT =
N->getValueType(0);
4036 GetExpandedInteger(
N->getOperand(0), LL, LH);
4037 GetExpandedInteger(
N->getOperand(1), RL, RH);
4047 LC = RTLIB::MUL_I16;
4048 else if (VT == MVT::i32)
4049 LC = RTLIB::MUL_I32;
4050 else if (VT == MVT::i64)
4051 LC = RTLIB::MUL_I64;
4052 else if (VT == MVT::i128)
4053 LC = RTLIB::MUL_I128;
4065 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4068 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4080 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4086 SplitInteger(Result,
Lo,
Hi);
4092 SplitInteger(Result,
Lo,
Hi);
4103 EVT VT =
N->getValueType(0);
4107 uint64_t Scale =
N->getConstantOperandVal(2);
4119 EVT BoolVT = getSetCCResultType(VT);
4144 SplitInteger(Result,
Lo,
Hi);
4150 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4154 GetExpandedInteger(LHS, LL, LH);
4155 GetExpandedInteger(RHS, RL, RH);
4159 if (!TLI.
expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4167 SplitInteger(LoTmp, Result[0], Result[1]);
4168 SplitInteger(HiTmp, Result[2], Result[3]);
4170 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4173 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4174 "the size of the current value type");
4196 if (Scale % NVTSize) {
4212 if (Scale == VTSize)
4233 EVT BoolNVT = getSetCCResultType(NVT);
4236 if (Scale < NVTSize) {
4243 }
else if (Scale == NVTSize) {
4246 }
else if (Scale < VTSize) {
4254 "(and saturation can't happen with Scale==VTSize).");
4261 if (Scale < NVTSize) {
4266 unsigned OverflowBits = VTSize - Scale + 1;
4267 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4268 "Extent of overflow bits must start within HL");
4285 }
else if (Scale == NVTSize) {
4298 }
else if (Scale < VTSize) {
4301 unsigned OverflowBits = VTSize - Scale + 1;
4328 N->getConstantOperandVal(2), DAG);
4332 N->getConstantOperandVal(2), TLI, DAG);
4333 SplitInteger(Res,
Lo,
Hi);
4336void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *
Node,
4339 "Node has unexpected Opcode");
4354 SDValue LHSL, LHSH, RHSL, RHSH;
4355 GetExpandedInteger(LHS, LHSL, LHSH);
4356 GetExpandedInteger(RHS, RHSL, RHSH);
4360 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
4362 Ovf =
Hi.getValue(1);
4369 SplitInteger(Sum,
Lo,
Hi);
4394 EVT VT =
LHS.getValueType();
4397 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
4401 EVT OType =
Node->getValueType(1);
4409void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4411 EVT VT =
N->getValueType(0);
4413 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4423 LC = RTLIB::SDIV_I16;
4424 else if (VT == MVT::i32)
4425 LC = RTLIB::SDIV_I32;
4426 else if (VT == MVT::i64)
4427 LC = RTLIB::SDIV_I64;
4428 else if (VT == MVT::i128)
4429 LC = RTLIB::SDIV_I128;
4430 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4434 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4437void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
4440 SDValue Shiftee =
N->getOperand(0);
4447 bool ShiftByByteMultiple =
4452 if (!ShiftByByteMultiple)
4456 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
4457 unsigned VTByteWidth = VTBitWidth / 8;
4459 "Shiftee type size is not a power of two!");
4460 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4461 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4467 Align StackSlotAlignment(1);
4475 cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex());
4480 unsigned WideningOpc =
4482 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4489 Ch = DAG.
getStore(Ch, dl,
Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4495 if (ShiftByByteMultiple)
4496 Flags.setExact(
true);
4508 bool WillIndexUpwards =
N->getOpcode() !=
ISD::SHL;
4510 WillIndexUpwards = !WillIndexUpwards;
4513 if (WillIndexUpwards) {
4517 StackPtr, DAG.
getConstant(VTByteWidth, dl, PtrTy), dl);
4518 ByteOffset = DAG.
getNegative(ByteOffset, dl, ShAmtVT);
4527 VT, dl, Ch, AdjStackPtr,
4532 if (!ShiftByByteMultiple) {
4535 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
4539 SplitInteger(Res,
Lo,
Hi);
4542void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4544 EVT VT =
N->getValueType(0);
4545 unsigned Opc =
N->getOpcode();
4551 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
4555 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
4574 const bool LegalOrCustom =
4578 unsigned ExpansionFactor = 1;
4580 for (
EVT TmpVT = NVT;;) {
4582 if (NewTMPVT == TmpVT)
4592 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
4594 if (LegalOrCustom &&
4598 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4604 SDValue ShiftOp =
N->getOperand(1);
4609 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4611 Hi =
Lo.getValue(1);
4621 LC = RTLIB::SHL_I16;
4622 else if (VT == MVT::i32)
4623 LC = RTLIB::SHL_I32;
4624 else if (VT == MVT::i64)
4625 LC = RTLIB::SHL_I64;
4626 else if (VT == MVT::i128)
4627 LC = RTLIB::SHL_I128;
4631 LC = RTLIB::SRL_I16;
4632 else if (VT == MVT::i32)
4633 LC = RTLIB::SRL_I32;
4634 else if (VT == MVT::i64)
4635 LC = RTLIB::SRL_I64;
4636 else if (VT == MVT::i128)
4637 LC = RTLIB::SRL_I128;
4642 LC = RTLIB::SRA_I16;
4643 else if (VT == MVT::i32)
4644 LC = RTLIB::SRA_I32;
4645 else if (VT == MVT::i64)
4646 LC = RTLIB::SRA_I64;
4647 else if (VT == MVT::i128)
4648 LC = RTLIB::SRA_I128;
4655 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
4658 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4662 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
4666void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
4671 if (
Op.getValueType().bitsLE(NVT)) {
4682 assert(getTypeAction(
Op.getValueType()) ==
4684 "Only know how to promote this result!");
4687 "Operand over promoted?");
4689 SplitInteger(Res,
Lo,
Hi);
4697void DAGTypeLegalizer::
4700 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4701 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
4723void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
4725 EVT VT =
N->getValueType(0);
4727 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4737 LC = RTLIB::SREM_I16;
4738 else if (VT == MVT::i32)
4739 LC = RTLIB::SREM_I32;
4740 else if (VT == MVT::i64)
4741 LC = RTLIB::SREM_I64;
4742 else if (VT == MVT::i128)
4743 LC = RTLIB::SREM_I128;
4744 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
4748 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4751void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
4763void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
4765 EVT VT =
N->getValueType(0);
4784 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4785 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4786 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4788 EVT BitVT =
N->getValueType(1);
4813 SplitInteger(Three,
Lo,
Hi);
4817 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4828 LC = RTLIB::MULO_I32;
4829 else if (VT == MVT::i64)
4830 LC = RTLIB::MULO_I64;
4831 else if (VT == MVT::i128)
4832 LC = RTLIB::MULO_I128;
4845 SplitInteger(
Mul, MulLo, MulHi);
4851 SplitInteger(MulLo,
Lo,
Hi);
4852 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4865 EVT ArgVT =
Op.getValueType();
4869 Entry.IsSExt =
true;
4870 Entry.IsZExt =
false;
4871 Args.push_back(Entry);
4877 Entry.IsSExt =
true;
4878 Entry.IsZExt =
false;
4879 Args.push_back(Entry);
4898 ReplaceValueWith(
SDValue(
N, 1), Ofl);
4901void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
4903 EVT VT =
N->getValueType(0);
4905 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4914 if (isa<ConstantSDNode>(
N->getOperand(1))) {
4917 if (isTypeLegal(NVT)) {
4919 GetExpandedInteger(
N->getOperand(0), InL, InH);
4931 LC = RTLIB::UDIV_I16;
4932 else if (VT == MVT::i32)
4933 LC = RTLIB::UDIV_I32;
4934 else if (VT == MVT::i64)
4935 LC = RTLIB::UDIV_I64;
4936 else if (VT == MVT::i128)
4937 LC = RTLIB::UDIV_I128;
4938 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
4941 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4944void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
4946 EVT VT =
N->getValueType(0);
4948 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4957 if (isa<ConstantSDNode>(
N->getOperand(1))) {
4960 if (isTypeLegal(NVT)) {
4962 GetExpandedInteger(
N->getOperand(0), InL, InH);
4974 LC = RTLIB::UREM_I16;
4975 else if (VT == MVT::i32)
4976 LC = RTLIB::UREM_I32;
4977 else if (VT == MVT::i64)
4978 LC = RTLIB::UREM_I64;
4979 else if (VT == MVT::i128)
4980 LC = RTLIB::UREM_I128;
4981 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
4984 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4987void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
4992 if (
Op.getValueType().bitsLE(NVT)) {
4999 assert(getTypeAction(
Op.getValueType()) ==
5001 "Only know how to promote this result!");
5004 "Operand over promoted?");
5006 SplitInteger(Res,
Lo,
Hi);
5014void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5017 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
5022 cast<AtomicSDNode>(
N)->getMemoryVT(), VTs,
N->getOperand(0),
5023 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(
N)->getMemOperand());
5029void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5034 SplitInteger(Res,
Lo,
Hi);
5037void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5043 N->getOperand(0),
N->getOperand(1));
5044 SplitInteger(Res,
Lo,
Hi);
5051 GetExpandedInteger(
N->getOperand(0), In3, In4);
5052 GetExpandedInteger(
N->getOperand(1), In1, In2);
5056 unsigned Opc =
N->getOpcode();
5059 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5076 Lo = DAG.
getNode(Opc,
DL, HalfVT, Select2, Select1, NewShAmt);
5077 Hi = DAG.
getNode(Opc,
DL, HalfVT, Select3, Select2, NewShAmt);
5082 EVT VT =
N->getValueType(0);
5092 SplitInteger(Res,
Lo,
Hi);
5103bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5107 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5110 switch (
N->getOpcode()) {
5113 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5114 N->dump(&DAG);
dbgs() <<
"\n";
5119 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5126 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5132 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(
N), OpNo);
break;
5139 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5145 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5148 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5150 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5151 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5152 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5157 if (!Res.
getNode())
return false;
5165 "Invalid operand expansion");
5167 ReplaceValueWith(
SDValue(
N, 0), Res);
5173void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5177 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5178 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5179 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5199 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5200 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5232 RHSLo, LowCC,
false, DagCombineInfo, dl);
5239 RHSHi, CCCode,
false, DagCombineInfo, dl);
5252 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5254 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5263 if (LHSHi == RHSHi) {
5276 if (HasSETCCCARRY) {
5279 bool FlipOperands =
false;
5307 false, DagCombineInfo, dl);
5316 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
5317 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
5318 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5330 N->getOperand(4)), 0);
5334 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5335 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
5336 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5347 N->getOperand(2),
N->getOperand(3),
5352 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5353 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
5354 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5359 "Unexpected setcc expansion!");
5375 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5376 GetExpandedInteger(LHS, LHSLo, LHSHi);
5377 GetExpandedInteger(RHS, RHSLo, RHSHi);
5390 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5400 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
5409 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5414 bool IsStrict =
N->isStrictFPOpcode();
5418 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
5419 EVT DstVT =
N->getValueType(0);
5422 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5423 "Don't know how to expand this XINT_TO_FP!");
5426 std::pair<SDValue, SDValue> Tmp =
5432 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
5433 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
5438 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
5441 return ExpandOp_NormalStore(
N, OpNo);
5444 assert(OpNo == 1 &&
"Can only expand the stored value so far");
5446 EVT VT =
N->getOperand(1).getValueType();
5457 if (
N->getMemoryVT().bitsLE(NVT)) {
5458 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5460 N->getMemoryVT(),
N->getOriginalAlign(), MMOFlags,
5466 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5469 N->getOriginalAlign(), MMOFlags, AAInfo);
5471 unsigned ExcessBits =
5479 N->getPointerInfo().getWithOffset(IncrementSize),
5480 NEVT,
N->getOriginalAlign(), MMOFlags, AAInfo);
5486 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5488 EVT ExtVT =
N->getMemoryVT();
5491 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5509 N->getOriginalAlign(), MMOFlags, AAInfo);
5515 N->getPointerInfo().getWithOffset(IncrementSize),
5517 N->getOriginalAlign(), MMOFlags, AAInfo);
5523 GetExpandedInteger(
N->getOperand(0), InL, InH);
5532 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
5533 cast<AtomicSDNode>(
N)->getMemOperand());
5537SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
5538 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5539 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5543 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
5548SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
5551 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5552 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5558SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
5561 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5562 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5571SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
5573 EVT OutVT =
N->getValueType(0);
5575 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5579 SDValue BaseIdx =
N->getOperand(1);
5606 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5615 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5619 "Promoted operand has an element type greater than result");
5632 InOp0 = GetPromotedInteger(
N->getOperand(0));
5639 for (
unsigned i = 0; i != OutNumElems; ++i) {
5655SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
5656 EVT OutVT =
N->getValueType(0);
5658 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5670 Vec = GetPromotedInteger(Vec);
5676SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
5679 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5685SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
5687 EVT VT =
N->getValueType(0);
5692 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5693 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5700 EVT OutVT =
N->getValueType(0);
5702 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5703 unsigned NumElems =
N->getNumOperands();
5711 for (
unsigned i = 0; i != NumElems; ++i) {
5713 EVT OpVT =
Op.getValueType();
5718 if (OpVT.
bitsLT(NOutVTElem)) {
5724 ExtOpc = NOutExtOpc;
5737 assert(!
N->getOperand(0).getValueType().isVector() &&
5738 "Input must be a scalar");
5740 EVT OutVT =
N->getValueType(0);
5742 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5747 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op);
5752 EVT OutVT =
N->getValueType(0);
5755 "Type must be promoted to a scalable vector type");
5756 const APInt &StepVal =
N->getConstantOperandAPInt(0);
5761SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
5764 EVT OutVT =
N->getValueType(0);
5766 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5768 unsigned NumOperands =
N->getNumOperands();
5773 SDUse *MaxSizedValue = std::max_element(
5775 EVT AVT = A.getValueType().getVectorElementType();
5776 EVT BVT = B.getValueType().getVectorElementType();
5777 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5783 for (
unsigned I = 0;
I < NumOperands; ++
I) {
5785 EVT OpVT =
Op.getValueType();
5787 Op = GetPromotedInteger(
Op);
5790 "Unhandled legalization type");
5807 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
5808 assert(NumElem * NumOperands == NumOutElem &&
5809 "Unexpected number of elements");
5813 for (
unsigned i = 0; i < NumOperands; ++i) {
5816 Op = GetPromotedInteger(
Op);
5817 EVT SclrTy =
Op.getValueType().getVectorElementType();
5818 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
5819 "Unexpected number of elements");
5821 for (
unsigned j = 0;
j < NumElem; ++
j) {
5831SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
5832 EVT VT =
N->getValueType(0);
5834 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
5842 if (getTypeAction(
N->getOperand(0).getValueType())
5846 switch(
N->getOpcode()) {
5848 Promoted = SExtPromotedInteger(
N->getOperand(0));
5851 Promoted = ZExtPromotedInteger(
N->getOperand(0));
5854 Promoted = GetPromotedInteger(
N->getOperand(0));
5859 return DAG.
getNode(
N->getOpcode(), dl, NVT, Promoted);
5863 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
5866SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
5867 EVT OutVT =
N->getValueType(0);
5869 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5874 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5877 NOutVTElem,
N->getOperand(1));
5879 V0, ConvElem,
N->getOperand(2));
5887 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->ops());
5895 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
5896 return DAG.
getNode(
N->getOpcode(),
DL, Start.getValueType(), Start,
5897 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
5900SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
5902 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5914SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
5919 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5929SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
5931 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5934 N->getValueType(0).getVectorNumElements());
5939SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
5942 EVT ResVT =
N->getValueType(0);
5943 unsigned NumElems =
N->getNumOperands();
5948 for (
unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
5950 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
5958 EVT RetSclrTy =
N->getValueType(0).getVectorElementType();
5964 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
5966 EVT SclrTy =
Incoming->getValueType(0).getVectorElementType();
5967 unsigned NumElem =
Incoming->getValueType(0).getVectorNumElements();
5969 for (
unsigned i=0; i<NumElem; ++i) {
5981SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
5994 for (
unsigned I = 0;
I < OpNo;
I++)
5997 EVT Ty =
Op.getValueType();
6009 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6014 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6020SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6033 for (
unsigned I = 0;
I < OpNo;
I++)
6036 EVT Ty =
Op.getValueType();
6048 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6053 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
This class is used to represent ISD::LOAD nodes.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
@ TypeScalarizeScalableVector
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew=0)
Returns the largest uint64_t less than or equal to Value and is Skew mod Align.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)