31#define DEBUG_TYPE "legalize-types"
41void DAGTypeLegalizer::PromoteIntegerResult(
SDNode *
N,
unsigned ResNo) {
46 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true)) {
51 switch (
N->getOpcode()) {
54 dbgs() <<
"PromoteIntegerResult #" << ResNo <<
": ";
55 N->dump(&DAG);
dbgs() <<
"\n";
62 case ISD::VP_BITREVERSE:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(
N);
break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(
N);
break;
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(
N);
break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(
N);
break;
79 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
80 case ISD::VP_CTTZ_ELTS:
81 Res = PromoteIntRes_VP_CttzElements(
N);
84 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(
N);
break;
85 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(
N));
break;
86 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(
N));
88 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(
N));
91 Res = PromoteIntRes_VECTOR_COMPRESS(
N);
97 Res = PromoteIntRes_Select(
N);
102 case ISD::SETCC: Res = PromoteIntRes_SETCC(
N);
break;
104 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
106 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(
N);
break;
109 case ISD::VP_SHL: Res = PromoteIntRes_SHL(
N);
break;
111 Res = PromoteIntRes_SIGN_EXTEND_INREG(
N);
break;
113 case ISD::VP_SRA: Res = PromoteIntRes_SRA(
N);
break;
115 case ISD::VP_SRL: Res = PromoteIntRes_SRL(
N);
break;
116 case ISD::VP_TRUNCATE:
118 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(
N);
break;
119 case ISD::VAARG: Res = PromoteIntRes_VAARG(
N);
break;
123 Res = PromoteIntRes_EXTRACT_SUBVECTOR(
N);
break;
125 Res = PromoteIntRes_INSERT_SUBVECTOR(
N);
break;
127 Res = PromoteIntRes_VECTOR_REVERSE(
N);
break;
129 Res = PromoteIntRes_VECTOR_SHUFFLE(
N);
break;
131 Res = PromoteIntRes_VECTOR_SPLICE(
N);
break;
134 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
N);
137 Res = PromoteIntRes_INSERT_VECTOR_ELT(
N);
break;
139 Res = PromoteIntRes_BUILD_VECTOR(
N);
143 case ISD::EXPERIMENTAL_VP_SPLAT:
144 Res = PromoteIntRes_ScalarOp(
N);
148 Res = PromoteIntRes_CONCAT_VECTORS(
N);
break;
153 Res = PromoteIntRes_EXTEND_VECTOR_INREG(
N);
break;
156 case ISD::VP_SIGN_EXTEND:
158 case ISD::VP_ZERO_EXTEND:
161 case ISD::VP_FP_TO_SINT:
162 case ISD::VP_FP_TO_UINT:
170 Res = PromoteIntRes_FP_TO_XINT_SAT(
N);
break;
174 Res = PromoteIntRes_FP_TO_FP16_BF16(
N);
178 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(
N);
193 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(
N);
break;
202 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(
N);
break;
211 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(
N);
break;
214 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(
N, ResNo);
break;
216 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(
N, ResNo);
break;
218 case ISD::UMULO: Res = PromoteIntRes_XMULO(
N, ResNo);
break;
234 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(
N);
236 case ISD::VP_SADDSAT:
237 case ISD::VP_UADDSAT:
238 case ISD::VP_SSUBSAT:
239 case ISD::VP_USUBSAT:
240 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(
N);
245 Res = PromoteIntRes_CMP(
N);
258 case ISD::ABS: Res = PromoteIntRes_ABS(
N);
break;
261 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(
N));
break;
275 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(
N));
break;
279 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(
N), ResNo);
291 Res = PromoteIntRes_VECREDUCE(
N);
294 case ISD::VP_REDUCE_ADD:
295 case ISD::VP_REDUCE_MUL:
296 case ISD::VP_REDUCE_AND:
297 case ISD::VP_REDUCE_OR:
298 case ISD::VP_REDUCE_XOR:
299 case ISD::VP_REDUCE_SMAX:
300 case ISD::VP_REDUCE_SMIN:
301 case ISD::VP_REDUCE_UMAX:
302 case ISD::VP_REDUCE_UMIN:
303 Res = PromoteIntRes_VP_REDUCE(
N);
307 Res = PromoteIntRes_FREEZE(
N);
312 Res = PromoteIntRes_Rotate(
N);
317 Res = PromoteIntRes_FunnelShift(
N);
322 Res = PromoteIntRes_VPFunnelShift(
N);
326 Res = PromoteIntRes_IS_FPCLASS(
N);
329 Res = PromoteIntRes_FFREXP(
N);
334 Res = PromoteIntRes_XRINT(
N);
338 Res = PromoteIntRes_PATCHPOINT(
N);
344 SetPromotedInteger(
SDValue(
N, ResNo), Res);
349 SDValue Op = DisintegrateMERGE_VALUES(
N, ResNo);
350 return GetPromotedInteger(
Op);
355 SDValue Op = SExtPromotedInteger(
N->getOperand(0));
357 Op.getValueType(),
Op,
N->getOperand(1));
362 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
364 Op.getValueType(),
Op,
N->getOperand(1));
370 N->getMemoryVT(), ResVT,
371 N->getChain(),
N->getBasePtr(),
390 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
400 SDValue Op2 = GetPromotedInteger(
N->getOperand(2));
403 N->getChain(),
N->getBasePtr(),
404 Op2,
N->getMemOperand());
415 EVT SVT = getSetCCResultType(
N->getOperand(2).getValueType());
426 N->getChain(),
N->getBasePtr(),
N->getOperand(2),
N->getOperand(3),
436 SDValue Op3 = GetPromotedInteger(
N->getOperand(3));
439 Op2 = SExtPromotedInteger(Op2);
442 Op2 = ZExtPromotedInteger(Op2);
445 Op2 = GetPromotedInteger(Op2);
454 N->getOpcode(),
SDLoc(
N),
N->getMemoryVT(), VTs,
N->getChain(),
455 N->getBasePtr(), Op2, Op3,
N->getMemOperand());
457 for (
unsigned i = 1, NumResults =
N->getNumValues(); i < NumResults; ++i)
466 EVT OutVT =
N->getValueType(0);
470 switch (getTypeAction(InVT)) {
497 BitConvertToInteger(GetScalarizedVector(InOp)));
506 GetSplitVector(
N->getOperand(0),
Lo,
Hi);
507 Lo = BitConvertToInteger(
Lo);
508 Hi = BitConvertToInteger(
Hi);
516 JoinIntegers(
Lo,
Hi));
550 if (isTypeLegal(WideOutVT)) {
551 InOp = DAG.
getBitcast(WideOutVT, GetWidenedVector(InOp));
561 CreateStackStoreLoad(InOp, OutVT));
565 SDValue V = GetPromotedInteger(
N->getOperand(0));
567 V.getValueType(), V);
571 SDValue Op = GetPromotedInteger(
N->getOperand(0));
572 EVT OVT =
N->getValueType(0);
573 EVT NVT =
Op.getValueType();
593 return DAG.
getNode(ISD::VP_SRL, dl, NVT,
594 DAG.
getNode(ISD::VP_BSWAP, dl, NVT,
Op, Mask, EVL), ShAmt,
599 SDValue Op = GetPromotedInteger(
N->getOperand(0));
600 EVT OVT =
N->getValueType(0);
601 EVT NVT =
Op.getValueType();
621 return DAG.
getNode(ISD::VP_SRL, dl, NVT,
622 DAG.
getNode(ISD::VP_BITREVERSE, dl, NVT,
Op, Mask, EVL),
631 N->getValueType(0)), JoinIntegers(
N->getOperand(0),
636 EVT VT =
N->getValueType(0);
645 assert(isa<ConstantSDNode>(Result) &&
"Didn't constant fold ext?");
650 EVT OVT =
N->getValueType(0);
666 unsigned CtlzOpcode =
N->getOpcode();
667 if (CtlzOpcode ==
ISD::CTLZ || CtlzOpcode == ISD::VP_CTLZ) {
672 if (!
N->isVPOpcode()) {
674 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
682 SDValue Op = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
683 return DAG.
getNode(ISD::VP_SUB, dl, NVT,
684 DAG.
getNode(
N->getOpcode(), dl, NVT,
Op, Mask, EVL),
685 ExtractLeadingBits, Mask, EVL);
688 CtlzOpcode == ISD::VP_CTLZ_ZERO_UNDEF) {
690 SDValue Op = GetPromotedInteger(
N->getOperand(0));
695 if (!
N->isVPOpcode()) {
697 return DAG.
getNode(CtlzOpcode, dl, NVT,
Op);
702 Op = DAG.
getNode(ISD::VP_SHL, dl, NVT,
Op, ShiftConst, Mask, EVL);
703 return DAG.
getNode(CtlzOpcode, dl, NVT,
Op, Mask, EVL);
709 EVT OVT =
N->getValueType(0);
726 if (!
N->isVPOpcode()) {
727 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
733 SDValue Op = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
739 SDValue Op = GetPromotedInteger(
N->getOperand(0));
740 EVT OVT =
N->getValueType(0);
741 EVT NVT =
Op.getValueType();
759 unsigned NewOpc =
N->getOpcode();
760 if (NewOpc ==
ISD::CTTZ || NewOpc == ISD::VP_CTTZ) {
772 N->getOperand(1),
N->getOperand(2));
773 NewOpc = ISD::VP_CTTZ_ZERO_UNDEF;
776 if (!
N->isVPOpcode())
778 return DAG.
getNode(NewOpc, dl, NVT,
Op,
N->getOperand(1),
N->getOperand(2));
781SDValue DAGTypeLegalizer::PromoteIntRes_VP_CttzElements(
SDNode *
N) {
784 return DAG.
getNode(
N->getOpcode(),
DL, NewVT,
N->ops());
787SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(
SDNode *
N) {
802 EVT SVT =
In.getValueType().getScalarType();
814 unsigned NewOpc =
N->getOpcode();
831 if (
N->getOpcode() == ISD::VP_FP_TO_UINT &&
834 NewOpc = ISD::VP_FP_TO_SINT;
837 if (
N->isStrictFPOpcode()) {
838 Res = DAG.
getNode(NewOpc, dl, {NVT, MVT::Other},
839 {
N->getOperand(0),
N->getOperand(1)});
843 }
else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
844 Res = DAG.
getNode(NewOpc, dl, NVT, {
N->getOperand(0),
N->getOperand(1),
847 Res = DAG.
getNode(NewOpc, dl, NVT,
N->getOperand(0));
859 N->getOpcode() == ISD::VP_FP_TO_UINT)
866SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(
SDNode *
N) {
870 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
874SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(
SDNode *
N) {
878 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
881SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(
SDNode *
N) {
886 N->getOperand(0),
N->getOperand(1));
894 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
902 DAG.
getNode(
N->getOpcode(), dl, {NVT, MVT::Other},
N->getOperand(0));
914 if (getTypeAction(
N->getOperand(0).getValueType())
916 SDValue Res = GetPromotedInteger(
N->getOperand(0));
934 if (
N->getNumOperands() != 1) {
935 assert(
N->getNumOperands() == 3 &&
"Unexpected number of operands!");
936 assert(
N->isVPOpcode() &&
"Expected VP opcode");
937 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0),
938 N->getOperand(1),
N->getOperand(2));
940 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
950 N->getMemoryVT(),
N->getMemOperand());
960 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
968 N->getOffset(),
N->getMask(), ExtPassThru,
969 N->getMemoryVT(),
N->getMemOperand(),
970 N->getAddressingMode(), ExtType,
971 N->isExpandingLoad());
980 SDValue ExtPassThru = GetPromotedInteger(
N->getPassThru());
982 "Gather result type and the passThru argument type should be the same");
989 SDValue Ops[] = {
N->getChain(), ExtPassThru,
N->getMask(),
N->getBasePtr(),
990 N->getIndex(),
N->getScale() };
992 N->getMemoryVT(), dl, Ops,
993 N->getMemOperand(),
N->getIndexType(),
1001SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_COMPRESS(
SDNode *
N) {
1002 SDValue Vec = GetPromotedInteger(
N->getOperand(0));
1003 SDValue Passthru = GetPromotedInteger(
N->getOperand(2));
1005 N->getOperand(1), Passthru);
1013 EVT VT =
N->getValueType(0);
1014 EVT SVT = getSetCCResultType(VT);
1015 SDValue Ops[3] = {
N->getOperand(0),
N->getOperand(1) };
1016 unsigned NumOps =
N->getNumOperands();
1017 assert(NumOps <= 3 &&
"Too many operands");
1019 Ops[2] = PromoteTargetBoolean(
N->getOperand(2), VT);
1027 ReplaceValueWith(
SDValue(
N, 0), Res);
1033template <
class MatchContextClass>
1045 MatchContextClass matcher(DAG, TLI,
N);
1048 unsigned Opcode = matcher.getRootBaseOpcode();
1052 SDValue Op1Promoted, Op2Promoted;
1054 Op1Promoted = GetPromotedInteger(Op1);
1055 Op2Promoted = ZExtPromotedInteger(Op2);
1057 Op1Promoted = ZExtPromotedInteger(Op1);
1058 Op2Promoted = ZExtPromotedInteger(Op2);
1060 Op1Promoted = SExtPromotedInteger(Op1);
1061 Op2Promoted = SExtPromotedInteger(Op2);
1070 matcher.getNode(
ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1071 return matcher.getNode(
ISD::UMIN, dl, PromotedType,
Add, SatMax);
1076 return matcher.getNode(
ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1081 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1094 "addition, subtraction or left shift");
1097 unsigned SHLAmount = NewBits - OldBits;
1104 matcher.
getNode(
ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1107 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1108 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1117 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1118 Result = matcher.getNode(
ISD::SMIN, dl, PromotedType, Result, SatMax);
1119 Result = matcher.getNode(
ISD::SMAX, dl, PromotedType, Result, SatMin);
1126 SDValue Op1Promoted, Op2Promoted;
1132 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1133 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1135 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1136 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1138 EVT OldType =
N->getOperand(0).getValueType();
1152 Op2Promoted,
N->getOperand(2));
1154 return DAG.
getNode(ShiftOp, dl, PromotedType, Result,
1157 return DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1162 unsigned SatW,
bool Signed,
1165 EVT VT = V.getValueType();
1192 EVT VT =
LHS.getValueType();
1210 assert(Res &&
"Expanding DIVFIX with wide type failed?");
1216 "Tried to saturate to more than the original type?");
1225 SDValue Op1Promoted, Op2Promoted;
1231 Op1Promoted = SExtPromotedInteger(
N->getOperand(0));
1232 Op2Promoted = SExtPromotedInteger(
N->getOperand(1));
1234 Op1Promoted = ZExtPromotedInteger(
N->getOperand(0));
1235 Op2Promoted = ZExtPromotedInteger(
N->getOperand(1));
1238 unsigned Scale =
N->getConstantOperandVal(2);
1247 N->getValueType(0).getScalarSizeInBits();
1252 SDValue Res = DAG.
getNode(
N->getOpcode(), dl, PromotedType, Op1Promoted,
1253 Op2Promoted,
N->getOperand(2));
1263 Op2Promoted, Scale, DAG)) {
1266 N->getValueType(0).getScalarSizeInBits(),
1274 N->getValueType(0).getScalarSizeInBits());
1277SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(
SDNode *
N,
unsigned ResNo) {
1279 return PromoteIntRes_Overflow(
N);
1283 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1284 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1285 EVT OVT =
N->getOperand(0).getValueType();
1286 EVT NVT =
LHS.getValueType();
1301 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1307 EVT PromotedResultTy =
1310 N->getOperand(0),
N->getOperand(1));
1316 SDValue LHS = GetPromotedInteger(
N->getOperand(1));
1317 SDValue RHS = GetPromotedInteger(
N->getOperand(2));
1319 unsigned Opcode =
N->getOpcode();
1320 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1327 SDValue LHS = GetPromotedInteger(
N->getOperand(2));
1328 SDValue RHS = GetPromotedInteger(
N->getOperand(3));
1330 LHS.getValueType(),
N->getOperand(0),
1331 N->getOperand(1), LHS, RHS,
N->getOperand(4));
1335 unsigned OpNo =
N->isStrictFPOpcode() ? 1 : 0;
1336 EVT InVT =
N->getOperand(OpNo).getValueType();
1339 EVT SVT = getSetCCResultType(InVT);
1347 SVT = getSetCCResultType(InVT);
1355 assert(SVT.
isVector() ==
N->getOperand(OpNo).getValueType().isVector() &&
1356 "Vector compare must return a vector result!");
1360 if (
N->isStrictFPOpcode()) {
1362 SDValue Opers[] = {
N->getOperand(0),
N->getOperand(1),
1363 N->getOperand(2),
N->getOperand(3)};
1364 SetCC = DAG.
getNode(
N->getOpcode(), dl, VTs, Opers,
N->getFlags());
1369 SetCC = DAG.
getNode(
N->getOpcode(), dl, SVT,
N->getOperand(0),
1370 N->getOperand(1),
N->getOperand(2),
N->getFlags());
1386 EVT VT =
N->getValueType(0);
1392 ReplaceValueWith(
SDValue(
N, 0), Res);
1397 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1399 if (
N->getOpcode() != ISD::VP_SHL) {
1401 RHS = ZExtPromotedInteger(RHS);
1409 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1414SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(
SDNode *
N) {
1415 SDValue Op = GetPromotedInteger(
N->getOperand(0));
1417 Op.getValueType(),
Op,
N->getOperand(1));
1420SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(
SDNode *
N) {
1424 SDValue LHS = GetPromotedInteger(
N->getOperand(0));
1425 SDValue RHS = GetPromotedInteger(
N->getOperand(1));
1426 if (
N->getNumOperands() == 2)
1428 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1429 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1431 N->getOperand(2),
N->getOperand(3));
1435 if (
N->getNumOperands() == 2) {
1437 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1438 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1441 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1442 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1446 SDValue LHS = VPSExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1447 SDValue RHS = VPSExtPromotedInteger(
N->getOperand(1), Mask, EVL);
1453 if (
N->getNumOperands() == 2) {
1455 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1456 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1459 assert(
N->getNumOperands() == 4 &&
"Unexpected number of operands!");
1460 assert(
N->isVPOpcode() &&
"Expected VP opcode");
1464 SDValue LHS = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1465 SDValue RHS = VPZExtPromotedInteger(
N->getOperand(1), Mask, EVL);
1476 SExtOrZExtPromotedOperands(LHS, RHS);
1479 LHS.getValueType(), LHS, RHS);
1484 if (
N->getOpcode() != ISD::VP_SRA) {
1486 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1488 RHS = ZExtPromotedInteger(RHS);
1495 SDValue LHS = VPSExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1497 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1504 if (
N->getOpcode() != ISD::VP_SRL) {
1506 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1508 RHS = ZExtPromotedInteger(RHS);
1515 SDValue LHS = VPZExtPromotedInteger(
N->getOperand(0), Mask, EVL);
1517 RHS = VPZExtPromotedInteger(RHS, Mask, EVL);
1525 ReplaceValueWith(
SDValue(
N, 0), Res);
1530 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1531 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1534 Amt = ZExtPromotedInteger(Amt);
1538 EVT OldVT =
N->getOperand(0).getValueType();
1539 EVT VT =
Lo.getValueType();
1540 unsigned Opcode =
N->getOpcode();
1554 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1579SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(
SDNode *
N) {
1580 SDValue Hi = GetPromotedInteger(
N->getOperand(0));
1581 SDValue Lo = GetPromotedInteger(
N->getOperand(1));
1586 Amt = VPZExtPromotedInteger(Amt, Mask, EVL);
1590 EVT OldVT =
N->getOperand(0).getValueType();
1591 EVT VT =
Lo.getValueType();
1592 unsigned Opcode =
N->getOpcode();
1593 bool IsFSHR = Opcode == ISD::VP_FSHR;
1598 Amt = DAG.
getNode(ISD::VP_UREM,
DL, AmtVT, Amt,
1606 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1609 Hi = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Hi, HiShift, Mask, EVL);
1612 Res = DAG.
getNode(IsFSHR ? ISD::VP_SRL : ISD::VP_SHL,
DL, VT, Res, Amt,
1615 Res = DAG.
getNode(ISD::VP_SRL,
DL, VT, Res, HiShift, Mask, EVL);
1621 Lo = DAG.
getNode(ISD::VP_SHL,
DL, VT,
Lo, ShiftOffset, Mask, EVL);
1626 Amt = DAG.
getNode(ISD::VP_ADD,
DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1644 Res = GetPromotedInteger(InOp);
1651 "Dst and Src must have the same number of elements");
1653 "Promoted vector type must be a power of two");
1656 GetSplitVector(InOp, EOp1, EOp2);
1664 assert(
N->getOpcode() == ISD::VP_TRUNCATE &&
1665 "Expected VP_TRUNCATE opcode");
1666 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1667 std::tie(MaskLo, MaskHi) = SplitMask(
N->getOperand(1));
1668 std::tie(EVLLo, EVLHi) =
1669 DAG.
SplitEVL(
N->getOperand(2),
N->getValueType(0), dl);
1670 EOp1 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1671 EOp2 = DAG.
getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1678 SDValue WideInOp = GetWidenedVector(InOp);
1683 N->getValueType(0).getScalarType(), NumElem);
1698 if (
N->getOpcode() == ISD::VP_TRUNCATE)
1699 return DAG.
getNode(ISD::VP_TRUNCATE, dl, NVT, Res,
N->getOperand(1),
1704SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(
SDNode *
N,
unsigned ResNo) {
1706 return PromoteIntRes_Overflow(
N);
1710 SDValue LHS = ZExtPromotedInteger(
N->getOperand(0));
1711 SDValue RHS = ZExtPromotedInteger(
N->getOperand(1));
1712 EVT OVT =
N->getOperand(0).getValueType();
1713 EVT NVT =
LHS.getValueType();
1727 ReplaceValueWith(
SDValue(
N, 1), Ofl);
1738 return PromoteIntRes_Overflow(
N);
1750 SDValue LHS = SExtPromotedInteger(
N->getOperand(0));
1751 SDValue RHS = SExtPromotedInteger(
N->getOperand(1));
1753 EVT ValueVTs[] = {
LHS.getValueType(),
N->getValueType(1)};
1757 LHS, RHS,
N->getOperand(2));
1767 assert(ResNo == 1 &&
"Don't know how to promote other results yet.");
1768 return PromoteIntRes_Overflow(
N);
1772 EVT OVT =
N->getValueType(0);
1785 SDValue Op0 = SExtPromotedInteger(
N->getOperand(0));
1789SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(
SDNode *
N,
unsigned ResNo) {
1792 return PromoteIntRes_Overflow(
N);
1796 EVT SmallVT =
LHS.getValueType();
1803 LHS = SExtPromotedInteger(LHS);
1804 RHS = SExtPromotedInteger(RHS);
1806 LHS = ZExtPromotedInteger(LHS);
1807 RHS = ZExtPromotedInteger(RHS);
1838 ReplaceValueWith(
SDValue(
N, 1), Overflow);
1844 N->getValueType(0)));
1850 const APInt &MulImm =
N->getConstantOperandAPInt(0);
1857 EVT VT =
N->getValueType(0);
1865 for (
unsigned i = 0; i < NumRegs; ++i) {
1866 Parts[i] = DAG.
getVAArg(RegVT, dl, Chain,
Ptr,
N->getOperand(2),
1867 N->getConstantOperandVal(3));
1873 std::reverse(Parts.begin(), Parts.end());
1878 for (
unsigned i = 1; i < NumRegs; ++i) {
1889 ReplaceValueWith(
SDValue(
N, 1), Chain);
1902bool DAGTypeLegalizer::PromoteIntegerOperand(
SDNode *
N,
unsigned OpNo) {
1905 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false)) {
1910 switch (
N->getOpcode()) {
1913 dbgs() <<
"PromoteIntegerOperand Op #" << OpNo <<
": ";
1914 N->dump(&DAG);
dbgs() <<
"\n";
1920 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(
N));
1923 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(
N, OpNo);
break;
1924 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(
N, OpNo);
break;
1930 Res = PromoteIntOp_INSERT_VECTOR_ELT(
N, OpNo);
1934 case ISD::EXPERIMENTAL_VP_SPLAT:
1935 Res = PromoteIntOp_ScalarOp(
N);
1938 case ISD::SELECT: Res = PromoteIntOp_SELECT(
N, OpNo);
break;
1941 case ISD::SETCC: Res = PromoteIntOp_SETCC(
N, OpNo);
break;
1943 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(
N);
break;
1944 case ISD::VP_SINT_TO_FP:
1947 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(
N),
1949 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(
N),
1951 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(
N),
1953 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(
N),
1955 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(
N),
1958 Res = PromoteIntOp_VECTOR_COMPRESS(
N, OpNo);
1960 case ISD::VP_TRUNCATE:
1964 case ISD::VP_UINT_TO_FP:
1969 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(
N);
break;
1977 case ISD::ROTR: Res = PromoteIntOp_Shift(
N);
break;
1980 case ISD::UCMP: Res = PromoteIntOp_CMP(
N);
break;
1983 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(
N);
break;
2009 case ISD::VP_REDUCE_ADD:
2010 case ISD::VP_REDUCE_MUL:
2011 case ISD::VP_REDUCE_AND:
2012 case ISD::VP_REDUCE_OR:
2013 case ISD::VP_REDUCE_XOR:
2014 case ISD::VP_REDUCE_SMAX:
2015 case ISD::VP_REDUCE_SMIN:
2016 case ISD::VP_REDUCE_UMAX:
2017 case ISD::VP_REDUCE_UMIN:
2018 Res = PromoteIntOp_VP_REDUCE(
N, OpNo);
2023 Res = PromoteIntOp_STACKMAP(
N, OpNo);
2026 Res = PromoteIntOp_PATCHPOINT(
N, OpNo);
2028 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
2029 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
2030 Res = PromoteIntOp_VP_STRIDED(
N, OpNo);
2032 case ISD::EXPERIMENTAL_VP_SPLICE:
2033 Res = PromoteIntOp_VP_SPLICE(
N, OpNo);
2038 if (!Res.
getNode())
return false;
2045 const bool IsStrictFp =
N->isStrictFPOpcode();
2047 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
2048 "Invalid operand expansion");
2052 ReplaceValueWith(
SDValue(
N, 0), Res);
2062void DAGTypeLegalizer::SExtOrZExtPromotedOperands(
SDValue &LHS,
SDValue &RHS) {
2063 SDValue OpL = GetPromotedInteger(LHS);
2064 SDValue OpR = GetPromotedInteger(RHS);
2070 unsigned OpLEffectiveBits =
2072 unsigned OpREffectiveBits =
2074 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2075 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2082 LHS = SExtPromotedInteger(LHS);
2083 RHS = SExtPromotedInteger(RHS);
2094 if (OpLEffectiveBits <=
LHS.getScalarValueSizeInBits() &&
2095 OpREffectiveBits <=
RHS.getScalarValueSizeInBits()) {
2102 LHS = ZExtPromotedInteger(LHS);
2103 RHS = ZExtPromotedInteger(RHS);
2108void DAGTypeLegalizer::PromoteSetCCOperands(
SDValue &LHS,
SDValue &RHS,
2117 LHS = SExtPromotedInteger(LHS);
2118 RHS = SExtPromotedInteger(RHS);
2123 "Unknown integer comparison!");
2125 SExtOrZExtPromotedOperands(LHS, RHS);
2129 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2134 SDValue Op1 = GetPromotedInteger(
N->getOperand(1));
2136 N->getChain(), Op1,
N->getBasePtr(),
N->getMemOperand());
2142 return CreateStackStoreLoad(
N->getOperand(0),
N->getValueType(0));
2145SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(
SDNode *
N,
unsigned OpNo) {
2146 assert(OpNo == 2 &&
"Don't know how to promote this operand!");
2150 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(1))->get());
2155 N->getOperand(1), LHS, RHS,
N->getOperand(4)),
2159SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(
SDNode *
N,
unsigned OpNo) {
2160 assert(OpNo == 1 &&
"only know how to promote condition");
2163 SDValue Cond = PromoteTargetBoolean(
N->getOperand(1), MVT::Other);
2167 N->getOperand(2)), 0);
2172 EVT OVT =
N->getOperand(0).getValueType();
2173 SDValue Lo = ZExtPromotedInteger(
N->getOperand(0));
2174 SDValue Hi = GetPromotedInteger(
N->getOperand(1));
2175 assert(
Lo.getValueType() ==
N->getValueType(0) &&
"Operand over promoted?");
2188 EVT VecVT =
N->getValueType(0);
2191 "Legal vector of one illegal element?");
2196 assert(
N->getOperand(0).getValueSizeInBits() >=
2197 N->getValueType(0).getScalarSizeInBits() &&
2198 "Type of inserted value narrower than vector element type!");
2201 for (
unsigned i = 0; i < NumElts; ++i)
2202 NewOps.
push_back(GetPromotedInteger(
N->getOperand(i)));
2207SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(
SDNode *
N,
2214 assert(
N->getOperand(1).getValueSizeInBits() >=
2215 N->getValueType(0).getScalarSizeInBits() &&
2216 "Type of inserted value narrower than vector element type!");
2218 GetPromotedInteger(
N->getOperand(1)),
2223 assert(OpNo == 2 &&
"Different operand and result vector types?");
2229 N->getOperand(1),
Idx), 0);
2233 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2234 if (
N->getOpcode() == ISD::EXPERIMENTAL_VP_SPLAT)
2243SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(
SDNode *
N,
unsigned OpNo) {
2244 assert(OpNo == 0 &&
"Only know how to promote the condition!");
2246 EVT OpTy =
N->getOperand(1).getValueType();
2249 if (
SDValue Res = WidenVSELECTMask(
N))
2251 Res,
N->getOperand(1),
N->getOperand(2));
2255 Cond = PromoteTargetBoolean(
Cond, OpVT);
2258 N->getOperand(2)), 0);
2261SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(
SDNode *
N,
unsigned OpNo) {
2262 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2266 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(4))->get());
2270 N->getOperand(3),
N->getOperand(4)), 0);
2273SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(
SDNode *
N,
unsigned OpNo) {
2274 assert(OpNo == 0 &&
"Don't know how to promote this operand!");
2278 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(
N->getOperand(2))->get());
2284 assert(
N->getOpcode() == ISD::VP_SETCC &&
"Expected VP_SETCC opcode");
2287 N->getOperand(3),
N->getOperand(4)),
2293 ZExtPromotedInteger(
N->getOperand(1))), 0);
2298 ? ZExtPromotedInteger(
N->getOperand(0))
2299 : SExtPromotedInteger(
N->getOperand(0));
2301 ? ZExtPromotedInteger(
N->getOperand(1))
2302 : SExtPromotedInteger(
N->getOperand(1));
2309 ZExtPromotedInteger(
N->getOperand(2))), 0);
2313 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2320SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(
SDNode *
N) {
2322 EVT VT =
N->getValueType(0);
2323 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2325 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2333 return DAG.
getNode(ISD::VP_SRA, dl, VT, Shl, ShAmt,
N->getOperand(1),
2338 if (
N->getOpcode() == ISD::VP_SINT_TO_FP)
2340 SExtPromotedInteger(
N->getOperand(0)),
2341 N->getOperand(1),
N->getOperand(2)),
2344 SExtPromotedInteger(
N->getOperand(0))), 0);
2347SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(
SDNode *
N) {
2349 SExtPromotedInteger(
N->getOperand(1))), 0);
2357 SDValue Val = GetPromotedInteger(
N->getValue());
2361 N->getMemoryVT(),
N->getMemOperand());
2372 Mask = PromoteTargetBoolean(Mask, DataVT);
2378 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2379 DataOp = GetPromotedInteger(DataOp);
2382 N->getOffset(), Mask,
N->getMemoryVT(),
2383 N->getMemOperand(),
N->getAddressingMode(),
2384 true,
N->isCompressingStore());
2389 assert(OpNo == 3 &&
"Only know how to promote the mask!");
2390 EVT DataVT =
N->getValueType(0);
2391 SDValue Mask = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2393 NewOps[OpNo] =
Mask;
2410 EVT DataVT =
N->getValueType(0);
2411 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2412 }
else if (OpNo == 4) {
2414 if (
N->isIndexSigned())
2416 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2418 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2420 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2434 bool TruncateStore =
N->isTruncatingStore();
2439 EVT DataVT =
N->getValue().getValueType();
2440 NewOps[OpNo] = PromoteTargetBoolean(
N->getOperand(OpNo), DataVT);
2441 }
else if (OpNo == 4) {
2443 if (
N->isIndexSigned())
2445 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2447 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2449 NewOps[OpNo] = GetPromotedInteger(
N->getOperand(OpNo));
2450 TruncateStore =
true;
2454 SDLoc(
N), NewOps,
N->getMemOperand(),
2455 N->getIndexType(), TruncateStore);
2460 assert(OpNo == 1 &&
"Can only promote VECTOR_COMPRESS mask.");
2463 SDValue Passthru =
N->getOperand(2);
2464 SDValue Mask = PromoteTargetBoolean(
N->getOperand(1), VT);
2469 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2470 if (
N->getOpcode() == ISD::VP_TRUNCATE)
2472 N->getOperand(1),
N->getOperand(2));
2477 if (
N->getOpcode() == ISD::VP_UINT_TO_FP)
2479 ZExtPromotedInteger(
N->getOperand(0)),
2480 N->getOperand(1),
N->getOperand(2)),
2483 ZExtPromotedInteger(
N->getOperand(0))), 0);
2486SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(
SDNode *
N) {
2488 ZExtPromotedInteger(
N->getOperand(1))), 0);
2493 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2498SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(
SDNode *
N) {
2500 EVT VT =
N->getValueType(0);
2501 SDValue Op = GetPromotedInteger(
N->getOperand(0));
2503 Op = DAG.
getNode(ISD::VP_ZERO_EXTEND, dl, VT,
Op,
N->getOperand(1),
2506 N->getOperand(0).getValueType());
2510 SDValue Op2 = ZExtPromotedInteger(
N->getOperand(2));
2515SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(
SDNode *
N) {
2517 SDValue Op = ZExtPromotedInteger(
N->getOperand(0));
2522 bool IsStrict =
N->isStrictFPOpcode();
2534 SDValue Op = SExtPromotedInteger(
N->getOperand(1));
2544 unsigned OpOffset = IsStrict ? 1 : 0;
2547 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2548 "POWI exponent should match with sizeof(int) when doing the libcall.");
2551 SDValue Ops[2] = {
N->getOperand(0 + OpOffset),
N->getOperand(1 + OpOffset)};
2552 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(
2553 DAG, LC,
N->getValueType(0), Ops, CallOptions,
SDLoc(
N), Chain);
2554 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
2556 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2561 switch (
N->getOpcode()) {
2569 case ISD::VP_REDUCE_ADD:
2570 case ISD::VP_REDUCE_MUL:
2571 case ISD::VP_REDUCE_AND:
2572 case ISD::VP_REDUCE_OR:
2573 case ISD::VP_REDUCE_XOR:
2577 case ISD::VP_REDUCE_SMAX:
2578 case ISD::VP_REDUCE_SMIN:
2582 case ISD::VP_REDUCE_UMAX:
2583 case ISD::VP_REDUCE_UMIN:
2593 return GetPromotedInteger(V);
2595 return SExtPromotedInteger(V);
2597 return ZExtPromotedInteger(V);
2603 SDValue Op = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
2605 EVT OrigEltVT =
N->getOperand(0).getValueType().getVectorElementType();
2606 EVT InVT =
Op.getValueType();
2608 EVT ResVT =
N->getValueType(0);
2609 unsigned Opcode =
N->getOpcode();
2629 Op = ZExtPromotedInteger(
N->getOperand(0));
2632 Op = SExtPromotedInteger(
N->getOperand(0));
2648 Op = ZExtPromotedInteger(
N->getOperand(0));
2651 Op = SExtPromotedInteger(
N->getOperand(0));
2665SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(
SDNode *
N,
unsigned OpNo) {
2672 NewOps[2] = PromoteTargetBoolean(
Op,
N->getOperand(1).getValueType());
2676 assert(OpNo == 1 &&
"Unexpected operand for promotion");
2678 Op = PromoteIntOpVectorReduction(
N,
Op);
2682 EVT VT =
N->getValueType(0);
2683 EVT EltVT =
Op.getValueType().getScalarType();
2698 SDValue Op = ZExtPromotedInteger(
N->getOperand(1));
2702SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
2705 SDValue Operand =
N->getOperand(OpNo);
2711SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
2714 SDValue Operand =
N->getOperand(OpNo);
2720SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
2721 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2722 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2725 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2730SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(
SDNode *
N,
unsigned OpNo) {
2734 NewOps[OpNo] = SExtPromotedInteger(
N->getOperand(OpNo));
2738 assert((OpNo == 4 || OpNo == 5) &&
"Unexpected operand for promotion");
2740 NewOps[OpNo] = ZExtPromotedInteger(
N->getOperand(OpNo));
2752void DAGTypeLegalizer::ExpandIntegerResult(
SDNode *
N,
unsigned ResNo) {
2758 if (CustomLowerNode(
N,
N->getValueType(ResNo),
true))
2761 switch (
N->getOpcode()) {
2764 dbgs() <<
"ExpandIntegerResult #" << ResNo <<
": ";
2765 N->dump(&DAG);
dbgs() <<
"\n";
2811 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(
N),
Lo,
Hi);
break;
2838 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(
N);
2839 SplitInteger(Tmp.first,
Lo,
Hi);
2840 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
2848 N->getOperand(0),
N->getOperand(1),
N->getOperand(2),
N->getOperand(3),
2857 SplitInteger(Tmp,
Lo,
Hi);
2936 ExpandIntRes_Rotate(
N,
Lo,
Hi);
2941 ExpandIntRes_FunnelShift(
N,
Lo,
Hi);
2945 ExpandIntRes_VSCALE(
N,
Lo,
Hi);
2955std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(
SDNode *
Node) {
2956 unsigned Opc =
Node->getOpcode();
2962 EVT RetVT =
Node->getValueType(0);
2970 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2971 "Unexpected atomic op or value type!");
2975 Node->getOperand(0));
2980void DAGTypeLegalizer::ExpandShiftByConstant(
SDNode *
N,
const APInt &Amt,
2985 GetExpandedInteger(
N->getOperand(0), InL, InH);
2996 unsigned VTBits =
N->getValueType(0).getSizeInBits();
3000 if (Amt.
uge(VTBits)) {
3002 }
else if (Amt.
ugt(NVTBits)) {
3006 }
else if (Amt == NVTBits) {
3023 if (Amt.
uge(VTBits)) {
3025 }
else if (Amt.
ugt(NVTBits)) {
3029 }
else if (Amt == NVTBits) {
3046 if (Amt.
uge(VTBits)) {
3049 }
else if (Amt.
ugt(NVTBits)) {
3054 }
else if (Amt == NVTBits) {
3074bool DAGTypeLegalizer::
3076 unsigned Opc =
N->getOpcode();
3084 "Expanded integer type size not a power of two!");
3091 if (((Known.
Zero | Known.
One) & HighBitMask) == 0)
3096 GetExpandedInteger(In, InL, InH);
3150 Lo = DAG.
getNode(Opc, dl, NVT, InL, Amt);
3163bool DAGTypeLegalizer::
3170 "Expanded integer type size not a power of two!");
3175 GetExpandedInteger(
N->getOperand(0), InL, InH);
3187 switch (
N->getOpcode()) {
3202 DAG.
getSelect(dl, NVT, isShort, HiS, HiL));
3218 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3234 DAG.
getSelect(dl, NVT, isShort, LoS, LoL));
3255void DAGTypeLegalizer::ExpandIntRes_MINMAX(
SDNode *
N,
3264 unsigned NumBits =
N->getValueType(0).getScalarSizeInBits();
3265 unsigned NumHalfBits = NumBits / 2;
3268 SDValue LHSL, LHSH, RHSL, RHSH;
3269 GetExpandedInteger(LHS, LHSL, LHSH);
3270 GetExpandedInteger(RHS, RHSL, RHSH);
3283 SDValue LHSL, LHSH, RHSL, RHSH;
3284 GetExpandedInteger(LHS, LHSL, LHSH);
3285 GetExpandedInteger(RHS, RHSL, RHSH);
3287 EVT CCT = getSetCCResultType(NVT);
3296 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3300 const APInt *RHSVal =
nullptr;
3301 if (
auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3302 RHSVal = &RHSConst->getAPIntValue();
3309 SDValue LHSL, LHSH, RHSL, RHSH;
3310 GetExpandedInteger(LHS, LHSL, LHSH);
3311 GetExpandedInteger(RHS, RHSL, RHSH);
3313 EVT CCT = getSetCCResultType(NVT);
3319 Hi = DAG.
getNode(
N->getOpcode(),
DL, NVT, {LHSH, RHSH});
3338 switch (
N->getOpcode()) {
3365 EVT VT =
N->getValueType(0);
3366 EVT CCT = getSetCCResultType(VT);
3369 SplitInteger(Result,
Lo,
Hi);
3374 SplitInteger(ExpandedCMP,
Lo,
Hi);
3377void DAGTypeLegalizer::ExpandIntRes_ADDSUB(
SDNode *
N,
3381 SDValue LHSL, LHSH, RHSL, RHSH;
3382 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3383 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3386 SDValue LoOps[2] = { LHSL, RHSL };
3387 SDValue HiOps[3] = { LHSH, RHSH };
3396 HiOps[2] =
Lo.getValue(1);
3402 HiOps[2] =
Lo.getValue(1);
3424 HiOps[2] =
Lo.getValue(1);
3428 HiOps[2] =
Lo.getValue(1);
3441 EVT OvfVT = getSetCCResultType(NVT);
3481 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3484 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3487 Cmp = DAG.
getSetCC(dl, getSetCCResultType(NVT),
Lo, LoOps[0],
3505 DAG.
getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3519void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(
SDNode *
N,
3522 SDValue LHSL, LHSH, RHSL, RHSH;
3524 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3525 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3527 SDValue LoOps[2] = { LHSL, RHSL };
3528 SDValue HiOps[3] = { LHSH, RHSH };
3532 HiOps[2] =
Lo.getValue(1);
3536 HiOps[2] =
Lo.getValue(1);
3542 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3545void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(
SDNode *
N,
3548 SDValue LHSL, LHSH, RHSL, RHSH;
3550 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3551 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3553 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3554 SDValue HiOps[3] = { LHSH, RHSH };
3556 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3557 HiOps[2] =
Lo.getValue(1);
3558 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3562 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3565void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(
SDNode *
N,
3573 unsigned CarryOp, NoCarryOp;
3575 switch(
N->getOpcode()) {
3595 SDValue LHSL, LHSH, RHSL, RHSH;
3596 GetExpandedInteger(LHS, LHSL, LHSH);
3597 GetExpandedInteger(RHS, RHSL, RHSH);
3599 SDValue LoOps[2] = { LHSL, RHSL };
3600 SDValue HiOps[3] = { LHSH, RHSH };
3602 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3603 HiOps[2] =
Lo.getValue(1);
3604 Hi = DAG.
getNode(CarryOp, dl, VTList, HiOps);
3606 Ovf =
Hi.getValue(1);
3611 SplitInteger(Sum,
Lo,
Hi);
3622 DAG.
getSetCC(dl,
N->getValueType(1), LHS,
3633 ReplaceValueWith(
SDValue(
N, 1), Ovf);
3639 SDValue LHSL, LHSH, RHSL, RHSH;
3641 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3642 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3644 SDValue LoOps[3] = { LHSL, RHSL,
N->getOperand(2) };
3647 Lo = DAG.
getNode(
N->getOpcode(), dl, VTList, LoOps);
3648 HiOps[2] =
Lo.getValue(1);
3649 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, HiOps);
3653 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3656void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(
SDNode *
N,
3659 SDValue LHSL, LHSH, RHSL, RHSH;
3661 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
3662 GetExpandedInteger(
N->getOperand(1), RHSL, RHSH);
3668 Lo = DAG.
getNode(CarryOp, dl, VTList, { LHSL, RHSL,
N->getOperand(2) });
3669 Hi = DAG.
getNode(
N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3673 ReplaceValueWith(
SDValue(
N, 1),
Hi.getValue(1));
3676void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(
SDNode *
N,
3681 if (
Op.getValueType().bitsLE(NVT)) {
3688 assert(getTypeAction(
Op.getValueType()) ==
3690 "Only know how to promote this result!");
3693 "Operand over promoted?");
3695 SplitInteger(Res,
Lo,
Hi);
3699void DAGTypeLegalizer::ExpandIntRes_AssertSext(
SDNode *
N,
3702 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3703 EVT NVT =
Lo.getValueType();
3704 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3708 if (NVTBits < EVTBits) {
3711 EVTBits - NVTBits)));
3721void DAGTypeLegalizer::ExpandIntRes_AssertZext(
SDNode *
N,
3724 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3725 EVT NVT =
Lo.getValueType();
3726 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
3730 if (NVTBits < EVTBits) {
3733 EVTBits - NVTBits)));
3741void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(
SDNode *
N,
3744 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3749void DAGTypeLegalizer::ExpandIntRes_BSWAP(
SDNode *
N,
3752 GetExpandedInteger(
N->getOperand(0),
Hi,
Lo);
3761 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3762 EVT NVT =
Lo.getValueType();
3768void DAGTypeLegalizer::ExpandIntRes_Constant(
SDNode *
N,
3772 auto Constant = cast<ConstantSDNode>(
N);
3774 bool IsTarget =
Constant->isTargetOpcode();
3775 bool IsOpaque =
Constant->isOpaque();
3786 GetExpandedInteger(N0,
Lo,
Hi);
3787 EVT NVT =
Lo.getValueType();
3818 EVT VT =
N->getValueType(0);
3822 SplitInteger(Neg, NegLo, NegHi);
3830void DAGTypeLegalizer::ExpandIntRes_CTLZ(
SDNode *
N,
3834 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3835 EVT NVT =
Lo.getValueType();
3850void DAGTypeLegalizer::ExpandIntRes_CTPOP(
SDNode *
N,
3854 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3855 EVT NVT =
Lo.getValueType();
3861void DAGTypeLegalizer::ExpandIntRes_CTTZ(
SDNode *
N,
3865 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
3866 EVT NVT =
Lo.getValueType();
3895 ReplaceValueWith(
SDValue(
N, 1), Chain);
3903 Chain =
Op.getValue(1);
3912 EVT VT =
N->getValueType(0);
3916 bool IsStrict =
N->isStrictFPOpcode();
3918 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3920 Op = GetPromotedFloat(
Op);
3923 EVT OFPVT =
Op.getValueType();
3925 Op = GetSoftPromotedHalf(
Op);
3929 SplitInteger(
Op,
Lo,
Hi);
3933 if (
Op.getValueType() == MVT::bf16) {
3940 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fp-to-xint conversion!");
3943 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, VT,
Op,
3944 CallOptions, dl, Chain);
3945 SplitInteger(Tmp.first,
Lo,
Hi);
3948 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
3954 SplitInteger(Res,
Lo,
Hi);
3960 bool IsStrict =
N->isStrictFPOpcode();
3961 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
3965 "Input type needs to be promoted!");
3967 EVT VT =
Op.getValueType();
3969 if (VT == MVT::f16) {
3979 LC = RTLIB::LROUND_F32;
3980 else if (VT == MVT::f64)
3981 LC = RTLIB::LROUND_F64;
3982 else if (VT == MVT::f80)
3983 LC = RTLIB::LROUND_F80;
3984 else if (VT == MVT::f128)
3985 LC = RTLIB::LROUND_F128;
3986 else if (VT == MVT::ppcf128)
3987 LC = RTLIB::LROUND_PPCF128;
3988 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lround input type!");
3992 LC = RTLIB::LRINT_F32;
3993 else if (VT == MVT::f64)
3994 LC = RTLIB::LRINT_F64;
3995 else if (VT == MVT::f80)
3996 LC = RTLIB::LRINT_F80;
3997 else if (VT == MVT::f128)
3998 LC = RTLIB::LRINT_F128;
3999 else if (VT == MVT::ppcf128)
4000 LC = RTLIB::LRINT_PPCF128;
4001 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected lrint input type!");
4005 LC = RTLIB::LLROUND_F32;
4006 else if (VT == MVT::f64)
4007 LC = RTLIB::LLROUND_F64;
4008 else if (VT == MVT::f80)
4009 LC = RTLIB::LLROUND_F80;
4010 else if (VT == MVT::f128)
4011 LC = RTLIB::LLROUND_F128;
4012 else if (VT == MVT::ppcf128)
4013 LC = RTLIB::LLROUND_PPCF128;
4014 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llround input type!");
4018 LC = RTLIB::LLRINT_F32;
4019 else if (VT == MVT::f64)
4020 LC = RTLIB::LLRINT_F64;
4021 else if (VT == MVT::f80)
4022 LC = RTLIB::LLRINT_F80;
4023 else if (VT == MVT::f128)
4024 LC = RTLIB::LLRINT_F128;
4025 else if (VT == MVT::ppcf128)
4026 LC = RTLIB::LLRINT_PPCF128;
4027 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected llrint input type!");
4031 EVT RetVT =
N->getValueType(0);
4035 std::pair<SDValue, SDValue> Tmp = TLI.
makeLibCall(DAG, LC, RetVT,
4036 Op, CallOptions, dl,
4038 SplitInteger(Tmp.first,
Lo,
Hi);
4040 if (
N->isStrictFPOpcode())
4041 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
4044void DAGTypeLegalizer::ExpandIntRes_LOAD(
LoadSDNode *
N,
4046 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_LOAD?");
4049 ExpandRes_NormalLoad(
N,
Lo,
Hi);
4055 EVT VT =
N->getValueType(0);
4066 if (
N->getMemoryVT().bitsLE(NVT)) {
4067 EVT MemVT =
N->getMemoryVT();
4070 N->getOriginalAlign(), MMOFlags, AAInfo);
4073 Ch =
Lo.getValue(1);
4078 unsigned LoSize =
Lo.getValueSizeInBits();
4093 N->getOriginalAlign(), MMOFlags, AAInfo);
4095 unsigned ExcessBits =
4103 N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
4104 N->getOriginalAlign(), MMOFlags, AAInfo);
4113 EVT MemVT =
N->getMemoryVT();
4116 unsigned ExcessBits = (EBytes - IncrementSize)*8;
4122 N->getOriginalAlign(), MMOFlags, AAInfo);
4128 N->getPointerInfo().getWithOffset(IncrementSize),
4130 N->getOriginalAlign(), MMOFlags, AAInfo);
4154 ReplaceValueWith(
SDValue(
N, 1), Ch);
4157void DAGTypeLegalizer::ExpandIntRes_Logical(
SDNode *
N,
4161 GetExpandedInteger(
N->getOperand(0), LL, LH);
4162 GetExpandedInteger(
N->getOperand(1), RL, RH);
4167void DAGTypeLegalizer::ExpandIntRes_MUL(
SDNode *
N,
4169 EVT VT =
N->getValueType(0);
4174 GetExpandedInteger(
N->getOperand(0), LL, LH);
4175 GetExpandedInteger(
N->getOperand(1), RL, RH);
4185 LC = RTLIB::MUL_I16;
4186 else if (VT == MVT::i32)
4187 LC = RTLIB::MUL_I32;
4188 else if (VT == MVT::i64)
4189 LC = RTLIB::MUL_I64;
4190 else if (VT == MVT::i128)
4191 LC = RTLIB::MUL_I128;
4203 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4206 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
4218 ReplaceValueWith(
SDValue(
N, 1),
R.getValue(2));
4223 SplitInteger(Result,
Lo,
Hi);
4229 SplitInteger(Result,
Lo,
Hi);
4235 SplitInteger(Result,
Lo,
Hi);
4246 EVT VT =
N->getValueType(0);
4250 uint64_t Scale =
N->getConstantOperandVal(2);
4262 EVT BoolVT = getSetCCResultType(VT);
4287 SplitInteger(Result,
Lo,
Hi);
4293 assert(Scale <= VTSize &&
"Scale can't be larger than the value type size.");
4297 GetExpandedInteger(LHS, LL, LH);
4298 GetExpandedInteger(RHS, RL, RH);
4302 if (!TLI.
expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
4310 SplitInteger(LoTmp, Result[0], Result[1]);
4311 SplitInteger(HiTmp, Result[2], Result[3]);
4313 assert(
Result.size() == 4 &&
"Unexpected number of partlets in the result");
4316 assert((VTSize == NVTSize * 2) &&
"Expected the new value type to be half "
4317 "the size of the current value type");
4339 if (Scale % NVTSize) {
4355 if (Scale == VTSize)
4376 EVT BoolNVT = getSetCCResultType(NVT);
4379 if (Scale < NVTSize) {
4386 }
else if (Scale == NVTSize) {
4389 }
else if (Scale < VTSize) {
4397 "(and saturation can't happen with Scale==VTSize).");
4404 if (Scale < NVTSize) {
4409 unsigned OverflowBits = VTSize - Scale + 1;
4410 assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
4411 "Extent of overflow bits must start within HL");
4428 }
else if (Scale == NVTSize) {
4441 }
else if (Scale < VTSize) {
4444 unsigned OverflowBits = VTSize - Scale + 1;
4471 N->getConstantOperandVal(2), DAG);
4475 N->getConstantOperandVal(2), TLI, DAG);
4476 SplitInteger(Res,
Lo,
Hi);
4479void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(
SDNode *
Node,
4482 "Node has unexpected Opcode");
4497 SDValue LHSL, LHSH, RHSL, RHSH;
4498 GetExpandedInteger(LHS, LHSL, LHSH);
4499 GetExpandedInteger(RHS, RHSL, RHSH);
4503 Hi = DAG.
getNode(CarryOp, dl, VTList, { LHSH, RHSH,
Lo.getValue(1) });
4505 Ovf =
Hi.getValue(1);
4512 SplitInteger(Sum,
Lo,
Hi);
4537 EVT VT =
LHS.getValueType();
4540 SignsMatch = DAG.
getNOT(dl, SignsMatch, VT);
4544 EVT OType =
Node->getValueType(1);
4552void DAGTypeLegalizer::ExpandIntRes_SDIV(
SDNode *
N,
4554 EVT VT =
N->getValueType(0);
4556 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4566 LC = RTLIB::SDIV_I16;
4567 else if (VT == MVT::i32)
4568 LC = RTLIB::SDIV_I32;
4569 else if (VT == MVT::i64)
4570 LC = RTLIB::SDIV_I64;
4571 else if (VT == MVT::i128)
4572 LC = RTLIB::SDIV_I128;
4573 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SDIV!");
4577 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4580void DAGTypeLegalizer::ExpandIntRes_ShiftThroughStack(
SDNode *
N,
SDValue &
Lo,
4583 SDValue Shiftee =
N->getOperand(0);
4590 bool ShiftByByteMultiple =
4595 if (!ShiftByByteMultiple)
4599 assert(VTBitWidth % 8 == 0 &&
"Shifting a not byte multiple value?");
4600 unsigned VTByteWidth = VTBitWidth / 8;
4602 "Shiftee type size is not a power of two!");
4603 unsigned StackSlotByteWidth = 2 * VTByteWidth;
4604 unsigned StackSlotBitWidth = 8 * StackSlotByteWidth;
4610 Align StackSlotAlignment(1);
4618 cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex());
4623 unsigned WideningOpc =
4625 Init = DAG.
getNode(WideningOpc, dl, StackSlotVT, Shiftee);
4632 Ch = DAG.
getStore(Ch, dl,
Init, StackPtr, StackPtrInfo, StackSlotAlignment);
4638 if (ShiftByByteMultiple)
4639 Flags.setExact(
true);
4651 bool WillIndexUpwards =
N->getOpcode() !=
ISD::SHL;
4653 WillIndexUpwards = !WillIndexUpwards;
4656 if (WillIndexUpwards) {
4660 StackPtr, DAG.
getConstant(VTByteWidth, dl, PtrTy), dl);
4661 ByteOffset = DAG.
getNegative(ByteOffset, dl, ShAmtVT);
4670 VT, dl, Ch, AdjStackPtr,
4675 if (!ShiftByByteMultiple) {
4678 Res = DAG.
getNode(
N->getOpcode(), dl, VT, Res, ShAmtRem);
4682 SplitInteger(Res,
Lo,
Hi);
4685void DAGTypeLegalizer::ExpandIntRes_Shift(
SDNode *
N,
4687 EVT VT =
N->getValueType(0);
4688 unsigned Opc =
N->getOpcode();
4694 return ExpandShiftByConstant(
N, CN->getAPIntValue(),
Lo,
Hi);
4698 if (ExpandShiftWithKnownAmountBit(
N,
Lo,
Hi))
4717 const bool LegalOrCustom =
4721 unsigned ExpansionFactor = 1;
4723 for (
EVT TmpVT = NVT;;) {
4725 if (NewTMPVT == TmpVT)
4735 return ExpandIntRes_ShiftThroughStack(
N,
Lo,
Hi);
4737 if (LegalOrCustom &&
4741 GetExpandedInteger(
N->getOperand(0), LHSL, LHSH);
4747 SDValue ShiftOp =
N->getOperand(1);
4752 SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4754 Hi =
Lo.getValue(1);
4764 LC = RTLIB::SHL_I16;
4765 else if (VT == MVT::i32)
4766 LC = RTLIB::SHL_I32;
4767 else if (VT == MVT::i64)
4768 LC = RTLIB::SHL_I64;
4769 else if (VT == MVT::i128)
4770 LC = RTLIB::SHL_I128;
4774 LC = RTLIB::SRL_I16;
4775 else if (VT == MVT::i32)
4776 LC = RTLIB::SRL_I32;
4777 else if (VT == MVT::i64)
4778 LC = RTLIB::SRL_I64;
4779 else if (VT == MVT::i128)
4780 LC = RTLIB::SRL_I128;
4785 LC = RTLIB::SRA_I16;
4786 else if (VT == MVT::i32)
4787 LC = RTLIB::SRA_I32;
4788 else if (VT == MVT::i64)
4789 LC = RTLIB::SRA_I64;
4790 else if (VT == MVT::i128)
4791 LC = RTLIB::SRA_I128;
4798 SDValue Ops[2] = {
N->getOperand(0), ShAmt};
4801 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4805 if (!ExpandShiftWithUnknownAmountBit(
N,
Lo,
Hi))
4809void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(
SDNode *
N,
4814 if (
Op.getValueType().bitsLE(NVT)) {
4825 assert(getTypeAction(
Op.getValueType()) ==
4827 "Only know how to promote this result!");
4830 "Operand over promoted?");
4832 SplitInteger(Res,
Lo,
Hi);
4840void DAGTypeLegalizer::
4843 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
4844 EVT EVT = cast<VTSDNode>(
N->getOperand(1))->getVT();
4866void DAGTypeLegalizer::ExpandIntRes_SREM(
SDNode *
N,
4868 EVT VT =
N->getValueType(0);
4870 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
4880 LC = RTLIB::SREM_I16;
4881 else if (VT == MVT::i32)
4882 LC = RTLIB::SREM_I32;
4883 else if (VT == MVT::i64)
4884 LC = RTLIB::SREM_I64;
4885 else if (VT == MVT::i128)
4886 LC = RTLIB::SREM_I128;
4887 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported SREM!");
4891 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
4894void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(
SDNode *
N,
4906void DAGTypeLegalizer::ExpandIntRes_XMULO(
SDNode *
N,
4908 EVT VT =
N->getValueType(0);
4927 SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4928 GetExpandedInteger(LHS, LHSLow, LHSHigh);
4929 GetExpandedInteger(RHS, RHSLow, RHSHigh);
4931 EVT BitVT =
N->getValueType(1);
4956 SplitInteger(Three,
Lo,
Hi);
4960 ReplaceValueWith(
SDValue(
N, 1), Overflow);
4971 LC = RTLIB::MULO_I32;
4972 else if (VT == MVT::i64)
4973 LC = RTLIB::MULO_I64;
4974 else if (VT == MVT::i128)
4975 LC = RTLIB::MULO_I128;
4988 SplitInteger(
Mul, MulLo, MulHi);
4994 SplitInteger(MulLo,
Lo,
Hi);
4995 ReplaceValueWith(
SDValue(
N, 1), Overflow);
5008 EVT ArgVT =
Op.getValueType();
5012 Entry.IsSExt =
true;
5013 Entry.IsZExt =
false;
5014 Args.push_back(Entry);
5020 Entry.IsSExt =
true;
5021 Entry.IsZExt =
false;
5022 Args.push_back(Entry);
5041 ReplaceValueWith(
SDValue(
N, 1), Ofl);
5044void DAGTypeLegalizer::ExpandIntRes_UDIV(
SDNode *
N,
5046 EVT VT =
N->getValueType(0);
5048 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5057 if (isa<ConstantSDNode>(
N->getOperand(1))) {
5060 if (isTypeLegal(NVT)) {
5062 GetExpandedInteger(
N->getOperand(0), InL, InH);
5074 LC = RTLIB::UDIV_I16;
5075 else if (VT == MVT::i32)
5076 LC = RTLIB::UDIV_I32;
5077 else if (VT == MVT::i64)
5078 LC = RTLIB::UDIV_I64;
5079 else if (VT == MVT::i128)
5080 LC = RTLIB::UDIV_I128;
5081 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UDIV!");
5084 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5087void DAGTypeLegalizer::ExpandIntRes_UREM(
SDNode *
N,
5089 EVT VT =
N->getValueType(0);
5091 SDValue Ops[2] = {
N->getOperand(0),
N->getOperand(1) };
5100 if (isa<ConstantSDNode>(
N->getOperand(1))) {
5103 if (isTypeLegal(NVT)) {
5105 GetExpandedInteger(
N->getOperand(0), InL, InH);
5117 LC = RTLIB::UREM_I16;
5118 else if (VT == MVT::i32)
5119 LC = RTLIB::UREM_I32;
5120 else if (VT == MVT::i64)
5121 LC = RTLIB::UREM_I64;
5122 else if (VT == MVT::i128)
5123 LC = RTLIB::UREM_I128;
5124 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unsupported UREM!");
5127 SplitInteger(TLI.
makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
Lo,
Hi);
5130void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(
SDNode *
N,
5135 if (
Op.getValueType().bitsLE(NVT)) {
5142 assert(getTypeAction(
Op.getValueType()) ==
5144 "Only know how to promote this result!");
5147 "Operand over promoted?");
5149 SplitInteger(Res,
Lo,
Hi);
5157void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(
SDNode *
N,
5160 EVT VT = cast<AtomicSDNode>(
N)->getMemoryVT();
5165 cast<AtomicSDNode>(
N)->getMemoryVT(), VTs,
N->getOperand(0),
5166 N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(
N)->getMemOperand());
5172void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(
SDNode *
N,
5177 SplitInteger(Res,
Lo,
Hi);
5180void DAGTypeLegalizer::ExpandIntRes_Rotate(
SDNode *
N,
5186 N->getOperand(0),
N->getOperand(1));
5187 SplitInteger(Res,
Lo,
Hi);
5194 GetExpandedInteger(
N->getOperand(0), In3, In4);
5195 GetExpandedInteger(
N->getOperand(1), In1, In2);
5199 unsigned Opc =
N->getOpcode();
5202 EVT ShAmtCCVT = getSetCCResultType(ShAmtVT);
5219 Lo = DAG.
getNode(Opc,
DL, HalfVT, Select2, Select1, NewShAmt);
5220 Hi = DAG.
getNode(Opc,
DL, HalfVT, Select3, Select2, NewShAmt);
5225 EVT VT =
N->getValueType(0);
5235 SplitInteger(Res,
Lo,
Hi);
5246bool DAGTypeLegalizer::ExpandIntegerOperand(
SDNode *
N,
unsigned OpNo) {
5250 if (CustomLowerNode(
N,
N->getOperand(OpNo).getValueType(),
false))
5253 switch (
N->getOpcode()) {
5256 dbgs() <<
"ExpandIntegerOperand Op #" << OpNo <<
": ";
5257 N->dump(&DAG);
dbgs() <<
"\n";
5262 case ISD::BR_CC: Res = ExpandIntOp_BR_CC(
N);
break;
5267 case ISD::EXPERIMENTAL_VP_SPLAT:
5270 case ISD::SETCC: Res = ExpandIntOp_SETCC(
N);
break;
5276 case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(
N), OpNo);
break;
5283 case ISD::ROTR: Res = ExpandIntOp_Shift(
N);
break;
5288 case ISD::UCMP: Res = ExpandIntOp_CMP(
N);
break;
5292 Res = ExpandIntOp_STACKMAP(
N, OpNo);
5295 Res = ExpandIntOp_PATCHPOINT(
N, OpNo);
5297 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
5298 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
5299 Res = ExpandIntOp_VP_STRIDED(
N, OpNo);
5304 if (!Res.
getNode())
return false;
5312 "Invalid operand expansion");
5314 ReplaceValueWith(
SDValue(
N, 0), Res);
5320void DAGTypeLegalizer::IntegerExpandSetCCOperands(
SDValue &NewLHS,
5324 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5325 GetExpandedInteger(NewLHS, LHSLo, LHSHi);
5326 GetExpandedInteger(NewRHS, RHSLo, RHSHi);
5346 if ((CCCode ==
ISD::SETLT && CST->isZero()) ||
5347 (CCCode ==
ISD::SETGT && CST->isAllOnes())) {
5379 RHSLo, LowCC,
false, DagCombineInfo, dl);
5386 RHSHi, CCCode,
false, DagCombineInfo, dl);
5399 if ((EqAllowed && (HiCmpC && HiCmpC->
isZero())) ||
5401 ((HiCmpC && HiCmpC->
isOne()) || (LoCmpC && LoCmpC->
isZero())))) {
5410 if (LHSHi == RHSHi) {
5423 if (HasSETCCCARRY) {
5426 bool FlipOperands =
false;
5454 false, DagCombineInfo, dl);
5463 SDValue NewLHS =
N->getOperand(2), NewRHS =
N->getOperand(3);
5464 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(1))->get();
5465 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5477 N->getOperand(4)), 0);
5481 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5482 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(4))->get();
5483 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5494 N->getOperand(2),
N->getOperand(3),
5499 SDValue NewLHS =
N->getOperand(0), NewRHS =
N->getOperand(1);
5500 ISD::CondCode CCCode = cast<CondCodeSDNode>(
N->getOperand(2))->get();
5501 IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode,
SDLoc(
N));
5506 "Unexpected setcc expansion!");
5522 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
5523 GetExpandedInteger(LHS, LHSLo, LHSHi);
5524 GetExpandedInteger(RHS, RHSLo, RHSHi);
5537 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5547 GetExpandedInteger(
N->getOperand(1),
Lo,
Hi);
5560 GetExpandedInteger(
N->getOperand(0),
Lo,
Hi);
5565 bool IsStrict =
N->isStrictFPOpcode();
5569 SDValue Op =
N->getOperand(IsStrict ? 1 : 0);
5570 EVT DstVT =
N->getValueType(0);
5573 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
5574 "Don't know how to expand this XINT_TO_FP!");
5577 std::pair<SDValue, SDValue> Tmp =
5583 ReplaceValueWith(
SDValue(
N, 1), Tmp.second);
5584 ReplaceValueWith(
SDValue(
N, 0), Tmp.first);
5589 assert(!
N->isAtomic() &&
"Should have been a ATOMIC_STORE?");
5592 return ExpandOp_NormalStore(
N, OpNo);
5595 assert(OpNo == 1 &&
"Can only expand the stored value so far");
5597 EVT VT =
N->getOperand(1).getValueType();
5608 if (
N->getMemoryVT().bitsLE(NVT)) {
5609 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5611 N->getMemoryVT(),
N->getOriginalAlign(), MMOFlags,
5617 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5620 N->getOriginalAlign(), MMOFlags, AAInfo);
5622 unsigned ExcessBits =
5630 N->getPointerInfo().getWithOffset(IncrementSize),
5631 NEVT,
N->getOriginalAlign(), MMOFlags, AAInfo);
5637 GetExpandedInteger(
N->getValue(),
Lo,
Hi);
5639 EVT ExtVT =
N->getMemoryVT();
5642 unsigned ExcessBits = (EBytes - IncrementSize)*8;
5660 N->getOriginalAlign(), MMOFlags, AAInfo);
5666 N->getPointerInfo().getWithOffset(IncrementSize),
5668 N->getOriginalAlign(), MMOFlags, AAInfo);
5674 GetExpandedInteger(
N->getOperand(0), InL, InH);
5683 N->getOperand(0),
N->getOperand(2),
N->getOperand(1),
5684 cast<AtomicSDNode>(
N)->getMemOperand());
5688SDValue DAGTypeLegalizer::ExpandIntOp_VP_STRIDED(
SDNode *
N,
unsigned OpNo) {
5689 assert((
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
5690 (
N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
5694 GetExpandedInteger(NewOps[OpNo], NewOps[OpNo],
Hi);
5699SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SPLICE(
SDNode *
N) {
5702 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5703 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5709SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(
SDNode *
N) {
5712 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5713 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5722SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(
SDNode *
N) {
5724 EVT OutVT =
N->getValueType(0);
5726 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5730 SDValue BaseIdx =
N->getOperand(1);
5757 SDValue Ops[] = {GetWidenedVector(InOp0), BaseIdx};
5766 SDValue Ops[] = { GetPromotedInteger(InOp0), BaseIdx };
5770 "Promoted operand has an element type greater than result");
5783 InOp0 = GetPromotedInteger(InOp0);
5791 for (
unsigned i = 0; i != OutNumElems; ++i) {
5796 N->getOperand(0),
Index);
5805SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_SUBVECTOR(
SDNode *
N) {
5806 EVT OutVT =
N->getValueType(0);
5808 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5820 Vec = GetPromotedInteger(Vec);
5826SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_REVERSE(
SDNode *
N) {
5829 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5835SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(
SDNode *
N) {
5837 EVT VT =
N->getValueType(0);
5842 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
5843 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
5850 EVT OutVT =
N->getValueType(0);
5852 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5853 unsigned NumElems =
N->getNumOperands();
5861 for (
unsigned i = 0; i != NumElems; ++i) {
5863 EVT OpVT =
Op.getValueType();
5868 if (OpVT.
bitsLT(NOutVTElem)) {
5874 ExtOpc = NOutExtOpc;
5887 assert(!
N->getOperand(0).getValueType().isVector() &&
5888 "Input must be a scalar");
5890 EVT OutVT =
N->getValueType(0);
5892 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5896 if (
N->isVPOpcode())
5897 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op,
N->getOperand(1),
5900 return DAG.
getNode(
N->getOpcode(), dl, NOutVT,
Op);
5905 EVT OutVT =
N->getValueType(0);
5908 "Type must be promoted to a scalable vector type");
5909 const APInt &StepVal =
N->getConstantOperandAPInt(0);
5914SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(
SDNode *
N) {
5917 EVT OutVT =
N->getValueType(0);
5919 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
5921 unsigned NumOperands =
N->getNumOperands();
5926 SDUse *MaxSizedValue = std::max_element(
5928 EVT AVT = A.getValueType().getVectorElementType();
5929 EVT BVT = B.getValueType().getVectorElementType();
5930 return AVT.getScalarSizeInBits() < BVT.getScalarSizeInBits();
5936 for (
unsigned I = 0;
I < NumOperands; ++
I) {
5938 EVT OpVT =
Op.getValueType();
5940 Op = GetPromotedInteger(
Op);
5943 "Unhandled legalization type");
5960 unsigned NumElem =
N->getOperand(0).getValueType().getVectorNumElements();
5961 assert(NumElem * NumOperands == NumOutElem &&
5962 "Unexpected number of elements");
5966 for (
unsigned i = 0; i < NumOperands; ++i) {
5969 Op = GetPromotedInteger(
Op);
5970 EVT SclrTy =
Op.getValueType().getVectorElementType();
5971 assert(NumElem ==
Op.getValueType().getVectorNumElements() &&
5972 "Unexpected number of elements");
5974 for (
unsigned j = 0;
j < NumElem; ++
j) {
5984SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(
SDNode *
N) {
5985 EVT VT =
N->getValueType(0);
5987 assert(NVT.
isVector() &&
"This type must be promoted to a vector type");
5995 if (getTypeAction(
N->getOperand(0).getValueType())
5999 switch(
N->getOpcode()) {
6001 Promoted = SExtPromotedInteger(
N->getOperand(0));
6004 Promoted = ZExtPromotedInteger(
N->getOperand(0));
6007 Promoted = GetPromotedInteger(
N->getOperand(0));
6012 return DAG.
getNode(
N->getOpcode(), dl, NVT, Promoted);
6016 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->getOperand(0));
6019SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(
SDNode *
N) {
6020 EVT OutVT =
N->getValueType(0);
6022 assert(NOutVT.
isVector() &&
"This type must be promoted to a vector type");
6027 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6030 NOutVTElem,
N->getOperand(1));
6032 V0, ConvElem,
N->getOperand(2));
6040 return DAG.
getNode(
N->getOpcode(), dl, NVT,
N->ops());
6048 SDValue Start = PromoteIntOpVectorReduction(
N,
N->getOperand(0));
6049 return DAG.
getNode(
N->getOpcode(),
DL, Start.getValueType(), Start,
6050 N->getOperand(1),
N->getOperand(2),
N->getOperand(3));
6057 assert(
N->getNumValues() == 3 &&
"Expected 3 values for PATCHPOINT");
6071SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(
SDNode *
N) {
6073 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6085SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_SUBVECTOR(
SDNode *
N) {
6090 SDValue V1 = GetPromotedInteger(
N->getOperand(1));
6100SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(
SDNode *
N) {
6102 SDValue V0 = GetPromotedInteger(
N->getOperand(0));
6105 N->getValueType(0).getVectorNumElements());
6110SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(
SDNode *
N) {
6113 EVT ResVT =
N->getValueType(0);
6114 unsigned NumElems =
N->getNumOperands();
6119 for (
unsigned OpIdx = 0; OpIdx < NumElems; ++OpIdx) {
6121 unsigned OpNumElts =
Op.getValueType().getVectorMinNumElements();
6129 EVT RetSclrTy =
N->getValueType(0).getVectorElementType();
6135 for (
unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
6137 EVT SclrTy =
Incoming->getValueType(0).getVectorElementType();
6138 unsigned NumElem =
Incoming->getValueType(0).getVectorNumElements();
6140 for (
unsigned i=0; i<NumElem; ++i) {
6152SDValue DAGTypeLegalizer::ExpandIntOp_STACKMAP(
SDNode *
N,
unsigned OpNo) {
6165 for (
unsigned I = 0;
I < OpNo;
I++)
6168 EVT Ty =
Op.getValueType();
6180 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6185 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
6191SDValue DAGTypeLegalizer::ExpandIntOp_PATCHPOINT(
SDNode *
N,
unsigned OpNo) {
6204 for (
unsigned I = 0;
I < OpNo;
I++)
6207 EVT Ty =
Op.getValueType();
6219 for (
unsigned I = OpNo + 1;
I <
N->getNumOperands();
I++)
6224 for (
unsigned ResNum = 0; ResNum <
N->getNumValues(); ResNum++)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static bool isSigned(unsigned int Opcode)
static SDValue SaturateWidenedDIVFIX(SDValue V, SDLoc &dl, unsigned SatW, bool Signed, const TargetLowering &TLI, SelectionDAG &DAG)
static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT, SDLoc DL, SelectionDAG &DAG)
static SDValue earlyExpandDIVFIX(SDNode *N, SDValue LHS, SDValue RHS, unsigned Scale, const TargetLowering &TLI, SelectionDAG &DAG, unsigned SatW=0)
static unsigned getExtendForIntVecReduction(SDNode *N)
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
unsigned countLeadingOnes() const
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
bool intersects(const APInt &RHS) const
This operation tests if there are any pairs of corresponding bits between this APInt and RHS that are...
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
unsigned countTrailingZeros() const
unsigned countLeadingZeros() const
APInt sext(unsigned width) const
Sign extend to a new width.
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
unsigned countTrailingOnes() const
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
This is an SDNode representing atomic operations.
const APInt & getValue() const
Return the constant as an APInt value reference.
const ConstantInt * getConstantIntValue() const
uint64_t getZExtValue() const
This is an important base class in LLVM.
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
This class represents an Operation in the Expression.
bool isLittleEndian() const
Layout endianness...
This class is used to represent ISD::LOAD nodes.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
Flags
Flags values. These may be or'd together.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Represents a use of a SDNode.
EVT getValueType() const
Convenience function for get().getValueType().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
uint64_t getScalarValueSizeInBits() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getVPZeroExtendInReg(SDValue Op, SDValue Mask, SDValue EVL, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT, unsigned Opcode)
Convert Op, which must be of integer type, to the integer type VT, by either any/sign/zero-extending ...
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue getExternalSymbol(const char *Sym, EVT VT)
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
const TargetLibraryInfo & getLibInfo() const
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
MachineFunction & getMachineFunction() const
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
ArrayRef< int > getMask() const
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
@ TypeScalarizeScalableVector
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT, std::optional< MVT > RegisterVT=std::nullopt) const
Return the number of registers that this ValueType will eventually require.
LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT, unsigned Scale) const
Some fixed point operations may be natively supported by the target but only for specific scales.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
virtual ISD::NodeType getExtendForAtomicCmpSwapArg() const
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
BooleanContent
Enum that describes how the target represents true/false values.
@ ZeroOrOneBooleanContent
@ UndefinedBooleanContent
@ ZeroOrNegativeOneBooleanContent
virtual ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
LegalizeAction getOperationAction(unsigned Op, EVT VT) const
Return how this operation should be treated: either it is legal, needs to be promoted to a larger siz...
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US][ADD|SUB]SAT.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandCTLZ(SDNode *N, SelectionDAG &DAG) const
Expand CTLZ/CTLZ_ZERO_UNDEF nodes.
SDValue expandBITREVERSE(SDNode *N, SelectionDAG &DAG) const
Expand BITREVERSE nodes.
SDValue expandCTTZ(SDNode *N, SelectionDAG &DAG) const
Expand CTTZ/CTTZ_ZERO_UNDEF nodes.
SDValue expandShlSat(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]SHLSAT.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandABS(SDNode *N, SelectionDAG &DAG, bool IsNegative=false) const
Expand ABS nodes.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue expandCTPOP(SDNode *N, SelectionDAG &DAG) const
Expand CTPOP nodes.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
SDValue expandBSWAP(SDNode *N, SelectionDAG &DAG) const
Expand BSWAP nodes.
bool expandDIVREMByConstant(SDNode *N, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, SDValue LL=SDValue(), SDValue LH=SDValue()) const
Attempt to expand an n-bit div/rem/divrem by constant using a n/2-bit urem by constant and other arit...
void forceExpandWideMUL(SelectionDAG &DAG, const SDLoc &dl, bool Signed, EVT WideVT, const SDValue LL, const SDValue LH, const SDValue RL, const SDValue RH, SDValue &Lo, SDValue &Hi) const
forceExpandWideMUL - Unconditionally expand a MUL into either a libcall or brute force via a wide mul...
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
SDValue expandFixedPointDiv(unsigned Opcode, const SDLoc &dl, SDValue LHS, SDValue RHS, unsigned Scale, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]DIVFIX[SAT].
SDValue expandROT(SDNode *N, bool AllowVectorOps, SelectionDAG &DAG) const
Expand rotations.
SDValue expandCMP(SDNode *Node, SelectionDAG &DAG) const
Method for building the DAG expansion of ISD::[US]CMP.
bool expandMUL_LOHI(unsigned Opcode, EVT VT, const SDLoc &dl, SDValue LHS, SDValue RHS, SmallVectorImpl< SDValue > &Result, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL or [US]MUL_LOHI of n-bit values into two or four nodes, respectively,...
SDValue expandAVG(SDNode *N, SelectionDAG &DAG) const
Expand vector/scalar AVGCEILS/AVGCEILU/AVGFLOORS/AVGFLOORU nodes.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
constexpr bool hasKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns true if there exists a value X where RHS.multiplyCoefficientBy(X) will result in a value whos...
constexpr ScalarTy getKnownScalarFactor(const FixedOrScalableQuantity &RHS) const
Returns a value X where RHS.multiplyCoefficientBy(X) will result in a value whose quantity matches ou...
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ MLOAD
Masked load and store - consecutive vector load and store operations with additional mask operand tha...
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ VECTOR_INTERLEAVE
VECTOR_INTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the same...
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ VECTOR_REVERSE
VECTOR_REVERSE(VECTOR) - Returns a vector, of the same type as VECTOR, whose elements are shuffled us...
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ MGATHER
Masked gather and scatter - load and store operations for a vector of random addresses with additiona...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ SCMP
[US]CMP - 3-way comparison of signed or unsigned integers.
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ VECTOR_COMPRESS
VECTOR_COMPRESS(Vec, Mask, Passthru) consecutively place vector elements based on mask e....
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ VECTOR_DEINTERLEAVE
VECTOR_DEINTERLEAVE(VEC1, VEC2) - Returns two vectors with all input and output vectors having the sa...
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isUnsignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs an unsigned comparison when used with intege...
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
This struct is a compact representation of a valid (non-zero power of two) alignment.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
ElementCount getVectorElementCount() const
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
uint64_t getScalarSizeInBits() const
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
bool isZero() const
Returns true if value is all zero.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)