56#define DEBUG_TYPE "legalizedag"
62struct FloatSignAsInt {
85class SelectionDAGLegalize {
97 EVT getSetCCResultType(
EVT VT)
const {
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
168 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
170 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
207 dbgs() <<
" with: "; New->dump(&DAG));
210 "Replacing one node with another that produces a different number "
214 UpdatedNodes->
insert(New);
220 dbgs() <<
" with: "; New->dump(&DAG));
224 UpdatedNodes->
insert(New.getNode());
225 ReplacedNode(Old.getNode());
232 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
236 UpdatedNodes->
insert(New[i].getNode());
243 dbgs() <<
" with: "; New->dump(&DAG));
247 UpdatedNodes->
insert(New.getNode());
248 ReplacedNode(Old.getNode());
258SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
263 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
265 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
267 if (NumEltsGrowth == 1)
268 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
271 for (
unsigned i = 0; i != NumMaskElts; ++i) {
273 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
280 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
281 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
282 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
319 TLI.ShouldShrinkFPConstant(OrigVT)) {
329 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
330 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
339 OrigVT, dl, DAG.getEntryNode(), CPIdx,
347 EVT VT =
CP->getValueType(0);
348 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
349 TLI.getPointerTy(DAG.getDataLayout()));
350 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
352 VT, dl, DAG.getEntryNode(), CPIdx,
361SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(
SDValue Vec,
379 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
383 DAG.getEntryNode(), dl, Tmp1, StackPtr,
386 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
389 Ch = DAG.getTruncStore(
390 Ch, dl, Tmp2, StackPtr2,
394 DAG.getMachineFunction(), SPFI));
415 for (
unsigned i = 0; i != NumElts; ++i)
416 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
418 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
421 return PerformInsertVectorEltInMemory(Vec, Val,
Idx, dl);
450 bitcastToAPInt().zextOrTrunc(32),
452 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
453 ST->getOriginalAlign(), MMOFlags, AAInfo);
461 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
462 ST->getOriginalAlign(), MMOFlags, AAInfo);
465 if (TLI.isTypeLegal(
MVT::i32) && !
ST->isVolatile()) {
472 if (DAG.getDataLayout().isBigEndian())
475 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
476 ST->getOriginalAlign(), MMOFlags, AAInfo);
478 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
479 ST->getPointerInfo().getWithOffset(4),
480 ST->getOriginalAlign(), MMOFlags, AAInfo);
489void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
498 if (!
ST->isTruncatingStore()) {
500 if (
SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
501 ReplaceNode(ST, OptStore);
506 MVT VT =
Value.getSimpleValueType();
507 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
509 case TargetLowering::Legal: {
512 EVT MemVT =
ST->getMemoryVT();
514 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
515 *
ST->getMemOperand())) {
518 ReplaceNode(
SDValue(ST, 0), Result);
523 case TargetLowering::Custom: {
530 case TargetLowering::Promote: {
533 "Can only promote stores to same size type");
536 ST->getOriginalAlign(), MMOFlags, AAInfo);
546 EVT StVT =
ST->getMemoryVT();
549 auto &
DL = DAG.getDataLayout();
551 if (StWidth != StSize) {
556 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
558 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
559 ST->getOriginalAlign(), MMOFlags, AAInfo);
565 unsigned LogStWidth =
Log2_32(StWidthBits);
567 unsigned RoundWidth = 1 << LogStWidth;
568 assert(RoundWidth < StWidthBits);
569 unsigned ExtraWidth = StWidthBits - RoundWidth;
570 assert(ExtraWidth < RoundWidth);
571 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
572 "Store size not an integral number of bytes!");
576 unsigned IncrementSize;
578 if (
DL.isLittleEndian()) {
581 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
582 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
585 IncrementSize = RoundWidth / 8;
589 DAG.getConstant(RoundWidth, dl,
590 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
591 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
592 ST->getPointerInfo().getWithOffset(IncrementSize),
593 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
600 DAG.getConstant(ExtraWidth, dl,
601 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
602 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
603 ST->getOriginalAlign(), MMOFlags, AAInfo);
606 IncrementSize = RoundWidth / 8;
608 DAG.getConstant(IncrementSize, dl,
609 Ptr.getValueType()));
611 ST->getPointerInfo().getWithOffset(IncrementSize),
612 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
619 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
621 case TargetLowering::Legal: {
622 EVT MemVT =
ST->getMemoryVT();
625 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
626 *
ST->getMemOperand())) {
628 ReplaceNode(
SDValue(ST, 0), Result);
632 case TargetLowering::Custom: {
638 case TargetLowering::Expand:
640 "Vector Stores are handled in LegalizeVectorOps");
645 if (TLI.isTypeLegal(StVT)) {
648 ST->getOriginalAlign(), MMOFlags, AAInfo);
653 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
656 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
657 ST->getOriginalAlign(), MMOFlags, AAInfo);
666void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
675 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
676 MVT VT =
Node->getSimpleValueType(0);
680 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
682 case TargetLowering::Legal: {
683 EVT MemVT =
LD->getMemoryVT();
687 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
688 *
LD->getMemOperand())) {
689 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
693 case TargetLowering::Custom:
694 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
700 case TargetLowering::Promote: {
701 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
703 "Can only promote loads to same size type");
705 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
713 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
714 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
716 UpdatedNodes->insert(RVal.
getNode());
717 UpdatedNodes->insert(RChain.
getNode());
725 EVT SrcVT =
LD->getMemoryVT();
739 TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
MVT::i1) ==
740 TargetLowering::Promote)) {
754 Chain,
Ptr,
LD->getPointerInfo(), NVT,
755 LD->getOriginalAlign(), MMOFlags, AAInfo);
763 Result, DAG.getValueType(SrcVT));
767 Result.getValueType(), Result,
768 DAG.getValueType(SrcVT));
776 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
778 unsigned RoundWidth = 1 << LogSrcWidth;
779 assert(RoundWidth < SrcWidthBits);
780 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
781 assert(ExtraWidth < RoundWidth);
782 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
783 "Load size not an integral number of bytes!");
787 unsigned IncrementSize;
788 auto &
DL = DAG.getDataLayout();
790 if (
DL.isLittleEndian()) {
794 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
798 IncrementSize = RoundWidth / 8;
800 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
801 LD->getPointerInfo().getWithOffset(IncrementSize),
802 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
812 DAG.getConstant(RoundWidth, dl,
813 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
821 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
822 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
826 IncrementSize = RoundWidth / 8;
829 LD->getPointerInfo().getWithOffset(IncrementSize),
830 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
840 DAG.getConstant(ExtraWidth, dl,
841 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
849 bool isCustom =
false;
850 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
853 case TargetLowering::Custom:
856 case TargetLowering::Legal:
868 EVT MemVT =
LD->getMemoryVT();
870 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
871 *
LD->getMemOperand())) {
872 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
877 case TargetLowering::Expand: {
878 EVT DestVT =
Node->getValueType(0);
883 if (TLI.isTypeLegal(SrcVT) ||
884 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
891 SrcVT,
LD->getMemOperand());
894 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
895 Chain =
Load.getValue(1);
909 Ptr, ISrcVT,
LD->getMemOperand());
911 Chain =
Result.getValue(1);
917 "Vector Loads are handled in LegalizeVectorOps");
924 "EXTLOAD should always be supported!");
928 Node->getValueType(0),
930 LD->getMemOperand());
935 Result, DAG.getValueType(SrcVT));
937 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
939 Chain =
Result.getValue(1);
950 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
952 UpdatedNodes->insert(
Value.getNode());
953 UpdatedNodes->insert(Chain.
getNode());
960void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
969 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
970 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
971 TargetLowering::TypeLegal &&
972 "Unexpected illegal type!");
975 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
976 TargetLowering::TypeLegal ||
979 "Unexpected illegal type!");
984 bool SimpleFinishLegalizing =
true;
985 switch (
Node->getOpcode()) {
993 Action = TLI.getOperationAction(
Node->getOpcode(),
994 Node->getValueType(0));
997 Action = TLI.getOperationAction(
Node->getOpcode(),
998 Node->getValueType(0));
999 if (Action != TargetLowering::Promote)
1011 Action = TLI.getOperationAction(
Node->getOpcode(),
1012 Node->getOperand(0).getValueType());
1024 Action = TLI.getOperationAction(
Node->getOpcode(),
1025 Node->getOperand(1).getValueType());
1028 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1029 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1033 Action = TLI.getOperationAction(
Node->getOpcode(),
1034 Node->getOperand(2).getValueType());
1043 unsigned Opc =
Node->getOpcode();
1048 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1050 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1054 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1056 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1057 Action = TLI.getCondCodeAction(CCCode, OpVT);
1058 if (Action == TargetLowering::Legal) {
1060 Action = TLI.getOperationAction(
Node->getOpcode(),
1061 Node->getValueType(0));
1063 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1071 SimpleFinishLegalizing =
false;
1078 SimpleFinishLegalizing =
false;
1091 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1092 if (Action == TargetLowering::Legal)
1093 Action = TargetLowering::Expand;
1103 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1104 if (Action == TargetLowering::Legal)
1105 Action = TargetLowering::Custom;
1117 Action = TargetLowering::Legal;
1120 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1121 if (Action == TargetLowering::Expand) {
1125 Node->getOperand(0));
1132 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1133 if (Action == TargetLowering::Expand) {
1137 Node->getOperand(0));
1151 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1161 unsigned Scale =
Node->getConstantOperandVal(2);
1162 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1163 Node->getValueType(0), Scale);
1167 Action = TLI.getOperationAction(
Node->getOpcode(),
1168 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1171 Action = TLI.getOperationAction(
Node->getOpcode(),
1172 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1174 case ISD::VP_SCATTER:
1175 Action = TLI.getOperationAction(
1177 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1180 Action = TLI.getOperationAction(
1182 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1184 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1185 Action = TLI.getOperationAction(
1187 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1203 Action = TLI.getOperationAction(
1204 Node->getOpcode(),
Node->getOperand(0).getValueType());
1208 case ISD::VP_REDUCE_FADD:
1209 case ISD::VP_REDUCE_FMUL:
1210 case ISD::VP_REDUCE_ADD:
1211 case ISD::VP_REDUCE_MUL:
1212 case ISD::VP_REDUCE_AND:
1213 case ISD::VP_REDUCE_OR:
1214 case ISD::VP_REDUCE_XOR:
1215 case ISD::VP_REDUCE_SMAX:
1216 case ISD::VP_REDUCE_SMIN:
1217 case ISD::VP_REDUCE_UMAX:
1218 case ISD::VP_REDUCE_UMIN:
1219 case ISD::VP_REDUCE_FMAX:
1220 case ISD::VP_REDUCE_FMIN:
1221 case ISD::VP_REDUCE_SEQ_FADD:
1222 case ISD::VP_REDUCE_SEQ_FMUL:
1223 Action = TLI.getOperationAction(
1224 Node->getOpcode(),
Node->getOperand(1).getValueType());
1228 Action = TLI.getCustomOperationAction(*
Node);
1230 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1235 if (SimpleFinishLegalizing) {
1237 switch (
Node->getOpcode()) {
1256 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1276 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1282 if (NewNode !=
Node) {
1283 ReplaceNode(
Node, NewNode);
1287 case TargetLowering::Legal:
1290 case TargetLowering::Custom:
1298 if (
Node->getNumValues() == 1) {
1303 "Type mismatch for custom legalized operation");
1311 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1316 "Type mismatch for custom legalized operation");
1320 ReplaceNode(
Node, ResultVals.
data());
1325 case TargetLowering::Expand:
1326 if (ExpandNode(
Node))
1329 case TargetLowering::LibCall:
1330 ConvertNodeToLibcall(
Node);
1332 case TargetLowering::Promote:
1338 switch (
Node->getOpcode()) {
1351 return LegalizeLoadOps(
Node);
1353 return LegalizeStoreOps(
Node);
1357SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1377 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1378 ST->getValue() != Vec)
1383 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1392 ST->hasPredecessor(
Op.getNode()))
1405 StackPtr = DAG.CreateStackTemporary(VecVT);
1406 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1411 Align ElementAlignment =
1412 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1413 DAG.getDataLayout().getPrefTypeAlign(
1414 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1416 if (
Op.getValueType().isVector()) {
1417 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1418 Op.getValueType(),
Idx);
1419 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1422 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1423 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1429 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1435 NewLoadOperands[0] = Ch;
1437 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1441SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1442 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1453 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1458 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1462 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT,
Idx);
1466 Ch, dl, Part, SubStackPtr,
1470 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1476 "Unexpected opcode!");
1482 EVT VT =
Node->getValueType(0);
1484 :
Node->getOperand(0).getValueType();
1486 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1487 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1494 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1498 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1499 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1502 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1504 if (
Node->getOperand(i).isUndef())
continue;
1506 unsigned Offset = TypeByteSize*i;
1511 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1515 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1520 if (!Stores.
empty())
1523 StoreChain = DAG.getEntryNode();
1526 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1532void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1535 EVT FloatVT =
Value.getValueType();
1537 State.FloatVT = FloatVT;
1540 if (TLI.isTypeLegal(IVT)) {
1543 State.SignBit = NumBits - 1;
1552 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1557 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1558 State.FloatPointerInfo);
1565 State.IntPointerInfo = State.FloatPointerInfo;
1568 unsigned ByteOffset = (NumBits / 8) - 1;
1575 State.IntPtr = IntPtr;
1576 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1577 State.IntPointerInfo,
MVT::i8);
1584SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1591 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1592 State.IntPointerInfo,
MVT::i8);
1593 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1594 State.FloatPointerInfo);
1603 FloatSignAsInt SignAsInt;
1604 getSignAsIntValue(SignAsInt,
DL, Sign);
1606 EVT IntVT = SignAsInt.IntValue.getValueType();
1607 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1613 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1614 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1617 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1619 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1623 FloatSignAsInt MagAsInt;
1624 getSignAsIntValue(MagAsInt,
DL, Mag);
1625 EVT MagVT = MagAsInt.IntValue.getValueType();
1626 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1631 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1632 EVT ShiftVT = IntVT;
1638 if (ShiftAmount > 0) {
1639 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1641 }
else if (ShiftAmount < 0) {
1642 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1652 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1658 FloatSignAsInt SignAsInt;
1659 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1660 EVT IntVT = SignAsInt.IntValue.getValueType();
1663 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1668 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1676 EVT FloatVT =
Value.getValueType();
1683 FloatSignAsInt ValueAsInt;
1684 getSignAsIntValue(ValueAsInt,
DL,
Value);
1685 EVT IntVT = ValueAsInt.IntValue.getValueType();
1686 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1689 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1692void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1694 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1695 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1696 " not tell us which reg is the stack pointer!");
1698 EVT VT =
Node->getValueType(0);
1706 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1709 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1711 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1719 if (Alignment > StackAlign)
1721 DAG.getConstant(-Alignment.
value(), dl, VT));
1722 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1724 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1736 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1744 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1747 if ((SrcVT.
bitsGT(SlotVT) &&
1748 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1749 (SlotVT.
bitsLT(DestVT) &&
1750 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1754 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1755 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1767 if (SrcVT.
bitsGT(SlotVT))
1768 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1772 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1776 if (SlotVT.
bitsEq(DestVT))
1777 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1780 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1793 SDValue Ch = DAG.getTruncStore(
1794 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1796 Node->getValueType(0).getVectorElementType());
1798 Node->getValueType(0), dl, Ch, StackPtr,
1805 unsigned NumElems =
Node->getNumOperands();
1807 EVT VT =
Node->getValueType(0);
1819 for (
unsigned i = 0; i < NumElems; ++i) {
1830 while (IntermedVals.
size() > 2) {
1831 NewIntermedVals.
clear();
1832 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1838 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1839 IntermedVals[i+1].second.
size());
1842 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1845 FinalIndices.
push_back(IntermedVals[i].second[j]);
1847 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1849 ShuffleVec[k] = NumElems + j;
1850 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1856 IntermedVals[i+1].first,
1861 std::make_pair(
Shuffle, std::move(FinalIndices)));
1866 if ((IntermedVals.
size() & 1) != 0)
1869 IntermedVals.
swap(NewIntermedVals);
1873 "Invalid number of intermediate vectors");
1874 SDValue Vec1 = IntermedVals[0].first;
1876 if (IntermedVals.
size() > 1)
1877 Vec2 = IntermedVals[1].first;
1882 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1883 ShuffleVec[IntermedVals[0].second[i]] = i;
1884 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1885 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1899 unsigned NumElems =
Node->getNumOperands();
1902 EVT VT =
Node->getValueType(0);
1903 EVT OpVT =
Node->getOperand(0).getValueType();
1908 bool isOnlyLowElement =
true;
1909 bool MoreThanTwoValues =
false;
1911 for (
unsigned i = 0; i < NumElems; ++i) {
1916 isOnlyLowElement =
false;
1917 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1922 }
else if (!Value2.
getNode()) {
1925 }
else if (V != Value1 && V != Value2) {
1926 MoreThanTwoValues =
true;
1931 return DAG.getUNDEF(VT);
1933 if (isOnlyLowElement)
1939 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1941 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1944 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
1963 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1964 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1966 VT, dl, DAG.getEntryNode(), CPIdx,
1972 for (
unsigned i = 0; i < NumElems; ++i) {
1973 if (
Node->getOperand(i).isUndef())
1978 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
1979 if (!MoreThanTwoValues) {
1981 for (
unsigned i = 0; i < NumElems; ++i) {
1985 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
1987 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
1994 Vec2 = DAG.getUNDEF(VT);
1997 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2007 return ExpandVectorBuildThroughStack(
Node);
2012 EVT VT =
Node->getValueType(0);
2015 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2027 EVT ArgVT =
Op.getValueType();
2031 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2032 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2033 Args.push_back(Entry);
2035 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2036 TLI.getPointerTy(DAG.getDataLayout()));
2038 EVT RetVT =
Node->getValueType(0);
2045 SDValue InChain = DAG.getEntryNode();
2050 const Function &
F = DAG.getMachineFunction().getFunction();
2052 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2053 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2058 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2061 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy,
Callee,
2063 .setTailCall(isTailCall)
2064 .setSExtResult(signExtend)
2065 .setZExtResult(!signExtend)
2066 .setIsPostTypeLegalization(
true);
2068 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2073 return DAG.getRoot();
2080void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2083 if (LC == RTLIB::UNKNOWN_LIBCALL)
2086 if (
Node->isStrictFPOpcode()) {
2087 EVT RetVT =
Node->getValueType(0);
2091 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2094 Node->getOperand(0));
2096 Results.push_back(Tmp.second);
2104void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2112 Call_F32, Call_F64, Call_F80,
2113 Call_F128, Call_PPCF128);
2124 switch (
Node->getSimpleValueType(0).SimpleTy) {
2126 case MVT::i8: LC = Call_I8;
break;
2127 case MVT::i16: LC = Call_I16;
break;
2128 case MVT::i32: LC = Call_I32;
break;
2129 case MVT::i64: LC = Call_I64;
break;
2137void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2144 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2146 Call_F32, Call_F64, Call_F80,
2147 Call_F128, Call_PPCF128);
2153SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2155 unsigned Opcode =
Node->getOpcode();
2159 switch (
Node->getSimpleValueType(0).SimpleTy) {
2161 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2162 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2163 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2164 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2171 SDValue InChain = DAG.getEntryNode();
2173 EVT RetVT =
Node->getValueType(0);
2179 EVT ArgVT =
Op.getValueType();
2185 Args.push_back(Entry);
2189 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2191 Entry.Ty =
RetTy->getPointerTo();
2194 Args.push_back(Entry);
2196 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2197 TLI.getPointerTy(DAG.getDataLayout()));
2203 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy,
Callee,
2208 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2220 switch (
Node->getSimpleValueType(0).SimpleTy) {
2222 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2223 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2224 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2225 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2249SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2252 switch (
Node->getSimpleValueType(0).SimpleTy) {
2254 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2255 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2256 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2257 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2264 SDValue InChain = DAG.getEntryNode();
2266 EVT RetVT =
Node->getValueType(0);
2273 Entry.Node =
Node->getOperand(0);
2275 Entry.IsSExt =
false;
2276 Entry.IsZExt =
false;
2277 Args.push_back(Entry);
2280 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2281 Entry.Node = SinPtr;
2282 Entry.Ty =
RetTy->getPointerTo();
2283 Entry.IsSExt =
false;
2284 Entry.IsZExt =
false;
2285 Args.push_back(Entry);
2288 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2289 Entry.Node = CosPtr;
2290 Entry.Ty =
RetTy->getPointerTo();
2291 Entry.IsSExt =
false;
2292 Entry.IsZExt =
false;
2293 Args.push_back(Entry);
2295 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2296 TLI.getPointerTy(DAG.getDataLayout()));
2300 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2304 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2320 EVT DestVT =
Node->getValueType(0);
2322 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2333 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2344 DAG.getConstant(0x80000000u, dl,
MVT::i32));
2350 if (DAG.getDataLayout().isBigEndian())
2353 SDValue MemChain = DAG.getEntryNode();
2356 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2368 SDValue Bias = DAG.getConstantFP(
2369 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2370 : llvm::bit_cast<double>(0x4330000000000000ULL),
2375 if (
Node->isStrictFPOpcode()) {
2377 {
Node->getOperand(0), Load, Bias});
2380 std::pair<SDValue, SDValue> ResultPair;
2382 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2383 Result = ResultPair.first;
2384 Chain = ResultPair.second;
2390 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2401 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2416 EVT SetCCVT = getSetCCResultType(SrcVT);
2418 SDValue SignBitTest = DAG.getSetCC(
2419 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2421 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2422 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2424 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2429 if (
Node->isStrictFPOpcode()) {
2432 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2434 {
Node->getOperand(0), InCvt });
2442 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2443 Fast->setFlags(Flags);
2444 Flags.setNoFPExcept(
true);
2452 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2456 if (!TLI.isOperationLegalOrCustom(
2463 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2465 "Cannot perform lossless SINT_TO_FP!");
2468 if (
Node->isStrictFPOpcode()) {
2470 {
Node->getOperand(0), Op0 });
2474 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2477 Four = DAG.getIntPtrConstant(4, dl);
2478 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2479 SignSet, Four, Zero);
2488 case MVT::i8 : FF = 0x43800000ULL;
break;
2489 case MVT::i16: FF = 0x47800000ULL;
break;
2490 case MVT::i32: FF = 0x4F800000ULL;
break;
2491 case MVT::i64: FF = 0x5F800000ULL;
break;
2493 if (DAG.getDataLayout().isLittleEndian())
2499 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2500 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2505 FudgeInReg = DAG.getLoad(
2506 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2515 LegalizeOp(
Load.getNode());
2519 if (
Node->isStrictFPOpcode()) {
2521 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2522 Chain =
Result.getValue(1);
2526 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2534void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2536 bool IsStrict =
N->isStrictFPOpcode();
2539 EVT DestVT =
N->getValueType(0);
2540 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2547 unsigned OpToUse = 0;
2555 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2563 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2578 dl, NewInTy, LegalOp)});
2585 DAG.getNode(OpToUse, dl, DestVT,
2587 dl, NewInTy, LegalOp)));
2595void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2597 bool IsStrict =
N->isStrictFPOpcode();
2600 EVT DestVT =
N->getValueType(0);
2601 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2603 EVT NewOutTy = DestVT;
2605 unsigned OpToUse = 0;
2615 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2620 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2630 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2632 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2647 unsigned Opcode =
Node->getOpcode();
2650 EVT NewOutTy =
Node->getValueType(0);
2655 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2662 Node->getOperand(1));
2668 EVT VT =
Op.getValueType();
2669 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2674 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
2680 DAG.getConstant(1ULL << (--i), dl, ShVT));
2685 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2688bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
2692 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2694 switch (
Node->getOpcode()) {
2696 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
2700 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
2705 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
2710 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
2714 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
2718 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
2722 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
2727 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
2730 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
2731 TLI.getPointerTy(DAG.getDataLayout()));
2739 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2745 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
2761 DAG.getConstant(0, dl,
Node->getValueType(0)));
2774 SDValue Swap = DAG.getAtomicCmpSwap(
2776 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
2777 cast<AtomicSDNode>(
Node)->getMemOperand());
2785 cast<AtomicSDNode>(
Node)->getMemoryVT(),
2786 Node->getOperand(0),
2787 Node->getOperand(1),
Node->getOperand(2),
2788 cast<AtomicSDNode>(
Node)->getMemOperand());
2797 SDValue Res = DAG.getAtomicCmpSwap(
2799 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
2800 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
2806 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
2807 EVT OuterType =
Node->getValueType(0);
2808 switch (TLI.getExtendForAtomicOps()) {
2811 DAG.getValueType(AtomicType));
2813 Node->getOperand(2), DAG.getValueType(AtomicType));
2818 DAG.getValueType(AtomicType));
2819 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
2823 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2824 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
2842 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
2846 EVT VT =
Node->getValueType(0);
2848 Results.push_back(DAG.getConstant(0, dl, VT));
2851 Results.push_back(DAG.getConstantFP(0, dl, VT));
2858 if (TLI.isStrictFPEnabled())
2862 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
2863 Node->getValueType(0))
2864 == TargetLowering::Legal)
2868 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
2869 Node->getValueType(0), dl,
2870 Node->getOperand(0)))) {
2872 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
2878 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
2879 Node->getValueType(0), dl)))
2885 if (TLI.isStrictFPEnabled())
2889 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
2890 Node->getValueType(0))
2891 == TargetLowering::Legal)
2895 if ((Tmp1 = EmitStackConvert(
2896 Node->getOperand(1),
Node->getOperand(1).getValueType(),
2897 Node->getValueType(0), dl,
Node->getOperand(0)))) {
2899 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
2904 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
2905 Node->getOperand(0).getValueType(),
2906 Node->getValueType(0), dl)))
2923 DAG.getConstant(16, dl,
2924 TLI.getShiftAmountTy(
MVT::i32, DAG.getDataLayout())));
2936 DAG.getIntPtrConstant(0, dl,
true));
2939 DAG.getConstant(16, dl,
2940 TLI.getShiftAmountTy(
MVT::i32, DAG.getDataLayout())));
2947 Op = DAG.getAnyExtOrTrunc(Op, dl,
Node->getValueType(0));
2953 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
2954 EVT VT =
Node->getValueType(0);
2964 SDValue One = DAG.getConstant(1, dl, VT);
2974 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2977 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2979 Node->getOperand(0), ShiftCst);
2986 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
2988 if (
Node->isStrictFPOpcode())
2995 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
2997 if (
Node->isStrictFPOpcode())
3002 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3006 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3008 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3013 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3017 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3019 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3022 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3028 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3038 if (
Node->getOperand(0).getValueType().getVectorNumElements() == 1)
3041 Node->getOperand(0));
3043 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3053 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3059 Results.push_back(ExpandINSERT_VECTOR_ELT(
Node->getOperand(0),
3060 Node->getOperand(1),
3061 Node->getOperand(2), dl));
3067 EVT VT =
Node->getValueType(0);
3071 if (!TLI.isTypeLegal(EltVT)) {
3072 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3077 if (NewEltVT.
bitsLT(EltVT)) {
3093 unsigned int factor =
3101 for (
unsigned fi = 0; fi < factor; ++fi)
3105 for (
unsigned fi = 0; fi < factor; ++fi)
3116 for (
unsigned i = 0; i != NumElems; ++i) {
3124 DAG.getVectorIdxConstant(
Idx, dl)));
3128 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3131 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3138 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3142 EVT OpTy =
Node->getOperand(0).getValueType();
3143 if (
Node->getConstantOperandVal(1)) {
3147 TLI.getShiftAmountTy(
3148 Node->getOperand(0).getValueType(),
3149 DAG.getDataLayout())));
3154 Node->getOperand(0));
3162 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3163 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3164 Node->getValueType(0)));
3167 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3174 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3175 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3176 Node->getOperand(1)));
3182 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3195 auto CNode = cast<ConstantSDNode>(
Node->getOperand(1));
3198 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3209 switch (
Node->getOpcode()) {
3216 Tmp1 =
Node->getOperand(0);
3217 Tmp2 =
Node->getOperand(1);
3218 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3224 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3230 EVT VT =
Node->getValueType(0);
3236 SDVTList VTs = DAG.getVTList(VT, VT);
3265 {
Node->getOperand(0),
Node->getOperand(1)});
3275 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3277 MVT SVT =
Op.getSimpleValueType();
3284 DAG.getIntPtrConstant(0, dl,
true));
3295 DAG.shouldOptForSize()))
3296 Results.push_back(ExpandConstantFP(CFP,
true));
3301 Results.push_back(ExpandConstant(CP));
3305 EVT VT =
Node->getValueType(0);
3306 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3307 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3316 EVT VT =
Node->getValueType(0);
3318 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3319 "Don't know how to expand this subtraction!");
3320 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3321 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3327 if (TLI.expandREM(
Node, Tmp1, DAG))
3334 EVT VT =
Node->getValueType(0);
3335 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3336 SDVTList VTs = DAG.getVTList(VT, VT);
3337 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3338 Node->getOperand(1));
3345 unsigned ExpandOpcode =
3347 EVT VT =
Node->getValueType(0);
3348 SDVTList VTs = DAG.getVTList(VT, VT);
3350 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3351 Node->getOperand(1));
3359 MVT VT =
LHS.getSimpleValueType();
3360 unsigned MULHOpcode =
3363 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3365 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3371 assert(TLI.isTypeLegal(HalfType));
3372 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3374 TargetLowering::MulExpansionKind::Always)) {
3375 for (
unsigned i = 0; i < 2; ++i) {
3378 SDValue Shift = DAG.getConstant(
3380 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3389 EVT VT =
Node->getValueType(0);
3390 SDVTList VTs = DAG.getVTList(VT, VT);
3396 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3397 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3398 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3399 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3400 unsigned OpToUse = 0;
3401 if (HasSMUL_LOHI && !HasMULHS) {
3403 }
else if (HasUMUL_LOHI && !HasMULHU) {
3405 }
else if (HasSMUL_LOHI) {
3407 }
else if (HasUMUL_LOHI) {
3411 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3412 Node->getOperand(1)));
3420 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3421 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3422 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3423 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3428 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3436 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3441 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3448 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3458 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3465 Node->getOperand(0),
3466 Node->getOperand(1),
3467 Node->getConstantOperandVal(2),
3490 EVT VT =
LHS.getValueType();
3494 EVT CarryType =
Node->getValueType(1);
3495 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3497 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3500 SDValue One = DAG.getConstant(1, dl, VT);
3502 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3511 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3512 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3514 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3520 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3526 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3534 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3542 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
3554 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3555 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3560 Tmp1 =
Node->getOperand(0);
3561 Tmp2 =
Node->getOperand(1);
3562 Tmp3 =
Node->getOperand(2);
3566 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3568 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3581 EVT PTy = TLI.getPointerTy(TD);
3583 unsigned EntrySize =
3584 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3591 Index = DAG.getNode(
3596 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
3605 if (TLI.isJumpTableRelative()) {
3610 TLI.getPICJumpTableRelocBase(Table, DAG));
3613 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, DAG);
3620 Tmp1 =
Node->getOperand(0);
3621 Tmp2 =
Node->getOperand(1);
3627 Node->getOperand(2));
3639 Node->getOperand(2));
3647 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
3652 unsigned Offset = IsStrict ? 1 : 0;
3661 bool Legalized = TLI.LegalizeSetCCCondCode(
3662 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
3663 Chain, IsSignaling);
3671 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
3675 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
3677 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
3678 Tmp2, Tmp3,
Node->getFlags());
3686 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
3689 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
3701 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
3706 EVT VT =
Node->getValueType(0);
3709 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
3710 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
3717 Tmp1 =
Node->getOperand(0);
3718 Tmp2 =
Node->getOperand(1);
3719 Tmp3 =
Node->getOperand(2);
3720 Tmp4 =
Node->getOperand(3);
3721 EVT VT =
Node->getValueType(0);
3731 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3733 EVT CCVT = getSetCCResultType(CmpVT);
3735 Results.push_back(DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4));
3740 bool Legalized =
false;
3748 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3758 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3764 Legalized = TLI.LegalizeSetCCCondCode(
3768 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
3779 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
3784 Tmp2, Tmp3, Tmp4,
CC);
3794 Tmp1 =
Node->getOperand(0);
3795 Tmp2 =
Node->getOperand(2);
3796 Tmp3 =
Node->getOperand(3);
3797 Tmp4 =
Node->getOperand(1);
3799 bool Legalized = TLI.LegalizeSetCCCondCode(
3800 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
3803 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
3808 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
3811 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
3816 Tmp2, Tmp3,
Node->getOperand(4));
3831 EVT VT =
Node->getValueType(0);
3837 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
3840 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
3843 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
3865 Results.push_back(TLI.expandVecReduce(
Node, DAG));
3882 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
3888 switch (
Node->getOpcode()) {
3890 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3891 Node->getValueType(0))
3892 == TargetLowering::Legal)
3896 if (TLI.getStrictFPOperationAction(
3899 if (TLI.getStrictFPOperationAction(
3903 EVT VT =
Node->getValueType(0);
3907 {Node->getOperand(0), Node->getOperand(1), Neg},
3922 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3923 Node->getOperand(1).getValueType())
3924 == TargetLowering::Legal)
3941void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
3946 unsigned Opc =
Node->getOpcode();
3955 .setChain(
Node->getOperand(0))
3958 DAG.getExternalSymbol(
"__sync_synchronize",
3959 TLI.getPointerTy(DAG.getDataLayout())),
3962 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3964 Results.push_back(CallResult.second);
3986 EVT RetVT =
Node->getValueType(0);
3989 if (TLI.getLibcallName(LC)) {
3996 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3997 "Unexpected atomic op or value type!");
4001 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4004 Node->getOperand(0));
4006 Results.push_back(Tmp.second);
4014 .setChain(
Node->getOperand(0))
4016 DAG.getExternalSymbol(
4017 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4019 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4021 Results.push_back(CallResult.second);
4026 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4027 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4028 RTLIB::FMIN_PPCF128,
Results);
4032 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4033 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4034 RTLIB::FMAX_PPCF128,
Results);
4038 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4039 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4040 RTLIB::SQRT_PPCF128,
Results);
4043 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4044 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4045 RTLIB::CBRT_PPCF128,
Results);
4049 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4050 RTLIB::SIN_F80, RTLIB::SIN_F128,
4055 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4056 RTLIB::COS_F80, RTLIB::COS_F128,
4065 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4066 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4070 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4071 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4075 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4076 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4080 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4081 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4085 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4086 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4090 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4091 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4092 RTLIB::TRUNC_PPCF128,
Results);
4096 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4097 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4098 RTLIB::FLOOR_PPCF128,
Results);
4102 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4103 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4104 RTLIB::CEIL_PPCF128,
Results);
4108 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4109 RTLIB::RINT_F80, RTLIB::RINT_F128,
4110 RTLIB::RINT_PPCF128,
Results);
4114 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4115 RTLIB::NEARBYINT_F64,
4116 RTLIB::NEARBYINT_F80,
4117 RTLIB::NEARBYINT_F128,
4118 RTLIB::NEARBYINT_PPCF128,
Results);
4122 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4126 RTLIB::ROUND_PPCF128,
Results);
4130 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4131 RTLIB::ROUNDEVEN_F64,
4132 RTLIB::ROUNDEVEN_F80,
4133 RTLIB::ROUNDEVEN_F128,
4134 RTLIB::ROUNDEVEN_PPCF128,
Results);
4139 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4140 if (!TLI.getLibcallName(LC)) {
4142 if (
Node->isStrictFPOpcode()) {
4145 {
Node->getValueType(0),
Node->getValueType(1)},
4146 {
Node->getOperand(0),
Node->getOperand(2)});
4149 {
Node->getValueType(0),
Node->getValueType(1)},
4156 Node->getOperand(1));
4158 Node->getValueType(0),
4163 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4164 bool ExponentHasSizeOfInt =
4165 DAG.getLibInfo().getIntSize() ==
4166 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4167 if (!ExponentHasSizeOfInt) {
4170 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4171 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4179 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4180 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4184 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4185 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4187 RTLIB::LROUND_PPCF128,
Results);
4191 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4192 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4193 RTLIB::LLROUND_F128,
4194 RTLIB::LLROUND_PPCF128,
Results);
4198 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4199 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4201 RTLIB::LRINT_PPCF128,
Results);
4205 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4206 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4208 RTLIB::LLRINT_PPCF128,
Results);
4212 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4213 RTLIB::DIV_F80, RTLIB::DIV_F128,
4218 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4219 RTLIB::REM_F80, RTLIB::REM_F128,
4224 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4225 RTLIB::FMA_F80, RTLIB::FMA_F128,
4230 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4231 RTLIB::ADD_F80, RTLIB::ADD_F128,
4236 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4237 RTLIB::MUL_F80, RTLIB::MUL_F128,
4242 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false));
4248 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4249 DAG, RTLIB::FPEXT_F16_F32,
MVT::f32,
Node->getOperand(1), CallOptions,
4252 Results.push_back(Tmp.second);
4259 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4260 Results.push_back(ExpandLibCall(LC,
Node,
false));
4266 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4267 Results.push_back(ExpandLibCall(LC,
Node,
false));
4275 bool IsStrict =
Node->isStrictFPOpcode();
4278 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
4279 EVT RVT =
Node->getValueType(0);
4296 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4301 NVT,
Node->getOperand(IsStrict ? 1 : 0));
4304 std::pair<SDValue, SDValue> Tmp =
4305 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4308 Results.push_back(Tmp.second);
4316 bool IsStrict =
Node->isStrictFPOpcode();
4321 EVT SVT =
Op.getValueType();
4322 EVT RVT =
Node->getValueType(0);
4339 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4343 std::pair<SDValue, SDValue> Tmp =
4344 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4349 Results.push_back(Tmp.second);
4360 bool IsStrict =
Node->isStrictFPOpcode();
4363 EVT VT =
Node->getValueType(0);
4364 assert(cast<ConstantSDNode>(
Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4365 "Unable to expand as libcall if it is not normal rounding");
4368 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4371 std::pair<SDValue, SDValue> Tmp =
4372 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions,
SDLoc(
Node), Chain);
4375 Results.push_back(Tmp.second);
4381 Node->getValueType(0)),
4391 Node->getValueType(0));
4392 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4395 std::pair<SDValue, SDValue> Tmp =
4396 TLI.makeLibCall(DAG, LC,
Node->getValueType(0),
Node->getOperand(1),
4399 Results.push_back(Tmp.second);
4404 ExpandFPLibCall(
Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4405 RTLIB::SUB_F80, RTLIB::SUB_F128,
4411 RTLIB::SREM_I16, RTLIB::SREM_I32,
4412 RTLIB::SREM_I64, RTLIB::SREM_I128));
4417 RTLIB::UREM_I16, RTLIB::UREM_I32,
4418 RTLIB::UREM_I64, RTLIB::UREM_I128));
4423 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4424 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4429 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4430 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4440 RTLIB::MUL_I16, RTLIB::MUL_I32,
4441 RTLIB::MUL_I64, RTLIB::MUL_I128));
4444 switch (
Node->getSimpleValueType(0).SimpleTy) {
4448 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32,
Node,
false));
4451 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64,
Node,
false));
4454 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128,
Node,
false));
4462 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
4471 MVT EltVT,
MVT NewEltVT) {
4478void SelectionDAGLegalize::PromoteNode(
SDNode *
Node) {
4481 MVT OVT =
Node->getSimpleValueType(0);
4487 OVT =
Node->getOperand(0).getSimpleValueType();
4493 OVT =
Node->getOperand(1).getSimpleValueType();
4496 OVT =
Node->getOperand(2).getSimpleValueType();
4497 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), OVT);
4499 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4500 switch (
Node->getOpcode()) {
4520 DAG.getConstant(TopBit, dl, NVT));
4524 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
4538 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
4541 DAG.getConstant(DiffBits, dl,
4542 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4555 Results.push_back(PromoteLegalFP_TO_INT_SAT(