56 #define DEBUG_TYPE "legalizedag"
62 struct FloatSignAsInt {
85 class SelectionDAGLegalize {
97 EVT getSetCCResultType(
EVT VT)
const {
108 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111 void LegalizeOp(
SDNode *Node);
116 void LegalizeLoadOps(
SDNode *Node);
117 void LegalizeStoreOps(
SDNode *Node);
149 void ExpandArgFPLibCall(
SDNode *Node,
164 void ExpandDYNAMIC_STACKALLOC(
SDNode *Node,
166 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
168 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
190 bool ExpandNode(
SDNode *Node);
191 void ConvertNodeToLibcall(
SDNode *Node);
192 void PromoteNode(
SDNode *Node);
205 dbgs() <<
" with: "; New->dump(&DAG));
208 "Replacing one node with another that produces a different number "
212 UpdatedNodes->
insert(New);
218 dbgs() <<
" with: "; New->dump(&DAG));
222 UpdatedNodes->
insert(New.getNode());
234 UpdatedNodes->
insert(New[
i].getNode());
241 dbgs() <<
" with: "; New->dump(&DAG));
245 UpdatedNodes->
insert(New.getNode());
256 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
261 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
263 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
265 if (NumEltsGrowth == 1)
266 return DAG.getVectorShuffle(NVT, dl, N1, N2,
Mask);
269 for (
unsigned i = 0;
i != NumMaskElts; ++
i) {
271 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
273 NewMask.push_back(-1);
275 NewMask.push_back(Idx * NumEltsGrowth +
j);
278 assert(NewMask.size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
279 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
280 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
317 TLI.ShouldShrinkFPConstant(OrigVT)) {
327 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
337 OrigVT, dl, DAG.getEntryNode(), CPIdx,
345 EVT VT =
CP->getValueType(0);
346 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
347 TLI.getPointerTy(DAG.getDataLayout()));
350 VT, dl, DAG.getEntryNode(), CPIdx,
359 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(
SDValue Vec,
377 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
381 DAG.getEntryNode(), dl, Tmp1, StackPtr,
384 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
387 Ch = DAG.getTruncStore(
388 Ch, dl, Tmp2, StackPtr2,
392 DAG.getMachineFunction(), SPFI));
413 for (
unsigned i = 0;
i != NumElts; ++
i)
414 ShufOps.push_back(
i != InsertPos->getZExtValue() ?
i : NumElts);
416 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
419 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
448 bitcastToAPInt().zextOrTrunc(32),
450 return DAG.getStore(Chain, dl, Con, Ptr,
ST->getPointerInfo(),
451 ST->getOriginalAlign(), MMOFlags, AAInfo);
459 return DAG.getStore(Chain, dl, Con, Ptr,
ST->getPointerInfo(),
460 ST->getOriginalAlign(), MMOFlags, AAInfo);
463 if (TLI.isTypeLegal(
MVT::i32) && !
ST->isVolatile()) {
470 if (DAG.getDataLayout().isBigEndian())
473 Lo = DAG.getStore(Chain, dl, Lo, Ptr,
ST->getPointerInfo(),
474 ST->getOriginalAlign(), MMOFlags, AAInfo);
476 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
477 ST->getPointerInfo().getWithOffset(4),
478 ST->getOriginalAlign(), MMOFlags, AAInfo);
487 void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *Node) {
496 if (!
ST->isTruncatingStore()) {
498 if (
SDNode *OptStore = OptimizeFloatStore(
ST).getNode()) {
499 ReplaceNode(
ST, OptStore);
504 MVT VT =
Value.getSimpleValueType();
505 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
510 EVT MemVT =
ST->getMemoryVT();
512 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
513 *
ST->getMemOperand())) {
524 if (Res && Res !=
SDValue(Node, 0))
525 ReplaceNode(
SDValue(Node, 0), Res);
531 "Can only promote stores to same size type");
534 ST->getOriginalAlign(), MMOFlags, AAInfo);
535 ReplaceNode(
SDValue(Node, 0), Result);
544 EVT StVT =
ST->getMemoryVT();
547 auto &
DL = DAG.getDataLayout();
549 if (StWidth != StSize) {
554 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
556 DAG.getTruncStore(Chain, dl,
Value, Ptr,
ST->getPointerInfo(), NVT,
557 ST->getOriginalAlign(), MMOFlags, AAInfo);
558 ReplaceNode(
SDValue(Node, 0), Result);
563 unsigned LogStWidth =
Log2_32(StWidthBits);
565 unsigned RoundWidth = 1 << LogStWidth;
566 assert(RoundWidth < StWidthBits);
567 unsigned ExtraWidth = StWidthBits - RoundWidth;
568 assert(ExtraWidth < RoundWidth);
569 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
570 "Store size not an integral number of bytes!");
574 unsigned IncrementSize;
576 if (
DL.isLittleEndian()) {
579 Lo = DAG.getTruncStore(Chain, dl,
Value, Ptr,
ST->getPointerInfo(),
580 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
583 IncrementSize = RoundWidth / 8;
584 Ptr = DAG.getMemBasePlusOffset(Ptr,
TypeSize::Fixed(IncrementSize), dl);
587 DAG.getConstant(RoundWidth, dl,
588 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
589 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
590 ST->getPointerInfo().getWithOffset(IncrementSize),
591 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
598 DAG.getConstant(ExtraWidth, dl,
599 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
600 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
ST->getPointerInfo(), RoundVT,
601 ST->getOriginalAlign(), MMOFlags, AAInfo);
604 IncrementSize = RoundWidth / 8;
606 DAG.getConstant(IncrementSize, dl,
608 Lo = DAG.getTruncStore(Chain, dl,
Value, Ptr,
609 ST->getPointerInfo().getWithOffset(IncrementSize),
610 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
615 ReplaceNode(
SDValue(Node, 0), Result);
617 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
620 EVT MemVT =
ST->getMemoryVT();
623 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
624 *
ST->getMemOperand())) {
632 if (Res && Res !=
SDValue(Node, 0))
633 ReplaceNode(
SDValue(Node, 0), Res);
638 "Vector Stores are handled in LegalizeVectorOps");
643 if (TLI.isTypeLegal(StVT)) {
645 Result = DAG.getStore(Chain, dl,
Value, Ptr,
ST->getPointerInfo(),
646 ST->getOriginalAlign(), MMOFlags, AAInfo);
651 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
654 DAG.getTruncStore(Chain, dl,
Value, Ptr,
ST->getPointerInfo(), StVT,
655 ST->getOriginalAlign(), MMOFlags, AAInfo);
658 ReplaceNode(
SDValue(Node, 0), Result);
664 void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *Node) {
673 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
674 MVT VT = Node->getSimpleValueType(0);
678 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
681 EVT MemVT =
LD->getMemoryVT();
685 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
686 *
LD->getMemOperand())) {
687 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(
LD, DAG);
692 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
699 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
701 "Can only promote loads to same size type");
703 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr,
LD->getMemOperand());
709 if (RChain.
getNode() != Node) {
710 assert(RVal.
getNode() != Node &&
"Load must be completely replaced");
711 DAG.ReplaceAllUsesOfValueWith(
SDValue(Node, 0), RVal);
712 DAG.ReplaceAllUsesOfValueWith(
SDValue(Node, 1), RChain);
714 UpdatedNodes->insert(RVal.
getNode());
715 UpdatedNodes->insert(RChain.
getNode());
723 EVT SrcVT =
LD->getMemoryVT();
737 TLI.getLoadExtAction(ExtType, Node->getValueType(0),
MVT::i1) ==
751 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
752 Chain, Ptr,
LD->getPointerInfo(), NVT,
753 LD->getOriginalAlign(), MMOFlags, AAInfo);
761 Result, DAG.getValueType(SrcVT));
765 Result.getValueType(), Result,
766 DAG.getValueType(SrcVT));
774 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
776 unsigned RoundWidth = 1 << LogSrcWidth;
777 assert(RoundWidth < SrcWidthBits);
778 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
779 assert(ExtraWidth < RoundWidth);
780 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
781 "Load size not an integral number of bytes!");
785 unsigned IncrementSize;
786 auto &
DL = DAG.getDataLayout();
788 if (
DL.isLittleEndian()) {
791 Lo = DAG.getExtLoad(
ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
792 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
796 IncrementSize = RoundWidth / 8;
797 Ptr = DAG.getMemBasePlusOffset(Ptr,
TypeSize::Fixed(IncrementSize), dl);
798 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
799 LD->getPointerInfo().getWithOffset(IncrementSize),
800 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
810 DAG.getConstant(RoundWidth, dl,
811 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
814 Value = DAG.getNode(
ISD::OR, dl, Node->getValueType(0), Lo, Hi);
819 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
820 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
824 IncrementSize = RoundWidth / 8;
825 Ptr = DAG.getMemBasePlusOffset(Ptr,
TypeSize::Fixed(IncrementSize), dl);
826 Lo = DAG.getExtLoad(
ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
827 LD->getPointerInfo().getWithOffset(IncrementSize),
828 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
838 DAG.getConstant(ExtraWidth, dl,
839 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
842 Value = DAG.getNode(
ISD::OR, dl, Node->getValueType(0), Lo, Hi);
847 bool isCustom =
false;
848 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
866 EVT MemVT =
LD->getMemoryVT();
868 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
869 *
LD->getMemOperand())) {
870 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(
LD, DAG);
876 EVT DestVT = Node->getValueType(0);
881 if (TLI.isTypeLegal(SrcVT) ||
882 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
888 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
889 SrcVT,
LD->getMemOperand());
892 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0),
Load);
893 Chain =
Load.getValue(1);
907 Ptr, ISrcVT,
LD->getMemOperand());
909 Chain =
Result.getValue(1);
915 "Vector Loads are handled in LegalizeVectorOps");
922 "EXTLOAD should always be supported!");
926 Node->getValueType(0),
928 LD->getMemOperand());
933 Result, DAG.getValueType(SrcVT));
935 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
937 Chain =
Result.getValue(1);
946 assert(
Value.getNode() != Node &&
"Load must be completely replaced");
948 DAG.ReplaceAllUsesOfValueWith(
SDValue(Node, 1), Chain);
950 UpdatedNodes->insert(
Value.getNode());
951 UpdatedNodes->insert(Chain.
getNode());
958 void SelectionDAGLegalize::LegalizeOp(
SDNode *Node) {
967 for (
unsigned i = 0,
e = Node->getNumValues();
i !=
e; ++
i)
968 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(
i)) ==
970 "Unexpected illegal type!");
972 for (
const SDValue &
Op : Node->op_values())
973 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
977 "Unexpected illegal type!");
982 bool SimpleFinishLegalizing =
true;
983 switch (Node->getOpcode()) {
988 Action = TLI.getOperationAction(Node->getOpcode(),
MVT::Other);
991 Action = TLI.getOperationAction(Node->getOpcode(),
992 Node->getValueType(0));
995 Action = TLI.getOperationAction(Node->getOpcode(),
996 Node->getValueType(0));
998 Action = TLI.getOperationAction(Node->getOpcode(),
MVT::Other);
1008 Action = TLI.getOperationAction(Node->getOpcode(),
1009 Node->getOperand(0).getValueType());
1021 Action = TLI.getOperationAction(Node->getOpcode(),
1022 Node->getOperand(1).getValueType());
1025 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1026 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1030 Action = TLI.getOperationAction(Node->getOpcode(),
1031 Node->getOperand(2).getValueType());
1039 unsigned Opc = Node->getOpcode();
1043 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1045 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1049 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1051 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1052 Action = TLI.getCondCodeAction(CCCode, OpVT);
1055 Action = TLI.getOperationAction(Node->getOpcode(),
1056 Node->getValueType(0));
1058 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1066 SimpleFinishLegalizing =
false;
1073 SimpleFinishLegalizing =
false;
1086 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1098 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1105 Action = TLI.getOperationAction(Node->getOpcode(),
MVT::i64);
1115 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1120 Node->getOperand(0));
1121 ReplaceNode(Node, NewVal.
getNode());
1127 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1132 Node->getOperand(0));
1133 ReplaceNode(Node, NewVal.
getNode());
1146 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1156 unsigned Scale = Node->getConstantOperandVal(2);
1157 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1158 Node->getValueType(0), Scale);
1162 Action = TLI.getOperationAction(Node->getOpcode(),
1163 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1166 Action = TLI.getOperationAction(Node->getOpcode(),
1167 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1169 case ISD::VP_SCATTER:
1170 Action = TLI.getOperationAction(
1172 cast<VPScatterSDNode>(Node)->getValue().getValueType());
1175 Action = TLI.getOperationAction(
1177 cast<VPStoreSDNode>(Node)->getValue().getValueType());
1179 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1180 Action = TLI.getOperationAction(
1182 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1198 Action = TLI.getOperationAction(
1199 Node->getOpcode(), Node->getOperand(0).getValueType());
1203 case ISD::VP_REDUCE_FADD:
1204 case ISD::VP_REDUCE_FMUL:
1205 case ISD::VP_REDUCE_ADD:
1206 case ISD::VP_REDUCE_MUL:
1207 case ISD::VP_REDUCE_AND:
1208 case ISD::VP_REDUCE_OR:
1209 case ISD::VP_REDUCE_XOR:
1210 case ISD::VP_REDUCE_SMAX:
1211 case ISD::VP_REDUCE_SMIN:
1212 case ISD::VP_REDUCE_UMAX:
1213 case ISD::VP_REDUCE_UMIN:
1214 case ISD::VP_REDUCE_FMAX:
1215 case ISD::VP_REDUCE_FMIN:
1216 case ISD::VP_REDUCE_SEQ_FADD:
1217 case ISD::VP_REDUCE_SEQ_FMUL:
1218 Action = TLI.getOperationAction(
1219 Node->getOpcode(), Node->getOperand(1).getValueType());
1223 Action = TLI.getCustomOperationAction(*Node);
1225 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1230 if (SimpleFinishLegalizing) {
1232 switch (Node->getOpcode()) {
1241 SDValue Op0 = Node->getOperand(0);
1242 SDValue Op1 = Node->getOperand(1);
1251 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1262 SDValue Op0 = Node->getOperand(0);
1263 SDValue Op1 = Node->getOperand(1);
1264 SDValue Op2 = Node->getOperand(2);
1271 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1277 if (NewNode != Node) {
1278 ReplaceNode(Node, NewNode);
1293 if (Node->getNumValues() == 1) {
1298 "Type mismatch for custom legalized operation");
1301 ReplaceNode(
SDValue(Node, 0), Res);
1306 for (
unsigned i = 0,
e = Node->getNumValues();
i !=
e; ++
i) {
1311 "Type mismatch for custom legalized operation");
1315 ReplaceNode(Node, ResultVals.data());
1321 if (ExpandNode(Node))
1325 ConvertNodeToLibcall(Node);
1333 switch (Node->getOpcode()) {
1346 return LegalizeLoadOps(Node);
1348 return LegalizeStoreOps(Node);
1352 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1368 Worklist.push_back(Idx.
getNode());
1372 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1373 ST->getValue() != Vec)
1378 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1387 ST->hasPredecessor(
Op.getNode()))
1400 StackPtr = DAG.CreateStackTemporary(VecVT);
1401 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1407 if (
Op.getValueType().isVector()) {
1408 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1409 Op.getValueType(), Idx);
1413 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1414 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1420 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1426 NewLoadOperands[0] = Ch;
1428 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1432 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1433 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1444 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1449 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1453 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1457 Ch, dl, Part, SubStackPtr,
1461 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1464 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(
SDNode* Node) {
1467 "Unexpected opcode!");
1473 EVT VT = Node->getValueType(0);
1475 : Node->getOperand(0).getValueType();
1477 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1478 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1485 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1489 bool Truncate = isa<BuildVectorSDNode>(Node) &&
1490 MemVT.
bitsLT(Node->getOperand(0).getValueType());
1493 for (
unsigned i = 0,
e = Node->getNumOperands();
i !=
e; ++
i) {
1495 if (Node->getOperand(
i).isUndef())
continue;
1497 unsigned Offset = TypeByteSize*
i;
1502 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1503 Node->getOperand(
i), Idx,
1506 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(
i),
1511 if (!Stores.empty())
1514 StoreChain = DAG.getEntryNode();
1517 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1523 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1526 EVT FloatVT =
Value.getValueType();
1528 State.FloatVT = FloatVT;
1531 if (TLI.isTypeLegal(IVT)) {
1534 State.SignBit = NumBits - 1;
1540 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(),
MVT::i8);
1543 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1548 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1549 State.FloatPointerInfo);
1556 State.IntPointerInfo = State.FloatPointerInfo;
1559 unsigned ByteOffset = (NumBits / 8) - 1;
1566 State.IntPtr = IntPtr;
1567 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1568 State.IntPointerInfo,
MVT::i8);
1575 SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1582 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1583 State.IntPointerInfo,
MVT::i8);
1584 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1585 State.FloatPointerInfo);
1588 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(
SDNode *Node)
const {
1590 SDValue Mag = Node->getOperand(0);
1591 SDValue Sign = Node->getOperand(1);
1594 FloatSignAsInt SignAsInt;
1595 getSignAsIntValue(SignAsInt,
DL, Sign);
1597 EVT IntVT = SignAsInt.IntValue.getValueType();
1598 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1604 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1605 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1608 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1610 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1614 FloatSignAsInt MagAsInt;
1615 getSignAsIntValue(MagAsInt,
DL, Mag);
1616 EVT MagVT = MagAsInt.IntValue.getValueType();
1617 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1622 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1623 EVT ShiftVT = IntVT;
1629 if (ShiftAmount > 0) {
1630 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1632 }
else if (ShiftAmount < 0) {
1633 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1643 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1646 SDValue SelectionDAGLegalize::ExpandFNEG(
SDNode *Node)
const {
1649 FloatSignAsInt SignAsInt;
1650 getSignAsIntValue(SignAsInt,
DL, Node->getOperand(0));
1651 EVT IntVT = SignAsInt.IntValue.getValueType();
1654 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1659 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1662 SDValue SelectionDAGLegalize::ExpandFABS(
SDNode *Node)
const {
1667 EVT FloatVT =
Value.getValueType();
1669 SDValue Zero = DAG.getConstantFP(0.0,
DL, FloatVT);
1674 FloatSignAsInt ValueAsInt;
1675 getSignAsIntValue(ValueAsInt,
DL,
Value);
1676 EVT IntVT = ValueAsInt.IntValue.getValueType();
1677 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1680 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1683 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode* Node,
1685 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1686 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1687 " not tell us which reg is the stack pointer!");
1689 EVT VT = Node->getValueType(0);
1692 SDValue Tmp3 = Node->getOperand(2);
1697 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1700 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1709 Tmp1 = DAG.
getNode(Opc, dl, VT, SP, Size);
1712 DAG.getConstant(-
Alignment.value(), dl, VT));
1713 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1715 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl,
true),
1716 DAG.getIntPtrConstant(0, dl,
true),
SDValue(), dl);
1728 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1734 unsigned SrcSize =
SrcOp.getValueSizeInBits();
1738 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1741 if ((SrcSize > SlotSize &&
1742 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1743 (SlotSize < DestSize &&
1744 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1748 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1749 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1761 if (SrcSize > SlotSize)
1762 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1765 assert(SrcSize == SlotSize &&
"Invalid store");
1767 DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1771 if (SlotSize == DestSize)
1772 return DAG.getLoad(DestVT, dl,
Store, FIPtr, PtrInfo, DestAlign);
1774 assert(SlotSize < DestSize &&
"Unknown extension!");
1779 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(
SDNode *Node) {
1788 SDValue Ch = DAG.getTruncStore(
1789 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1791 Node->getValueType(0).getVectorElementType());
1793 Node->getValueType(0), dl, Ch, StackPtr,
1800 unsigned NumElems = Node->getNumOperands();
1802 EVT VT = Node->getValueType(0);
1814 for (
unsigned i = 0;
i < NumElems; ++
i) {
1825 while (IntermedVals.size() > 2) {
1826 NewIntermedVals.
clear();
1827 for (
unsigned i = 0,
e = (IntermedVals.size() & ~1u);
i <
e;
i += 2) {
1833 FinalIndices.
reserve(IntermedVals[
i].second.size() +
1834 IntermedVals[
i+1].second.size());
1837 for (
unsigned j = 0,
f = IntermedVals[
i].second.size();
j !=
f;
1840 FinalIndices.push_back(IntermedVals[
i].second[
j]);
1842 for (
unsigned j = 0,
f = IntermedVals[
i+1].second.size();
j !=
f;
1844 ShuffleVec[k] = NumElems +
j;
1845 FinalIndices.push_back(IntermedVals[
i+1].second[
j]);
1851 IntermedVals[
i+1].first,
1855 NewIntermedVals.push_back(
1856 std::make_pair(Shuffle,
std::move(FinalIndices)));
1861 if ((IntermedVals.size() & 1) != 0)
1862 NewIntermedVals.push_back(IntermedVals.back());
1864 IntermedVals.
swap(NewIntermedVals);
1867 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1868 "Invalid number of intermediate vectors");
1869 SDValue Vec1 = IntermedVals[0].first;
1871 if (IntermedVals.size() > 1)
1872 Vec2 = IntermedVals[1].first;
1877 for (
unsigned i = 0,
e = IntermedVals[0].second.size();
i !=
e; ++
i)
1878 ShuffleVec[IntermedVals[0].second[
i]] =
i;
1879 for (
unsigned i = 0,
e = IntermedVals[1].second.size();
i !=
e; ++
i)
1880 ShuffleVec[IntermedVals[1].second[
i]] = NumElems +
i;
1893 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(
SDNode *Node) {
1894 unsigned NumElems = Node->getNumOperands();
1897 EVT VT = Node->getValueType(0);
1898 EVT OpVT = Node->getOperand(0).getValueType();
1903 bool isOnlyLowElement =
true;
1904 bool MoreThanTwoValues =
false;
1906 for (
unsigned i = 0;
i < NumElems; ++
i) {
1911 isOnlyLowElement =
false;
1912 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1917 }
else if (!Value2.
getNode()) {
1920 }
else if (V != Value1 && V != Value2) {
1921 MoreThanTwoValues =
true;
1926 return DAG.getUNDEF(VT);
1928 if (isOnlyLowElement)
1934 for (
unsigned i = 0,
e = NumElems;
i !=
e; ++
i) {
1936 dyn_cast<ConstantFPSDNode>(Node->getOperand(
i))) {
1937 CV.push_back(
const_cast<ConstantFP *
>(V->getConstantFPValue()));
1939 dyn_cast<ConstantSDNode>(Node->getOperand(
i))) {
1941 CV.push_back(
const_cast<ConstantInt *
>(V->getConstantIntValue()));
1951 assert(Node->getOperand(
i).isUndef());
1958 DAG.getConstantPool(
CP, TLI.getPointerTy(DAG.getDataLayout()));
1961 VT, dl, DAG.getEntryNode(), CPIdx,
1967 for (
unsigned i = 0;
i < NumElems; ++
i) {
1968 if (Node->getOperand(
i).isUndef())
1970 DefinedValues.
insert(Node->getOperand(
i));
1973 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
1974 if (!MoreThanTwoValues) {
1976 for (
unsigned i = 0;
i < NumElems; ++
i) {
1980 ShuffleVec[
i] = V == Value1 ? 0 : NumElems;
1982 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1989 Vec2 = DAG.getUNDEF(VT);
1992 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2002 return ExpandVectorBuildThroughStack(Node);
2005 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(
SDNode *Node) {
2007 EVT VT = Node->getValueType(0);
2008 SDValue SplatVal = Node->getOperand(0);
2010 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2020 TargetLowering::ArgListEntry Entry;
2021 for (
const SDValue &
Op : Node->op_values()) {
2022 EVT ArgVT =
Op.getValueType();
2026 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2027 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2028 Args.push_back(Entry);
2030 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2031 TLI.getPointerTy(DAG.getDataLayout()));
2033 EVT RetVT = Node->getValueType(0);
2040 SDValue InChain = DAG.getEntryNode();
2045 const Function &
F = DAG.getMachineFunction().getFunction();
2047 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2048 (RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2053 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2054 CLI.setDebugLoc(
SDLoc(Node))
2056 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2058 .setTailCall(isTailCall)
2059 .setSExtResult(signExtend)
2060 .setZExtResult(!signExtend)
2061 .setIsPostTypeLegalization(
true);
2063 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2068 return DAG.getRoot();
2075 void SelectionDAGLegalize::ExpandFPLibCall(
SDNode* Node,
2078 if (LC == RTLIB::UNKNOWN_LIBCALL)
2081 if (Node->isStrictFPOpcode()) {
2082 EVT RetVT = Node->getValueType(0);
2086 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2089 Node->getOperand(0));
2091 Results.push_back(Tmp.second);
2093 SDValue Tmp = ExpandLibCall(LC, Node,
false);
2099 void SelectionDAGLegalize::ExpandFPLibCall(
SDNode* Node,
2107 Call_F32, Call_F64, Call_F80,
2108 Call_F128, Call_PPCF128);
2109 ExpandFPLibCall(Node, LC,
Results);
2112 SDValue SelectionDAGLegalize::ExpandIntLibCall(
2117 switch (Node->getSimpleValueType(0).SimpleTy) {
2123 case MVT::i8: LC = Call_I8;
break;
2124 case MVT::i16: LC = Call_I16;
break;
2125 case MVT::i32: LC = Call_I32;
break;
2126 case MVT::i64: LC = Call_I64;
break;
2129 return ExpandLibCall(LC, Node, isSigned);
2134 void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode* Node,
2141 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2143 Call_F32, Call_F64, Call_F80,
2144 Call_F128, Call_PPCF128);
2145 ExpandFPLibCall(Node, LC,
Results);
2150 SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *Node,
2152 unsigned Opcode = Node->getOpcode();
2156 switch (Node->getSimpleValueType(0).SimpleTy) {
2159 LC = isSigned ? RTLIB::SDIVREM_IEXT : RTLIB::UDIVREM_IEXT;
2162 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2163 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2164 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2165 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2166 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2172 SDValue InChain = DAG.getEntryNode();
2174 EVT RetVT = Node->getValueType(0);
2178 TargetLowering::ArgListEntry Entry;
2179 for (
const SDValue &
Op : Node->op_values()) {
2180 EVT ArgVT =
Op.getValueType();
2184 Entry.IsSExt = isSigned;
2185 Entry.IsZExt = !isSigned;
2186 Args.push_back(Entry);
2190 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2193 Entry.IsSExt = isSigned;
2194 Entry.IsZExt = !isSigned;
2195 Args.push_back(Entry);
2197 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2198 TLI.getPointerTy(DAG.getDataLayout()));
2204 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2206 .setSExtResult(isSigned)
2207 .setZExtResult(!isSigned);
2209 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2221 switch (Node->getSimpleValueType(0).SimpleTy) {
2223 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2224 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2225 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2226 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2234 unsigned OtherOpcode = Node->getOpcode() ==
ISD::FSIN
2237 SDValue Op0 = Node->getOperand(0);
2250 SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *Node,
2253 switch (Node->getSimpleValueType(0).SimpleTy) {
2255 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2256 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2257 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2258 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2265 SDValue InChain = DAG.getEntryNode();
2267 EVT RetVT = Node->getValueType(0);
2271 TargetLowering::ArgListEntry Entry;
2274 Entry.Node = Node->getOperand(0);
2276 Entry.IsSExt =
false;
2277 Entry.IsZExt =
false;
2278 Args.push_back(Entry);
2281 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2282 Entry.Node = SinPtr;
2284 Entry.IsSExt =
false;
2285 Entry.IsZExt =
false;
2286 Args.push_back(Entry);
2289 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2290 Entry.Node = CosPtr;
2292 Entry.IsSExt =
false;
2293 Entry.IsZExt =
false;
2294 Args.push_back(Entry);
2296 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2297 TLI.getPointerTy(DAG.getDataLayout()));
2301 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2302 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2305 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2317 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(
SDNode *Node,
2321 EVT DestVT = Node->getValueType(0);
2323 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2324 SDValue Op0 = Node->getOperand(OpNo);
2334 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2345 DAG.getConstant(0x80000000u, dl,
MVT::i32));
2351 if (DAG.getDataLayout().isBigEndian())
2354 SDValue MemChain = DAG.getEntryNode();
2357 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2376 if (Node->isStrictFPOpcode()) {
2378 {Node->getOperand(0),
Load, Bias});
2381 std::pair<SDValue, SDValue> ResultPair;
2383 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2384 Result = ResultPair.first;
2385 Chain = ResultPair.second;
2391 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2402 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2417 EVT SetCCVT = getSetCCResultType(SrcVT);
2419 SDValue SignBitTest = DAG.getSetCC(
2420 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2422 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2423 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2425 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2430 if (Node->isStrictFPOpcode()) {
2433 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2435 { Node->getOperand(0), InCvt });
2437 {
Fast.getValue(1), Fast, Fast });
2444 Fast->setFlags(Flags);
2453 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2457 if (!TLI.isOperationLegalOrCustom(
2466 "Cannot perform lossless SINT_TO_FP!");
2469 if (Node->isStrictFPOpcode()) {
2471 { Node->getOperand(0), Op0 });
2475 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2477 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2478 Four = DAG.getIntPtrConstant(4, dl);
2480 SignSet, Four, Zero);
2489 case MVT::i8 : FF = 0x43800000ULL;
break;
2490 case MVT::i16: FF = 0x47800000ULL;
break;
2491 case MVT::i32: FF = 0x4F800000ULL;
break;
2492 case MVT::i64: FF = 0x5F800000ULL;
break;
2494 if (DAG.getDataLayout().isLittleEndian())
2500 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2506 FudgeInReg = DAG.getLoad(
2507 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2516 LegalizeOp(
Load.getNode());
2520 if (Node->isStrictFPOpcode()) {
2522 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2523 Chain =
Result.getValue(1);
2527 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2535 void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2537 bool IsStrict =
N->isStrictFPOpcode();
2540 EVT DestVT =
N->getValueType(0);
2541 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2548 unsigned OpToUse = 0;
2556 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2564 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2579 dl, NewInTy, LegalOp)});
2586 DAG.getNode(OpToUse, dl, DestVT,
2588 dl, NewInTy, LegalOp)));
2596 void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2598 bool IsStrict =
N->isStrictFPOpcode();
2601 EVT DestVT =
N->getValueType(0);
2602 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2604 EVT NewOutTy = DestVT;
2606 unsigned OpToUse = 0;
2616 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2621 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2631 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2633 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2646 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(
SDNode *Node,
2648 unsigned Opcode = Node->getOpcode();
2651 EVT NewOutTy = Node->getValueType(0);
2656 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2662 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2663 Node->getOperand(1));
2664 return DAG.getNode(
ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2669 EVT VT =
Op.getValueType();
2670 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2675 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
2681 DAG.getConstant(1ULL << (--
i), dl, ShVT));
2686 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2689 bool SelectionDAGLegalize::ExpandNode(
SDNode *Node) {
2693 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2695 switch (Node->getOpcode()) {
2697 if ((Tmp1 = TLI.expandABS(Node, DAG)))
2701 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
2706 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
2711 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
2715 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
2719 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
2723 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
2728 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2731 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2732 TLI.getPointerTy(DAG.getDataLayout()));
2740 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2746 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2747 Results.push_back(Node->getOperand(0));
2756 Results.push_back(Node->getOperand(0));
2761 Results.append(Node->getNumValues() - 1,
2762 DAG.getConstant(0, dl, Node->getValueType(0)));
2763 Results.push_back(Node->getOperand(0));
2769 Results.push_back(Node->getOperand(0));
2773 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2775 SDValue Swap = DAG.getAtomicCmpSwap(
2777 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2778 cast<AtomicSDNode>(Node)->getMemOperand());
2786 cast<AtomicSDNode>(Node)->getMemoryVT(),
2787 Node->getOperand(0),
2788 Node->getOperand(1), Node->getOperand(2),
2789 cast<AtomicSDNode>(Node)->getMemOperand());
2798 SDValue Res = DAG.getAtomicCmpSwap(
2800 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2801 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2807 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2808 EVT OuterType = Node->getValueType(0);
2809 switch (TLI.getExtendForAtomicOps()) {
2812 DAG.getValueType(AtomicType));
2814 Node->getOperand(2), DAG.getValueType(AtomicType));
2819 DAG.getValueType(AtomicType));
2820 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2824 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2825 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2840 ExpandDYNAMIC_STACKALLOC(Node,
Results);
2843 for (
unsigned i = 0;
i < Node->getNumValues();
i++)
2844 Results.push_back(Node->getOperand(
i));
2847 EVT VT = Node->getValueType(0);
2849 Results.push_back(DAG.getConstant(0, dl, VT));
2852 Results.push_back(DAG.getConstantFP(0, dl, VT));
2859 if (TLI.isStrictFPEnabled())
2863 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2864 Node->getValueType(0))
2869 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
2870 Node->getValueType(0), dl,
2871 Node->getOperand(0)))) {
2872 ReplaceNode(Node, Tmp1.
getNode());
2873 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
2879 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2880 Node->getValueType(0), dl)))
2886 if (TLI.isStrictFPEnabled())
2890 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2891 Node->getValueType(0))
2896 if ((Tmp1 = EmitStackConvert(
2897 Node->getOperand(1), Node->getOperand(1).getValueType(),
2898 Node->getValueType(0), dl, Node->getOperand(0)))) {
2899 ReplaceNode(Node, Tmp1.
getNode());
2900 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
2905 if ((Tmp1 = EmitStackConvert(Node->getOperand(0),
2906 Node->getOperand(0).getValueType(),
2907 Node->getValueType(0), dl)))
2911 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2912 EVT VT = Node->getValueType(0);
2922 SDValue One = DAG.getConstant(1, dl, VT);
2924 SDValue Zero = DAG.getConstant(0, dl, VT);
2932 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2935 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2937 Node->getOperand(0), ShiftCst);
2938 Tmp1 = DAG.
getNode(
ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2944 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
2946 if (Node->isStrictFPOpcode())
2953 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
2955 if (Node->isStrictFPOpcode())
2960 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2964 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
2965 ReplaceNode(Node, Tmp1.
getNode());
2966 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
2971 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
2975 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
2977 DAG.ReplaceAllUsesOfValueWith(
SDValue(Node,1), Tmp2);
2979 ReplaceNodeWithValue(
SDValue(Node, 0), Tmp1);
2980 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
2986 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
2989 Results.push_back(DAG.expandVAArg(Node));
2993 Results.push_back(DAG.expandVACopy(Node));
2996 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2999 Node->getOperand(0));
3001 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(Node, 0));
3005 Results.push_back(ExpandExtractFromVectorThroughStack(
SDValue(Node, 0)));
3008 Results.push_back(ExpandInsertToVectorThroughStack(
SDValue(Node, 0)));
3011 Results.push_back(ExpandVectorBuildThroughStack(Node));
3014 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3017 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3018 Node->getOperand(1),
3019 Node->getOperand(2), dl));
3025 EVT VT = Node->getValueType(0);
3027 SDValue Op0 = Node->getOperand(0);
3028 SDValue Op1 = Node->getOperand(1);
3029 if (!TLI.isTypeLegal(EltVT)) {
3030 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3035 if (NewEltVT.
bitsLT(EltVT)) {
3051 unsigned int factor =
3059 for (
unsigned fi = 0; fi < factor; ++fi)
3060 NewMask.push_back(
Mask[
i]);
3063 for (
unsigned fi = 0; fi < factor; ++fi)
3064 NewMask.push_back(
Mask[
i]*factor+fi);
3074 for (
unsigned i = 0;
i != NumElems; ++
i) {
3076 Ops.push_back(DAG.getUNDEF(EltVT));
3079 unsigned Idx =
Mask[
i];
3082 DAG.getVectorIdxConstant(Idx, dl)));
3086 DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3089 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3096 Results.push_back(TLI.expandVectorSplice(Node, DAG));
3100 EVT OpTy = Node->getOperand(0).getValueType();
3101 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3105 TLI.getShiftAmountTy(
3106 Node->getOperand(0).getValueType(),
3107 DAG.getDataLayout())));
3112 Node->getOperand(0));
3120 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3121 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3122 Node->getValueType(0)));
3125 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3126 Results.push_back(Node->getOperand(0));
3132 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3133 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3134 Node->getOperand(1)));
3136 Results.push_back(Node->getOperand(0));
3140 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3144 Results.push_back(ExpandFCOPYSIGN(Node));
3147 Results.push_back(ExpandFNEG(Node));
3150 Results.push_back(ExpandFABS(Node));
3153 auto CNode = cast<ConstantSDNode>(Node->getOperand(1));
3156 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3157 Test, Node->getFlags(),
SDLoc(Node), DAG))
3167 switch (Node->getOpcode()) {
3174 Tmp1 = Node->getOperand(0);
3175 Tmp2 = Node->getOperand(1);
3176 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3182 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3188 EVT VT = Node->getValueType(0);
3194 SDVTList VTs = DAG.getVTList(VT, VT);
3206 if (Node->getValueType(0) !=
MVT::f32) {
3217 if (Node->getValueType(0) !=
MVT::f32) {
3223 {Node->getOperand(0), Node->getOperand(1)});
3233 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3235 MVT SVT =
Op.getSimpleValueType();
3241 DAG.getIntPtrConstant(0, dl));
3251 if (!TLI.isFPImmLegal(CFP->
getValueAPF(), Node->getValueType(0),
3252 DAG.shouldOptForSize()))
3253 Results.push_back(ExpandConstantFP(CFP,
true));
3262 EVT VT = Node->getValueType(0);
3263 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3264 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3273 EVT VT = Node->getValueType(0);
3275 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3276 "Don't know how to expand this subtraction!");
3277 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3278 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3279 Results.push_back(DAG.getNode(
ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3284 if (TLI.expandREM(Node, Tmp1, DAG))
3289 bool isSigned = Node->getOpcode() ==
ISD::SDIV;
3291 EVT VT = Node->getValueType(0);
3292 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3293 SDVTList VTs = DAG.getVTList(VT, VT);
3294 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3295 Node->getOperand(1));
3302 unsigned ExpandOpcode =
3304 EVT VT = Node->getValueType(0);
3305 SDVTList VTs = DAG.getVTList(VT, VT);
3307 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3308 Node->getOperand(1));
3316 MVT VT =
LHS.getSimpleValueType();
3317 unsigned MULHOpcode =
3320 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3322 Results.push_back(DAG.getNode(MULHOpcode, dl, VT,
LHS,
RHS));
3328 assert(TLI.isTypeLegal(HalfType));
3329 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl,
LHS,
RHS, Halves,
3332 for (
unsigned i = 0;
i < 2; ++
i) {
3337 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3346 EVT VT = Node->getValueType(0);
3347 SDVTList VTs = DAG.getVTList(VT, VT);
3353 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3354 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3355 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3356 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3357 unsigned OpToUse = 0;
3358 if (HasSMUL_LOHI && !HasMULHS) {
3360 }
else if (HasUMUL_LOHI && !HasMULHU) {
3362 }
else if (HasSMUL_LOHI) {
3364 }
else if (HasUMUL_LOHI) {
3368 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3369 Node->getOperand(1)));
3377 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3378 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3379 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3385 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3393 if (
SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3398 if (
SDValue Expanded = TLI.expandROT(Node,
true , DAG))
3405 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3409 Results.push_back(TLI.expandShlSat(Node, DAG));
3415 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3421 if (
SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(),
SDLoc(Node),
3422 Node->getOperand(0),
3423 Node->getOperand(1),
3424 Node->getConstantOperandVal(2),
3441 SDValue Carry = Node->getOperand(2);
3447 EVT VT =
LHS.getValueType();
3451 EVT CarryType = Node->getValueType(1);
3452 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3454 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum,
LHS, CC);
3457 SDValue One = DAG.getConstant(1, dl, VT);
3459 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3466 SDValue Zero = DAG.getConstant(0, dl, VT);
3468 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3469 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3471 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3477 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3483 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3491 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3499 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3511 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3512 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3517 Tmp1 = Node->getOperand(0);
3518 Tmp2 = Node->getOperand(1);
3519 Tmp3 = Node->getOperand(2);
3523 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3525 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3533 SDValue Chain = Node->getOperand(0);
3534 SDValue Table = Node->getOperand(1);
3538 EVT PTy = TLI.getPointerTy(TD);
3540 unsigned EntrySize =
3541 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3548 Index = DAG.getNode(
3553 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
3562 if (TLI.isJumpTableRelative()) {
3567 TLI.getPICJumpTableRelocBase(Table, DAG));
3570 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, DAG);
3577 Tmp1 = Node->getOperand(0);
3578 Tmp2 = Node->getOperand(1);
3584 Node->getOperand(2));
3590 cast<ConstantSDNode>(Tmp2.
getOperand(1))->getZExtValue() == 1))
3598 Node->getOperand(2));
3606 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
3611 unsigned Offset = IsStrict ? 1 : 0;
3612 Tmp1 = Node->getOperand(0 + Offset);
3613 Tmp2 = Node->getOperand(1 + Offset);
3614 Tmp3 = Node->getOperand(2 + Offset);
3617 Mask = Node->getOperand(3 + Offset);
3618 EVL = Node->getOperand(4 + Offset);
3620 bool Legalized = TLI.LegalizeSetCCCondCode(
3621 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3,
Mask, EVL, NeedInvert, dl,
3622 Chain, IsSignaling);
3629 Tmp1 = DAG.
getNode(Node->getOpcode(), dl, Node->getVTList(),
3630 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
3633 Tmp1 = DAG.
getNode(Node->getOpcode(), dl, Node->getValueType(0),
3634 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
3636 Tmp1 = DAG.
getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
3637 Tmp2, Tmp3, Node->getFlags());
3645 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
3660 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
3665 EVT VT = Node->getValueType(0);
3668 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
3669 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
3676 Tmp1 = Node->getOperand(0);
3677 Tmp2 = Node->getOperand(1);
3678 Tmp3 = Node->getOperand(2);
3679 Tmp4 = Node->getOperand(3);
3680 EVT VT = Node->getValueType(0);
3682 SDValue CC = Node->getOperand(4);
3690 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3692 EVT CCVT = getSetCCResultType(CmpVT);
3694 Results.push_back(DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4));
3699 bool Legalized =
false;
3707 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3717 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3723 Legalized = TLI.LegalizeSetCCCondCode(
3724 DAG, getSetCCResultType(Tmp1.
getValueType()), Tmp1, Tmp2, CC,
3727 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
3738 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3743 Tmp2, Tmp3, Tmp4, CC);
3753 Tmp1 = Node->getOperand(0);
3754 Tmp2 = Node->getOperand(2);
3755 Tmp3 = Node->getOperand(3);
3756 Tmp4 = Node->getOperand(1);
3758 bool Legalized = TLI.LegalizeSetCCCondCode(
3759 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
3762 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
3767 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
3770 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3775 Tmp2, Tmp3, Node->getOperand(4));
3781 Results.push_back(ExpandBUILD_VECTOR(Node));
3784 Results.push_back(ExpandSPLAT_VECTOR(Node));
3790 EVT VT = Node->getValueType(0);
3796 for (
unsigned Idx = 0; Idx < NumElem; Idx++) {
3799 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
3802 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
3803 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3807 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3824 Results.push_back(TLI.expandVecReduce(Node, DAG));
3841 if (!TLI.isStrictFPEnabled() &&
Results.
empty() && Node->isStrictFPOpcode()) {
3847 switch (Node->getOpcode()) {
3849 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3850 Node->getValueType(0))
3855 if (TLI.getStrictFPOperationAction(
3858 if (TLI.getStrictFPOperationAction(
3862 EVT VT = Node->getValueType(0);
3866 {Node->getOperand(0), Node->getOperand(1), Neg},
3881 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3882 Node->getOperand(1).getValueType())
3896 ReplaceNode(Node,
Results.data());
3900 void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *Node) {
3905 unsigned Opc = Node->getOpcode();
3914 .setChain(Node->getOperand(0))
3917 DAG.getExternalSymbol(
"__sync_synchronize",
3918 TLI.getPointerTy(DAG.getDataLayout())),
3921 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3923 Results.push_back(CallResult.second);
3943 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
3945 EVT RetVT = Node->getValueType(0);
3948 if (TLI.getLibcallName(LC)) {
3950 Ops.
append(Node->op_begin() + 2, Node->op_end());
3951 Ops.push_back(Node->getOperand(1));
3955 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3956 "Unexpected atomic op or value type!");
3958 Ops.
append(Node->op_begin() + 1, Node->op_end());
3960 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3963 Node->getOperand(0));
3965 Results.push_back(Tmp.second);
3973 .setChain(Node->getOperand(0))
3975 DAG.getExternalSymbol(
3976 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3978 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3980 Results.push_back(CallResult.second);
3985 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3986 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3987 RTLIB::FMIN_PPCF128,
Results);
3991 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3992 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3993 RTLIB::FMAX_PPCF128,
Results);
3997 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3998 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3999 RTLIB::SQRT_PPCF128,
Results);
4002 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4003 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4004 RTLIB::CBRT_PPCF128,
Results);
4008 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4009 RTLIB::SIN_F80, RTLIB::SIN_F128,
4014 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4015 RTLIB::COS_F80, RTLIB::COS_F128,
4020 ExpandSinCosLibCall(Node,
Results);
4024 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4025 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4029 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4030 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4034 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4035 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4039 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4040 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4044 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4045 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4049 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4050 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4051 RTLIB::TRUNC_PPCF128,
Results);
4055 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4056 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4057 RTLIB::FLOOR_PPCF128,
Results);
4061 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4062 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4063 RTLIB::CEIL_PPCF128,
Results);
4067 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4068 RTLIB::RINT_F80, RTLIB::RINT_F128,
4069 RTLIB::RINT_PPCF128,
Results);
4073 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4074 RTLIB::NEARBYINT_F64,
4075 RTLIB::NEARBYINT_F80,
4076 RTLIB::NEARBYINT_F128,
4077 RTLIB::NEARBYINT_PPCF128,
Results);
4081 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4085 RTLIB::ROUND_PPCF128,
Results);
4089 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4090 RTLIB::ROUNDEVEN_F64,
4091 RTLIB::ROUNDEVEN_F80,
4092 RTLIB::ROUNDEVEN_F128,
4093 RTLIB::ROUNDEVEN_PPCF128,
Results);
4098 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4099 if (!TLI.getLibcallName(LC)) {
4101 if (Node->isStrictFPOpcode()) {
4104 {Node->getValueType(0), Node->getValueType(1)},
4105 {Node->getOperand(0), Node->getOperand(2)});
4108 {Node->getValueType(0), Node->getValueType(1)},
4109 {
Exponent.getValue(1), Node->getOperand(1), Exponent});
4115 Node->getOperand(1));
4117 Node->getValueType(0),
4118 Node->getOperand(0), Exponent));
4122 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4123 bool ExponentHasSizeOfInt =
4124 DAG.getLibInfo().getIntSize() ==
4125 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4126 if (!ExponentHasSizeOfInt) {
4129 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4130 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4133 ExpandFPLibCall(Node, LC,
Results);
4138 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4139 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4143 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4144 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4146 RTLIB::LROUND_PPCF128,
Results);
4150 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4151 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4152 RTLIB::LLROUND_F128,
4153 RTLIB::LLROUND_PPCF128,
Results);
4157 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4158 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4160 RTLIB::LRINT_PPCF128,
Results);
4164 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4165 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4167 RTLIB::LLRINT_PPCF128,
Results);
4171 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4172 RTLIB::DIV_F80, RTLIB::DIV_F128,
4177 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4178 RTLIB::REM_F80, RTLIB::REM_F128,
4183 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4184 RTLIB::FMA_F80, RTLIB::FMA_F128,
4189 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4190 RTLIB::ADD_F80, RTLIB::ADD_F128,
4195 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4196 RTLIB::MUL_F80, RTLIB::MUL_F128,
4200 if (Node->getValueType(0) ==
MVT::f32) {
4201 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node,
false));
4205 if (Node->getValueType(0) ==
MVT::f32) {
4207 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4208 DAG, RTLIB::FPEXT_F16_F32,
MVT::f32, Node->getOperand(1), CallOptions,
4209 SDLoc(Node), Node->getOperand(0));
4211 Results.push_back(Tmp.second);
4218 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4219 Results.push_back(ExpandLibCall(LC, Node,
false));
4227 bool IsStrict = Node->isStrictFPOpcode();
4230 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4231 EVT RVT = Node->getValueType(0);
4248 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4253 NVT, Node->getOperand(IsStrict ? 1 : 0));
4256 std::pair<SDValue, SDValue> Tmp =
4257 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, dl, Chain);
4260 Results.push_back(Tmp.second);
4268 bool IsStrict = Node->isStrictFPOpcode();
4272 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4273 EVT SVT =
Op.getValueType();
4274 EVT RVT = Node->getValueType(0);
4291 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4295 std::pair<SDValue, SDValue> Tmp =
4296 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
4301 Results.push_back(Tmp.second);
4312 bool IsStrict = Node->isStrictFPOpcode();
4313 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4315 EVT VT = Node->getValueType(0);
4316 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4317 "Unable to expand as libcall if it is not normal rounding");
4320 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4323 std::pair<SDValue, SDValue> Tmp =
4324 TLI.makeLibCall(DAG, LC, VT,
Op, CallOptions,
SDLoc(Node), Chain);
4327 Results.push_back(Tmp.second);
4333 Node->getValueType(0)),
4342 : RTLIB::
getFPEXT(Node->getOperand(1).getValueType(),
4343 Node->getValueType(0));
4344 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4347 std::pair<SDValue, SDValue> Tmp =
4348 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4349 CallOptions,
SDLoc(Node), Node->getOperand(0));
4351 Results.push_back(Tmp.second);
4356 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4357 RTLIB::SUB_F80, RTLIB::SUB_F128,
4361 Results.push_back(ExpandIntLibCall(
4362 Node,
true, RTLIB::SREM_I8, RTLIB::SREM_I16, RTLIB::SREM_I32,
4363 RTLIB::SREM_I64, RTLIB::SREM_I128, RTLIB::SREM_IEXT));
4366 Results.push_back(ExpandIntLibCall(
4367 Node,
false, RTLIB::UREM_I8, RTLIB::UREM_I16, RTLIB::UREM_I32,
4368 RTLIB::UREM_I64, RTLIB::UREM_I128, RTLIB::UREM_IEXT));
4371 Results.push_back(ExpandIntLibCall(
4372 Node,
true, RTLIB::SDIV_I8, RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4373 RTLIB::SDIV_I64, RTLIB::SDIV_I128, RTLIB::SDIV_IEXT));
4376 Results.push_back(ExpandIntLibCall(
4377 Node,
false, RTLIB::UDIV_I8, RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4378 RTLIB::UDIV_I64, RTLIB::UDIV_I128, RTLIB::UDIV_IEXT));
4383 ExpandDivRemLibCall(Node,
Results);
4386 Results.push_back(ExpandIntLibCall(
4387 Node,
false, RTLIB::MUL_I8, RTLIB::MUL_I16, RTLIB::MUL_I32,
4388 RTLIB::MUL_I64, RTLIB::MUL_I128, RTLIB::MUL_IEXT));
4391 switch (Node->getSimpleValueType(0).SimpleTy) {
4395 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node,
false));
4398 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node,
false));
4401 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node,
false));
4409 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
4410 ReplaceNode(Node,
Results.data());
4418 MVT EltVT,
MVT NewEltVT) {
4425 void SelectionDAGLegalize::PromoteNode(
SDNode *Node) {
4428 MVT OVT = Node->getSimpleValueType(0);
4434 OVT = Node->getOperand(0).getSimpleValueType();
4440 OVT = Node->getOperand(1).getSimpleValueType();
4443 OVT = Node->getOperand(2).getSimpleValueType();
4444 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4446 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4447 switch (Node->getOpcode()) {
4467 DAG.getConstant(TopBit, dl, NVT));
4471 Tmp1 = DAG.
getNode(Node->getOpcode(), dl, NVT, Tmp1);
4485 Tmp1 = DAG.
getNode(Node->getOpcode(), dl, NVT, Tmp1);
4488 DAG.getConstant(DiffBits, dl,
4489 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4498 PromoteLegalFP_TO_INT(Node, dl,
Results);
4502 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
4508 PromoteLegalINT_TO_FP(Node, dl,
Results);
4511 SDValue Chain = Node->getOperand(0);
4512 SDValue Ptr = Node->getOperand(1);
4519 &&
"VAARG promotion is supported only for vectors or integer types");
4524 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4525 Node->getConstantOperandVal(3));
4528 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
4532 DAG.ReplaceAllUsesOfValueWith(
SDValue(Node, 0), Tmp2);
4533 DAG.ReplaceAllUsesOfValueWith(
SDValue(Node, 1), Chain);
4535 UpdatedNodes->insert(Tmp2.
getNode());
4536 UpdatedNodes->insert(Chain.
getNode());
4549 unsigned ExtOp, TruncOp;
4556 switch (Node->getOpcode()) {
4572 Tmp1 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(0));
4573 Tmp2 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(1));
4575 Tmp1 = DAG.
getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4576 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4584 Tmp1 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(0));
4585 Tmp2 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(1));
4588 auto &
DL = DAG.getDataLayout();
4592 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(
DL, NVT)));
4598 unsigned ExtOp, TruncOp;
4599 if (Node->getValueType(0).isVector() ||
4600 Node->getValueType(0).getSizeInBits() == NVT.
getSizeInBits()) {
4603 }
else if (Node->getValueType(0).isInteger()) {
4610 Tmp1 = Node->getOperand(0);
4612 Tmp2 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(1));
4613 Tmp3 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(2));
4615 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4618 Tmp1 = DAG.
getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4620 Tmp1 = DAG.
getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4621 DAG.getIntPtrConstant(0, dl));
4633 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2,
Mask);
4642 Node->getOperand(2));
4650 MVT CVT = Node->getSimpleValueType(0);
4651 assert(CVT == OVT &&
"not handled");
4659 if (TLI.isCondCodeLegal(CCCode, CVT)) {
4660 Tmp1 = Node->getOperand(0);
4661 Tmp2 = Node->getOperand(1);
4663 Tmp1 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(0));
4664 Tmp2 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(1));
4667 Tmp3 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(2));
4668 Tmp4 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(3));
4678 DAG.getIntPtrConstant(0, dl));
4688 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
4691 if (Node->isStrictFPOpcode()) {
4692 SDValue InChain = Node->getOperand(0);
4693 std::tie(Tmp1, std::ignore) =
4694 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
4695 std::tie(Tmp2, std::ignore) =
4696 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
4698 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
4700 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
4701 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
4706 Tmp1 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(0));
4707 Tmp2 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(1));
4709 Tmp2, Node->getOperand(2), Node->getFlags()));
4716 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4719 Tmp1 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(2));
4720 Tmp2 = DAG.
getNode(ExtOp, dl, NVT, Node->getOperand(3));
4722 Node->getOperand(0), Node->getOperand(1),
4723 Tmp1, Tmp2, Node->getOperand(4)));
4736 Tmp3 = DAG.
getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4739 Tmp3, DAG.getIntPtrConstant(0, dl)));
4750 {Node->getOperand(0), Node->getOperand(1)});
4752 {Node->getOperand(0), Node->getOperand(2)});
4755 Tmp1 = DAG.
getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4756 {Tmp3, Tmp1, Tmp2});
4758 {Tmp1.
getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
4768 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4769 DAG.getIntPtrConstant(0, dl)));
4773 {Node->getOperand(0), Node->getOperand(1)});
4775 {Node->getOperand(0), Node->getOperand(2)});
4777 {Node->getOperand(0), Node->getOperand(3)});
4780 Tmp4 = DAG.
getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4781 {Tmp4, Tmp1, Tmp2, Tmp3});
4783 {Tmp4.
getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
4790 Tmp2 = Node->getOperand(1);
4791 Tmp3 = DAG.
getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4800 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4805 {Node->getOperand(0), Node->getOperand(1)});
4807 {Tmp1.
getValue(1), Tmp1, Node->getOperand(2)});
4809 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4831 Tmp2 = DAG.
getNode(Node->getOpcode(), dl, NVT, Tmp1);
4833 Tmp2, DAG.getIntPtrConstant(0, dl)));
4851 {Node->getOperand(0), Node->getOperand(1)});
4855 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4870 "Invalid promote type for build_vector");
4876 for (
unsigned I = 0,
E = Node->getNumOperands();
I !=
E; ++
I) {
4903 "Invalid promote type for extract_vector_elt");
4909 SDValue Idx = Node->getOperand(1);
4912 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4918 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
4919 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
4924 NewOps.push_back(Elt);
4927 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4949 "Invalid promote type for insert_vector_elt");
4955 SDValue Val = Node->getOperand(1);
4956 SDValue Idx = Node->getOperand(2);
4960 SDValue Factor = DAG.getConstant(NewEltsPerOldElt,
SDLoc(), IdxVT);
4967 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
4968 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
4972 CastVal, IdxOffset);
4975 NewVec, Elt, InEltIdx);
4993 SDValue Val = Node->getOperand(0);
5000 NewElts.push_back(CastVal);
5002 NewElts.push_back(
Undef);
5014 "unexpected promotion type");
5016 "unexpected atomic_swap with illegal type");
5021 { AM->getChain(), AM->getBasePtr(), CastVal },