57#define DEBUG_TYPE "legalizedag"
63struct FloatSignAsInt {
86class SelectionDAGLegalize {
98 EVT getSetCCResultType(
EVT VT)
const {
109 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
172 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
174 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
214 dbgs() <<
" with: "; New->dump(&DAG));
217 "Replacing one node with another that produces a different number "
221 UpdatedNodes->
insert(New);
227 dbgs() <<
" with: "; New->dump(&DAG));
231 UpdatedNodes->
insert(New.getNode());
232 ReplacedNode(Old.getNode());
239 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
243 UpdatedNodes->
insert(New[i].getNode());
250 dbgs() <<
" with: "; New->dump(&DAG));
254 UpdatedNodes->
insert(New.getNode());
255 ReplacedNode(Old.getNode());
265SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
270 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
272 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
274 if (NumEltsGrowth == 1)
275 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
278 for (
unsigned i = 0; i != NumMaskElts; ++i) {
280 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
287 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
288 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
289 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
308 assert((VT == MVT::f64 || VT == MVT::f32) &&
"Invalid type expansion");
310 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
320 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
326 TLI.ShouldShrinkFPConstant(OrigVT)) {
329 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
337 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
338 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
347 OrigVT, dl, DAG.getEntryNode(), CPIdx,
355 EVT VT =
CP->getValueType(0);
356 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
357 TLI.getPointerTy(DAG.getDataLayout()));
358 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
360 VT, dl, DAG.getEntryNode(), CPIdx,
369SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(
SDValue Vec,
387 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
391 DAG.getEntryNode(), dl, Tmp1, StackPtr,
394 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
397 Ch = DAG.getTruncStore(
398 Ch, dl, Tmp2, StackPtr2,
402 DAG.getMachineFunction(), SPFI));
423 for (
unsigned i = 0; i != NumElts; ++i)
424 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
426 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
429 return PerformInsertVectorEltInMemory(Vec, Val,
Idx, dl);
456 TLI.isTypeLegal(MVT::i32)) {
458 bitcastToAPInt().zextOrTrunc(32),
459 SDLoc(CFP), MVT::i32);
460 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
461 ST->getOriginalAlign(), MMOFlags, AAInfo);
466 if (TLI.isTypeLegal(MVT::i64)) {
468 zextOrTrunc(64),
SDLoc(CFP), MVT::i64);
469 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
470 ST->getOriginalAlign(), MMOFlags, AAInfo);
473 if (TLI.isTypeLegal(MVT::i32) && !
ST->isVolatile()) {
480 if (DAG.getDataLayout().isBigEndian())
483 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
484 ST->getOriginalAlign(), MMOFlags, AAInfo);
486 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
487 ST->getPointerInfo().getWithOffset(4),
488 ST->getOriginalAlign(), MMOFlags, AAInfo);
497void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
506 if (!
ST->isTruncatingStore()) {
508 if (
SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
509 ReplaceNode(ST, OptStore);
514 MVT VT =
Value.getSimpleValueType();
515 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
517 case TargetLowering::Legal: {
520 EVT MemVT =
ST->getMemoryVT();
522 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
523 *
ST->getMemOperand())) {
526 ReplaceNode(
SDValue(ST, 0), Result);
531 case TargetLowering::Custom: {
538 case TargetLowering::Promote: {
541 "Can only promote stores to same size type");
544 ST->getOriginalAlign(), MMOFlags, AAInfo);
554 EVT StVT =
ST->getMemoryVT();
557 auto &
DL = DAG.getDataLayout();
559 if (StWidth != StSize) {
564 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
566 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
567 ST->getOriginalAlign(), MMOFlags, AAInfo);
573 unsigned LogStWidth =
Log2_32(StWidthBits);
575 unsigned RoundWidth = 1 << LogStWidth;
576 assert(RoundWidth < StWidthBits);
577 unsigned ExtraWidth = StWidthBits - RoundWidth;
578 assert(ExtraWidth < RoundWidth);
579 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
580 "Store size not an integral number of bytes!");
584 unsigned IncrementSize;
586 if (
DL.isLittleEndian()) {
589 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
590 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
593 IncrementSize = RoundWidth / 8;
598 DAG.getConstant(RoundWidth, dl,
599 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
600 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
601 ST->getPointerInfo().getWithOffset(IncrementSize),
602 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
609 DAG.getConstant(ExtraWidth, dl,
610 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
611 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
612 ST->getOriginalAlign(), MMOFlags, AAInfo);
615 IncrementSize = RoundWidth / 8;
617 DAG.getConstant(IncrementSize, dl,
618 Ptr.getValueType()));
620 ST->getPointerInfo().getWithOffset(IncrementSize),
621 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
628 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
630 case TargetLowering::Legal: {
631 EVT MemVT =
ST->getMemoryVT();
634 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
635 *
ST->getMemOperand())) {
637 ReplaceNode(
SDValue(ST, 0), Result);
641 case TargetLowering::Custom: {
647 case TargetLowering::Expand:
649 "Vector Stores are handled in LegalizeVectorOps");
654 if (TLI.isTypeLegal(StVT)) {
657 ST->getOriginalAlign(), MMOFlags, AAInfo);
662 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
665 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
666 ST->getOriginalAlign(), MMOFlags, AAInfo);
675void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
684 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
685 MVT VT =
Node->getSimpleValueType(0);
689 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
691 case TargetLowering::Legal: {
692 EVT MemVT =
LD->getMemoryVT();
696 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
697 *
LD->getMemOperand())) {
698 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
702 case TargetLowering::Custom:
703 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
709 case TargetLowering::Promote: {
710 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
712 "Can only promote loads to same size type");
714 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
722 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
723 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
725 UpdatedNodes->insert(RVal.
getNode());
726 UpdatedNodes->insert(RChain.
getNode());
734 EVT SrcVT =
LD->getMemoryVT();
748 TLI.getLoadExtAction(ExtType,
Node->getValueType(0), MVT::i1) ==
749 TargetLowering::Promote)) {
763 Chain,
Ptr,
LD->getPointerInfo(), NVT,
764 LD->getOriginalAlign(), MMOFlags, AAInfo);
772 Result, DAG.getValueType(SrcVT));
776 Result.getValueType(), Result,
777 DAG.getValueType(SrcVT));
785 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
787 unsigned RoundWidth = 1 << LogSrcWidth;
788 assert(RoundWidth < SrcWidthBits);
789 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
790 assert(ExtraWidth < RoundWidth);
791 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
792 "Load size not an integral number of bytes!");
796 unsigned IncrementSize;
797 auto &
DL = DAG.getDataLayout();
799 if (
DL.isLittleEndian()) {
803 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
807 IncrementSize = RoundWidth / 8;
810 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
811 LD->getPointerInfo().getWithOffset(IncrementSize),
812 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
822 DAG.getConstant(RoundWidth, dl,
823 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
831 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
832 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
836 IncrementSize = RoundWidth / 8;
840 LD->getPointerInfo().getWithOffset(IncrementSize),
841 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
851 DAG.getConstant(ExtraWidth, dl,
852 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
860 bool isCustom =
false;
861 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
864 case TargetLowering::Custom:
867 case TargetLowering::Legal:
879 EVT MemVT =
LD->getMemoryVT();
881 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
882 *
LD->getMemOperand())) {
883 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
888 case TargetLowering::Expand: {
889 EVT DestVT =
Node->getValueType(0);
895 (TLI.isTypeLegal(SrcVT) ||
896 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
903 SrcVT,
LD->getMemOperand());
906 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
907 Chain =
Load.getValue(1);
921 Ptr, ISrcVT,
LD->getMemOperand());
923 Chain =
Result.getValue(1);
929 "Vector Loads are handled in LegalizeVectorOps");
936 "EXTLOAD should always be supported!");
940 Node->getValueType(0),
942 LD->getMemOperand());
947 Result, DAG.getValueType(SrcVT));
949 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
951 Chain =
Result.getValue(1);
962 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
964 UpdatedNodes->insert(
Value.getNode());
965 UpdatedNodes->insert(Chain.
getNode());
972void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
981 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
982 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
983 TargetLowering::TypeLegal &&
984 "Unexpected illegal type!");
987 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
988 TargetLowering::TypeLegal ||
991 "Unexpected illegal type!");
996 bool SimpleFinishLegalizing =
true;
997 switch (
Node->getOpcode()) {
1002 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
1005 Action = TLI.getOperationAction(
Node->getOpcode(),
1006 Node->getValueType(0));
1009 Action = TLI.getOperationAction(
Node->getOpcode(),
1010 Node->getValueType(0));
1011 if (Action != TargetLowering::Promote)
1012 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
1016 Action = TLI.getOperationAction(
Node->getOpcode(),
1017 Node->getOperand(1).getValueType());
1028 Action = TLI.getOperationAction(
Node->getOpcode(),
1029 Node->getOperand(0).getValueType());
1041 Action = TLI.getOperationAction(
Node->getOpcode(),
1042 Node->getOperand(1).getValueType());
1045 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1046 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1050 Action = TLI.getOperationAction(
Node->getOpcode(),
1051 Node->getOperand(1).getValueType());
1060 unsigned Opc =
Node->getOpcode();
1065 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1067 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1071 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1073 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1074 Action = TLI.getCondCodeAction(CCCode, OpVT);
1075 if (Action == TargetLowering::Legal) {
1077 Action = TLI.getOperationAction(
Node->getOpcode(),
1078 Node->getValueType(0));
1080 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1088 SimpleFinishLegalizing =
false;
1095 SimpleFinishLegalizing =
false;
1108 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1109 if (Action == TargetLowering::Legal)
1110 Action = TargetLowering::Expand;
1120 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1121 if (Action == TargetLowering::Legal)
1122 Action = TargetLowering::Custom;
1127 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::i64);
1134 Action = TargetLowering::Legal;
1137 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1138 if (Action == TargetLowering::Expand) {
1142 Node->getOperand(0));
1149 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1150 if (Action == TargetLowering::Expand) {
1154 Node->getOperand(0));
1168 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1178 unsigned Scale =
Node->getConstantOperandVal(2);
1179 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1180 Node->getValueType(0), Scale);
1184 Action = TLI.getOperationAction(
Node->getOpcode(),
1185 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1188 Action = TLI.getOperationAction(
Node->getOpcode(),
1189 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1191 case ISD::VP_SCATTER:
1192 Action = TLI.getOperationAction(
1194 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1197 Action = TLI.getOperationAction(
1199 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1201 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1202 Action = TLI.getOperationAction(
1204 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1222 Action = TLI.getOperationAction(
1223 Node->getOpcode(),
Node->getOperand(0).getValueType());
1227 case ISD::VP_REDUCE_FADD:
1228 case ISD::VP_REDUCE_FMUL:
1229 case ISD::VP_REDUCE_ADD:
1230 case ISD::VP_REDUCE_MUL:
1231 case ISD::VP_REDUCE_AND:
1232 case ISD::VP_REDUCE_OR:
1233 case ISD::VP_REDUCE_XOR:
1234 case ISD::VP_REDUCE_SMAX:
1235 case ISD::VP_REDUCE_SMIN:
1236 case ISD::VP_REDUCE_UMAX:
1237 case ISD::VP_REDUCE_UMIN:
1238 case ISD::VP_REDUCE_FMAX:
1239 case ISD::VP_REDUCE_FMIN:
1240 case ISD::VP_REDUCE_SEQ_FADD:
1241 case ISD::VP_REDUCE_SEQ_FMUL:
1242 Action = TLI.getOperationAction(
1243 Node->getOpcode(),
Node->getOperand(1).getValueType());
1247 Action = TLI.getCustomOperationAction(*
Node);
1249 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1254 if (SimpleFinishLegalizing) {
1256 switch (
Node->getOpcode()) {
1275 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1295 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1301 if (NewNode !=
Node) {
1302 ReplaceNode(
Node, NewNode);
1306 case TargetLowering::Legal:
1309 case TargetLowering::Custom:
1317 if (
Node->getNumValues() == 1) {
1321 Node->getValueType(0) == MVT::Glue) &&
1322 "Type mismatch for custom legalized operation");
1330 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1334 Node->getValueType(i) == MVT::Glue) &&
1335 "Type mismatch for custom legalized operation");
1339 ReplaceNode(
Node, ResultVals.
data());
1344 case TargetLowering::Expand:
1345 if (ExpandNode(
Node))
1348 case TargetLowering::LibCall:
1349 ConvertNodeToLibcall(
Node);
1351 case TargetLowering::Promote:
1357 switch (
Node->getOpcode()) {
1370 return LegalizeLoadOps(
Node);
1372 return LegalizeStoreOps(
Node);
1376SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1396 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1397 ST->getValue() != Vec)
1402 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1411 ST->hasPredecessor(
Op.getNode()))
1424 StackPtr = DAG.CreateStackTemporary(VecVT);
1425 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1430 Align ElementAlignment =
1431 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1432 DAG.getDataLayout().getPrefTypeAlign(
1433 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1435 if (
Op.getValueType().isVector()) {
1436 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1437 Op.getValueType(),
Idx);
1438 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1441 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1442 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1448 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1454 NewLoadOperands[0] = Ch;
1456 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1460SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1461 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1472 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1477 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1481 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT,
Idx);
1485 Ch, dl, Part, SubStackPtr,
1489 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1495 "Unexpected opcode!");
1501 EVT VT =
Node->getValueType(0);
1503 :
Node->getOperand(0).getValueType();
1505 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1506 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1513 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1517 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1518 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1521 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1523 if (
Node->getOperand(i).isUndef())
continue;
1525 unsigned Offset = TypeByteSize*i;
1531 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1535 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1540 if (!Stores.
empty())
1543 StoreChain = DAG.getEntryNode();
1546 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1552void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1555 EVT FloatVT =
Value.getValueType();
1557 State.FloatVT = FloatVT;
1560 if (TLI.isTypeLegal(IVT)) {
1563 State.SignBit = NumBits - 1;
1569 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1572 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1577 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1578 State.FloatPointerInfo);
1585 State.IntPointerInfo = State.FloatPointerInfo;
1588 unsigned ByteOffset = (NumBits / 8) - 1;
1595 State.IntPtr = IntPtr;
1596 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1597 State.IntPointerInfo, MVT::i8);
1604SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1611 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1612 State.IntPointerInfo, MVT::i8);
1613 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1614 State.FloatPointerInfo);
1623 FloatSignAsInt SignAsInt;
1624 getSignAsIntValue(SignAsInt,
DL, Sign);
1626 EVT IntVT = SignAsInt.IntValue.getValueType();
1627 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1633 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1634 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1637 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1639 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1643 FloatSignAsInt MagAsInt;
1644 getSignAsIntValue(MagAsInt,
DL, Mag);
1645 EVT MagVT = MagAsInt.IntValue.getValueType();
1646 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1651 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1652 EVT ShiftVT = IntVT;
1658 if (ShiftAmount > 0) {
1659 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1661 }
else if (ShiftAmount < 0) {
1662 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1672 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1678 FloatSignAsInt SignAsInt;
1679 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1680 EVT IntVT = SignAsInt.IntValue.getValueType();
1683 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1688 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1696 EVT FloatVT =
Value.getValueType();
1703 FloatSignAsInt ValueAsInt;
1704 getSignAsIntValue(ValueAsInt,
DL,
Value);
1705 EVT IntVT = ValueAsInt.IntValue.getValueType();
1706 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1709 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1712void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1714 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1715 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1716 " not tell us which reg is the stack pointer!");
1718 EVT VT =
Node->getValueType(0);
1726 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1729 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1731 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1739 if (Alignment > StackAlign)
1741 DAG.getConstant(-Alignment.
value(), dl, VT));
1742 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1744 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1756 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1764 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1767 if ((SrcVT.
bitsGT(SlotVT) &&
1768 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1769 (SlotVT.
bitsLT(DestVT) &&
1770 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1774 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1775 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1787 if (SrcVT.
bitsGT(SlotVT))
1788 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1792 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1796 if (SlotVT.
bitsEq(DestVT))
1797 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1800 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1813 SDValue Ch = DAG.getTruncStore(
1814 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1816 Node->getValueType(0).getVectorElementType());
1818 Node->getValueType(0), dl, Ch, StackPtr,
1825 unsigned NumElems =
Node->getNumOperands();
1827 EVT VT =
Node->getValueType(0);
1839 for (
unsigned i = 0; i < NumElems; ++i) {
1850 while (IntermedVals.
size() > 2) {
1851 NewIntermedVals.
clear();
1852 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1858 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1859 IntermedVals[i+1].second.
size());
1862 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1865 FinalIndices.
push_back(IntermedVals[i].second[j]);
1867 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1869 ShuffleVec[k] = NumElems + j;
1870 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1876 IntermedVals[i+1].first,
1881 std::make_pair(Shuffle, std::move(FinalIndices)));
1886 if ((IntermedVals.
size() & 1) != 0)
1889 IntermedVals.
swap(NewIntermedVals);
1893 "Invalid number of intermediate vectors");
1894 SDValue Vec1 = IntermedVals[0].first;
1896 if (IntermedVals.
size() > 1)
1897 Vec2 = IntermedVals[1].first;
1902 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1903 ShuffleVec[IntermedVals[0].second[i]] = i;
1904 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1905 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1919 unsigned NumElems =
Node->getNumOperands();
1922 EVT VT =
Node->getValueType(0);
1923 EVT OpVT =
Node->getOperand(0).getValueType();
1928 bool isOnlyLowElement =
true;
1929 bool MoreThanTwoValues =
false;
1931 for (
unsigned i = 0; i < NumElems; ++i) {
1936 isOnlyLowElement =
false;
1937 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1942 }
else if (!Value2.
getNode()) {
1945 }
else if (V != Value1 && V != Value2) {
1946 MoreThanTwoValues =
true;
1951 return DAG.getUNDEF(VT);
1953 if (isOnlyLowElement)
1959 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1961 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1964 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
1983 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1984 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1986 VT, dl, DAG.getEntryNode(), CPIdx,
1992 for (
unsigned i = 0; i < NumElems; ++i) {
1993 if (
Node->getOperand(i).isUndef())
1998 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
1999 if (!MoreThanTwoValues) {
2001 for (
unsigned i = 0; i < NumElems; ++i) {
2005 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2007 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
2014 Vec2 = DAG.getUNDEF(VT);
2017 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2027 return ExpandVectorBuildThroughStack(
Node);
2032 EVT VT =
Node->getValueType(0);
2035 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2046 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2047 TLI.getPointerTy(DAG.getDataLayout()));
2049 EVT RetVT =
Node->getValueType(0);
2056 SDValue InChain = DAG.getEntryNode();
2061 const Function &
F = DAG.getMachineFunction().getFunction();
2063 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2064 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2069 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2072 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2074 .setTailCall(isTailCall)
2075 .setSExtResult(signExtend)
2076 .setZExtResult(!signExtend)
2077 .setIsPostTypeLegalization(
true);
2079 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2084 return {DAG.getRoot(), DAG.getRoot()};
2096 EVT ArgVT =
Op.getValueType();
2100 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2101 Entry.IsZExt = !Entry.IsSExt;
2102 Args.push_back(Entry);
2105 return ExpandLibCall(LC,
Node, std::move(Args),
isSigned);
2108void SelectionDAGLegalize::ExpandFrexpLibCall(
2111 EVT VT =
Node->getValueType(0);
2112 EVT ExpVT =
Node->getValueType(1);
2120 FPArgEntry.
Node = FPOp;
2121 FPArgEntry.
Ty = ArgTy;
2123 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2125 PtrArgEntry.
Node = StackSlot;
2126 PtrArgEntry.
Ty = PointerType::get(*DAG.getContext(),
2127 DAG.getDataLayout().getAllocaAddrSpace());
2132 auto [
Call, Chain] = ExpandLibCall(LC,
Node, std::move(Args),
false);
2136 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2140 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2142 LoadExp.
getValue(1), DAG.getRoot());
2143 DAG.setRoot(OutputChain);
2149void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2152 if (LC == RTLIB::UNKNOWN_LIBCALL)
2155 if (
Node->isStrictFPOpcode()) {
2156 EVT RetVT =
Node->getValueType(0);
2160 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2163 Node->getOperand(0));
2165 Results.push_back(Tmp.second);
2167 SDValue Tmp = ExpandLibCall(LC,
Node,
false).first;
2173void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2181 Call_F32, Call_F64, Call_F80,
2182 Call_F128, Call_PPCF128);
2193 switch (
Node->getSimpleValueType(0).SimpleTy) {
2195 case MVT::i8: LC = Call_I8;
break;
2196 case MVT::i16: LC = Call_I16;
break;
2197 case MVT::i32: LC = Call_I32;
break;
2198 case MVT::i64: LC = Call_I64;
break;
2199 case MVT::i128: LC = Call_I128;
break;
2206void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2213 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2215 Call_F32, Call_F64, Call_F80,
2216 Call_F128, Call_PPCF128);
2222SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2228 switch (
Node->getSimpleValueType(0).SimpleTy) {
2230 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2231 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2232 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2233 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2234 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2240 SDValue InChain = DAG.getEntryNode();
2242 EVT RetVT =
Node->getValueType(0);
2248 EVT ArgVT =
Op.getValueType();
2254 Args.push_back(Entry);
2258 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2260 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2263 Args.push_back(Entry);
2265 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2266 TLI.getPointerTy(DAG.getDataLayout()));
2272 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2277 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2289 switch (
Node->getSimpleValueType(0).SimpleTy) {
2291 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2292 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2293 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2294 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2295 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2318SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2321 switch (
Node->getSimpleValueType(0).SimpleTy) {
2323 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2324 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2325 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2326 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2327 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2333 SDValue InChain = DAG.getEntryNode();
2335 EVT RetVT =
Node->getValueType(0);
2342 Entry.Node =
Node->getOperand(0);
2344 Entry.IsSExt =
false;
2345 Entry.IsZExt =
false;
2346 Args.push_back(Entry);
2349 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2350 Entry.Node = SinPtr;
2351 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2352 Entry.IsSExt =
false;
2353 Entry.IsZExt =
false;
2354 Args.push_back(Entry);
2357 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2358 Entry.Node = CosPtr;
2359 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2360 Entry.IsSExt =
false;
2361 Entry.IsZExt =
false;
2362 Args.push_back(Entry);
2364 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2365 TLI.getPointerTy(DAG.getDataLayout()));
2369 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2370 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2373 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2383 EVT VT =
Node->getValueType(0);
2386 EVT ExpVT =
N.getValueType();
2388 if (AsIntVT ==
EVT())
2401 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2406 const int Precision = APFloat::semanticsPrecision(FltSem);
2408 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2409 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2411 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2413 const APFloat One(FltSem,
"1.0");
2414 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2418 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2427 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2433 DAG.getSetCC(dl, SetCCVT,
N, DoubleMaxExp,
ISD::SETUGT);
2435 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2447 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2448 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2453 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2458 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2462 SDValue ScaleDownTwice = DAG.getSetCC(
2463 dl, SetCCVT,
N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2475 DAG.getNode(
ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small,
X));
2479 DAG.getNode(
ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small,
N));
2484 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2485 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2488 ExponentShiftAmt, NUW_NSW);
2490 return DAG.getNode(
ISD::FMUL, dl, VT, NewX, AsFP);
2497 EVT ExpVT =
Node->getValueType(1);
2499 if (AsIntVT ==
EVT())
2504 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2521 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2525 SDValue SmallestNormalizedInt = DAG.getConstant(
2531 DAG.getConstant(
APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2537 FractSignMaskVal.
setBit(BitSize - 1);
2540 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2542 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2544 const APFloat One(FltSem,
"1.0");
2548 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2550 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2554 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2560 SDValue AddNegSmallestNormal =
2562 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2566 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt,
ISD::SETULT);
2568 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2583 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2586 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2589 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2595 const APFloat Half(FltSem,
"0.5");
2596 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2609 return DAG.getMergeValues({Result0, Result1}, dl);
2620 EVT DestVT =
Node->getValueType(0);
2622 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2628 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2629 (DestVT.
bitsLE(MVT::f64) ||
2633 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2637 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2644 DAG.getConstant(0x80000000u, dl, MVT::i32));
2647 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2650 if (DAG.getDataLayout().isBigEndian())
2653 SDValue MemChain = DAG.getEntryNode();
2656 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2669 SDValue Bias = DAG.getConstantFP(
2670 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2671 : llvm::bit_cast<double>(0x4330000000000000ULL),
2676 if (
Node->isStrictFPOpcode()) {
2678 {
Node->getOperand(0), Load, Bias});
2681 std::pair<SDValue, SDValue> ResultPair;
2683 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2684 Result = ResultPair.first;
2685 Chain = ResultPair.second;
2691 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2700 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2701 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2702 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2717 EVT SetCCVT = getSetCCResultType(SrcVT);
2719 SDValue SignBitTest = DAG.getSetCC(
2720 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2722 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2723 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2725 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2730 if (
Node->isStrictFPOpcode()) {
2733 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2735 {
Node->getOperand(0), InCvt });
2743 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2744 Fast->setFlags(Flags);
2745 Flags.setNoFPExcept(
true);
2753 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2757 if (!TLI.isOperationLegalOrCustom(
2764 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2766 "Cannot perform lossless SINT_TO_FP!");
2769 if (
Node->isStrictFPOpcode()) {
2771 {
Node->getOperand(0), Op0 });
2775 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2778 Four = DAG.getIntPtrConstant(4, dl);
2779 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2780 SignSet, Four, Zero);
2789 case MVT::i8 : FF = 0x43800000ULL;
break;
2790 case MVT::i16: FF = 0x47800000ULL;
break;
2791 case MVT::i32: FF = 0x4F800000ULL;
break;
2792 case MVT::i64: FF = 0x5F800000ULL;
break;
2794 if (DAG.getDataLayout().isLittleEndian())
2800 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2801 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2805 if (DestVT == MVT::f32)
2806 FudgeInReg = DAG.getLoad(
2807 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2816 LegalizeOp(
Load.getNode());
2820 if (
Node->isStrictFPOpcode()) {
2822 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2823 Chain =
Result.getValue(1);
2827 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2835void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2837 bool IsStrict =
N->isStrictFPOpcode();
2840 EVT DestVT =
N->getValueType(0);
2841 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2848 unsigned OpToUse = 0;
2856 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2864 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2876 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
2879 dl, NewInTy, LegalOp)});
2886 DAG.getNode(OpToUse, dl, DestVT,
2888 dl, NewInTy, LegalOp)));
2896void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2898 bool IsStrict =
N->isStrictFPOpcode();
2901 EVT DestVT =
N->getValueType(0);
2902 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2904 EVT NewOutTy = DestVT;
2906 unsigned OpToUse = 0;
2916 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2921 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2930 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2931 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2933 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2951 EVT NewOutTy =
Node->getValueType(0);
2956 if (TLI.isOperationLegalOrCustom(
Opcode, NewOutTy))
2963 Node->getOperand(1));
2969 EVT VT =
Op.getValueType();
2970 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2975 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
2981 DAG.getConstant(1ULL << (--i), dl, ShVT));
2986 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2989bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
2993 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2995 switch (
Node->getOpcode()) {
2997 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
3002 if ((Tmp1 = TLI.expandABD(
Node, DAG)))
3006 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
3011 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
3016 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
3020 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
3024 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
3028 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3033 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3036 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
3037 TLI.getPointerTy(DAG.getDataLayout()));
3045 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3051 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
3067 DAG.getConstant(0, dl,
Node->getValueType(0)));
3073 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3079 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3080 SDValue Swap = DAG.getAtomicCmpSwap(
3082 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
3083 cast<AtomicSDNode>(
Node)->getMemOperand());
3092 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3093 cast<AtomicSDNode>(
Node)->getMemOperand());
3101 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3102 SDValue Res = DAG.getAtomicCmpSwap(
3104 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3105 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
3111 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
3112 EVT OuterType =
Node->getValueType(0);
3113 switch (TLI.getExtendForAtomicOps()) {
3116 DAG.getValueType(AtomicType));
3118 Node->getOperand(2), DAG.getValueType(AtomicType));
3123 DAG.getValueType(AtomicType));
3124 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3128 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3129 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3145 EVT VT =
Node->getValueType(0);
3149 cast<VTSDNode>(
RHS->getOperand(1))->getVT() == AN->
getMemoryVT())
3150 RHS =
RHS->getOperand(0);
3154 Node->getOperand(0),
Node->getOperand(1),
3164 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3168 EVT VT =
Node->getValueType(0);
3170 Results.push_back(DAG.getConstant(0, dl, VT));
3173 Results.push_back(DAG.getConstantFP(0, dl, VT));
3180 if (TLI.isStrictFPEnabled())
3184 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3185 Node->getValueType(0))
3186 == TargetLowering::Legal)
3190 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3191 Node->getValueType(0), dl,
3192 Node->getOperand(0)))) {
3194 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3200 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3201 Node->getValueType(0), dl)))
3207 if (TLI.isStrictFPEnabled())
3211 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3212 Node->getValueType(0))
3213 == TargetLowering::Legal)
3217 if ((Tmp1 = EmitStackConvert(
3218 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3219 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3221 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3226 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
3227 Node->getOperand(0).getValueType(),
3228 Node->getValueType(0), dl)))
3237 if (
Op.getValueType() == MVT::bf16) {
3241 Op = DAG.getAnyExtOrTrunc(
Op, dl, MVT::i32);
3245 DAG.getConstant(16, dl,
3246 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3249 if (
Node->getValueType(0) != MVT::f32)
3256 if (
Op.getValueType() != MVT::f32)
3258 DAG.getIntPtrConstant(0, dl,
true));
3261 DAG.getConstant(16, dl,
3262 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3265 if (
Node->getValueType(0) == MVT::bf16) {
3269 Op = DAG.getAnyExtOrTrunc(
Op, dl,
Node->getValueType(0));
3275 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
3276 EVT VT =
Node->getValueType(0);
3286 SDValue One = DAG.getConstant(1, dl, VT);
3296 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3299 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3301 Node->getOperand(0), ShiftCst);
3308 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
3310 if (
Node->isStrictFPOpcode())
3317 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
3319 if (
Node->isStrictFPOpcode())
3324 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3328 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3330 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3335 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3339 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3341 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3344 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3350 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3360 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3363 Node->getOperand(0));
3365 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3375 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3381 Results.push_back(ExpandINSERT_VECTOR_ELT(
Node->getOperand(0),
3382 Node->getOperand(1),
3383 Node->getOperand(2), dl));
3389 EVT VT =
Node->getValueType(0);
3393 if (!TLI.isTypeLegal(EltVT)) {
3394 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3399 if (NewEltVT.
bitsLT(EltVT)) {
3415 unsigned int factor =
3423 for (
unsigned fi = 0; fi < factor; ++fi)
3427 for (
unsigned fi = 0; fi < factor; ++fi)
3438 for (
unsigned i = 0; i != NumElems; ++i) {
3446 DAG.getVectorIdxConstant(
Idx, dl)));
3450 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3453 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3460 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3464 EVT OpTy =
Node->getOperand(0).getValueType();
3465 if (
Node->getConstantOperandVal(1)) {
3469 TLI.getShiftAmountTy(
3470 Node->getOperand(0).getValueType(),
3471 DAG.getDataLayout())));
3476 Node->getOperand(0));
3484 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3485 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3486 Node->getValueType(0)));
3489 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3496 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3497 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3498 Node->getOperand(1)));
3504 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3517 auto CNode = cast<ConstantSDNode>(
Node->getOperand(1));
3520 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3531 switch (
Node->getOpcode()) {
3538 Tmp1 =
Node->getOperand(0);
3539 Tmp2 =
Node->getOperand(1);
3540 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3546 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3552 EVT VT =
Node->getValueType(0);
3558 SDVTList VTs = DAG.getVTList(VT, VT);
3568 EVT VT =
Node->getValueType(0);
3572 if (TLI.getLibcallName(LC))
3578 Results.push_back(Expanded.getValue(1));
3587 if (TLI.getLibcallName(LC))
3592 Results.push_back(Expanded.getValue(1));
3600 if (
Node->getValueType(0) != MVT::f32) {
3611 if (
Node->getValueType(0) != MVT::f32) {
3617 {
Node->getOperand(0),
Node->getOperand(1)});
3619 {
Node->getValueType(0), MVT::Other},
3627 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3629 MVT SVT =
Op.getSimpleValueType();
3630 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3636 DAG.getIntPtrConstant(0, dl,
true));
3647 DAG.shouldOptForSize()))
3648 Results.push_back(ExpandConstantFP(CFP,
true));
3653 Results.push_back(ExpandConstant(CP));
3657 EVT VT =
Node->getValueType(0);
3658 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3659 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3668 EVT VT =
Node->getValueType(0);
3670 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3671 "Don't know how to expand this subtraction!");
3672 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3673 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3679 if (TLI.expandREM(
Node, Tmp1, DAG))
3686 EVT VT =
Node->getValueType(0);
3687 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3688 SDVTList VTs = DAG.getVTList(VT, VT);
3689 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3690 Node->getOperand(1));
3697 unsigned ExpandOpcode =
3699 EVT VT =
Node->getValueType(0);
3700 SDVTList VTs = DAG.getVTList(VT, VT);
3702 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3703 Node->getOperand(1));
3711 MVT VT =
LHS.getSimpleValueType();
3712 unsigned MULHOpcode =
3715 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3717 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3723 assert(TLI.isTypeLegal(HalfType));
3724 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3726 TargetLowering::MulExpansionKind::Always)) {
3727 for (
unsigned i = 0; i < 2; ++i) {
3730 SDValue Shift = DAG.getConstant(
3732 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3741 EVT VT =
Node->getValueType(0);
3742 SDVTList VTs = DAG.getVTList(VT, VT);
3748 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3749 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3750 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3751 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3752 unsigned OpToUse = 0;
3753 if (HasSMUL_LOHI && !HasMULHS) {
3755 }
else if (HasUMUL_LOHI && !HasMULHU) {
3757 }
else if (HasSMUL_LOHI) {
3759 }
else if (HasUMUL_LOHI) {
3763 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3764 Node->getOperand(1)));
3772 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3773 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3774 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3775 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3780 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3788 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3793 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3800 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3810 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3817 Node->getOperand(0),
3818 Node->getOperand(1),
3819 Node->getConstantOperandVal(2),
3842 EVT VT =
LHS.getValueType();
3846 EVT CarryType =
Node->getValueType(1);
3847 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3849 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3852 SDValue One = DAG.getConstant(1, dl, VT);
3854 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3863 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3864 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3866 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3872 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3878 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3886 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3894 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
3906 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3907 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3912 Tmp1 =
Node->getOperand(0);
3913 Tmp2 =
Node->getOperand(1);
3914 Tmp3 =
Node->getOperand(2);
3918 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3920 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3931 int JTI = cast<JumpTableSDNode>(Table.
getNode())->getIndex();
3934 EVT PTy = TLI.getPointerTy(TD);
3936 unsigned EntrySize =
3937 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3944 Index = DAG.getNode(
3949 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
3958 if (TLI.isJumpTableRelative()) {
3963 TLI.getPICJumpTableRelocBase(Table, DAG));
3966 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, JTI, DAG);
3973 Tmp1 =
Node->getOperand(0);
3974 Tmp2 =
Node->getOperand(1);
3980 Node->getOperand(2));
3992 Node->getOperand(2));
4000 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
4005 unsigned Offset = IsStrict ? 1 : 0;
4014 bool Legalized = TLI.LegalizeSetCCCondCode(
4015 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4016 Chain, IsSignaling);
4024 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4028 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
4030 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4031 Tmp2, Tmp3,
Node->getFlags());
4039 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
4042 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
4054 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4059 EVT VT =
Node->getValueType(0);
4062 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
4063 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
4070 Tmp1 =
Node->getOperand(0);
4071 Tmp2 =
Node->getOperand(1);
4072 Tmp3 =
Node->getOperand(2);
4073 Tmp4 =
Node->getOperand(3);
4074 EVT VT =
Node->getValueType(0);
4084 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4086 EVT CCVT = getSetCCResultType(CmpVT);
4088 Results.push_back(DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4));
4093 bool Legalized =
false;
4101 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4111 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4117 Legalized = TLI.LegalizeSetCCCondCode(
4121 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4132 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
4137 Tmp2, Tmp3, Tmp4,
CC);
4147 Tmp1 =
Node->getOperand(0);
4148 Tmp2 =
Node->getOperand(2);
4149 Tmp3 =
Node->getOperand(3);
4150 Tmp4 =
Node->getOperand(1);
4152 bool Legalized = TLI.LegalizeSetCCCondCode(
4153 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
4156 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4161 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4164 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4169 Tmp2, Tmp3,
Node->getOperand(4));
4184 EVT VT =
Node->getValueType(0);
4190 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4193 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
4196 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
4220 Results.push_back(TLI.expandVecReduce(
Node, DAG));
4237 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
4243 switch (
Node->getOpcode()) {
4245 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4246 Node->getValueType(0))
4247 == TargetLowering::Legal)
4251 if (TLI.getStrictFPOperationAction(
4254 if (TLI.getStrictFPOperationAction(
4258 EVT VT =
Node->getValueType(0);
4262 {Node->getOperand(0), Node->getOperand(1), Neg},
4277 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4278 Node->getOperand(1).getValueType())
4279 == TargetLowering::Legal)
4296void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
4301 unsigned Opc =
Node->getOpcode();
4310 .setChain(
Node->getOperand(0))
4313 DAG.getExternalSymbol(
"__sync_synchronize",
4314 TLI.getPointerTy(DAG.getDataLayout())),
4317 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4319 Results.push_back(CallResult.second);
4341 EVT RetVT =
Node->getValueType(0);
4344 if (TLI.getLibcallName(LC)) {
4351 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4352 "Unexpected atomic op or value type!");
4356 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4359 Node->getOperand(0));
4361 Results.push_back(Tmp.second);
4369 .setChain(
Node->getOperand(0))
4371 DAG.getExternalSymbol(
4372 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4374 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4376 Results.push_back(CallResult.second);
4381 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4382 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4383 RTLIB::FMIN_PPCF128,
Results);
4390 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4391 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4392 RTLIB::FMAX_PPCF128,
Results);
4396 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4397 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4398 RTLIB::SQRT_PPCF128,
Results);
4401 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4402 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4403 RTLIB::CBRT_PPCF128,
Results);
4407 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4408 RTLIB::SIN_F80, RTLIB::SIN_F128,
4413 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4414 RTLIB::COS_F80, RTLIB::COS_F128,
4423 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4424 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4428 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4429 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4433 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4434 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4438 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4439 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4443 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4444 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4447 ExpandFPLibCall(
Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4448 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128,
Results);
4452 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4453 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4454 RTLIB::TRUNC_PPCF128,
Results);
4458 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4459 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4460 RTLIB::FLOOR_PPCF128,
Results);
4464 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4465 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4466 RTLIB::CEIL_PPCF128,
Results);
4470 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4471 RTLIB::RINT_F80, RTLIB::RINT_F128,
4472 RTLIB::RINT_PPCF128,
Results);
4476 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4477 RTLIB::NEARBYINT_F64,
4478 RTLIB::NEARBYINT_F80,
4479 RTLIB::NEARBYINT_F128,
4480 RTLIB::NEARBYINT_PPCF128,
Results);
4484 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4488 RTLIB::ROUND_PPCF128,
Results);
4492 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4493 RTLIB::ROUNDEVEN_F64,
4494 RTLIB::ROUNDEVEN_F80,
4495 RTLIB::ROUNDEVEN_F128,
4496 RTLIB::ROUNDEVEN_PPCF128,
Results);
4500 ExpandFPLibCall(
Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4501 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128,
Results);
4510 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4511 if (!TLI.getLibcallName(LC)) {
4513 if (
Node->isStrictFPOpcode()) {
4516 {
Node->getValueType(0),
Node->getValueType(1)},
4517 {
Node->getOperand(0),
Node->getOperand(2)});
4520 {
Node->getValueType(0),
Node->getValueType(1)},
4527 Node->getOperand(1));
4529 Node->getValueType(0),
4534 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4535 bool ExponentHasSizeOfInt =
4536 DAG.getLibInfo().getIntSize() ==
4537 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4538 if (!ExponentHasSizeOfInt) {
4541 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4542 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4550 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4551 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4555 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4556 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4558 RTLIB::LROUND_PPCF128,
Results);
4562 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4563 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4564 RTLIB::LLROUND_F128,
4565 RTLIB::LLROUND_PPCF128,
Results);
4569 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4570 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4572 RTLIB::LRINT_PPCF128,
Results);
4576 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4577 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4579 RTLIB::LLRINT_PPCF128,
Results);
4583 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4584 RTLIB::DIV_F80, RTLIB::DIV_F128,
4589 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4590 RTLIB::REM_F80, RTLIB::REM_F128,
4595 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4596 RTLIB::FMA_F80, RTLIB::FMA_F128,
4601 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4602 RTLIB::ADD_F80, RTLIB::ADD_F128,
4607 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4608 RTLIB::MUL_F80, RTLIB::MUL_F128,
4612 if (
Node->getValueType(0) == MVT::f32) {
4613 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false).first);
4617 if (
Node->getValueType(0) == MVT::f32) {
4619 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4620 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
4623 Results.push_back(Tmp.second);
4630 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4631 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4637 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4638 Results.push_back(ExpandLibCall(LC,
Node,
false).first);