58#define DEBUG_TYPE "legalizedag"
64struct FloatSignAsInt {
87class SelectionDAGLegalize {
99 EVT getSetCCResultType(
EVT VT)
const {
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
166 void getSignAsIntValue(FloatSignAsInt &State,
const SDLoc &
DL,
168 SDValue modifySignAsInt(
const FloatSignAsInt &State,
const SDLoc &
DL,
215 dbgs() <<
" with: "; New->dump(&DAG));
218 "Replacing one node with another that produces a different number "
222 UpdatedNodes->
insert(New);
228 dbgs() <<
" with: "; New->dump(&DAG));
232 UpdatedNodes->
insert(New.getNode());
233 ReplacedNode(Old.getNode());
240 for (
unsigned i = 0, e = Old->
getNumValues(); i != e; ++i) {
244 UpdatedNodes->
insert(New[i].getNode());
251 dbgs() <<
" with: "; New->dump(&DAG));
255 UpdatedNodes->
insert(New.getNode());
256 ReplacedNode(Old.getNode());
266 bool isObjectScalable) {
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
274 ObjectSize, MFI.getObjectAlign(FI));
281SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
286 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
288 assert(NumEltsGrowth &&
"Cannot promote to vector type with fewer elts!");
290 if (NumEltsGrowth == 1)
291 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
294 for (
unsigned i = 0; i != NumMaskElts; ++i) {
296 for (
unsigned j = 0;
j != NumEltsGrowth; ++
j) {
303 assert(NewMask.
size() == NumDestElts &&
"Non-integer NumEltsGrowth?");
304 assert(TLI.isShuffleMaskLegal(NewMask, NVT) &&
"Shuffle not legal?");
305 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
324 assert((VT == MVT::f64 || VT == MVT::f32) &&
"Invalid type expansion");
326 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
336 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
342 TLI.ShouldShrinkFPConstant(OrigVT)) {
345 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
353 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
354 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
363 OrigVT, dl, DAG.getEntryNode(), CPIdx,
371 EVT VT =
CP->getValueType(0);
372 SDValue CPIdx = DAG.getConstantPool(
CP->getConstantIntValue(),
373 TLI.getPointerTy(DAG.getDataLayout()));
374 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
376 VT, dl, DAG.getEntryNode(), CPIdx,
402 for (
unsigned i = 0; i != NumElts; ++i)
403 ShufOps.
push_back(i != InsertPos->getZExtValue() ? i : NumElts);
405 return DAG.getVectorShuffle(Vec.
getValueType(), dl, Vec, ScVec, ShufOps);
408 return ExpandInsertToVectorThroughStack(
Op);
435 TLI.isTypeLegal(MVT::i32)) {
437 bitcastToAPInt().zextOrTrunc(32),
438 SDLoc(CFP), MVT::i32);
439 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
440 ST->getOriginalAlign(), MMOFlags, AAInfo);
446 if (TLI.isTypeLegal(MVT::i64)) {
448 zextOrTrunc(64),
SDLoc(CFP), MVT::i64);
449 return DAG.getStore(Chain, dl, Con,
Ptr,
ST->getPointerInfo(),
450 ST->getOriginalAlign(), MMOFlags, AAInfo);
453 if (TLI.isTypeLegal(MVT::i32) && !
ST->isVolatile()) {
460 if (DAG.getDataLayout().isBigEndian())
463 Lo = DAG.getStore(Chain, dl,
Lo,
Ptr,
ST->getPointerInfo(),
464 ST->getOriginalAlign(), MMOFlags, AAInfo);
466 Hi = DAG.getStore(Chain, dl,
Hi,
Ptr,
467 ST->getPointerInfo().getWithOffset(4),
468 ST->getOriginalAlign(), MMOFlags, AAInfo);
477void SelectionDAGLegalize::LegalizeStoreOps(
SDNode *
Node) {
486 if (!
ST->isTruncatingStore()) {
488 if (
SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
489 ReplaceNode(ST, OptStore);
494 MVT VT =
Value.getSimpleValueType();
495 switch (TLI.getOperationAction(
ISD::STORE, VT)) {
497 case TargetLowering::Legal: {
500 EVT MemVT =
ST->getMemoryVT();
502 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
503 *
ST->getMemOperand())) {
506 ReplaceNode(
SDValue(ST, 0), Result);
511 case TargetLowering::Custom: {
518 case TargetLowering::Promote: {
521 "Can only promote stores to same size type");
524 ST->getOriginalAlign(), MMOFlags, AAInfo);
534 EVT StVT =
ST->getMemoryVT();
537 auto &
DL = DAG.getDataLayout();
539 if (StWidth != StSize) {
544 Value = DAG.getZeroExtendInReg(
Value, dl, StVT);
546 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), NVT,
547 ST->getOriginalAlign(), MMOFlags, AAInfo);
553 unsigned LogStWidth =
Log2_32(StWidthBits);
555 unsigned RoundWidth = 1 << LogStWidth;
556 assert(RoundWidth < StWidthBits);
557 unsigned ExtraWidth = StWidthBits - RoundWidth;
558 assert(ExtraWidth < RoundWidth);
559 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
560 "Store size not an integral number of bytes!");
564 unsigned IncrementSize;
566 if (
DL.isLittleEndian()) {
569 Lo = DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(),
570 RoundVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
573 IncrementSize = RoundWidth / 8;
578 DAG.getConstant(RoundWidth, dl,
579 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
580 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
581 ST->getPointerInfo().getWithOffset(IncrementSize),
582 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
589 DAG.getConstant(ExtraWidth, dl,
590 TLI.getShiftAmountTy(
Value.getValueType(),
DL)));
591 Hi = DAG.getTruncStore(Chain, dl,
Hi,
Ptr,
ST->getPointerInfo(), RoundVT,
592 ST->getOriginalAlign(), MMOFlags, AAInfo);
595 IncrementSize = RoundWidth / 8;
597 DAG.getConstant(IncrementSize, dl,
598 Ptr.getValueType()));
600 ST->getPointerInfo().getWithOffset(IncrementSize),
601 ExtraVT,
ST->getOriginalAlign(), MMOFlags, AAInfo);
608 switch (TLI.getTruncStoreAction(
ST->getValue().getValueType(), StVT)) {
610 case TargetLowering::Legal: {
611 EVT MemVT =
ST->getMemoryVT();
614 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
615 *
ST->getMemOperand())) {
617 ReplaceNode(
SDValue(ST, 0), Result);
621 case TargetLowering::Custom: {
627 case TargetLowering::Expand:
629 "Vector Stores are handled in LegalizeVectorOps");
634 if (TLI.isTypeLegal(StVT)) {
637 ST->getOriginalAlign(), MMOFlags, AAInfo);
642 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
645 DAG.getTruncStore(Chain, dl,
Value,
Ptr,
ST->getPointerInfo(), StVT,
646 ST->getOriginalAlign(), MMOFlags, AAInfo);
655void SelectionDAGLegalize::LegalizeLoadOps(
SDNode *
Node) {
664 LLVM_DEBUG(
dbgs() <<
"Legalizing non-extending load operation\n");
665 MVT VT =
Node->getSimpleValueType(0);
669 switch (TLI.getOperationAction(
Node->getOpcode(), VT)) {
671 case TargetLowering::Legal: {
672 EVT MemVT =
LD->getMemoryVT();
676 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(),
DL, MemVT,
677 *
LD->getMemOperand())) {
678 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
682 case TargetLowering::Custom:
683 if (
SDValue Res = TLI.LowerOperation(RVal, DAG)) {
689 case TargetLowering::Promote: {
690 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VT);
692 "Can only promote loads to same size type");
694 SDValue Res = DAG.getLoad(NVT, dl, Chain,
Ptr,
LD->getMemOperand());
702 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), RVal);
703 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), RChain);
705 UpdatedNodes->insert(RVal.
getNode());
706 UpdatedNodes->insert(RChain.
getNode());
714 EVT SrcVT =
LD->getMemoryVT();
728 TLI.getLoadExtAction(ExtType,
Node->getValueType(0), MVT::i1) ==
729 TargetLowering::Promote)) {
743 Chain,
Ptr,
LD->getPointerInfo(), NVT,
744 LD->getOriginalAlign(), MMOFlags, AAInfo);
752 Result, DAG.getValueType(SrcVT));
756 Result.getValueType(), Result,
757 DAG.getValueType(SrcVT));
765 unsigned LogSrcWidth =
Log2_32(SrcWidthBits);
767 unsigned RoundWidth = 1 << LogSrcWidth;
768 assert(RoundWidth < SrcWidthBits);
769 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
770 assert(ExtraWidth < RoundWidth);
771 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
772 "Load size not an integral number of bytes!");
776 unsigned IncrementSize;
777 auto &
DL = DAG.getDataLayout();
779 if (
DL.isLittleEndian()) {
783 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
787 IncrementSize = RoundWidth / 8;
790 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
791 LD->getPointerInfo().getWithOffset(IncrementSize),
792 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
802 DAG.getConstant(RoundWidth, dl,
803 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
811 Hi = DAG.getExtLoad(ExtType, dl,
Node->getValueType(0), Chain,
Ptr,
812 LD->getPointerInfo(), RoundVT,
LD->getOriginalAlign(),
816 IncrementSize = RoundWidth / 8;
820 LD->getPointerInfo().getWithOffset(IncrementSize),
821 ExtraVT,
LD->getOriginalAlign(), MMOFlags, AAInfo);
831 DAG.getConstant(ExtraWidth, dl,
832 TLI.getShiftAmountTy(
Hi.getValueType(),
DL)));
840 bool isCustom =
false;
841 switch (TLI.getLoadExtAction(ExtType,
Node->getValueType(0),
844 case TargetLowering::Custom:
847 case TargetLowering::Legal:
859 EVT MemVT =
LD->getMemoryVT();
861 if (!TLI.allowsMemoryAccess(*DAG.getContext(),
DL, MemVT,
862 *
LD->getMemOperand())) {
863 std::tie(
Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
868 case TargetLowering::Expand: {
869 EVT DestVT =
Node->getValueType(0);
875 (TLI.isTypeLegal(SrcVT) ||
876 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
883 SrcVT,
LD->getMemOperand());
886 Value = DAG.getNode(ExtendOp, dl,
Node->getValueType(0), Load);
887 Chain =
Load.getValue(1);
896 if (SVT == MVT::f16 || SVT == MVT::bf16) {
902 Ptr, ISrcVT,
LD->getMemOperand());
906 Chain =
Result.getValue(1);
912 "Vector Loads are handled in LegalizeVectorOps");
919 "EXTLOAD should always be supported!");
923 Node->getValueType(0),
925 LD->getMemOperand());
930 Result, DAG.getValueType(SrcVT));
932 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
934 Chain =
Result.getValue(1);
945 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
947 UpdatedNodes->insert(
Value.getNode());
948 UpdatedNodes->insert(Chain.
getNode());
955void SelectionDAGLegalize::LegalizeOp(
SDNode *
Node) {
964 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i)
965 assert(TLI.getTypeAction(*DAG.getContext(),
Node->getValueType(i)) ==
966 TargetLowering::TypeLegal &&
967 "Unexpected illegal type!");
970 assert((TLI.getTypeAction(*DAG.getContext(),
Op.getValueType()) ==
971 TargetLowering::TypeLegal ||
974 "Unexpected illegal type!");
979 bool SimpleFinishLegalizing =
true;
980 switch (
Node->getOpcode()) {
985 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
988 Action = TLI.getOperationAction(
Node->getOpcode(),
989 Node->getValueType(0));
992 Action = TLI.getOperationAction(
Node->getOpcode(),
993 Node->getValueType(0));
994 if (Action != TargetLowering::Promote)
995 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::Other);
999 Action = TLI.getOperationAction(
Node->getOpcode(),
1000 Node->getOperand(1).getValueType());
1011 Action = TLI.getOperationAction(
Node->getOpcode(),
1012 Node->getOperand(0).getValueType());
1025 Action = TLI.getOperationAction(
Node->getOpcode(),
1026 Node->getOperand(1).getValueType());
1029 EVT InnerType = cast<VTSDNode>(
Node->getOperand(1))->getVT();
1030 Action = TLI.getOperationAction(
Node->getOpcode(), InnerType);
1034 Action = TLI.getOperationAction(
Node->getOpcode(),
1035 Node->getOperand(1).getValueType());
1044 unsigned Opc =
Node->getOpcode();
1049 : (Opc ==
ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1051 unsigned CompareOperand = Opc ==
ISD::BR_CC ? 2
1055 MVT OpVT =
Node->getOperand(CompareOperand).getSimpleValueType();
1057 cast<CondCodeSDNode>(
Node->getOperand(CCOperand))->get();
1058 Action = TLI.getCondCodeAction(CCCode, OpVT);
1059 if (Action == TargetLowering::Legal) {
1061 Action = TLI.getOperationAction(
Node->getOpcode(),
1062 Node->getValueType(0));
1064 Action = TLI.getOperationAction(
Node->getOpcode(), OpVT);
1072 SimpleFinishLegalizing =
false;
1079 SimpleFinishLegalizing =
false;
1092 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1093 if (Action == TargetLowering::Legal)
1094 Action = TargetLowering::Expand;
1104 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1105 if (Action == TargetLowering::Legal)
1106 Action = TargetLowering::Custom;
1112 Action = TLI.getOperationAction(
Node->getOpcode(), MVT::i64);
1119 Action = TargetLowering::Legal;
1122 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1123 if (Action == TargetLowering::Expand) {
1127 Node->getOperand(0));
1134 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1135 if (Action == TargetLowering::Expand) {
1139 Node->getOperand(0));
1153 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1163 unsigned Scale =
Node->getConstantOperandVal(2);
1164 Action = TLI.getFixedPointOperationAction(
Node->getOpcode(),
1165 Node->getValueType(0), Scale);
1169 Action = TLI.getOperationAction(
Node->getOpcode(),
1170 cast<MaskedScatterSDNode>(
Node)->getValue().getValueType());
1173 Action = TLI.getOperationAction(
Node->getOpcode(),
1174 cast<MaskedStoreSDNode>(
Node)->getValue().getValueType());
1176 case ISD::VP_SCATTER:
1177 Action = TLI.getOperationAction(
1179 cast<VPScatterSDNode>(
Node)->getValue().getValueType());
1182 Action = TLI.getOperationAction(
1184 cast<VPStoreSDNode>(
Node)->getValue().getValueType());
1186 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1187 Action = TLI.getOperationAction(
1189 cast<VPStridedStoreSDNode>(
Node)->getValue().getValueType());
1207 Action = TLI.getOperationAction(
1208 Node->getOpcode(),
Node->getOperand(0).getValueType());
1212 case ISD::VP_REDUCE_FADD:
1213 case ISD::VP_REDUCE_FMUL:
1214 case ISD::VP_REDUCE_ADD:
1215 case ISD::VP_REDUCE_MUL:
1216 case ISD::VP_REDUCE_AND:
1217 case ISD::VP_REDUCE_OR:
1218 case ISD::VP_REDUCE_XOR:
1219 case ISD::VP_REDUCE_SMAX:
1220 case ISD::VP_REDUCE_SMIN:
1221 case ISD::VP_REDUCE_UMAX:
1222 case ISD::VP_REDUCE_UMIN:
1223 case ISD::VP_REDUCE_FMAX:
1224 case ISD::VP_REDUCE_FMIN:
1225 case ISD::VP_REDUCE_FMAXIMUM:
1226 case ISD::VP_REDUCE_FMINIMUM:
1227 case ISD::VP_REDUCE_SEQ_FADD:
1228 case ISD::VP_REDUCE_SEQ_FMUL:
1229 Action = TLI.getOperationAction(
1230 Node->getOpcode(),
Node->getOperand(1).getValueType());
1232 case ISD::VP_CTTZ_ELTS:
1233 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
1234 Action = TLI.getOperationAction(
Node->getOpcode(),
1235 Node->getOperand(0).getValueType());
1239 Action = TLI.getCustomOperationAction(*
Node);
1241 Action = TLI.getOperationAction(
Node->getOpcode(),
Node->getValueType(0));
1246 if (SimpleFinishLegalizing) {
1248 switch (
Node->getOpcode()) {
1267 NewNode = DAG.UpdateNodeOperands(
Node, Op0, SAO);
1287 NewNode = DAG.UpdateNodeOperands(
Node, Op0, Op1, SAO);
1293 if (NewNode !=
Node) {
1294 ReplaceNode(
Node, NewNode);
1298 case TargetLowering::Legal:
1301 case TargetLowering::Custom:
1309 if (
Node->getNumValues() == 1) {
1313 Node->getValueType(0) == MVT::Glue) &&
1314 "Type mismatch for custom legalized operation");
1322 for (
unsigned i = 0, e =
Node->getNumValues(); i != e; ++i) {
1326 Node->getValueType(i) == MVT::Glue) &&
1327 "Type mismatch for custom legalized operation");
1331 ReplaceNode(
Node, ResultVals.
data());
1336 case TargetLowering::Expand:
1337 if (ExpandNode(
Node))
1340 case TargetLowering::LibCall:
1341 ConvertNodeToLibcall(
Node);
1343 case TargetLowering::Promote:
1349 switch (
Node->getOpcode()) {
1362 return LegalizeLoadOps(
Node);
1364 return LegalizeStoreOps(
Node);
1368SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(
SDValue Op) {
1388 if (
ST->isIndexed() ||
ST->isTruncatingStore() ||
1389 ST->getValue() != Vec)
1394 if (!
ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1403 ST->hasPredecessor(
Op.getNode()))
1416 StackPtr = DAG.CreateStackTemporary(VecVT);
1419 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1423 Align ElementAlignment =
1424 std::min(cast<StoreSDNode>(Ch)->
getAlign(),
1425 DAG.getDataLayout().getPrefTypeAlign(
1426 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1428 if (
Op.getValueType().isVector()) {
1429 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1430 Op.getValueType(),
Idx);
1431 NewLoad = DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr,
1434 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1435 NewLoad = DAG.getExtLoad(
ISD::EXTLOAD, dl,
Op.getValueType(), Ch, StackPtr,
1441 DAG.ReplaceAllUsesOfValueWith(Ch,
SDValue(NewLoad.
getNode(), 1));
1447 NewLoadOperands[0] = Ch;
1449 SDValue(DAG.UpdateNodeOperands(NewLoad.
getNode(), NewLoadOperands), 0);
1453SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(
SDValue Op) {
1454 assert(
Op.getValueType().isVector() &&
"Non-vector insert subvector!");
1465 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1470 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1473 Idx = DAG.getFreeze(
Idx);
1478 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, PartVT,
Idx);
1482 Ch, dl, Part, SubStackPtr,
1486 TLI.getVectorElementPointer(DAG, StackPtr, VecVT,
Idx);
1489 Ch = DAG.getTruncStore(
1490 Ch, dl, Part, SubStackPtr,
1496 return DAG.getLoad(
Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1502 "Unexpected opcode!");
1508 EVT VT =
Node->getValueType(0);
1510 :
Node->getOperand(0).getValueType();
1512 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1513 int FI = cast<FrameIndexSDNode>(FIPtr.
getNode())->getIndex();
1520 assert(TypeByteSize > 0 &&
"Vector element type too small for stack store!");
1524 bool Truncate = isa<BuildVectorSDNode>(
Node) &&
1525 MemVT.
bitsLT(
Node->getOperand(0).getValueType());
1528 for (
unsigned i = 0, e =
Node->getNumOperands(); i != e; ++i) {
1530 if (
Node->getOperand(i).isUndef())
continue;
1532 unsigned Offset = TypeByteSize*i;
1538 Stores.
push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1542 Stores.
push_back(DAG.getStore(DAG.getEntryNode(), dl,
Node->getOperand(i),
1547 if (!Stores.
empty())
1550 StoreChain = DAG.getEntryNode();
1553 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1559void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1562 EVT FloatVT =
Value.getValueType();
1564 State.FloatVT = FloatVT;
1567 if (TLI.isTypeLegal(IVT)) {
1570 State.SignBit = NumBits - 1;
1576 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1579 int FI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
1584 State.Chain = DAG.getStore(DAG.getEntryNode(),
DL,
Value, State.FloatPtr,
1585 State.FloatPointerInfo);
1592 State.IntPointerInfo = State.FloatPointerInfo;
1595 unsigned ByteOffset = (NumBits / 8) - 1;
1602 State.IntPtr = IntPtr;
1603 State.IntValue = DAG.getExtLoad(
ISD::EXTLOAD,
DL, LoadTy, State.Chain, IntPtr,
1604 State.IntPointerInfo, MVT::i8);
1611SDValue SelectionDAGLegalize::modifySignAsInt(
const FloatSignAsInt &State,
1618 SDValue Chain = DAG.getTruncStore(State.Chain,
DL, NewIntValue, State.IntPtr,
1619 State.IntPointerInfo, MVT::i8);
1620 return DAG.getLoad(State.FloatVT,
DL, Chain, State.FloatPtr,
1621 State.FloatPointerInfo);
1630 FloatSignAsInt SignAsInt;
1631 getSignAsIntValue(SignAsInt,
DL, Sign);
1633 EVT IntVT = SignAsInt.IntValue.getValueType();
1634 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1640 if (TLI.isOperationLegalOrCustom(
ISD::FABS, FloatVT) &&
1641 TLI.isOperationLegalOrCustom(
ISD::FNEG, FloatVT)) {
1644 SDValue Cond = DAG.getSetCC(
DL, getSetCCResultType(IntVT), SignBit,
1646 return DAG.getSelect(
DL, FloatVT,
Cond, NegValue, AbsValue);
1650 FloatSignAsInt MagAsInt;
1651 getSignAsIntValue(MagAsInt,
DL, Mag);
1652 EVT MagVT = MagAsInt.IntValue.getValueType();
1653 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask,
DL, MagVT);
1658 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1659 EVT ShiftVT = IntVT;
1665 if (ShiftAmount > 0) {
1666 SDValue ShiftCnst = DAG.getConstant(ShiftAmount,
DL, ShiftVT);
1668 }
else if (ShiftAmount < 0) {
1669 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount,
DL, ShiftVT);
1679 return modifySignAsInt(MagAsInt,
DL, CopiedSign);
1685 FloatSignAsInt SignAsInt;
1686 getSignAsIntValue(SignAsInt,
DL,
Node->getOperand(0));
1687 EVT IntVT = SignAsInt.IntValue.getValueType();
1690 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask,
DL, IntVT);
1695 return modifySignAsInt(SignAsInt,
DL, SignFlip);
1703 EVT FloatVT =
Value.getValueType();
1710 FloatSignAsInt ValueAsInt;
1711 getSignAsIntValue(ValueAsInt,
DL,
Value);
1712 EVT IntVT = ValueAsInt.IntValue.getValueType();
1713 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask,
DL, IntVT);
1716 return modifySignAsInt(ValueAsInt,
DL, ClearedSign);
1719void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(
SDNode*
Node,
1721 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1722 assert(SPReg &&
"Target cannot require DYNAMIC_STACKALLOC expansion and"
1723 " not tell us which reg is the stack pointer!");
1725 EVT VT =
Node->getValueType(0);
1733 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1736 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1738 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1746 if (Alignment > StackAlign)
1748 DAG.getConstant(-Alignment.
value(), dl, VT));
1749 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1);
1751 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0,
SDValue(), dl);
1763 return EmitStackConvert(
SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1771 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1774 if ((SrcVT.
bitsGT(SlotVT) &&
1775 !TLI.isTruncStoreLegalOrCustom(
SrcOp.getValueType(), SlotVT)) ||
1776 (SlotVT.
bitsLT(DestVT) &&
1777 !TLI.isLoadExtLegalOrCustom(
ISD::EXTLOAD, DestVT, SlotVT)))
1781 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1782 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1794 if (SrcVT.
bitsGT(SlotVT))
1795 Store = DAG.getTruncStore(Chain, dl,
SrcOp, FIPtr, PtrInfo,
1799 Store = DAG.getStore(Chain, dl,
SrcOp, FIPtr, PtrInfo, SrcAlign);
1803 if (SlotVT.
bitsEq(DestVT))
1804 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1807 return DAG.getExtLoad(
ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1820 SDValue Ch = DAG.getTruncStore(
1821 DAG.getEntryNode(), dl,
Node->getOperand(0), StackPtr,
1823 Node->getValueType(0).getVectorElementType());
1825 Node->getValueType(0), dl, Ch, StackPtr,
1832 unsigned NumElems =
Node->getNumOperands();
1834 EVT VT =
Node->getValueType(0);
1846 for (
unsigned i = 0; i < NumElems; ++i) {
1857 while (IntermedVals.
size() > 2) {
1858 NewIntermedVals.
clear();
1859 for (
unsigned i = 0, e = (IntermedVals.
size() & ~1u); i < e; i += 2) {
1865 FinalIndices.
reserve(IntermedVals[i].second.
size() +
1866 IntermedVals[i+1].second.
size());
1869 for (
unsigned j = 0, f = IntermedVals[i].second.
size(); j != f;
1872 FinalIndices.
push_back(IntermedVals[i].second[j]);
1874 for (
unsigned j = 0, f = IntermedVals[i+1].second.
size(); j != f;
1876 ShuffleVec[k] = NumElems + j;
1877 FinalIndices.
push_back(IntermedVals[i+1].second[j]);
1883 IntermedVals[i+1].first,
1888 std::make_pair(Shuffle, std::move(FinalIndices)));
1893 if ((IntermedVals.
size() & 1) != 0)
1896 IntermedVals.
swap(NewIntermedVals);
1900 "Invalid number of intermediate vectors");
1901 SDValue Vec1 = IntermedVals[0].first;
1903 if (IntermedVals.
size() > 1)
1904 Vec2 = IntermedVals[1].first;
1909 for (
unsigned i = 0, e = IntermedVals[0].second.
size(); i != e; ++i)
1910 ShuffleVec[IntermedVals[0].second[i]] = i;
1911 for (
unsigned i = 0, e = IntermedVals[1].second.
size(); i != e; ++i)
1912 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1926 unsigned NumElems =
Node->getNumOperands();
1929 EVT VT =
Node->getValueType(0);
1930 EVT OpVT =
Node->getOperand(0).getValueType();
1935 bool isOnlyLowElement =
true;
1936 bool MoreThanTwoValues =
false;
1938 for (
unsigned i = 0; i < NumElems; ++i) {
1943 isOnlyLowElement =
false;
1944 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1949 }
else if (!Value2.
getNode()) {
1952 }
else if (V != Value1 && V != Value2) {
1953 MoreThanTwoValues =
true;
1958 return DAG.getUNDEF(VT);
1960 if (isOnlyLowElement)
1966 for (
unsigned i = 0, e = NumElems; i !=
e; ++i) {
1968 dyn_cast<ConstantFPSDNode>(
Node->getOperand(i))) {
1971 dyn_cast<ConstantSDNode>(
Node->getOperand(i))) {
1990 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1991 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1993 VT, dl, DAG.getEntryNode(), CPIdx,
1999 for (
unsigned i = 0; i < NumElems; ++i) {
2000 if (
Node->getOperand(i).isUndef())
2005 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.
size())) {
2006 if (!MoreThanTwoValues) {
2008 for (
unsigned i = 0; i < NumElems; ++i) {
2012 ShuffleVec[i] =
V == Value1 ? 0 : NumElems;
2014 if (TLI.isShuffleMaskLegal(ShuffleVec,
Node->getValueType(0))) {
2021 Vec2 = DAG.getUNDEF(VT);
2024 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2034 return ExpandVectorBuildThroughStack(
Node);
2039 EVT VT =
Node->getValueType(0);
2042 return DAG.getSplatBuildVector(VT,
DL, SplatVal);
2053 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2054 TLI.getPointerTy(DAG.getDataLayout()));
2056 EVT RetVT =
Node->getValueType(0);
2063 SDValue InChain = DAG.getEntryNode();
2068 const Function &
F = DAG.getMachineFunction().getFunction();
2070 TLI.isInTailCallPosition(DAG,
Node, TCChain) &&
2071 (
RetTy ==
F.getReturnType() ||
F.getReturnType()->isVoidTy());
2076 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT,
isSigned);
2079 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2081 .setTailCall(isTailCall)
2082 .setSExtResult(signExtend)
2083 .setZExtResult(!signExtend)
2084 .setIsPostTypeLegalization(
true);
2086 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2091 return {DAG.getRoot(), DAG.getRoot()};
2103 EVT ArgVT =
Op.getValueType();
2107 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT,
isSigned);
2108 Entry.IsZExt = !Entry.IsSExt;
2109 Args.push_back(Entry);
2112 return ExpandLibCall(LC,
Node, std::move(Args),
isSigned);
2115void SelectionDAGLegalize::ExpandFrexpLibCall(
2118 EVT VT =
Node->getValueType(0);
2119 EVT ExpVT =
Node->getValueType(1);
2127 FPArgEntry.
Node = FPOp;
2128 FPArgEntry.
Ty = ArgTy;
2130 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2132 PtrArgEntry.
Node = StackSlot;
2133 PtrArgEntry.
Ty = PointerType::get(*DAG.getContext(),
2134 DAG.getDataLayout().getAllocaAddrSpace());
2139 auto [
Call, Chain] = ExpandLibCall(LC,
Node, std::move(Args),
false);
2143 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2147 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2149 LoadExp.
getValue(1), DAG.getRoot());
2150 DAG.setRoot(OutputChain);
2156void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2159 if (LC == RTLIB::UNKNOWN_LIBCALL)
2162 if (
Node->isStrictFPOpcode()) {
2163 EVT RetVT =
Node->getValueType(0);
2167 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2170 Node->getOperand(0));
2172 Results.push_back(Tmp.second);
2174 SDValue Tmp = ExpandLibCall(LC,
Node,
false).first;
2180void SelectionDAGLegalize::ExpandFPLibCall(
SDNode*
Node,
2188 Call_F32, Call_F64, Call_F80,
2189 Call_F128, Call_PPCF128);
2200 switch (
Node->getSimpleValueType(0).SimpleTy) {
2202 case MVT::i8: LC = Call_I8;
break;
2203 case MVT::i16: LC = Call_I16;
break;
2204 case MVT::i32: LC = Call_I32;
break;
2205 case MVT::i64: LC = Call_I64;
break;
2206 case MVT::i128: LC = Call_I128;
break;
2213void SelectionDAGLegalize::ExpandArgFPLibCall(
SDNode*
Node,
2220 EVT InVT =
Node->getOperand(
Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2222 Call_F32, Call_F64, Call_F80,
2223 Call_F128, Call_PPCF128);
2229SelectionDAGLegalize::ExpandDivRemLibCall(
SDNode *
Node,
2231 unsigned Opcode =
Node->getOpcode();
2235 switch (
Node->getSimpleValueType(0).SimpleTy) {
2237 case MVT::i8: LC=
isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8;
break;
2238 case MVT::i16: LC=
isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16;
break;
2239 case MVT::i32: LC=
isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
break;
2240 case MVT::i64: LC=
isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64;
break;
2241 case MVT::i128: LC=
isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128;
break;
2247 SDValue InChain = DAG.getEntryNode();
2249 EVT RetVT =
Node->getValueType(0);
2255 EVT ArgVT =
Op.getValueType();
2261 Args.push_back(Entry);
2265 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2267 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2270 Args.push_back(Entry);
2272 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2273 TLI.getPointerTy(DAG.getDataLayout()));
2279 .setLibCallee(TLI.getLibcallCallingConv(LC),
RetTy, Callee,
2284 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2296 switch (
Node->getSimpleValueType(0).SimpleTy) {
2298 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2299 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2300 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2301 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2302 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2325SelectionDAGLegalize::ExpandSinCosLibCall(
SDNode *
Node,
2328 switch (
Node->getSimpleValueType(0).SimpleTy) {
2330 case MVT::f32: LC = RTLIB::SINCOS_F32;
break;
2331 case MVT::f64: LC = RTLIB::SINCOS_F64;
break;
2332 case MVT::f80: LC = RTLIB::SINCOS_F80;
break;
2333 case MVT::f128: LC = RTLIB::SINCOS_F128;
break;
2334 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128;
break;
2340 SDValue InChain = DAG.getEntryNode();
2342 EVT RetVT =
Node->getValueType(0);
2349 Entry.Node =
Node->getOperand(0);
2351 Entry.IsSExt =
false;
2352 Entry.IsZExt =
false;
2353 Args.push_back(Entry);
2356 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2357 Entry.Node = SinPtr;
2358 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2359 Entry.IsSExt =
false;
2360 Entry.IsZExt =
false;
2361 Args.push_back(Entry);
2364 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2365 Entry.Node = CosPtr;
2366 Entry.Ty = PointerType::getUnqual(
RetTy->getContext());
2367 Entry.IsSExt =
false;
2368 Entry.IsZExt =
false;
2369 Args.push_back(Entry);
2371 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2372 TLI.getPointerTy(DAG.getDataLayout()));
2376 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2377 TLI.getLibcallCallingConv(LC),
Type::getVoidTy(*DAG.getContext()), Callee,
2380 std::pair<SDValue, SDValue>
CallInfo = TLI.LowerCallTo(CLI);
2390 EVT VT =
Node->getValueType(0);
2393 EVT ExpVT =
N.getValueType();
2395 if (AsIntVT ==
EVT())
2408 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2413 const int Precision = APFloat::semanticsPrecision(FltSem);
2415 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2416 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2418 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2420 const APFloat One(FltSem,
"1.0");
2421 APFloat ScaleUpK =
scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2425 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2434 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2440 DAG.getSetCC(dl, SetCCVT,
N, DoubleMaxExp,
ISD::SETUGT);
2442 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2454 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2455 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2460 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2465 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2469 SDValue ScaleDownTwice = DAG.getSetCC(
2470 dl, SetCCVT,
N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2482 DAG.getNode(
ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small,
X));
2486 DAG.getNode(
ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small,
N));
2491 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2492 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2495 ExponentShiftAmt, NUW_NSW);
2497 return DAG.getNode(
ISD::FMUL, dl, VT, NewX, AsFP);
2504 EVT ExpVT =
Node->getValueType(1);
2506 if (AsIntVT ==
EVT())
2511 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2528 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2532 SDValue SmallestNormalizedInt = DAG.getConstant(
2538 DAG.getConstant(
APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2544 FractSignMaskVal.
setBit(BitSize - 1);
2547 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2549 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2551 const APFloat One(FltSem,
"1.0");
2555 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2557 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2561 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2567 SDValue AddNegSmallestNormal =
2569 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2573 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt,
ISD::SETULT);
2575 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2590 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2593 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2596 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2603 SDValue FPHalf = DAG.getConstant(
Half.bitcastToAPInt(), dl, AsIntVT);
2616 return DAG.getMergeValues({Result0, Result1}, dl);
2627 EVT DestVT =
Node->getValueType(0);
2629 unsigned OpNo =
Node->isStrictFPOpcode() ? 1 : 0;
2635 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2636 (DestVT.
bitsLE(MVT::f64) ||
2640 LLVM_DEBUG(
dbgs() <<
"32-bit [signed|unsigned] integer to float/double "
2644 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2651 DAG.getConstant(0x80000000u, dl, MVT::i32));
2654 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2657 if (DAG.getDataLayout().isBigEndian())
2660 SDValue MemChain = DAG.getEntryNode();
2663 SDValue Store1 = DAG.getStore(MemChain, dl,
Lo, StackSlot,
2676 SDValue Bias = DAG.getConstantFP(
2677 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2678 : llvm::bit_cast<double>(0x4330000000000000ULL),
2683 if (
Node->isStrictFPOpcode()) {
2685 {
Node->getOperand(0), Load, Bias});
2688 std::pair<SDValue, SDValue> ResultPair;
2690 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2691 Result = ResultPair.first;
2692 Chain = ResultPair.second;
2698 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2707 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2708 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2709 LLVM_DEBUG(
dbgs() <<
"Converting unsigned i32/i64 to f32/f64\n");
2724 EVT SetCCVT = getSetCCResultType(SrcVT);
2726 SDValue SignBitTest = DAG.getSetCC(
2727 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT),
ISD::SETLT);
2729 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2730 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2732 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2737 if (
Node->isStrictFPOpcode()) {
2740 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest,
Or, Op0);
2742 {
Node->getOperand(0), InCvt });
2750 Flags.setNoFPExcept(
Node->getFlags().hasNoFPExcept());
2751 Fast->setFlags(Flags);
2752 Flags.setNoFPExcept(
true);
2760 return DAG.getSelect(dl, DestVT, SignBitTest, Slow,
Fast);
2764 if (!TLI.isOperationLegalOrCustom(
2771 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2773 "Cannot perform lossless SINT_TO_FP!");
2776 if (
Node->isStrictFPOpcode()) {
2778 {
Node->getOperand(0), Op0 });
2782 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2785 Four = DAG.getIntPtrConstant(4, dl);
2786 SDValue CstOffset = DAG.getSelect(dl,
Zero.getValueType(),
2787 SignSet, Four, Zero);
2796 case MVT::i8 : FF = 0x43800000ULL;
break;
2797 case MVT::i16: FF = 0x47800000ULL;
break;
2798 case MVT::i32: FF = 0x4F800000ULL;
break;
2799 case MVT::i64: FF = 0x5F800000ULL;
break;
2801 if (DAG.getDataLayout().isLittleEndian())
2803 Constant *FudgeFactor = ConstantInt::get(
2807 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2808 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2812 if (DestVT == MVT::f32)
2813 FudgeInReg = DAG.getLoad(
2814 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2823 LegalizeOp(
Load.getNode());
2827 if (
Node->isStrictFPOpcode()) {
2829 { Tmp1.
getValue(1), Tmp1, FudgeInReg });
2830 Chain =
Result.getValue(1);
2834 return DAG.getNode(
ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2842void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2844 bool IsStrict =
N->isStrictFPOpcode();
2847 EVT DestVT =
N->getValueType(0);
2848 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2855 unsigned OpToUse = 0;
2863 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2871 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2883 DAG.
getNode(OpToUse, dl, {DestVT, MVT::Other},
2886 dl, NewInTy, LegalOp)});
2893 DAG.getNode(OpToUse, dl, DestVT,
2895 dl, NewInTy, LegalOp)));
2903void SelectionDAGLegalize::PromoteLegalFP_TO_INT(
SDNode *
N,
const SDLoc &dl,
2905 bool IsStrict =
N->isStrictFPOpcode();
2908 EVT DestVT =
N->getValueType(0);
2909 SDValue LegalOp =
N->getOperand(IsStrict ? 1 : 0);
2911 EVT NewOutTy = DestVT;
2913 unsigned OpToUse = 0;
2923 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2928 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2937 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2938 Operation = DAG.getNode(OpToUse, dl, VTs,
N->getOperand(0), LegalOp);
2940 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2955 unsigned Opcode =
Node->getOpcode();
2958 EVT NewOutTy =
Node->getValueType(0);
2963 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2970 Node->getOperand(1));
2976 EVT VT =
Op.getValueType();
2977 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2982 if (TLI.isOperationLegalOrPromote(
ISD::CTPOP, VT)) {
2988 DAG.getConstant(1ULL << (--i), dl, ShVT));
2993 return DAG.getNode(
ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2997 MVT VecVT =
Node->getOperand(1).getSimpleValueType();
2998 MVT NewVecVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), VecVT);
2999 MVT ScalarVT =
Node->getSimpleValueType(0);
3007 assert(
Node->getOperand(0).getValueType().isFloatingPoint() &&
3008 "Only FP promotion is supported");
3012 for (
unsigned j = 1;
j !=
Node->getNumOperands(); ++
j)
3013 if (
Node->getOperand(j).getValueType().isVector() &&
3018 assert(
Node->getOperand(j).getValueType().isFloatingPoint() &&
3019 "Only FP promotion is supported");
3031 DAG.getIntPtrConstant(0,
DL,
true));
3034bool SelectionDAGLegalize::ExpandNode(
SDNode *
Node) {
3038 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3040 switch (
Node->getOpcode()) {
3042 if ((Tmp1 = TLI.expandABS(
Node, DAG)))
3047 if ((Tmp1 = TLI.expandABD(
Node, DAG)))
3051 if ((Tmp1 = TLI.expandCTPOP(
Node, DAG)))
3056 if ((Tmp1 = TLI.expandCTLZ(
Node, DAG)))
3061 if ((Tmp1 = TLI.expandCTTZ(
Node, DAG)))
3065 if ((Tmp1 = TLI.expandBITREVERSE(
Node, DAG)))
3069 if ((Tmp1 = TLI.expandBSWAP(
Node, DAG)))
3073 Results.push_back(ExpandPARITY(
Node->getOperand(0), dl));
3078 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3081 SDValue CfaArg = DAG.getSExtOrTrunc(
Node->getOperand(0), dl,
3082 TLI.getPointerTy(DAG.getDataLayout()));
3090 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3096 Results.push_back(DAG.getConstant(1, dl,
Node->getValueType(0)));
3113 DAG.getConstant(0, dl,
Node->getValueType(0)));
3119 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3125 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3126 SDValue Swap = DAG.getAtomicCmpSwap(
3128 Node->getOperand(0),
Node->getOperand(1), Zero, Zero,
3129 cast<AtomicSDNode>(
Node)->getMemOperand());
3138 Node->getOperand(0),
Node->getOperand(2),
Node->getOperand(1),
3139 cast<AtomicSDNode>(
Node)->getMemOperand());
3147 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
3148 SDValue Res = DAG.getAtomicCmpSwap(
3150 Node->getOperand(0),
Node->getOperand(1),
Node->getOperand(2),
3151 Node->getOperand(3), cast<MemSDNode>(
Node)->getMemOperand());
3157 EVT AtomicType = cast<AtomicSDNode>(
Node)->getMemoryVT();
3158 EVT OuterType =
Node->getValueType(0);
3159 switch (TLI.getExtendForAtomicOps()) {
3162 DAG.getValueType(AtomicType));
3164 Node->getOperand(2), DAG.getValueType(AtomicType));
3169 DAG.getValueType(AtomicType));
3170 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3174 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3175 RHS = DAG.getZeroExtendInReg(
Node->getOperand(2), dl, AtomicType);
3191 EVT VT =
Node->getValueType(0);
3195 cast<VTSDNode>(
RHS->getOperand(1))->getVT() == AN->
getMemoryVT())
3196 RHS =
RHS->getOperand(0);
3200 Node->getOperand(0),
Node->getOperand(1),
3210 for (
unsigned i = 0; i <
Node->getNumValues(); i++)
3214 EVT VT =
Node->getValueType(0);
3216 Results.push_back(DAG.getConstant(0, dl, VT));
3219 Results.push_back(DAG.getConstantFP(0, dl, VT));
3226 if (TLI.isStrictFPEnabled())
3230 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3231 Node->getValueType(0))
3232 == TargetLowering::Legal)
3236 if ((Tmp1 = EmitStackConvert(
Node->getOperand(1),
Node->getValueType(0),
3237 Node->getValueType(0), dl,
3238 Node->getOperand(0)))) {
3240 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_ROUND node\n");
3245 if ((Tmp1 = TLI.expandFP_ROUND(
Node, DAG))) {
3253 if ((Tmp1 = EmitStackConvert(
Node->getOperand(0),
Node->getValueType(0),
3254 Node->getValueType(0), dl)))
3260 if (TLI.isStrictFPEnabled())
3264 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
3265 Node->getValueType(0))
3266 == TargetLowering::Legal)
3270 if ((Tmp1 = EmitStackConvert(
3271 Node->getOperand(1),
Node->getOperand(1).getValueType(),
3272 Node->getValueType(0), dl,
Node->getOperand(0)))) {
3274 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_EXTEND node\n");
3280 EVT SrcVT =
Op.getValueType();
3281 EVT DstVT =
Node->getValueType(0);
3287 if ((Tmp1 = EmitStackConvert(
Op, SrcVT, DstVT, dl)))
3297 if (
Op.getValueType() == MVT::bf16) {
3301 Op = DAG.getAnyExtOrTrunc(
Op, dl, MVT::i32);
3305 DAG.getConstant(16, dl,
3306 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3309 if (
Node->getValueType(0) != MVT::f32)
3316 if (
Op.getValueType() != MVT::f32)
3318 DAG.getIntPtrConstant(0, dl,
true));
3320 if (!DAG.isKnownNeverSNaN(
Op)) {
3325 DAG.getConstant(16, dl,
3326 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3329 if (
Node->getValueType(0) == MVT::bf16) {
3333 Op = DAG.getAnyExtOrTrunc(
Op, dl,
Node->getValueType(0));
3339 EVT ExtraVT = cast<VTSDNode>(
Node->getOperand(1))->getVT();
3340 EVT VT =
Node->getValueType(0);
3350 SDValue One = DAG.getConstant(1, dl, VT);
3360 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3363 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3365 Node->getOperand(0), ShiftCst);
3372 if (TLI.expandUINT_TO_FP(
Node, Tmp1, Tmp2, DAG)) {
3374 if (
Node->isStrictFPOpcode())
3381 if ((Tmp1 = ExpandLegalINT_TO_FP(
Node, Tmp2))) {
3383 if (
Node->isStrictFPOpcode())
3388 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG))
3392 if (TLI.expandFP_TO_SINT(
Node, Tmp1, DAG)) {
3394 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_SINT node\n");
3399 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG))
3403 if (TLI.expandFP_TO_UINT(
Node, Tmp1, Tmp2, DAG)) {
3405 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node,1), Tmp2);
3408 LLVM_DEBUG(
dbgs() <<
"Successfully expanded STRICT_FP_TO_UINT node\n");
3414 Results.push_back(TLI.expandFP_TO_INT_SAT(
Node, DAG));
3424 if (
Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3427 Node->getOperand(0));
3429 Tmp1 = ExpandExtractFromVectorThroughStack(
SDValue(
Node, 0));
3439 Results.push_back(ExpandVectorBuildThroughStack(
Node));
3451 EVT VT =
Node->getValueType(0);
3455 if (!TLI.isTypeLegal(EltVT)) {
3456 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3461 if (NewEltVT.
bitsLT(EltVT)) {
3477 unsigned int factor =
3485 for (
unsigned fi = 0; fi < factor; ++fi)
3489 for (
unsigned fi = 0; fi < factor; ++fi)
3500 for (
unsigned i = 0; i != NumElems; ++i) {
3508 DAG.getVectorIdxConstant(
Idx, dl)));
3512 DAG.getVectorIdxConstant(
Idx - NumElems, dl)));
3515 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3522 Results.push_back(TLI.expandVectorSplice(
Node, DAG));
3526 EVT OpTy =
Node->getOperand(0).getValueType();
3527 if (
Node->getConstantOperandVal(1)) {
3531 TLI.getShiftAmountTy(
3532 Node->getOperand(0).getValueType(),
3533 DAG.getDataLayout())));
3538 Node->getOperand(0));
3546 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3547 Results.push_back(DAG.getCopyFromReg(
Node->getOperand(0), dl, SP,
3548 Node->getValueType(0)));
3551 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
3558 if (
Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3559 Results.push_back(DAG.getCopyToReg(
Node->getOperand(0), dl, SP,
3560 Node->getOperand(1)));
3566 Results.push_back(DAG.getConstant(0, dl,
Node->getValueType(0)));
3581 TLI.expandIS_FPCLASS(
Node->getValueType(0),
Node->getOperand(0),
3592 switch (
Node->getOpcode()) {
3599 Tmp1 =
Node->getOperand(0);
3600 Tmp2 =
Node->getOperand(1);
3601 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3607 if (
SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(
Node, DAG))
3613 if (
SDValue Expanded = TLI.expandFMINIMUM_FMAXIMUM(
Node, DAG))
3619 EVT VT =
Node->getValueType(0);
3625 SDVTList VTs = DAG.getVTList(VT, VT);
3635 EVT VT =
Node->getValueType(0);
3639 if (TLI.getLibcallName(LC))
3645 Results.push_back(Expanded.getValue(1));
3654 if (TLI.getLibcallName(LC))
3659 Results.push_back(Expanded.getValue(1));
3667 if (
Node->getValueType(0) != MVT::f32) {
3679 if (
Node->getValueType(0) != MVT::f32) {
3684 {Node->getOperand(0), Node->getOperand(1)});
3686 {
Node->getValueType(0), MVT::Other},
3694 if (!TLI.useSoftFloat() &&
TM.Options.UnsafeFPMath) {
3696 MVT SVT =
Op.getSimpleValueType();
3697 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3703 DAG.getIntPtrConstant(0, dl,
true));
3714 DAG.shouldOptForSize()))
3715 Results.push_back(ExpandConstantFP(CFP,
true));
3720 Results.push_back(ExpandConstant(CP));
3724 EVT VT =
Node->getValueType(0);
3725 if (TLI.isOperationLegalOrCustom(
ISD::FADD, VT) &&
3726 TLI.isOperationLegalOrCustom(
ISD::FNEG, VT)) {
3735 EVT VT =
Node->getValueType(0);
3737 TLI.isOperationLegalOrCustom(
ISD::XOR, VT) &&
3738 "Don't know how to expand this subtraction!");
3739 Tmp1 = DAG.getNOT(dl,
Node->getOperand(1), VT);
3740 Tmp1 = DAG.
getNode(
ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3746 if (TLI.expandREM(
Node, Tmp1, DAG))
3753 EVT VT =
Node->getValueType(0);
3754 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3755 SDVTList VTs = DAG.getVTList(VT, VT);
3756 Tmp1 = DAG.
getNode(DivRemOpc, dl, VTs,
Node->getOperand(0),
3757 Node->getOperand(1));
3764 unsigned ExpandOpcode =
3766 EVT VT =
Node->getValueType(0);
3767 SDVTList VTs = DAG.getVTList(VT, VT);
3769 Tmp1 = DAG.
getNode(ExpandOpcode, dl, VTs,
Node->getOperand(0),
3770 Node->getOperand(1));
3778 MVT VT =
LHS.getSimpleValueType();
3779 unsigned MULHOpcode =
3782 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3784 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3790 assert(TLI.isTypeLegal(HalfType));
3791 if (TLI.expandMUL_LOHI(
Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3793 TargetLowering::MulExpansionKind::Always)) {
3794 for (
unsigned i = 0; i < 2; ++i) {
3797 SDValue Shift = DAG.getConstant(
3799 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3808 EVT VT =
Node->getValueType(0);
3809 SDVTList VTs = DAG.getVTList(VT, VT);
3815 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::SMUL_LOHI, VT);
3816 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(
ISD::UMUL_LOHI, VT);
3817 bool HasMULHS = TLI.isOperationLegalOrCustom(
ISD::MULHS, VT);
3818 bool HasMULHU = TLI.isOperationLegalOrCustom(
ISD::MULHU, VT);
3819 unsigned OpToUse = 0;
3820 if (HasSMUL_LOHI && !HasMULHS) {
3822 }
else if (HasUMUL_LOHI && !HasMULHU) {
3824 }
else if (HasSMUL_LOHI) {
3826 }
else if (HasUMUL_LOHI) {
3830 Results.push_back(DAG.getNode(OpToUse, dl, VTs,
Node->getOperand(0),
3831 Node->getOperand(1)));
3839 TLI.isOperationLegalOrCustom(
ISD::SHL, VT) &&
3840 TLI.isOperationLegalOrCustom(
ISD::OR, VT) &&
3841 TLI.expandMUL(
Node,
Lo,
Hi, HalfType, DAG,
3842 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3847 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3855 if (
SDValue Expanded = TLI.expandFunnelShift(
Node, DAG))
3860 if (
SDValue Expanded = TLI.expandROT(
Node,
true , DAG))
3867 Results.push_back(TLI.expandAddSubSat(
Node, DAG));
3877 Results.push_back(TLI.expandFixedPointMul(
Node, DAG));
3884 Node->getOperand(0),
3885 Node->getOperand(1),
3886 Node->getConstantOperandVal(2),
3909 EVT VT =
LHS.getValueType();
3913 EVT CarryType =
Node->getValueType(1);
3914 EVT SetCCType = getSetCCResultType(
Node->getValueType(0));
3916 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS,
CC);
3919 SDValue One = DAG.getConstant(1, dl, VT);
3921 DAG.
getNode(
ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3930 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero,
ISD::SETEQ)
3931 : DAG.getSetCC(dl, SetCCType, Sum, Zero,
ISD::SETEQ);
3933 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3939 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3945 TLI.expandSADDSUBO(
Node, Result, Overflow, DAG);
3953 TLI.expandUADDSUBO(
Node, Result, Overflow, DAG);
3961 if (TLI.expandMULO(
Node, Result, Overflow, DAG)) {
3973 DAG.getConstant(
PairTy.getSizeInBits() / 2, dl,
3974 TLI.getShiftAmountTy(
PairTy, DAG.getDataLayout())));
3979 Tmp1 =
Node->getOperand(0);
3980 Tmp2 =
Node->getOperand(1);
3981 Tmp3 =
Node->getOperand(2);
3985 cast<CondCodeSDNode>(Tmp1.
getOperand(2))->get());
3987 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3998 int JTI = cast<JumpTableSDNode>(Table.
getNode())->getIndex();
4001 EVT PTy = TLI.getPointerTy(TD);
4003 unsigned EntrySize =
4004 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
4011 Index = DAG.getNode(
4016 DAG.getConstant(EntrySize, dl,
Index.getValueType()));
4025 if (TLI.isJumpTableRelative()) {
4030 TLI.getPICJumpTableRelocBase(Table, DAG));
4033 Tmp1 = TLI.expandIndirectJTBranch(dl,
LD.getValue(1),
Addr, JTI, DAG);
4040 Tmp1 =
Node->getOperand(0);
4041 Tmp2 =
Node->getOperand(1);
4047 Node->getOperand(2));
4059 Node->getOperand(2));
4067 bool IsVP =
Node->getOpcode() == ISD::VP_SETCC;
4072 unsigned Offset = IsStrict ? 1 : 0;
4081 bool Legalized = TLI.LegalizeSetCCCondCode(
4082 DAG,
Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4083 Chain, IsSignaling);
4091 {Chain, Tmp1, Tmp2, Tmp3},
Node->getFlags());
4095 {Tmp1, Tmp2, Tmp3, Mask, EVL},
Node->getFlags());
4097 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl,
Node->getValueType(0), Tmp1,
4098 Tmp2, Tmp3,
Node->getFlags());
4106 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->
getValueType(0));
4109 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->
getValueType(0));
4121 assert(!IsStrict &&
"Don't know how to expand for strict nodes.");
4126 EVT VT =
Node->getValueType(0);
4129 DAG.getBoolConstant(
true, dl, VT, Tmp1VT),
4130 DAG.getBoolConstant(
false, dl, VT, Tmp1VT), Tmp3);
4137 Tmp1 =
Node->getOperand(0);
4138 Tmp2 =
Node->getOperand(1);
4139 Tmp3 =
Node->getOperand(2);
4140 Tmp4 =
Node->getOperand(3);
4141 EVT VT =
Node->getValueType(0);
4151 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4153 EVT CCVT = getSetCCResultType(CmpVT);
4155 Results.push_back(DAG.getSelect(dl, VT,
Cond, Tmp3, Tmp4));
4160 bool Legalized =
false;
4168 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4178 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4184 Legalized = TLI.LegalizeSetCCCondCode(
4188 assert(Legalized &&
"Can't legalize SELECT_CC with legal condition!");
4199 Tmp1, Tmp2, Tmp3, Tmp4,
CC);
4204 Tmp2, Tmp3, Tmp4,
CC);
4214 Tmp1 =
Node->getOperand(0);
4215 Tmp2 =
Node->getOperand(2);
4216 Tmp3 =
Node->getOperand(3);
4217 Tmp4 =
Node->getOperand(1);
4219 bool Legalized = TLI.LegalizeSetCCCondCode(
4220 DAG, getSetCCResultType(Tmp2.
getValueType()), Tmp2, Tmp3, Tmp4,
4223 assert(Legalized &&
"Can't legalize BR_CC with legal condition!");
4228 assert(!NeedInvert &&
"Don't know how to invert BR_CC!");
4231 Tmp4, Tmp2, Tmp3,
Node->getOperand(4));
4236 Tmp2, Tmp3,
Node->getOperand(4));
4251 EVT VT =
Node->getValueType(0);
4257 for (
unsigned Idx = 0;
Idx < NumElem;
Idx++) {
4260 Node->getOperand(0), DAG.getVectorIdxConstant(
Idx, dl));
4263 Node->getOperand(1), DAG.getVectorIdxConstant(
Idx, dl));
4287 Results.push_back(TLI.expandVecReduce(
Node, DAG));
4289 case ISD::VP_CTTZ_ELTS:
4290 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
4291 Results.push_back(TLI.expandVPCTTZElements(
Node, DAG));
4308 if (!TLI.isStrictFPEnabled() &&
Results.
empty() &&
Node->isStrictFPOpcode()) {
4314 switch (
Node->getOpcode()) {
4316 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4317 Node->getValueType(0))
4318 == TargetLowering::Legal)
4322 if (TLI.getStrictFPOperationAction(
4325 if (TLI.getStrictFPOperationAction(
4329 EVT VT =
Node->getValueType(0);
4333 {Node->getOperand(0), Node->getOperand(1), Neg},
4348 if (TLI.getStrictFPOperationAction(
Node->getOpcode(),
4349 Node->getOperand(1).getValueType())
4350 == TargetLowering::Legal)
4367void SelectionDAGLegalize::ConvertNodeToLibcall(
SDNode *
Node) {
4372 unsigned Opc =
Node->getOpcode();
4381 .setChain(
Node->getOperand(0))
4384 DAG.getExternalSymbol(
"__sync_synchronize",
4385 TLI.getPointerTy(DAG.getDataLayout())),
4388 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4390 Results.push_back(CallResult.second);
4412 EVT RetVT =
Node->getValueType(0);
4415 if (TLI.getLibcallName(LC)) {
4422 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4423 "Unexpected atomic op or value type!");
4427 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4430 Node->getOperand(0));
4432 Results.push_back(Tmp.second);
4440 .setChain(
Node->getOperand(0))
4442 DAG.getExternalSymbol(
4443 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4445 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4447 Results.push_back(CallResult.second);
4452 ExpandFPLibCall(
Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4453 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4454 RTLIB::FMIN_PPCF128,
Results);
4461 ExpandFPLibCall(
Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4462 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4463 RTLIB::FMAX_PPCF128,
Results);
4467 ExpandFPLibCall(
Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4468 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4469 RTLIB::SQRT_PPCF128,
Results);
4472 ExpandFPLibCall(
Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4473 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4474 RTLIB::CBRT_PPCF128,
Results);
4478 ExpandFPLibCall(
Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4479 RTLIB::SIN_F80, RTLIB::SIN_F128,
4484 ExpandFPLibCall(
Node, RTLIB::COS_F32, RTLIB::COS_F64,
4485 RTLIB::COS_F80, RTLIB::COS_F128,
4494 ExpandFPLibCall(
Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4495 RTLIB::LOG_F128, RTLIB::LOG_PPCF128,
Results);
4499 ExpandFPLibCall(
Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4500 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128,
Results);
4504 ExpandFPLibCall(
Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4505 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128,
Results);
4509 ExpandFPLibCall(
Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4510 RTLIB::EXP_F128, RTLIB::EXP_PPCF128,
Results);
4514 ExpandFPLibCall(
Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4515 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128,
Results);
4518 ExpandFPLibCall(
Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4519 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128,
Results);
4523 ExpandFPLibCall(
Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4524 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4525 RTLIB::TRUNC_PPCF128,
Results);
4529 ExpandFPLibCall(
Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4530 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4531 RTLIB::FLOOR_PPCF128,
Results);
4535 ExpandFPLibCall(
Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4536 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4537 RTLIB::CEIL_PPCF128,
Results);
4541 ExpandFPLibCall(
Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4542 RTLIB::RINT_F80, RTLIB::RINT_F128,
4543 RTLIB::RINT_PPCF128,
Results);
4547 ExpandFPLibCall(
Node, RTLIB::NEARBYINT_F32,
4548 RTLIB::NEARBYINT_F64,
4549 RTLIB::NEARBYINT_F80,
4550 RTLIB::NEARBYINT_F128,
4551 RTLIB::NEARBYINT_PPCF128,
Results);
4555 ExpandFPLibCall(
Node, RTLIB::ROUND_F32,
4559 RTLIB::ROUND_PPCF128,
Results);
4563 ExpandFPLibCall(
Node, RTLIB::ROUNDEVEN_F32,
4564 RTLIB::ROUNDEVEN_F64,
4565 RTLIB::ROUNDEVEN_F80,
4566 RTLIB::ROUNDEVEN_F128,
4567 RTLIB::ROUNDEVEN_PPCF128,
Results);
4571 ExpandFPLibCall(
Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4572 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128,
Results);
4581 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected fpowi.");
4582 if (!TLI.getLibcallName(LC)) {
4584 if (
Node->isStrictFPOpcode()) {
4587 {
Node->getValueType(0),
Node->getValueType(1)},
4588 {
Node->getOperand(0),
Node->getOperand(2)});
4591 {
Node->getValueType(0),
Node->getValueType(1)},
4598 Node->getOperand(1));
4600 Node->getValueType(0),
4605 unsigned Offset =
Node->isStrictFPOpcode() ? 1 : 0;
4606 bool ExponentHasSizeOfInt =
4607 DAG.getLibInfo().getIntSize() ==
4608 Node->getOperand(1 +
Offset).getValueType().getSizeInBits();
4609 if (!ExponentHasSizeOfInt) {
4612 DAG.getContext()->emitError(
"POWI exponent does not match sizeof(int)");
4613 Results.push_back(DAG.getUNDEF(
Node->getValueType(0)));
4621 ExpandFPLibCall(
Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4622 RTLIB::POW_F128, RTLIB::POW_PPCF128,
Results);
4626 ExpandArgFPLibCall(
Node, RTLIB::LROUND_F32,
4627 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4629 RTLIB::LROUND_PPCF128,
Results);
4633 ExpandArgFPLibCall(
Node, RTLIB::LLROUND_F32,
4634 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4635 RTLIB::LLROUND_F128,
4636 RTLIB::LLROUND_PPCF128,
Results);
4640 ExpandArgFPLibCall(
Node, RTLIB::LRINT_F32,
4641 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4643 RTLIB::LRINT_PPCF128,
Results);
4647 ExpandArgFPLibCall(
Node, RTLIB::LLRINT_F32,
4648 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4650 RTLIB::LLRINT_PPCF128,
Results);
4654 ExpandFPLibCall(
Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4655 RTLIB::DIV_F80, RTLIB::DIV_F128,
4660 ExpandFPLibCall(
Node, RTLIB::REM_F32, RTLIB::REM_F64,
4661 RTLIB::REM_F80, RTLIB::REM_F128,
4666 ExpandFPLibCall(
Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4667 RTLIB::FMA_F80, RTLIB::FMA_F128,
4672 ExpandFPLibCall(
Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4673 RTLIB::ADD_F80, RTLIB::ADD_F128,
4678 ExpandFPLibCall(
Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4679 RTLIB::MUL_F80, RTLIB::MUL_F128,
4683 if (
Node->getValueType(0) == MVT::f32) {
4684 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32,
Node,
false).first);
4688 if (
Node->getValueType(0) == MVT::f32) {
4690 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4691 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32,
Node->getOperand(1),
4694 Results.push_back(Tmp.second);
4698 if (
Node->getValueType(0) == MVT::f32) {
4700 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4701 DAG, RTLIB::FPEXT_F16_F32, MVT::f32,
Node->getOperand(1), CallOptions,
4704 Results.push_back(Tmp.second);
4711 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_fp16");
4712 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4718 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to expand fp_to_bf16");
4719 Results.push_back(ExpandLibCall(LC,
Node,
false).first);
4727 bool IsStrict =
Node->isStrictFPOpcode();
4730 EVT SVT =
Node->getOperand(IsStrict ? 1 : 0).getValueType();
4731 EVT RVT =
Node->getValueType(0);
4739 for (
unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4740 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4748 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4753 NVT,
Node->getOperand(IsStrict ? 1 : 0));
4756 std::pair<SDValue, SDValue> Tmp =
4757 TLI.makeLibCall(DAG, LC, RVT,
Op, CallOptions, dl, Chain);
4760 Results.push_back(Tmp.second);
4768 bool IsStrict =
Node->isStrictFPOpcode();
4773 EVT SVT =
Op.getValueType();
4774 EVT RVT =
Node->getValueType(0);
4782 for (
unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4783 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4791 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4795 std::pair<SDValue, SDValue> Tmp =
4796 TLI.makeLibCall(DAG, LC, NVT,
Op, CallOptions, dl, Chain);
4801 Results.push_back(Tmp.second);
4812 bool IsStrict =
Node->isStrictFPOpcode();
4815 EVT VT =
Node->getValueType(0);
4816 assert(cast<ConstantSDNode>(
Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4817 "Unable to expand as libcall if it is not normal rounding");
4820 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4823 std::pair<SDValue, SDValue> Tmp =
4824 TLI.makeLibCall(DAG, LC, VT,
Op, CallOptions,
SDLoc(
Node), Chain);
4827 Results.push_back(Tmp.second);
4833 Node->getValueType(0)),
4834 Node,
false).first);
4847 Node->getValueType(0));
4849 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
"Unable to legalize as libcall");
4852 std::pair<SDValue, SDValue> Tmp =
4853 TLI.makeLibCall(DAG, LC,
Node->getValueType(0),
Node->getOperand(1),
4856 Results.push_back(Tmp.second);
4861 ExpandFPLibCall(
Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4862 RTLIB::SUB_F80, RTLIB::SUB_F128,
4868 RTLIB::SREM_I16, RTLIB::SREM_I32,
4869 RTLIB::SREM_I64, RTLIB::SREM_I128));
4874 RTLIB::UREM_I16, RTLIB::UREM_I32,
4875 RTLIB::UREM_I64, RTLIB::UREM_I128));
4880 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4881 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4886 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4887 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4897 RTLIB::MUL_I16, RTLIB::MUL_I32,
4898 RTLIB::MUL_I64, RTLIB::MUL_I128));
4901 switch (
Node->getSimpleValueType(0).SimpleTy) {
4905 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32,
Node,
false).first);
4908 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64,
Node,
false).first);
4911 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128,
Node,
false).first);
4918 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4921 DAG.makeStateFunctionCall(RTLIB::FESETENV,
Ptr, Chain, dl));
4928 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4935 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4941 EVT ModeVT =
Node->getValueType(0);
4943 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4944 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4945 Node->getOperand(0), dl);
4947 ModeVT, dl, Chain, StackPtr,
4957 EVT ModeVT =
Mode.getValueType();
4959 int SPFI = cast<FrameIndexSDNode>(
StackPtr.getNode())->getIndex();
4960 SDValue StInst = DAG.getStore(
4961 Node->getOperand(0), dl, Mode, StackPtr,
4964 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4972 EVT PtrTy = TLI.getPointerTy(
DL);
4974 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4975 Node->getOperand(0), dl));
4982 LLVM_DEBUG(
dbgs() <<
"Successfully converted node to libcall\n");
4991 MVT EltVT,
MVT NewEltVT) {
4993 MVT MidVT = OldEltsPerNewElt == 1
5000void SelectionDAGLegalize::PromoteNode(
SDNode *
Node) {
5003 MVT OVT =
Node->getSimpleValueType(0);
5009 OVT =
Node->getOperand(0).getSimpleValueType();
5016 Node->getOpcode() == ISD::VP_REDUCE_FADD ||
5017 Node->getOpcode() == ISD::VP_REDUCE_FMUL ||
5018 Node->getOpcode() == ISD::VP_REDUCE_FMAX ||
5019 Node->getOpcode() == ISD::VP_REDUCE_FMIN ||
5020 Node->getOpcode() == ISD::VP_REDUCE_FMAXIMUM ||
5021 Node->getOpcode() == ISD::VP_REDUCE_FMINIMUM ||
5022 Node->getOpcode() == ISD::VP_REDUCE_SEQ_FADD)
5023 OVT =
Node->getOperand(1).getSimpleValueType();
5026 OVT =
Node->getOperand(2).getSimpleValueType();
5027 MVT NVT = TLI.getTypeToPromoteTo(
Node->getOpcode(), OVT);
5029 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
5030 switch (
Node->getOpcode()) {
5043 unsigned NewOpc =
Node->getOpcode();
5051 DAG.getConstant(TopBit, dl, NVT));
5056 Tmp1 = DAG.
getNode(NewOpc, dl, NVT, Tmp1);
5070 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5073 DAG.getConstant(DiffBits, dl,
5074 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5087 Results.push_back(PromoteLegalFP_TO_INT_SAT(
Node, dl));
5104 &&
"VAARG promotion is supported only for vectors or integer types");
5109 Tmp1 = DAG.getVAArg(NVT, dl, Chain,
Ptr,
Node->getOperand(2),
5110 Node->getConstantOperandVal(3));
5113 Tmp2 = DAG.
getNode(TruncOp, dl, OVT, Tmp1);
5117 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 0), Tmp2);
5118 DAG.ReplaceAllUsesOfValueWith(
SDValue(
Node, 1), Chain);
5120 UpdatedNodes->insert(Tmp2.
getNode());
5121 UpdatedNodes->insert(Chain.
getNode());
5138 unsigned ExtOp, TruncOp;
5145 switch (
Node->getOpcode()) {
5161 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5170 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5171 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5173 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5174 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5182 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5183 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5186 auto &
DL = DAG.getDataLayout();
5190 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(
DL, NVT)));
5196 unsigned ExtOp, TruncOp;
5197 if (
Node->getValueType(0).isVector() ||
5201 }
else if (
Node->getValueType(0).isInteger()) {
5208 Tmp1 =
Node->getOperand(0);
5210 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5211 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5213 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5216 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1);
5218 Tmp1 = DAG.
getNode(TruncOp, dl,
Node->getValueType(0), Tmp1,
5219 DAG.getIntPtrConstant(0, dl));
5231 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5240 Node->getOperand(2));
5248 MVT CVT =
Node->getSimpleValueType(0);
5249 assert(CVT == OVT &&
"not handled");
5257 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5258 Tmp1 =
Node->getOperand(0);
5259 Tmp2 =
Node->getOperand(1);
5261 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5262 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5265 Tmp3 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5266 Tmp4 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5276 DAG.getIntPtrConstant(0, dl,
true));
5288 TLI.isSExtCheaperThanZExt(
Node->getOperand(0).getValueType(), NVT))
5293 if (
Node->isStrictFPOpcode()) {
5295 std::tie(Tmp1, std::ignore) =
5296 DAG.getStrictFPExtendOrRound(
Node->getOperand(1), InChain, dl, NVT);
5297 std::tie(Tmp2, std::ignore) =
5298 DAG.getStrictFPExtendOrRound(
Node->getOperand(2), InChain, dl, NVT);
5300 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5301 SDVTList VTs = DAG.getVTList(
Node->getValueType(0), MVT::Other);
5302 Results.push_back(DAG.getNode(
Node->getOpcode(), dl, VTs,
5303 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5308 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(0));
5309 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(1));
5311 Tmp2,
Node->getOperand(2),
Node->getFlags()));
5318 cast<CondCodeSDNode>(
Node->getOperand(1))->get();
5321 Tmp1 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(2));
5322 Tmp2 = DAG.
getNode(ExtOp, dl, NVT,
Node->getOperand(3));
5324 Node->getOperand(0),
Node->getOperand(1),
5325 Tmp1, Tmp2,
Node->getOperand(4)));
5340 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5344 DAG.getIntPtrConstant(0, dl,
true)));
5355 {
Node->getOperand(0),
Node->getOperand(1)});
5357 {
Node->getOperand(0),
Node->getOperand(2)});
5360 Tmp1 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5361 {Tmp3, Tmp1, Tmp2});
5363 {Tmp1.
getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5373 DAG.getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5374 DAG.getIntPtrConstant(0, dl,
true)));
5378 {
Node->getOperand(0),
Node->getOperand(1)});
5380 {
Node->getOperand(0),
Node->getOperand(2)});
5382 {
Node->getOperand(0),
Node->getOperand(3)});
5385 Tmp4 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5386 {Tmp4, Tmp1, Tmp2, Tmp3});
5388 {Tmp4.
getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5396 Tmp2 =
Node->getOperand(1);
5397 Tmp3 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5407 DAG.getIntPtrConstant(isTrunc, dl,
true)));
5412 {
Node->getOperand(0),
Node->getOperand(1)});
5413 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5416 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5426 DAG.getIntPtrConstant(0, dl,
true)));
5451 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5454 DAG.getIntPtrConstant(0, dl,
true)));
5472 {
Node->getOperand(0),
Node->getOperand(1)});
5473 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, {NVT, MVT::Other},
5476 {Tmp2.
getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5491 "Invalid promote type for build_vector");
5497 for (
unsigned I = 0, E =
Node->getNumOperands();
I != E; ++
I) {
5526 "Invalid promote type for extract_vector_elt");
5533 EVT IdxVT =
Idx.getValueType();
5535 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5541 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5542 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5550 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5572 "Invalid promote type for insert_vector_elt");
5580 EVT IdxVT =
Idx.getValueType();
5583 SDValue Factor = DAG.getConstant(NewEltsPerOldElt,
SDLoc(), IdxVT);
5590 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
5591 SDValue IdxOffset = DAG.getConstant(
I, SL, IdxVT);
5595 CastVal, IdxOffset);
5598 NewVec, Elt, InEltIdx);
5638 "unexpected promotion type");
5640 "unexpected atomic_swap with illegal type");
5664 "unexpected promotion type");
5666 "unexpected atomic_load with illegal type");
5677 MVT ScalarType =
Scalar.getSimpleValueType();
5681 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5686 Tmp2 = DAG.
getNode(
Node->getOpcode(), dl, NVT, Tmp1);
5689 DAG.getIntPtrConstant(0, dl,
true)));
5692 case ISD::VP_REDUCE_FADD:
5693 case ISD::VP_REDUCE_FMUL:
5694 case ISD::VP_REDUCE_FMAX:
5695 case ISD::VP_REDUCE_FMIN:
5696 case ISD::VP_REDUCE_FMAXIMUM:
5697 case ISD::VP_REDUCE_FMINIMUM:
5698 case ISD::VP_REDUCE_SEQ_FADD:
5724 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes);
5731 bool AnyLegalized =
false;
5742 if (LegalizedNodes.
insert(
N).second) {
5743 AnyLegalized =
true;
5764 SelectionDAGLegalize
Legalizer(*
this, LegalizedNodes, &UpdatedNodes);
5771 return LegalizedNodes.
count(
N);
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
mir Rename Register Operands
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
support::ulittle16_t & Lo
support::ulittle16_t & Hi
DEMANGLE_DUMP_METHOD void dump() const
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
APInt bitcastToAPInt() const
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This is an SDNode representing atomic operations.
const SDValue & getBasePtr() const
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
const BasicBlock & back() const
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
const TargetLowering & getTargetLoweringInfo() const
allnodes_const_iterator allnodes_begin() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void swap(SmallVectorImpl &RHS)
void push_back(const T &Elt)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
constexpr ScalarTy getFixedValue() const
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
@ SET_FPENV
Sets the current floating-point environment.
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
@ RESET_FPENV
Set floating-point environment to default state.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
@ FADD
Simple binary floating point operators.
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
@ SIGN_EXTEND
Conversion operators.
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ SSUBO
Same for subtraction.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
@ SMULO
Same for multiplication.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ GET_FPENV_MEM
Gets the current floating-point environment.
@ STRICT_FADD
Constrained versions of the binary floating point operators.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
@ SET_FPENV_MEM
Sets the current floating point environment.
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)