24#include "llvm/IR/IntrinsicsAMDGPU.h"
32#include "AMDGPUGenCallingConv.inc"
35 "amdgpu-bypass-slow-div",
36 cl::desc(
"Skip 64-bit divide for dynamic 32-bit values"),
45 if (StoreSize % 32 == 0)
201 {MVT::v2i8, MVT::v4i8, MVT::v2i16, MVT::v3i16, MVT::v4i16})
401 {MVT::f16, MVT::f32, MVT::f64},
Expand);
431 {MVT::v2f16, MVT::v3f16, MVT::v4f16, MVT::v16f16, MVT::v2f32, MVT::v3f32,
432 MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32, MVT::v8f32, MVT::v16f32,
433 MVT::v2f64, MVT::v3f64, MVT::v4f64, MVT::v8f64, MVT::v16f64},
440 {MVT::v3i32, MVT::v3f32, MVT::v4i32, MVT::v4f32,
441 MVT::v5i32, MVT::v5f32, MVT::v6i32, MVT::v6f32,
442 MVT::v7i32, MVT::v7f32, MVT::v8i32, MVT::v8f32,
443 MVT::v9i32, MVT::v9f32, MVT::v10i32, MVT::v10f32,
444 MVT::v11i32, MVT::v11f32, MVT::v12i32, MVT::v12f32},
450 {MVT::v2f16, MVT::v2bf16, MVT::v2i16, MVT::v4f16, MVT::v4bf16,
451 MVT::v4i16, MVT::v2f32, MVT::v2i32, MVT::v3f32, MVT::v3i32,
452 MVT::v4f32, MVT::v4i32, MVT::v5f32, MVT::v5i32, MVT::v6f32,
453 MVT::v6i32, MVT::v7f32, MVT::v7i32, MVT::v8f32, MVT::v8i32,
454 MVT::v9f32, MVT::v9i32, MVT::v10i32, MVT::v10f32, MVT::v11i32,
455 MVT::v11f32, MVT::v12i32, MVT::v12f32, MVT::v16f16, MVT::v16bf16,
456 MVT::v16i16, MVT::v16f32, MVT::v16i32, MVT::v32f32, MVT::v32i32,
457 MVT::v2f64, MVT::v2i64, MVT::v3f64, MVT::v3i64, MVT::v4f64,
458 MVT::v4i64, MVT::v8f64, MVT::v8i64, MVT::v16f64, MVT::v16i64,
459 MVT::v32i16, MVT::v32f16, MVT::v32bf16},
465 const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 };
466 for (
MVT VT : ScalarIntVTs) {
505 for (
auto VT : {MVT::i8, MVT::i16})
509 MVT::v2i32, MVT::v3i32, MVT::v4i32, MVT::v5i32, MVT::v6i32, MVT::v7i32,
510 MVT::v9i32, MVT::v10i32, MVT::v11i32, MVT::v12i32};
512 for (
MVT VT : VectorIntTypes) {
530 MVT::v2f32, MVT::v3f32, MVT::v4f32, MVT::v5f32, MVT::v6f32, MVT::v7f32,
531 MVT::v9f32, MVT::v10f32, MVT::v11f32, MVT::v12f32};
533 for (
MVT VT : FloatVectorTypes) {
638 const auto Flags =
Op.getNode()->getFlags();
639 if (Flags.hasNoSignedZeros())
688 unsigned Opc =
N->getOpcode();
709 return (
N->getNumOperands() > 2 &&
N->getOpcode() !=
ISD::SELECT) ||
718 return N->getValueType(0) == MVT::f32;
725 if (isa<MemSDNode>(
N))
728 switch (
N->getOpcode()) {
743 switch (
N->getConstantOperandVal(0)) {
744 case Intrinsic::amdgcn_interp_p1:
745 case Intrinsic::amdgcn_interp_p2:
746 case Intrinsic::amdgcn_interp_mov:
747 case Intrinsic::amdgcn_interp_p1_f16:
748 case Intrinsic::amdgcn_interp_p2_f16:
768 unsigned NumMayIncreaseSize = 0;
769 MVT VT =
N->getValueType(0).getScalarType().getSimpleVT();
774 for (
const SDNode *U :
N->uses()) {
809 bool ForCodeSize)
const {
811 return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 ||
818 return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
835 EVT OldVT =
N->getValueType(0);
843 if (OldSize >= 32 && NewSize < 32 && MN->
getAlign() >=
Align(4) &&
858 return (OldSize < 32);
873 if ((LScalarSize >= CastScalarSize) && (CastScalarSize < 32))
878 CastTy, MMO, &
Fast) &&
894 switch (
N->getOpcode()) {
899 unsigned IntrID =
N->getConstantOperandVal(0);
917 switch (
Op.getOpcode()) {
927 EVT VT =
Op.getValueType();
952 return VT == MVT::f32 || VT == MVT::f64 ||
953 (Subtarget->
has16BitInsts() && (VT == MVT::f16 || VT == MVT::bf16));
960 return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f16 || VT == MVT::bf16;
984 unsigned SrcSize = Source.getSizeInBits();
987 return DestSize < SrcSize && DestSize % 32 == 0 ;
993 unsigned SrcSize = Source->getScalarSizeInBits();
997 return SrcSize >= 32;
999 return DestSize < SrcSize && DestSize % 32 == 0;
1003 unsigned SrcSize = Src->getScalarSizeInBits();
1007 return DestSize >= 32;
1009 return SrcSize == 32 && DestSize == 64;
1018 if (Src == MVT::i16)
1019 return Dest == MVT::i32 ||Dest == MVT::i64 ;
1021 return Src == MVT::i32 && Dest == MVT::i64;
1038 "Expected shift op");
1046 if (
N->getValueType(0) == MVT::i32 &&
N->use_size() == 1 &&
1047 (
N->use_begin()->getOpcode() ==
ISD::SRA ||
1048 N->use_begin()->getOpcode() ==
ISD::SRL))
1055 auto *RHSLd = dyn_cast<LoadSDNode>(
RHS);
1056 auto *LHS0 = dyn_cast<LoadSDNode>(
LHS.getOperand(0));
1057 auto *LHS1 = dyn_cast<ConstantSDNode>(
LHS.getOperand(1));
1058 return LHS0 && LHS1 && RHSLd && LHS0->getExtensionType() ==
ISD::ZEXTLOAD &&
1059 LHS1->getAPIntValue() == LHS0->getMemoryVT().getScalarSizeInBits() &&
1064 return !(IsShiftAndLoad(
LHS,
RHS) || IsShiftAndLoad(
RHS,
LHS));
1084 return CC_AMDGPU_CS_CHAIN;
1088 return CC_AMDGPU_Func;
1113 return RetCC_SI_Shader;
1115 return RetCC_SI_Gfx;
1119 return RetCC_AMDGPU_Func;
1158 const unsigned ExplicitOffset = ST.getExplicitKernelArgOffset();
1165 unsigned InIndex = 0;
1168 const bool IsByRef = Arg.hasByRefAttr();
1169 Type *BaseArgTy = Arg.getType();
1170 Type *MemArgTy = IsByRef ? Arg.getParamByRefType() : BaseArgTy;
1171 Align Alignment =
DL.getValueOrABITypeAlignment(
1172 IsByRef ? Arg.getParamAlign() : std::nullopt, MemArgTy);
1173 MaxAlign = std::max(Alignment, MaxAlign);
1174 uint64_t AllocSize =
DL.getTypeAllocSize(MemArgTy);
1176 uint64_t ArgOffset =
alignTo(ExplicitArgOffset, Alignment) + ExplicitOffset;
1177 ExplicitArgOffset =
alignTo(ExplicitArgOffset, Alignment) + AllocSize;
1190 for (
unsigned Value = 0, NumValues = ValueVTs.
size();
1228 }
else if (RegisterVT.
isVector()) {
1231 assert(MemoryBits % NumElements == 0);
1235 MemoryBits / NumElements);
1257 unsigned PartOffset = 0;
1258 for (
unsigned i = 0; i != NumRegs; ++i) {
1260 BasePartOffset + PartOffset,
1299 int ClobberedFI)
const {
1302 int64_t LastByte = FirstByte + MFI.
getObjectSize(ClobberedFI) - 1;
1311 if (
LoadSDNode *L = dyn_cast<LoadSDNode>(U)) {
1313 if (FI->getIndex() < 0) {
1315 int64_t InLastByte = InFirstByte;
1318 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
1319 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
1341 FuncName =
G->getSymbol();
1343 FuncName =
G->getGlobal()->getName();
1375 switch (
Op.getOpcode()) {
1379 "instruction is not implemented yet!");
1424 switch (
N->getOpcode()) {
1470 if (std::optional<uint32_t>
Address =
1479 GV->
getName() !=
"llvm.amdgcn.module.lds") {
1483 Fn,
"local memory global used by non-kernel function",
1501 "Do not know what to do with an non-zero offset");
1517 EVT VT =
Op.getValueType();
1519 unsigned OpBitSize =
Op.getOperand(0).getValueType().getSizeInBits();
1520 if (OpBitSize >= 32 && OpBitSize % 32 == 0) {
1521 unsigned NewNumElt = OpBitSize / 32;
1522 EVT NewEltVT = (NewNumElt == 1) ? MVT::i32
1524 MVT::i32, NewNumElt);
1525 for (
const SDUse &U :
Op->ops()) {
1531 Args.push_back(NewIn);
1541 for (
const SDUse &U :
Op->ops())
1551 unsigned Start =
Op.getConstantOperandVal(1);
1552 EVT VT =
Op.getValueType();
1553 EVT SrcVT =
Op.getOperand(0).getValueType();
1558 assert(NumElt % 2 == 0 && NumSrcElt % 2 == 0 &&
"expect legal types");
1562 EVT NewVT = NumElt == 2
1674 if ((
LHS == True &&
RHS == False) || (
LHS == False &&
RHS == True))
1693 if (
LHS == NegTrue && CFalse && CRHS) {
1707std::pair<SDValue, SDValue>
1719 return std::pair(
Lo,
Hi);
1748 HiVT = NumElts - LoNumElts == 1
1751 return std::pair(LoVT, HiVT);
1756std::pair<SDValue, SDValue>
1758 const EVT &LoVT,
const EVT &HiVT,
1762 N.getValueType().getVectorNumElements() &&
1763 "More vector elements requested than available!");
1769 return std::pair(
Lo,
Hi);
1775 EVT VT =
Op.getValueType();
1787 SDValue BasePtr = Load->getBasePtr();
1788 EVT MemVT = Load->getMemoryVT();
1793 EVT LoMemVT, HiMemVT;
1801 Align BaseAlign = Load->getAlign();
1805 Load->getChain(), BasePtr, SrcValue, LoMemVT,
1806 BaseAlign, Load->getMemOperand()->getFlags());
1809 DAG.
getExtLoad(Load->getExtensionType(), SL, HiVT, Load->getChain(),
1811 HiMemVT, HiAlign, Load->getMemOperand()->getFlags());
1835 EVT VT =
Op.getValueType();
1836 SDValue BasePtr = Load->getBasePtr();
1837 EVT MemVT = Load->getMemoryVT();
1840 Align BaseAlign = Load->getAlign();
1845 if (NumElements != 3 ||
1846 (BaseAlign <
Align(8) &&
1850 assert(NumElements == 3);
1857 Load->getExtensionType(), SL, WideVT, Load->getChain(), BasePtr, SrcValue,
1858 WideMemVT, BaseAlign, Load->getMemOperand()->getFlags());
1869 SDValue Val = Store->getValue();
1877 EVT MemVT = Store->getMemoryVT();
1878 SDValue Chain = Store->getChain();
1879 SDValue BasePtr = Store->getBasePtr();
1883 EVT LoMemVT, HiMemVT;
1893 Align BaseAlign = Store->getAlign();
1898 DAG.
getTruncStore(Chain, SL,
Lo, BasePtr, SrcValue, LoMemVT, BaseAlign,
1899 Store->getMemOperand()->getFlags());
1902 HiMemVT, HiAlign, Store->getMemOperand()->getFlags());
1913 EVT VT =
Op.getValueType();
1916 MVT IntVT = MVT::i32;
1917 MVT FltVT = MVT::f32;
1920 if (LHSSignBits < 9)
1924 if (RHSSignBits < 9)
1928 unsigned SignBits = std::min(LHSSignBits, RHSSignBits);
1929 unsigned DivBits = BitSize - SignBits;
1973 bool UseFmadFtz =
false;
1974 if (Subtarget->
isGCN()) {
2029 EVT VT =
Op.getValueType();
2031 assert(VT == MVT::i64 &&
"LowerUDIVREM64 expects an i64");
2104 std::tie(Mulhi1_Lo, Mulhi1_Hi) =
2117 std::tie(Mulhi2_Lo, Mulhi2_Hi) =
2131 std::tie(Mul3_Lo, Mul3_Hi) = DAG.
SplitScalar(Mul3,
DL, HalfVT, HalfVT);
2210 for (
unsigned i = 0; i < halfBitWidth; ++i) {
2211 const unsigned bitPos = halfBitWidth - i - 1;
2242 EVT VT =
Op.getValueType();
2244 if (VT == MVT::i64) {
2250 if (VT == MVT::i32) {
2297 EVT VT =
Op.getValueType();
2305 if (VT == MVT::i32) {
2310 if (VT == MVT::i64 &&
2357 EVT VT =
Op.getValueType();
2358 auto Flags =
Op->getFlags();
2396 const unsigned FractBits = 52;
2397 const unsigned ExpBits = 11;
2413 assert(
Op.getValueType() == MVT::f64);
2423 const unsigned FractBits = 52;
2435 = DAG.
getConstant((UINT64_C(1) << FractBits) - 1, SL, MVT::i64);
2460 assert(
Op.getValueType() == MVT::f64);
2493 auto VT =
Op.getValueType();
2494 auto Arg =
Op.getOperand(0u);
2506 EVT VT =
Op.getValueType();
2557 switch (Src.getOpcode()) {
2559 return Src.getOperand(0).getValueType() == MVT::f16;
2564 unsigned IntrinsicID = Src.getConstantOperandVal(0);
2565 switch (IntrinsicID) {
2566 case Intrinsic::amdgcn_frexp_mant:
2581 if (Flags.hasApproximateFuncs())
2600 EVT VT = Src.getValueType();
2611 return IsLtSmallestNormal;
2617 EVT VT = Src.getValueType();
2630std::pair<SDValue, SDValue>
2651 return {ScaledInput, IsLtSmallestNormal};
2662 EVT VT =
Op.getValueType();
2666 if (VT == MVT::f16) {
2675 auto [ScaledInput, IsLtSmallestNormal] =
2698 EVT VT =
Op.getValueType();
2706 if (VT == MVT::f16 || Flags.hasApproximateFuncs() ||
2732 const float c_log10 = 0x1.344134p-2f;
2733 const float cc_log10 = 0x1.09f79ep-26f;
2736 const float c_log = 0x1.62e42ep-1f;
2737 const float cc_log = 0x1.efa39ep-25f;
2749 const float ch_log10 = 0x1.344000p-2f;
2750 const float ct_log10 = 0x1.3509f6p-18f;
2753 const float ch_log = 0x1.62e000p-1f;
2754 const float ct_log = 0x1.0bfbe8p-15f;
2771 const bool IsFiniteOnly = (Flags.hasNoNaNs() ||
Options.NoNaNsFPMath) &&
2772 (Flags.hasNoInfs() ||
Options.NoInfsFPMath);
2775 if (!IsFiniteOnly) {
2801 EVT VT = Src.getValueType();
2805 double Log2BaseInverted =
2808 if (VT == MVT::f32) {
2818 ScaledResultOffset, Zero, Flags);
2833 return DAG.
getNode(
ISD::FMUL, SL, VT, Log2Operand, Log2BaseInvertedOperand,
2842 EVT VT =
Op.getValueType();
2846 if (VT == MVT::f16) {
2891 EVT VT =
X.getValueType();
2899 SL, VT,
Mul, Flags);
2931 const EVT VT =
X.getValueType();
2981 EVT VT =
Op.getValueType();
3044 const float cc_exp = 0x1.4ae0bep-26f;
3045 const float c_exp10 = 0x1.a934f0p+1f;
3046 const float cc_exp10 = 0x1.2f346ep-24f;
3056 const float ch_exp = 0x1.714000p+0f;
3057 const float cl_exp = 0x1.47652ap-12f;
3059 const float ch_exp10 = 0x1.a92000p+1f;
3060 const float cl_exp10 = 0x1.4f0978p-11f;
3075 PL =
getMad(DAG, SL, VT, XH, CL, Mad0, Flags);
3090 DAG.
getConstantFP(IsExp10 ? -0x1.66d3e8p+5f : -0x1.9d1da0p+6f, SL, VT);
3100 if (!Flags.hasNoInfs() && !
Options.NoInfsFPMath) {
3102 DAG.
getConstantFP(IsExp10 ? 0x1.344136p+5f : 0x1.62e430p+6f, SL, VT);
3124 auto Opc =
Op.getOpcode();
3125 auto Arg =
Op.getOperand(0u);
3126 auto ResultVT =
Op.getValueType();
3128 if (ResultVT != MVT::i8 && ResultVT != MVT::i16)
3132 assert(ResultVT == Arg.getValueType());
3134 const uint64_t NumBits = ResultVT.getFixedSizeInBits();
3141 NewOp = DAG.
getNode(Opc, SL, MVT::i32, NewOp);
3144 NewOp = DAG.
getNode(Opc, SL, MVT::i32, NewOp);
3161 bool Is64BitScalar = !Src->isDivergent() && Src.getValueType() == MVT::i64;
3163 if (Src.getValueType() == MVT::i32 || Is64BitScalar) {
3177 Op.getValueType().getScalarSizeInBits(), SL, MVT::i32);
3197 OprLo = DAG.
getNode(AddOpc, SL, MVT::i32, OprLo, Const32);
3199 OprHi = DAG.
getNode(AddOpc, SL, MVT::i32, OprHi, Const32);
3316 if (Subtarget->
isGCN())
3359 EVT DestVT =
Op.getValueType();
3361 EVT SrcVT = Src.getValueType();
3363 if (SrcVT == MVT::i16) {
3364 if (DestVT == MVT::f16)
3373 if (DestVT == MVT::bf16) {
3380 if (SrcVT != MVT::i64)
3395 if (DestVT == MVT::f32)
3398 assert(DestVT == MVT::f64);
3404 EVT DestVT =
Op.getValueType();
3407 EVT SrcVT = Src.getValueType();
3409 if (SrcVT == MVT::i16) {
3410 if (DestVT == MVT::f16)
3419 if (DestVT == MVT::bf16) {
3426 if (SrcVT != MVT::i64)
3444 if (DestVT == MVT::f32)
3447 assert(DestVT == MVT::f64);
3456 EVT SrcVT = Src.getValueType();
3458 assert(SrcVT == MVT::f32 || SrcVT == MVT::f64);
3471 if (
Signed && SrcVT == MVT::f32) {
3484 if (SrcVT == MVT::f64) {
3486 llvm::bit_cast<double>(UINT64_C( 0x3df0000000000000)), SL,
3489 llvm::bit_cast<double>(UINT64_C( 0xc1f0000000000000)), SL,
3493 llvm::bit_cast<float>(UINT32_C( 0x2f800000)), SL, SrcVT);
3495 llvm::bit_cast<float>(UINT32_C( 0xcf800000)), SL, SrcVT);
3506 SL, MVT::i32, FloorMul);
3512 if (
Signed && SrcVT == MVT::f32) {
3542 const unsigned ExpMask = 0x7ff;
3543 const unsigned ExpBiasf64 = 1023;
3544 const unsigned ExpBiasf16 = 15;
3628 unsigned OpOpcode =
Op.getOpcode();
3629 EVT SrcVT = Src.getValueType();
3630 EVT DestVT =
Op.getValueType();
3633 if (SrcVT == MVT::f16 && DestVT == MVT::i16)
3636 if (SrcVT == MVT::bf16) {
3639 return DAG.
getNode(
Op.getOpcode(),
DL, DestVT, PromotedSrc);
3643 if (DestVT == MVT::i16 && (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
3650 if (DestVT != MVT::i64)
3653 if (SrcVT == MVT::f16 ||
3660 return DAG.
getNode(Ext,
DL, MVT::i64, FpToInt32);
3663 if (SrcVT == MVT::f32 || SrcVT == MVT::f64)
3671 EVT ExtraVT = cast<VTSDNode>(
Op.getOperand(1))->getVT();
3672 MVT VT =
Op.getSimpleValueType();
3686 for (
unsigned I = 0;
I < NElts; ++
I)
3701 EVT VT =
Op.getValueType();
3715 unsigned NewOpcode = Node24->
getOpcode();
3719 case Intrinsic::amdgcn_mul_i24:
3722 case Intrinsic::amdgcn_mul_u24:
3725 case Intrinsic::amdgcn_mulhi_i24:
3728 case Intrinsic::amdgcn_mulhi_u24:
3743 if (DemandedLHS || DemandedRHS)
3745 DemandedLHS ? DemandedLHS :
LHS,
3746 DemandedRHS ? DemandedRHS :
RHS);
3758template <
typename IntTy>
3761 if (Width +
Offset < 32) {
3763 IntTy Result =
static_cast<IntTy
>(Shl) >> (32 - Width);
3772 if (
MemSDNode *M = dyn_cast<MemSDNode>(U)) {
3773 if (M->isVolatile())
3921 EVT ExtVT = cast<VTSDNode>(N1)->getVT();
3925 EVT SrcVT = Src.getValueType();
3926 if (SrcVT.
bitsGE(ExtVT)) {
3937 unsigned IID =
N->getConstantOperandVal(0);
3939 case Intrinsic::amdgcn_mul_i24:
3940 case Intrinsic::amdgcn_mul_u24:
3941 case Intrinsic::amdgcn_mulhi_i24:
3942 case Intrinsic::amdgcn_mulhi_u24:
3944 case Intrinsic::amdgcn_fract:
3945 case Intrinsic::amdgcn_rsq:
3946 case Intrinsic::amdgcn_rcp_legacy:
3947 case Intrinsic::amdgcn_rsq_legacy:
3948 case Intrinsic::amdgcn_rsq_clamp: {
3951 return Src.isUndef() ? Src :
SDValue();
3953 case Intrinsic::amdgcn_frexp_exp: {
3959 if (PeekSign == Src)
3996 EVT VT =
N->getValueType(0);
4003 unsigned RHSVal =
RHS->getZExtValue();
4010 switch (
LHS->getOpcode()) {
4018 if (VT == MVT::i32 && RHSVal == 16 &&
X.getValueType() == MVT::i16 &&
4034 EVT XVT =
X.getValueType();
4064 if (
N->getValueType(0) != MVT::i64)
4073 unsigned RHSVal =
RHS->getZExtValue();
4099 auto *
RHS = dyn_cast<ConstantSDNode>(
N->getOperand(1));
4103 EVT VT =
N->getValueType(0);
4105 unsigned ShiftAmt =
RHS->getZExtValue();
4112 if (
auto *Mask = dyn_cast<ConstantSDNode>(
LHS.getOperand(1))) {
4113 unsigned MaskIdx, MaskLen;
4114 if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
4115 MaskIdx == ShiftAmt) {
4149 EVT VT =
N->getValueType(0);
4154 SDValue Vec = Src.getOperand(0);
4174 if (2 * K->getZExtValue() == Src.getValueType().getScalarSizeInBits()) {
4196 EVT SrcVT = Src.getValueType();
4201 SDValue Amt = Src.getOperand(1);
4208 const unsigned MaxCstSize =
4243 return DAG.
getNode(MulOpc, SL, MVT::i32, N0, N1);
4267 EVT VT =
N->getValueType(0);
4273 if (!
N->isDivergent())
4295 if (V.hasOneUse() ||
all_of(V->uses(), [](
const SDNode *U) ->
bool {
4296 return U->getOpcode() == ISD::MUL;
4305 if (
SDValue MulOper = IsFoldableAdd(N0)) {
4310 if (
SDValue MulOper = IsFoldableAdd(N1)) {
4351 if (
N->getValueType(0) != MVT::i32)
4371 unsigned LoOpcode, HiOpcode;
4394 EVT VT =
N->getValueType(0);
4405 if (Subtarget->
hasSMulHi() && !
N->isDivergent())
4427 EVT VT =
N->getValueType(0);
4438 if (Subtarget->
hasSMulHi() && !
N->isDivergent())
4461 unsigned Opc)
const {
4462 EVT VT =
Op.getValueType();
4465 LegalVT != MVT::i16))
4502 return getFFBX_U32(DAG, CmpLHS, SL, Opc);
4513 return getFFBX_U32(DAG, CmpLHS, SL, Opc);
4531 return DAG.
getNode(
Op, SL, VT, NewSelect);
4549 EVT VT =
N.getValueType();
4576 bool ShouldFoldNeg =
true;
4581 ShouldFoldNeg =
false;
4583 ShouldFoldNeg =
false;
4586 if (ShouldFoldNeg) {
4610 Cond, NewLHS, NewRHS);
4612 return DAG.
getNode(
LHS.getOpcode(), SL, VT, NewSelect);
4628 EVT VT =
N->getValueType(0);
4636 if (
Cond.hasOneUse()) {
4646 getSetCCInverse(cast<CondCodeSDNode>(
CC)->
get(),
LHS.getValueType());
4748 EVT VT =
N->getValueType(0);
4867 for (
unsigned I = 0;
I < 3; ++
I)
4933 EVT SrcVT = Src.getValueType();
4969 Ops.
back() = CastBack;
5019 EVT SrcVT = Src.getValueType();
5033 const auto *CFP = dyn_cast<ConstantFPSDNode>(
N->getOperand(0));
5038 const APFloat &Val = CFP->getValueAPF();
5048 switch(
N->getOpcode()) {
5052 EVT DestVT =
N->getValueType(0);
5064 EVT SrcVT = Src.getValueType();
5100 const APInt &Val =
C->getValueAPF().bitcastToAPInt();
5158 assert(!
N->getValueType(0).isVector() &&
5159 "Vector handling of BFE not implemented");
5172 SDValue BitsFrom =
N->getOperand(0);
5177 if (OffsetVal == 0) {
5179 unsigned SignBits =
Signed ? (32 - WidthVal + 1) : (32 - WidthVal);
5182 if (OpSignBits >= SignBits)
5202 return constantFoldBFE<int32_t>(DAG,
5203 CVal->getSExtValue(),
5209 return constantFoldBFE<uint32_t>(DAG,
5210 CVal->getZExtValue(),
5216 if ((OffsetVal + WidthVal) >= 32 &&
5217 !(Subtarget->
hasSDWA() && OffsetVal == 16 && WidthVal == 16)) {
5220 BitsFrom, ShiftVal);
5226 OffsetVal + WidthVal);
5256 EVT VT =
N->getValueType(0);
5263 if (N0CFP && N1CFP && N2CFP) {
5264 const auto FTZ = [](
const APFloat &V) {
5265 if (V.isDenormal()) {
5266 APFloat Zero(V.getSemantics(), 0);
5267 return V.isNegative() ? -Zero : Zero;
5294 bool RawReg)
const {
5299 if (!
MRI.isLiveIn(Reg)) {
5300 VReg =
MRI.createVirtualRegister(RC);
5301 MRI.addLiveIn(Reg, VReg);
5303 VReg =
MRI.getLiveInVirtReg(Reg);
5365 assert(Arg &&
"Attempting to load missing argument");
5374 unsigned Mask = Arg.
getMask();
5375 unsigned Shift = llvm::countr_zero<unsigned>(Mask);
5387 alignTo(ExplicitKernArgSize, Alignment) + ExplicitArgOffset;
5407#define NODE_NAME_CASE(node) case AMDGPUISD::node: return #node;
5578 int &RefinementSteps,
5579 bool &UseOneConstNR,
5580 bool Reciprocal)
const {
5583 if (VT == MVT::f32) {
5584 RefinementSteps = 0;
5596 int &RefinementSteps)
const {
5599 if (VT == MVT::f32) {
5605 RefinementSteps = 0;
5617 case Intrinsic::amdgcn_workitem_id_x:
5619 case Intrinsic::amdgcn_workitem_id_y:
5621 case Intrinsic::amdgcn_workitem_id_z:
5634 unsigned Opc =
Op.getOpcode();
5677 LHSKnown = LHSKnown.
trunc(24);
5678 RHSKnown = RHSKnown.
trunc(24);
5683 unsigned MaxValBits = LHSValBits + RHSValBits;
5684 if (MaxValBits > 32)
5686 unsigned SignBits = 32 - MaxValBits + 1;
5694 if ((LHSNonNegative && RHSNonNegative) || (LHSNegative && RHSNegative))
5696 else if ((LHSNegative && RHSPositive) || (LHSPositive && RHSNegative))
5701 unsigned MaxValBits = LHSValBits + RHSValBits;
5702 if (MaxValBits >= 32)
5717 for (
unsigned I = 0;
I < 32;
I += 8) {
5718 unsigned SelBits = Sel & 0xff;
5723 }
else if (SelBits < 7) {
5724 SelBits = (SelBits & 3) * 8;
5727 }
else if (SelBits == 0x0c) {
5728 Known.
Zero |= 0xFFull <<
I;
5729 }
else if (SelBits > 0x0c) {
5730 Known.
One |= 0xFFull <<
I;
5745 auto GA = cast<GlobalAddressSDNode>(
Op.getOperand(0).getNode());
5776 unsigned IID =
Op.getConstantOperandVal(0);
5778 case Intrinsic::amdgcn_workitem_id_x:
5779 case Intrinsic::amdgcn_workitem_id_y:
5780 case Intrinsic::amdgcn_workitem_id_z: {
5795 unsigned Depth)
const {
5796 switch (
Op.getOpcode()) {
5808 return std::max(SignBits, Op0SignBits);
5813 return Width ? 32 - (Width->
getZExtValue() & 0x1f) : 1;
5847 return std::min({Tmp0, Tmp1, Tmp2});
5857 unsigned Depth)
const {
5863 switch (
MI->getOpcode()) {
5864 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SBYTE:
5866 case AMDGPU::G_AMDGPU_BUFFER_LOAD_SSHORT:
5868 case AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE:
5870 case AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT:
5872 case AMDGPU::G_AMDGPU_SMED3:
5873 case AMDGPU::G_AMDGPU_UMED3: {
5874 auto [Dst, Src0, Src1, Src2] =
MI->getFirst4Regs();
5875 unsigned Tmp2 =
Analysis.computeNumSignBits(Src2, DemandedElts,
Depth + 1);
5878 unsigned Tmp1 =
Analysis.computeNumSignBits(Src1, DemandedElts,
Depth + 1);
5881 unsigned Tmp0 =
Analysis.computeNumSignBits(Src0, DemandedElts,
Depth + 1);
5884 return std::min({Tmp0, Tmp1, Tmp2});
5894 unsigned Depth)
const {
5895 unsigned Opcode =
Op.getOpcode();
5958 unsigned IntrinsicID =
Op.getConstantOperandVal(0);
5960 switch (IntrinsicID) {
5961 case Intrinsic::amdgcn_cubeid:
5964 case Intrinsic::amdgcn_frexp_mant: {
5969 case Intrinsic::amdgcn_cvt_pkrtz: {
5975 case Intrinsic::amdgcn_rcp:
5976 case Intrinsic::amdgcn_rsq:
5977 case Intrinsic::amdgcn_rcp_legacy:
5978 case Intrinsic::amdgcn_rsq_legacy:
5979 case Intrinsic::amdgcn_rsq_clamp: {
5986 case Intrinsic::amdgcn_trig_preop:
5987 case Intrinsic::amdgcn_fdot2:
5990 case Intrinsic::amdgcn_fma_legacy:
6007 return MRI.hasOneNonDBGUse(N0);
6021 unsigned ValSize =
DL.getTypeSizeInBits(RMW->
getType());
6022 if (ValSize == 32 || ValSize == 64)
6027 if (
auto *IntTy = dyn_cast<IntegerType>(RMW->
getType())) {
6028 unsigned Size = IntTy->getBitWidth();
6043 using namespace PatternMatch;
6045 for (
auto &
Op :
I->operands()) {
6047 if (
any_of(Ops, [&](
Use *U) {
return U->get() ==
Op.get(); }))
6054 return !Ops.
empty();
unsigned const MachineRegisterInfo * MRI
static LLVM_READONLY bool hasSourceMods(const MachineInstr &MI)
static bool isInv2Pi(const APFloat &APF)
static LLVM_READONLY bool opMustUseVOP3Encoding(const MachineInstr &MI, const MachineRegisterInfo &MRI)
returns true if the operation will definitely need to use a 64-bit encoding, and thus will use a VOP3...
static unsigned inverseMinMax(unsigned Opc)
static SDValue extractF64Exponent(SDValue Hi, const SDLoc &SL, SelectionDAG &DAG)
static unsigned workitemIntrinsicDim(unsigned ID)
static int getOrCreateFixedStackObject(MachineFrameInfo &MFI, unsigned Size, int64_t Offset)
static SDValue constantFoldBFE(SelectionDAG &DAG, IntTy Src0, uint32_t Offset, uint32_t Width, const SDLoc &DL)
static SDValue getMad(SelectionDAG &DAG, const SDLoc &SL, EVT VT, SDValue X, SDValue Y, SDValue C, SDNodeFlags Flags=SDNodeFlags())
static SDValue getAddOneOp(const SDNode *V)
If V is an add of a constant 1, returns the other operand.
#define NODE_NAME_CASE(node)
static LLVM_READONLY bool selectSupportsSourceMods(const SDNode *N)
Return true if v_cndmask_b32 will support fabs/fneg source modifiers for the type for ISD::SELECT.
static cl::opt< bool > AMDGPUBypassSlowDiv("amdgpu-bypass-slow-div", cl::desc("Skip 64-bit divide for dynamic 32-bit values"), cl::init(true))
static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, SDValue N0, SDValue N1, unsigned Size, bool Signed)
static bool fnegFoldsIntoOp(const SDNode *N)
static bool isI24(SDValue Op, SelectionDAG &DAG)
static bool isCttzOpc(unsigned Opc)
static bool isU24(SDValue Op, SelectionDAG &DAG)
static SDValue peekFPSignOps(SDValue Val)
static bool valueIsKnownNeverF32Denorm(SDValue Src)
Return true if it's known that Src can never be an f32 denormal value.
static SDValue distributeOpThroughSelect(TargetLowering::DAGCombinerInfo &DCI, unsigned Op, const SDLoc &SL, SDValue Cond, SDValue N1, SDValue N2)
static SDValue peekFNeg(SDValue Val)
static SDValue simplifyMul24(SDNode *Node24, TargetLowering::DAGCombinerInfo &DCI)
static bool isCtlzOpc(unsigned Opc)
static LLVM_READNONE bool fnegFoldsIntoOpcode(unsigned Opc)
static bool hasVolatileUser(SDNode *Val)
Interface definition of the TargetLowering class that is common to all AMD GPUs.
Contains the definition of a TargetInstrInfo class that is common to all AMD GPUs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
block Block Frequency Analysis
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
static cl::opt< unsigned > CostThreshold("dfa-cost-threshold", cl::desc("Maximum cost accepted for the transformation"), cl::Hidden, cl::init(50))
static Error getAddrSpace(StringRef R, unsigned &AddrSpace)
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Provides analysis for querying information about KnownBits during GISel passes.
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
static bool isUniformMMO(const MachineMemOperand *MMO)
uint64_t getExplicitKernArgSize() const
static std::optional< uint32_t > getLDSAbsoluteAddress(const GlobalValue &GV)
unsigned allocateLDSGlobal(const DataLayout &DL, const GlobalVariable &GV)
bool isModuleEntryFunction() const
bool hasFminFmaxLegacy() const
Align getAlignmentForImplicitArgPtr() const
bool hasMadMacF32Insts() const
unsigned getMaxWorkitemID(const Function &Kernel, unsigned Dimension) const
Return the maximum workitem ID value in the function, for the given (0, 1, 2) dimension.
bool has16BitInsts() const
bool hasFastFMAF32() const
unsigned getExplicitKernelArgOffset() const
Returns the offset in bytes from the start of the input buffer of the first explicit kernel argument.
static const AMDGPUSubtarget & get(const MachineFunction &MF)
bool hasInv2PiInlineImm() const
static unsigned numBitsSigned(SDValue Op, SelectionDAG &DAG)
SDValue combineFMinMaxLegacy(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, DAGCombinerInfo &DCI) const
Generate Min/Max node.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const
EVT getTypeForExtReturn(LLVMContext &Context, EVT VT, ISD::NodeType ExtendKind) const override
Return the type that should be used to zero or sign extend a zeroext/signext integer return value.
SDValue SplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Split a vector load into 2 loads of half the vector.
SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const
SDValue performLoadCombine(SDNode *N, DAGCombinerInfo &DCI) const
void analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl< ISD::InputArg > &Ins) const
The SelectionDAGBuilder will automatically promote function arguments with illegal types.
SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const
SDValue storeStackInputValue(SelectionDAG &DAG, const SDLoc &SL, SDValue Chain, SDValue ArgVal, int64_t Offset) const
bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AS) const override
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool shouldCombineMemoryType(EVT VT) const
SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL, unsigned Opc, SDValue LHS, uint32_t ValLo, uint32_t ValHi) const
Split the 64-bit value LHS into two 32-bit components, and perform the binary operation Opc to it wit...
SDValue lowerUnhandledCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals, StringRef Reason) const
SDValue performAssertSZExtCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isTruncateFree(EVT Src, EVT Dest) const override
bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override
SDValue LowerFCEIL(SDValue Op, SelectionDAG &DAG) const
TargetLowering::NegatibleCost getConstantNegateCost(const ConstantFPSDNode *C) const
SDValue LowerFLOGUnsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, bool IsLog10, SDNodeFlags Flags) const
SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const
AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isSDNodeAlwaysUniform(const SDNode *N) const override
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount through its operand,...
SDValue LowerFREM(SDValue Op, SelectionDAG &DAG) const
Split a vector store into multiple scalar stores.
SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const
bool isFNegFree(EVT VT) const override
Return true if an fneg operation is free to the point where it is never worthwhile to replace it with...
SDValue LowerFLOG10(SDValue Op, SelectionDAG &DAG) const
SDValue LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, bool Signed) const
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const
SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG, MachineFrameInfo &MFI, int ClobberedFI) const
bool isConstantCheaperToNegate(SDValue N) const
bool isReassocProfitable(MachineRegisterInfo &MRI, Register N0, Register N1) const override
static bool needsDenormHandlingF32(const SelectionDAG &DAG, SDValue Src, SDNodeFlags Flags)
uint32_t getImplicitParameterOffset(const MachineFunction &MF, const ImplicitParameter Param) const
Helper function that returns the byte offset of the given type of implicit parameter.
SDValue LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const
SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue performFNegCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const
virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op, SelectionDAG &DAG) const
bool isConstantCostlierToNegate(SDValue N) const
SDValue loadInputValue(SelectionDAG &DAG, const TargetRegisterClass *RC, EVT VT, const SDLoc &SL, const ArgDescriptor &Arg) const
SDValue LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const
SDValue lowerFEXP10Unsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, SDNodeFlags Flags) const
Emit approx-funcs appropriate lowering for exp10.
SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
SDValue performCtlz_CttzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS, SDValue RHS, DAGCombinerInfo &DCI) const
SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const
bool isSelectSupported(SelectSupportKind) const override
bool isZExtFree(Type *Src, Type *Dest) const override
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
SDValue lowerFEXP2(SDValue Op, SelectionDAG &DAG) const
SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue lowerFEXP(SDValue Op, SelectionDAG &DAG) const
SDValue getIsLtSmallestNormal(SelectionDAG &DAG, SDValue Op, SDNodeFlags Flags) const
bool mayIgnoreSignedZero(SDValue Op) const
SDValue getIsFinite(SelectionDAG &DAG, SDValue Op, SDNodeFlags Flags) const
bool isLoadBitCastBeneficial(EVT, EVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const final
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtType, EVT ExtVT) const override
Return true if it is profitable to reduce a load to a smaller type.
MVT getVectorIdxTy(const DataLayout &) const override
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
std::pair< SDValue, SDValue > splitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HighVT, SelectionDAG &DAG) const
Split a vector value into two parts of types LoVT and HiVT.
SDValue LowerFLOGCommon(SDValue Op, SelectionDAG &DAG) const
SDValue foldFreeOpFromSelect(TargetLowering::DAGCombinerInfo &DCI, SDValue N) const
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Whether it is profitable to sink the operands of an Instruction I to the basic block of I.
SDValue LowerINT_TO_FP32(SDValue Op, SelectionDAG &DAG, bool Signed) const
bool isFAbsFree(EVT VT) const override
Return true if an fabs operation is free to the point where it is never worthwhile to replace it with...
SDValue loadStackInputValue(SelectionDAG &DAG, EVT VT, const SDLoc &SL, int64_t Offset) const
Similar to CreateLiveInRegister, except value maybe loaded from a stack slot rather than passed in a ...
bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
SDValue LowerFLOG2(SDValue Op, SelectionDAG &DAG) const
static EVT getEquivalentMemType(LLVMContext &Context, EVT VT)
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const override
Hooks for building estimates in place of slower divisions and square roots.
unsigned computeNumSignBitsForTargetInstr(GISelKnownBits &Analysis, Register R, const APInt &DemandedElts, const MachineRegisterInfo &MRI, unsigned Depth=0) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
SDValue performTruncateCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const
static SDValue stripBitcast(SDValue Val)
SDValue CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC, Register Reg, EVT VT, const SDLoc &SL, bool RawReg=false) const
Helper function that adds Reg to the LiveIn list of the DAG's MachineFunction.
SDValue SplitVectorStore(SDValue Op, SelectionDAG &DAG) const
Split a vector store into 2 stores of half the vector.
SDValue LowerCTLZ_CTTZ(SDValue Op, SelectionDAG &DAG) const
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, NegatibleCost &Cost, unsigned Depth) const override
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
std::pair< SDValue, SDValue > split64BitValue(SDValue Op, SelectionDAG &DAG) const
Return 64-bit value Op as two 32-bit integers.
SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const override
Return a reciprocal estimate value for the input operand.
AMDGPUTargetLowering(const TargetMachine &TM, const AMDGPUSubtarget &STI)
SDValue LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const
static CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg)
std::pair< SDValue, SDValue > getScaledLogInput(SelectionDAG &DAG, const SDLoc SL, SDValue Op, SDNodeFlags Flags) const
If denormal handling is required return the scaled input to FLOG2, and the check for denormal range.
static CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg)
Selects the correct CCAssignFn for a given CallingConvention value.
static bool allUsesHaveSourceMods(const SDNode *N, unsigned CostThreshold=4)
SDValue LowerFROUNDEVEN(SDValue Op, SelectionDAG &DAG) const
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const override
If SNaN is false,.
static unsigned numBitsUnsigned(SDValue Op, SelectionDAG &DAG)
SDValue lowerFEXPUnsafe(SDValue Op, const SDLoc &SL, SelectionDAG &DAG, SDNodeFlags Flags) const
SDValue LowerFTRUNC(SDValue Op, SelectionDAG &DAG) const
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const
static bool allowApproxFunc(const SelectionDAG &DAG, SDNodeFlags Flags)
bool ShouldShrinkFPConstant(EVT VT) const override
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
SDValue performStoreCombine(SDNode *N, DAGCombinerInfo &DCI) const
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
SDValue performRcpCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue getLoHalf64(SDValue Op, SelectionDAG &DAG) const
SDValue lowerCTLZResults(SDValue Op, SelectionDAG &DAG) const
SDValue performFAbsCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerFP_TO_INT64(SDValue Op, SelectionDAG &DAG, bool Signed) const
static bool shouldFoldFNegIntoSrc(SDNode *FNeg, SDValue FNegSrc)
SDValue LowerFRINT(SDValue Op, SelectionDAG &DAG) const
SDValue performIntrinsicWOChainCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue LowerUDIVREM(SDValue Op, SelectionDAG &DAG) const
SDValue performMulLoHiCombine(SDNode *N, DAGCombinerInfo &DCI) const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG, SmallVectorImpl< SDValue > &Results) const
SDValue WidenOrSplitVectorLoad(SDValue Op, SelectionDAG &DAG) const
Widen a suitably aligned v3 load.
std::pair< EVT, EVT > getSplitDestVTs(const EVT &VT, SelectionDAG &DAG) const
Split a vector type into two parts.
SDValue getHiHalf64(SDValue Op, SelectionDAG &DAG) const
SDValue combineFMinMaxLegacyImpl(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, SDValue True, SDValue False, SDValue CC, DAGCombinerInfo &DCI) const
bool bitwiseIsEqual(const APFloat &RHS) const
opStatus add(const APFloat &RHS, roundingMode RM)
const fltSemantics & getSemantics() const
opStatus multiply(const APFloat &RHS, roundingMode RM)
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
This class represents an incoming formal argument to a Function.
an instruction that atomically reads a memory location, combines it with another value,...
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
BinOp getOperation() const
CCState - This class holds information needed while lowering arguments and return values.
MachineFunction & getMachineFunction() const
LLVMContext & getContext() const
void addLoc(const CCValAssign &V)
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
const APFloat & getValueAPF() const
bool isNegative() const
Return true if the value is negative.
uint64_t getZExtValue() const
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
bool print(raw_ostream &OS, DIDumpOptions DumpOpts, const DWARFExpression *Expr, DWARFUnit *U) const
A parsed version of the target data layout string in and methods for querying it.
Diagnostic information for unsupported feature in backend.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
iterator_range< arg_iterator > args()
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Module * getParent()
Get the module that this global value is contained inside of...
const Function * getFunction() const
Return the function this instruction belongs to.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
static auto integer_fixedlen_vector_valuetypes()
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static auto integer_valuetypes()
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
int64_t getObjectOffset(int ObjectIdx) const
Return the assigned stack offset of the specified object from the incoming stack pointer.
int getObjectIndexBegin() const
Return the minimum frame object index.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
DenormalMode getDenormalMode(const fltSemantics &FPType) const
Returns the denormal handling type for the default rounding mode of the function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
Flags getFlags() const
Return the raw flags of the source value,.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This is an abstract virtual class for memory operations.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
LLVMContext & getContext() const
Get the global data context.
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
Represents one node in the SelectionDAG.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
iterator_range< use_iterator > uses()
SDNodeFlags getFlags() const
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
op_iterator op_end() const
op_iterator op_begin() const
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
unsigned getNumOperands() const
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
SIModeRegisterDefaults getMode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
const TargetLowering & getTargetLoweringInfo() const
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getRegister(unsigned Reg, EVT VT)
bool isConstantValueOfAnyType(SDValue N) const
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
LLVMContext * getContext() const
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class is used to represent ISD::STORE nodes.
const SDValue & getBasePtr() const
const SDValue & getValue() const
StringRef - Represent a constant reference to a string, i.e.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
void setMaxDivRemBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum div/rem the backend supports.
bool PredictableSelectIsExpensive
Tells the code generator that select is more expensive than a branch if the branch is usually predict...
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
unsigned MaxGluedStoresPerMemcpy
Specify max number of store instructions to glue in inlined memcpy.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth)
Tells the code generator which bitwidths to bypass.
void setMaxLargeFPConvertBitWidthSupported(unsigned SizeInBits)
Set the size in bits of the maximum fp convert the backend supports.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
SelectSupportKind
Enum that describes what type of support for selects the target has.
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL) const
Returns the type for the shift amount of a shift opcode.
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
void setSupportsUnalignedAtomics(bool UnalignedSupported)
Sets whether unaligned atomic operations are supported.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const
Return true if it is profitable to reduce a load to a smaller type.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT)
If Opc/OrigVT is specified as being promoted, the promotion code defaults to trying a larger integer/...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned GatherAllAliasesMaxDepth
Depth that GatherAllAliases should continue looking for chain dependencies when trying to find a more...
NegatibleCost
Enum that specifies when a float negation is beneficial.
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
void setHasMultipleConditionRegisters(bool hasManyRegs=true)
Tells the code generator that the target has multiple (allocatable) condition registers that can be u...
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
void setJumpIsExpensive(bool isExpensive=true)
Tells the code generator not to expand logic operations on comparison predicates into separate sequen...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
SDValue scalarizeVectorStore(StoreSDNode *ST, SelectionDAG &DAG) const
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
SDValue expandUnalignedStore(StoreSDNode *ST, SelectionDAG &DAG) const
Expands an unaligned store to 2 half-size stores for integer values, and possibly more for vectors.
bool ShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const
Check to see if the specified operand of the specified instruction is a constant integer.
std::pair< SDValue, SDValue > expandUnalignedLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Expands an unaligned load to 2 half-size loads for an integer, and possibly more for vectors.
virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOps, bool OptForSize, NegatibleCost &Cost, unsigned Depth=0) const
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
std::pair< SDValue, SDValue > scalarizeVectorLoad(LoadSDNode *LD, SelectionDAG &DAG) const
Turn load of vector type into a load of the individual elements.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
Primary interface to the complete machine description for the target machine.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
StringRef getName() const
Return a constant reference to the value's name.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ CONSTANT_ADDRESS_32BIT
Address space for 32-bit constant memory.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ CONSTANT_ADDRESS
Address space for constant memory (VTX2).
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ FIRST_MEM_OPCODE_NUMBER
bool isIntrinsicAlwaysUniform(unsigned IntrID)
@ AMDGPU_CS
Used for Mesa/AMDPAL compute shaders.
@ AMDGPU_VS
Used for Mesa vertex shaders, or AMDPAL last shader stage before rasterization (vertex shader if tess...
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ AMDGPU_Gfx
Used for AMD graphics targets.
@ AMDGPU_CS_ChainPreserve
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_HS
Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
@ AMDGPU_GS
Used for Mesa/AMDPAL geometry shaders.
@ AMDGPU_CS_Chain
Used on AMDGPUs to give the middle-end more control over argument placement.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
@ SPIR_KERNEL
Used for SPIR kernel functions.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ AMDGPU_ES
Used for AMDPAL shader stage before geometry shader if geometry is in use.
@ AMDGPU_LS
Used for AMDPAL vertex shader if tessellation is in use.
@ C
The default llvm calling convention, compatible with C.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
@ BSWAP
Byte Swap and Counting operators.
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
@ ADD
Simple integer binary arithmetic operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
@ FADD
Simple binary floating point operators.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
@ SIGN_EXTEND
Conversion operators.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ SHL
Shift and rotation operations.
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
@ INLINEASM_BR
INLINEASM_BR - Branching version of inline asm. Used by asm-goto.
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
@ AND
Bitwise operators - logical and, logical or, logical xor.
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
@ INLINEASM
INLINEASM - Represents an inline asm block.
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool match(Val *V, const Pattern &P)
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
MaybeAlign getAlign(const Function &F, unsigned Index)
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
constexpr uint32_t Hi_32(uint64_t Value)
Return the high 32 bits of a 64 bit value.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
constexpr uint32_t Lo_32(uint64_t Value)
Return the low 32 bits of a 64 bit value.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
@ Mul
Product of integers.
@ And
Bitwise or logical AND of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
DWARFExpression::Operation Op
void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl< EVT > &ValueVTs, SmallVectorImpl< EVT > *MemVTs, SmallVectorImpl< TypeSize > *Offsets=nullptr, TypeSize StartingOffset=TypeSize::getZero())
ComputeValueVTs - Given an LLVM IR type, compute a sequence of EVTs that represent all the individual...
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
constexpr unsigned BitWidth
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
APFloat neg(APFloat X)
Returns the negated value of the argument.
unsigned Log2(Align A)
Returns the log2 of the alignment.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static const fltSemantics & IEEEsingle() LLVM_READNONE
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
static const fltSemantics & IEEEhalf() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
MCRegister getRegister() const
unsigned getStackOffset() const
DenormalModeKind Input
Denormal treatment kind for floating point instruction inputs in the default floating-point environme...
@ PreserveSign
The sign of a flushed-to-zero number is preserved in the sign of 0.
static constexpr DenormalMode getPreserveSign()
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
EVT getPow2VectorType(LLVMContext &Context) const
Widens the length of the given vector EVT up to the nearest power of 2 and returns that type.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
bool isByteSized() const
Return true if the bit size is a multiple of 8.
uint64_t getScalarSizeInBits() const
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
bool isPow2VectorType() const
Returns true if the given vector is a power of 2.
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
EVT getRoundIntegerType(LLVMContext &Context) const
Rounds the bit-width of the given integer EVT up to the nearest power of two (and at least to eight),...
bool isVector() const
Return true if this is a vector value type.
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
bool isNonNegative() const
Returns true if this value is known to be non-negative.
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
bool isUnknown() const
Returns true if we don't know any bits.
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
unsigned getBitWidth() const
Get the bit width of this value.
void resetAll()
Resets the known state of all bits.
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
bool isNegative() const
Returns true if this value is known to be negative.
unsigned countMaxSignificantBits() const
Returns the maximum number of bits needed to represent all possible signed values with these known bi...
This class contains a discriminated union of information about pointers in memory operands,...
bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
static MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
MachinePointerInfo getWithOffset(int64_t O) const
These are IR-level optimization flags that may be propagated to SDNodes.
void setAllowContract(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
DenormalMode FP32Denormals
If this is set, neither input or output denormals are flushed for most f32 instructions.
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
bool isBeforeLegalizeOps() const
CombineLevel getDAGCombineLevel()
void AddToWorklist(SDNode *N)
bool isCalledByLegalizer() const
bool isBeforeLegalize() const
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
void CommitTargetLoweringOpt(const TargetLoweringOpt &TLO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...