24 #include "llvm/IR/IntrinsicsS390.h"
31 #define DEBUG_TYPE "systemz-lower"
37 : Op0(Op0In), Op1(Op1In), Chain(ChainIn),
38 Opcode(0), ICmpType(0), CCValid(0), CCMask(0) {}
747 if (SplatBitSize > 64)
754 OpVals.push_back(((
unsigned) SignedValue));
762 if (
TII->isRxSBGMask(
Value, SplatBitSize, Start, End)) {
766 OpVals.push_back(Start - (64 - SplatBitSize));
767 OpVals.push_back(End - (64 - SplatBitSize));
786 if (tryValue(SplatBitsZ |
Upper | Lower))
793 return tryValue(SplatBitsZ | Middle);
808 unsigned HalfSize =
Width / 2;
813 if (HighValue != LowValue || 8 > HalfSize)
816 SplatBits = HighValue;
820 SplatBitSize =
Width;
828 BVN->
isConstantSplat(IntBits, SplatUndef, SplatBitSize, HasAnyUndefs, 128,
832 BVN->
isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs, 8,
837 bool ForCodeSize)
const {
883 LongDisplacement(LongDispl), IndexReg(IdxReg) {}
906 switch (II->getIntrinsicID()) {
908 case Intrinsic::memset:
909 case Intrinsic::memmove:
915 if (isa<LoadInst>(
I) &&
I->hasOneUse()) {
916 auto *SingleUser = cast<Instruction>(*
I->user_begin());
917 if (SingleUser->getParent() ==
I->getParent()) {
918 if (isa<ICmpInst>(SingleUser)) {
919 if (
auto *
C = dyn_cast<ConstantInt>(SingleUser->getOperand(1)))
920 if (
C->getBitWidth() <= 64 &&
924 }
else if (isa<StoreInst>(SingleUser))
928 }
else if (
auto *StoreI = dyn_cast<StoreInst>(
I)) {
929 if (
auto *LoadI = dyn_cast<LoadInst>(StoreI->getValueOperand()))
930 if (LoadI->hasOneUse() && LoadI->getParent() ==
I->getParent())
935 if (HasVector && (isa<LoadInst>(
I) || isa<StoreInst>(
I))) {
943 Type *MemAccessTy = (isa<LoadInst>(
I) ?
I->getType() :
944 I->getOperand(0)->getType());
946 bool IsVectorAccess = MemAccessTy->
isVectorTy();
950 if (!IsVectorAccess && isa<StoreInst>(
I)) {
951 Value *DataOp =
I->getOperand(0);
952 if (isa<ExtractElementInst>(DataOp))
953 IsVectorAccess =
true;
958 if (!IsVectorAccess && isa<LoadInst>(
I) &&
I->hasOneUse()) {
959 User *LoadUser = *
I->user_begin();
960 if (isa<InsertElementInst>(LoadUser))
961 IsVectorAccess =
true;
964 if (IsFPAccess || IsVectorAccess)
992 return AM.
Scale == 0;
999 std::vector<EVT> &MemOps,
unsigned Limit,
const MemOp &
Op,
unsigned DstAS,
1000 unsigned SrcAS,
const AttributeList &FuncAttributes)
const {
1001 const int MVCFastLen = 16;
1004 if (
Op.isMemcpy() &&
Op.allowOverlap() &&
Op.size() <= MVCFastLen)
1006 if (
Op.isMemset() &&
Op.size() - 1 <= MVCFastLen)
1008 if (
Op.isZeroMemset())
1012 SrcAS, FuncAttributes);
1023 unsigned FromBits =
FromType->getPrimitiveSizeInBits().getFixedSize();
1025 return FromBits > ToBits;
1033 return FromBits > ToBits;
1042 if (Constraint.
size() == 1) {
1043 switch (Constraint[0]) {
1069 }
else if (Constraint.
size() == 2 && Constraint[0] ==
'Z') {
1070 switch (Constraint[1]) {
1086 const char *constraint)
const {
1088 Value *CallOperandVal =
info.CallOperandVal;
1091 if (!CallOperandVal)
1095 switch (*constraint) {
1109 if (
type->isFloatingPointTy())
1114 if ((
type->isVectorTy() ||
type->isFloatingPointTy()) &&
1120 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1126 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1127 if (isUInt<12>(
C->getZExtValue()))
1132 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1138 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1139 if (isInt<20>(
C->getSExtValue()))
1144 if (
auto *
C = dyn_cast<ConstantInt>(CallOperandVal))
1145 if (
C->getZExtValue() == 0x7fffffff)
1155 static std::pair<unsigned, const TargetRegisterClass *>
1157 const unsigned *Map,
unsigned Size) {
1158 assert(*(Constraint.
end()-1) ==
'}' &&
"Missing '}'");
1159 if (isdigit(Constraint[2])) {
1163 if (!
Failed && Index < Size && Map[Index])
1164 return std::make_pair(Map[Index], RC);
1166 return std::make_pair(0U,
nullptr);
1169 std::pair<unsigned, const TargetRegisterClass *>
1172 if (Constraint.
size() == 1) {
1174 switch (Constraint[0]) {
1179 return std::make_pair(0U, &SystemZ::GR64BitRegClass);
1181 return std::make_pair(0U, &SystemZ::GR128BitRegClass);
1182 return std::make_pair(0U, &SystemZ::GR32BitRegClass);
1186 return std::make_pair(0U, &SystemZ::ADDR64BitRegClass);
1188 return std::make_pair(0U, &SystemZ::ADDR128BitRegClass);
1189 return std::make_pair(0U, &SystemZ::ADDR32BitRegClass);
1192 return std::make_pair(0U, &SystemZ::GRH32BitRegClass);
1197 return std::make_pair(0U, &SystemZ::FP64BitRegClass);
1199 return std::make_pair(0U, &SystemZ::FP128BitRegClass);
1200 return std::make_pair(0U, &SystemZ::FP32BitRegClass);
1206 return std::make_pair(0U, &SystemZ::VR32BitRegClass);
1208 return std::make_pair(0U, &SystemZ::VR64BitRegClass);
1209 return std::make_pair(0U, &SystemZ::VR128BitRegClass);
1214 if (Constraint.
size() > 0 && Constraint[0] ==
'{') {
1219 if (Constraint[1] ==
'r') {
1229 if (Constraint[1] ==
'f') {
1231 return std::make_pair(
1242 if (Constraint[1] ==
'v') {
1244 return std::make_pair(
1265 .
Case(
"r15", SystemZ::R15D)
1274 std::vector<SDValue> &Ops,
1277 if (Constraint.length() == 1) {
1278 switch (Constraint[0]) {
1280 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1283 Op.getValueType()));
1287 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1288 if (isUInt<12>(
C->getZExtValue()))
1290 Op.getValueType()));
1294 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1297 Op.getValueType()));
1301 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1302 if (isInt<20>(
C->getSExtValue()))
1304 Op.getValueType()));
1308 if (
auto *
C = dyn_cast<ConstantSDNode>(
Op))
1309 if (
C->getZExtValue() == 0x7fffffff)
1311 Op.getValueType()));
1322 #include "SystemZGenCallingConv.inc"
1326 static const MCPhysReg ScratchRegs[] = { SystemZ::R0D, SystemZ::R1D,
1332 Type *ToType)
const {
1349 for (
unsigned i = 0;
i <
Ins.size(); ++
i)
1354 for (
unsigned i = 0;
i < Outs.size(); ++
i)
1447 (NumParts == 2 && PartVT ==
MVT::i64))) &&
1448 "Unknown handling of i128 value.");
1449 if (ValueVT ==
MVT::i128 && NumParts == 1) {
1462 (NumParts == 2 && PartVT ==
MVT::i64))) &&
1463 "Unknown handling of i128 value.");
1464 if (ValueVT ==
MVT::i128 && NumParts == 1)
1491 unsigned NumFixedGPRs = 0;
1492 unsigned NumFixedFPRs = 0;
1493 for (
unsigned I = 0,
E = ArgLocs.size();
I !=
E; ++
I) {
1506 RC = &SystemZ::GR32BitRegClass;
1510 RC = &SystemZ::GR64BitRegClass;
1514 RC = &SystemZ::FP32BitRegClass;
1518 RC = &SystemZ::FP64BitRegClass;
1522 RC = &SystemZ::FP128BitRegClass;
1530 RC = &SystemZ::VR128BitRegClass;
1559 ArgValue = DAG.
getLoad(LocVT,
DL, Chain, FIN,
1570 unsigned ArgIndex =
Ins[
I].OrigArgIndex;
1572 while (
I + 1 !=
E &&
Ins[
I + 1].OrigArgIndex == ArgIndex) {
1574 unsigned PartOffset =
Ins[
I + 1].PartOffset;
1598 int64_t RegSaveOffset =
1613 &SystemZ::FP64BitRegClass);
1636 for (
unsigned I = 0,
E = ArgLocs.size();
I !=
E; ++
I) {
1643 if (
Reg == SystemZ::R6H ||
Reg == SystemZ::R6L ||
Reg == SystemZ::R6D)
1645 if (Outs[
I].Flags.isSwiftSelf() || Outs[
I].Flags.isSwiftError())
1707 for (
unsigned I = 0,
E = ArgLocs.size();
I !=
E; ++
I) {
1713 unsigned ArgIndex = Outs[
I].OrigArgIndex;
1715 if (
I + 1 !=
E && Outs[
I + 1].OrigArgIndex == ArgIndex) {
1717 Type *OrigArgType = CLI.
Args[Outs[
I].OrigArgIndex].Ty;
1723 SlotVT = Outs[
I].ArgVT;
1726 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1727 MemOpChains.push_back(
1732 assert (Outs[
I].PartOffset == 0);
1733 while (
I + 1 !=
E && Outs[
I + 1].OrigArgIndex == ArgIndex) {
1734 SDValue PartValue = OutVals[
I + 1];
1735 unsigned PartOffset = Outs[
I + 1].PartOffset;
1738 MemOpChains.push_back(
1742 SlotVT.
getStoreSize()) &&
"Not enough space for argument part!");
1745 ArgValue = SpillSlot;
1756 RegsToPass.push_back(std::make_pair(VA.
getLocReg(), ArgValue));
1762 if (!StackPtr.getNode())
1773 MemOpChains.push_back(
1784 RegsToPass.push_back(std::make_pair(SystemZ::R3D, ShadowArgValue));
1790 if (!MemOpChains.empty())
1798 if (
auto *
G = dyn_cast<GlobalAddressSDNode>(
Callee)) {
1801 }
else if (
auto *
E = dyn_cast<ExternalSymbolSDNode>(
Callee)) {
1804 }
else if (IsTailCall) {
1811 for (
unsigned I = 0,
E = RegsToPass.size();
I !=
E; ++
I) {
1813 RegsToPass[
I].second, Glue);
1819 Ops.push_back(Chain);
1824 for (
unsigned I = 0,
E = RegsToPass.size();
I !=
E; ++
I)
1826 RegsToPass[
I].second.getValueType()));
1831 assert(
Mask &&
"Missing call preserved mask for calling convention");
1836 Ops.push_back(Glue);
1855 CCState RetCCInfo(CallConv, IsVarArg, MF, RetLocs, Ctx);
1859 for (
unsigned I = 0,
E = RetLocs.size();
I !=
E; ++
I) {
1881 bool DoesNotReturn,
bool IsReturnValueUsed)
const {
1888 Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.
getContext());
1891 Args.push_back(Entry);
1921 for (
auto &Out : Outs)
1927 return RetCCInfo.
CheckReturn(Outs, RetCC_SystemZ);
1948 if (RetLocs.empty())
1957 RetOps.push_back(Chain);
1958 for (
unsigned I = 0,
E = RetLocs.size();
I !=
E; ++
I) {
1978 RetOps.push_back(Glue);
1987 unsigned &CCValid) {
1988 unsigned Id = cast<ConstantSDNode>(
Op.getOperand(1))->getZExtValue();
1990 case Intrinsic::s390_tbegin:
1995 case Intrinsic::s390_tbegin_nofloat:
2000 case Intrinsic::s390_tend:
2014 unsigned Id = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
2016 case Intrinsic::s390_vpkshs:
2017 case Intrinsic::s390_vpksfs:
2018 case Intrinsic::s390_vpksgs:
2023 case Intrinsic::s390_vpklshs:
2024 case Intrinsic::s390_vpklsfs:
2025 case Intrinsic::s390_vpklsgs:
2030 case Intrinsic::s390_vceqbs:
2031 case Intrinsic::s390_vceqhs:
2032 case Intrinsic::s390_vceqfs:
2033 case Intrinsic::s390_vceqgs:
2038 case Intrinsic::s390_vchbs:
2039 case Intrinsic::s390_vchhs:
2040 case Intrinsic::s390_vchfs:
2041 case Intrinsic::s390_vchgs:
2046 case Intrinsic::s390_vchlbs:
2047 case Intrinsic::s390_vchlhs:
2048 case Intrinsic::s390_vchlfs:
2049 case Intrinsic::s390_vchlgs:
2054 case Intrinsic::s390_vtm:
2059 case Intrinsic::s390_vfaebs:
2060 case Intrinsic::s390_vfaehs:
2061 case Intrinsic::s390_vfaefs:
2066 case Intrinsic::s390_vfaezbs:
2067 case Intrinsic::s390_vfaezhs:
2068 case Intrinsic::s390_vfaezfs:
2073 case Intrinsic::s390_vfeebs:
2074 case Intrinsic::s390_vfeehs:
2075 case Intrinsic::s390_vfeefs:
2080 case Intrinsic::s390_vfeezbs:
2081 case Intrinsic::s390_vfeezhs:
2082 case Intrinsic::s390_vfeezfs:
2087 case Intrinsic::s390_vfenebs:
2088 case Intrinsic::s390_vfenehs:
2089 case Intrinsic::s390_vfenefs:
2094 case Intrinsic::s390_vfenezbs:
2095 case Intrinsic::s390_vfenezhs:
2096 case Intrinsic::s390_vfenezfs:
2101 case Intrinsic::s390_vistrbs:
2102 case Intrinsic::s390_vistrhs:
2103 case Intrinsic::s390_vistrfs:
2108 case Intrinsic::s390_vstrcbs:
2109 case Intrinsic::s390_vstrchs:
2110 case Intrinsic::s390_vstrcfs:
2115 case Intrinsic::s390_vstrczbs:
2116 case Intrinsic::s390_vstrczhs:
2117 case Intrinsic::s390_vstrczfs:
2122 case Intrinsic::s390_vstrsb:
2123 case Intrinsic::s390_vstrsh:
2124 case Intrinsic::s390_vstrsf:
2129 case Intrinsic::s390_vstrszb:
2130 case Intrinsic::s390_vstrszh:
2131 case Intrinsic::s390_vstrszf:
2136 case Intrinsic::s390_vfcedbs:
2137 case Intrinsic::s390_vfcesbs:
2142 case Intrinsic::s390_vfchdbs:
2143 case Intrinsic::s390_vfchsbs:
2148 case Intrinsic::s390_vfchedbs:
2149 case Intrinsic::s390_vfchesbs:
2154 case Intrinsic::s390_vftcidb:
2155 case Intrinsic::s390_vftcisb:
2160 case Intrinsic::s390_tdc:
2174 unsigned NumOps =
Op.getNumOperands();
2177 Ops.push_back(
Op.getOperand(0));
2178 for (
unsigned I = 2;
I < NumOps; ++
I)
2179 Ops.push_back(
Op.getOperand(
I));
2181 assert(
Op->getNumValues() == 2 &&
"Expected only CC result and chain");
2187 return Intr.getNode();
2194 unsigned NumOps =
Op.getNumOperands();
2197 for (
unsigned I = 1;
I < NumOps; ++
I)
2198 Ops.push_back(
Op.getOperand(
I));
2201 return Intr.getNode();
2211 case ISD::SET##X: return SystemZ::CCMASK_CMP_##X; \
2212 case ISD::SETO##X: return SystemZ::CCMASK_CMP_##X; \
2213 case ISD::SETU##X: return SystemZ::CCMASK_CMP_UO | SystemZ::CCMASK_CMP_##X
2238 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1.getNode());
2242 int64_t
Value = ConstOp1->getSExtValue();
2258 if (!
C.Op0.hasOneUse() ||
2264 auto *
Load = cast<LoadSDNode>(
C.Op0);
2265 unsigned NumBits =
Load->getMemoryVT().getSizeInBits();
2266 if ((NumBits != 8 && NumBits != 16) ||
2267 NumBits !=
Load->getMemoryVT().getStoreSizeInBits())
2272 auto *ConstOp1 = cast<ConstantSDNode>(
C.Op1);
2277 int64_t SignedValue = ConstOp1->getSExtValue();
2284 }
else if (NumBits == 8) {
2311 Load->getExtensionType() != ExtType) {
2313 Load->getBasePtr(),
Load->getPointerInfo(),
2314 Load->getMemoryVT(),
Load->getAlignment(),
2315 Load->getMemOperand()->getFlags());
2322 Value != ConstOp1->getZExtValue())
2329 auto *
Load = dyn_cast<LoadSDNode>(
Op.getNode());
2335 switch (
Load->getExtensionType()) {
2358 if (isa<ConstantFPSDNode>(
C.Op1))
2363 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1);
2364 if (ConstOp1 && ConstOp1->getZExtValue() == 0)
2393 unsigned Opcode0 =
C.Op0.getOpcode();
2401 cast<ConstantSDNode>(
C.Op0.getOperand(1))->getZExtValue() == 0xffffffff)
2416 ((
N->getOperand(0) ==
C.Op0 &&
N->getOperand(1) ==
C.Op1) ||
2417 (
N->getOperand(0) ==
C.Op1 &&
N->getOperand(1) ==
C.Op0))) {
2435 auto *
C1 = dyn_cast<ConstantFPSDNode>(
C.Op1);
2436 if (
C1 &&
C1->isZero()) {
2458 cast<ConstantSDNode>(
C.Op1)->getZExtValue() == 0) {
2459 auto *
C1 = dyn_cast<ConstantSDNode>(
C.Op0.getOperand(1));
2460 if (
C1 &&
C1->getZExtValue() == 32) {
2465 cast<VTSDNode>(
N->getOperand(1))->getVT() ==
MVT::i32) {
2480 C.Op0.getOperand(0).getOpcode() ==
ISD::LOAD &&
2482 cast<ConstantSDNode>(
C.Op1)->getZExtValue() == 0) {
2483 auto *L = cast<LoadSDNode>(
C.Op0.getOperand(0));
2484 if (L->getMemoryVT().getStoreSizeInBits().getFixedSize() <=
2485 C.Op0.getValueSizeInBits().getFixedSize()) {
2486 unsigned Type = L->getExtensionType();
2489 C.Op0 =
C.Op0.getOperand(0);
2499 auto *
Shift = dyn_cast<ConstantSDNode>(
N.getOperand(1));
2504 if (Amount >=
N.getValueSizeInBits())
2519 unsigned ICmpType) {
2520 assert(
Mask != 0 &&
"ANDs with zero should have been removed by now");
2543 if (EffectivelyUnsigned && CmpVal > 0 && CmpVal <=
Low) {
2549 if (EffectivelyUnsigned && CmpVal <
Low) {
2557 if (CmpVal ==
Mask) {
2563 if (EffectivelyUnsigned && CmpVal >=
Mask -
Low && CmpVal <
Mask) {
2569 if (EffectivelyUnsigned && CmpVal >
Mask -
Low && CmpVal <=
Mask) {
2577 if (EffectivelyUnsigned && CmpVal >=
Mask -
High && CmpVal <
High) {
2583 if (EffectivelyUnsigned && CmpVal >
Mask -
High && CmpVal <=
High) {
2612 auto *ConstOp1 = dyn_cast<ConstantSDNode>(
C.Op1);
2615 uint64_t CmpVal = ConstOp1->getZExtValue();
2622 NewC.Op0 =
C.Op0.getOperand(0);
2623 NewC.Op1 =
C.Op0.getOperand(1);
2624 Mask = dyn_cast<ConstantSDNode>(NewC.Op1);
2627 MaskVal =
Mask->getZExtValue();
2632 if (NewC.Op0.getValueType() !=
MVT::i64 ||
2647 MaskVal = -(CmpVal & -CmpVal);
2655 unsigned BitSize = NewC.Op0.getValueSizeInBits();
2656 unsigned NewCCMask, ShiftVal;
2658 NewC.Op0.getOpcode() ==
ISD::SHL &&
2660 (MaskVal >> ShiftVal != 0) &&
2661 ((CmpVal >> ShiftVal) << ShiftVal) == CmpVal &&
2663 MaskVal >> ShiftVal,
2666 NewC.Op0 = NewC.Op0.getOperand(0);
2667 MaskVal >>= ShiftVal;
2669 NewC.Op0.getOpcode() ==
ISD::SRL &&
2671 (MaskVal << ShiftVal != 0) &&
2672 ((CmpVal << ShiftVal) >> ShiftVal) == CmpVal &&
2674 MaskVal << ShiftVal,
2677 NewC.Op0 = NewC.Op0.getOperand(0);
2678 MaskVal <<= ShiftVal;
2689 if (
Mask &&
Mask->getZExtValue() == MaskVal)
2694 C.CCMask = NewCCMask;
2704 auto *
Mask = dyn_cast<ConstantSDNode>(
C.Op0.getOperand(1));
2708 if ((~Known.
Zero).getZExtValue() & ~
Mask->getZExtValue())
2711 C.Op0 =
C.Op0.getOperand(0);
2723 C.CCValid = CCValid;
2726 C.CCMask = CC < 4 ? 1 << (3 - CC) : 0;
2729 C.CCMask = CC < 4 ? ~(1 << (3 - CC)) : -1;
2733 C.CCMask = CC < 4 ? ~0U << (4 - CC) : -1;
2736 C.CCMask = CC < 4 ? ~(~0U << (4 - CC)) : 0;
2740 C.CCMask = CC < 4 ? ~0U << (3 - CC) : -1;
2743 C.CCMask = CC < 4 ? ~(~0U << (3 - CC)) : 0;
2746 C.CCMask &= CCValid;
2754 bool IsSignaling =
false) {
2758 unsigned Opcode, CCValid;
2768 Comparison
C(CmpOp0, CmpOp1, Chain);
2770 if (
C.Op0.getValueType().isFloatingPoint()) {
2774 else if (!IsSignaling)
2816 if (!
C.Op1.getNode()) {
2818 switch (
C.Op0.getOpcode()) {
2824 return SDValue(Node, Node->getNumValues() - 1);
2840 return DAG.
getNode(
C.Opcode,
DL, VTs,
C.Chain,
C.Op0,
C.Op1);
2877 unsigned CCValid,
unsigned CCMask) {
2960 int Mask[] = { Start, -1, Start + 1, -1 };
2994 SDValue Ops[2] = { Res, NewChain };
3003 return DAG.
getNode(Opcode,
DL, VTs, Chain, CmpOp0, CmpOp1);
3005 return DAG.
getNode(Opcode,
DL, VT, CmpOp0, CmpOp1);
3018 bool IsSignaling)
const {
3021 assert (!IsSignaling || Chain);
3024 bool Invert =
false;
3032 assert(IsFP &&
"Unexpected integer comparison");
3034 DL, VT, CmpOp1, CmpOp0, Chain);
3036 DL, VT, CmpOp0, CmpOp1, Chain);
3040 LT.getValue(1),
GE.getValue(1));
3049 assert(IsFP &&
"Unexpected integer comparison");
3051 DL, VT, CmpOp1, CmpOp0, Chain);
3053 DL, VT, CmpOp0, CmpOp1, Chain);
3057 LT.getValue(1),
GT.getValue(1));
3066 Cmp = getVectorCmp(DAG, Opcode,
DL, VT, CmpOp0, CmpOp1, Chain);
3070 Cmp = getVectorCmp(DAG, Opcode,
DL, VT, CmpOp1, CmpOp0, Chain);
3075 Chain =
Cmp.getValue(1);
3083 if (Chain && Chain.
getNode() !=
Cmp.getNode()) {
3096 EVT VT =
Op.getValueType();
3098 return lowerVectorSETCC(DAG,
DL, VT, CC, CmpOp0, CmpOp1);
3100 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1, CC,
DL));
3107 bool IsSignaling)
const {
3113 EVT VT =
Op.getNode()->getValueType(0);
3115 SDValue Res = lowerVectorSETCC(DAG,
DL, VT, CC, CmpOp0, CmpOp1,
3116 Chain, IsSignaling);
3120 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1, CC,
DL, Chain, IsSignaling));
3135 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1, CC,
DL));
3148 cast<ConstantSDNode>(Neg.
getOperand(0))->getZExtValue() == 0 &&
3174 Comparison
C(
getCmp(DAG, CmpOp0, CmpOp1, CC,
DL));
3183 cast<ConstantSDNode>(
C.Op1)->getZExtValue() == 0) {
3191 SDValue Ops[] = {TrueOp, FalseOp,
3202 int64_t
Offset = Node->getOffset();
3217 if (Offset != 0 && (Offset & 1) == 0) {
3262 Chain = DAG.
getCopyToReg(Chain,
DL, SystemZ::R2D, GOTOffset, Glue);
3267 Ops.push_back(Chain);
3269 Node->getValueType(0),
3274 Ops.push_back(DAG.
getRegister(SystemZ::R2D, PtrVT));
3275 Ops.push_back(DAG.
getRegister(SystemZ::R12D, PtrVT));
3281 assert(
Mask &&
"Missing call preserved mask for calling convention");
3285 Ops.push_back(Glue);
3289 Chain = DAG.
getNode(Opcode,
DL, NodeTys, Ops);
3296 SDValue SystemZTargetLowering::lowerThreadPointer(
const SDLoc &
DL,
3328 SDValue TP = lowerThreadPointer(
DL, DAG);
3412 int64_t
Offset = Node->getOffset();
3436 if (
CP->isMachineConstantPoolEntry())
3455 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
3462 int BackChainIdx = TFL->getOrCreateFramePointerSaveIndex(MF);
3483 unsigned Depth = cast<ConstantSDNode>(
Op.getOperand(0))->getZExtValue();
3500 EVT InVT =
In.getValueType();
3501 EVT ResVT =
Op.getValueType();
3506 if (
auto *LoadN = dyn_cast<LoadSDNode>(
In))
3509 LoadN->getBasePtr(), LoadN->getMemOperand());
3550 return lowerVASTART_XPLINK(
Op, DAG);
3552 return lowerVASTART_ELF(
Op, DAG);
3567 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
3581 const Value *SV = cast<SrcValueSDNode>(
Op.getOperand(2))->getValue();
3585 const unsigned NumFields = 4;
3596 for (
unsigned I = 0;
I < NumFields; ++
I) {
3601 MemOps[
I] = DAG.
getStore(Chain,
DL, Fields[
I], FieldAddr,
3613 const Value *DstSV = cast<SrcValueSDNode>(
Op.getOperand(3))->getValue();
3614 const Value *SrcSV = cast<SrcValueSDNode>(
Op.getOperand(4))->getValue();
3620 Align(8),
false,
false,
3626 SystemZTargetLowering::lowerDYNAMIC_STACKALLOC(
SDValue Op,
3629 return lowerDYNAMIC_STACKALLOC_XPLINK(
Op, DAG);
3631 return lowerDYNAMIC_STACKALLOC_ELF(
Op, DAG);
3635 SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_XPLINK(
SDValue Op,
3648 (RealignOpt ? cast<ConstantSDNode>(
Align)->getZExtValue() : 0);
3658 if (ExtraAlignSpace)
3662 bool IsSigned =
false;
3663 bool DoesNotReturn =
false;
3664 bool IsReturnValueUsed =
false;
3665 EVT VT =
Op.getValueType();
3676 Register SPReg = Regs.getStackPointerRegister();
3687 if (ExtraAlignSpace) {
3699 SystemZTargetLowering::lowerDYNAMIC_STACKALLOC_ELF(
SDValue Op,
3714 (RealignOpt ? cast<ConstantSDNode>(
Align)->getZExtValue() : 0);
3733 if (ExtraAlignSpace)
3767 Chain = DAG.
getStore(Chain,
DL, Backchain, getBackchainAddress(NewSP, DAG),
3774 SDValue SystemZTargetLowering::lowerGET_DYNAMIC_AREA_OFFSET(
3783 EVT VT =
Op.getValueType();
3790 Op.getOperand(1), Ops[1], Ops[0]);
3796 Op.getOperand(0),
Op.getOperand(1), Ops[1], Ops[0]);
3820 LL, RL, Ops[1], Ops[0]);
3831 EVT VT =
Op.getValueType();
3838 Op.getOperand(1), Ops[1], Ops[0]);
3844 Op.getOperand(0),
Op.getOperand(1), Ops[1], Ops[0]);
3852 EVT VT =
Op.getValueType();
3872 EVT VT =
Op.getValueType();
3879 Op.getOperand(0),
Op.getOperand(1), Ops[1], Ops[0]);
3887 SDValue Ops[] = {
Op.getOperand(0),
Op.getOperand(1)};
3896 if ((Masks[0] >> 32) == 0xffffffff &&
uint32_t(Masks[1]) == 0xffffffff)
3898 else if ((Masks[1] >> 32) == 0xffffffff &&
uint32_t(Masks[0]) == 0xffffffff)
3913 int64_t
Value = int32_t(cast<ConstantSDNode>(LowOp)->getZExtValue());
3945 unsigned BaseOp = 0;
3946 unsigned CCValid = 0;
3947 unsigned CCMask = 0;
3949 switch (
Op.getOpcode()) {
4000 MVT VT =
N->getSimpleValueType(0);
4010 unsigned BaseOp = 0;
4011 unsigned CCValid = 0;
4012 unsigned CCMask = 0;
4014 switch (
Op.getOpcode()) {
4051 EVT VT =
Op.getValueType();
4053 Op =
Op.getOperand(0);
<