47#include "llvm/IR/IntrinsicsPowerPC.h"
71#define DEBUG_TYPE "ppc-isel"
72#define PASS_NAME "PowerPC DAG->DAG Pattern Instruction Selection"
75 "Number of (sext(setcc)) nodes expanded into GPR sequence.");
77 "Number of (zext(setcc)) nodes expanded into GPR sequence.");
79 "Number of sign extensions for compare inputs added.");
81 "Number of zero extensions for compare inputs added.");
83 "Number of logical ops on i1 values calculated in GPR.");
85 "Number of compares not eliminated as they have non-extending uses.");
87 "Number of compares lowered to setb.");
95 cl::desc(
"use aggressive ppc isel for bit permutations"),
98 "ppc-bit-perm-rewriter-stress-rotates",
99 cl::desc(
"stress rotate selection in aggressive ppc isel for "
104 "ppc-use-branch-hint",
cl::init(
true),
105 cl::desc(
"Enable static hinting of branches on ppc"),
110 cl::desc(
"Enable tls optimization peephole"),
119 cl::desc(
"Specify the types of comparisons to emit GPR-only code for."),
125 "Only comparisons where inputs don't need [sz]ext."),
128 "Only i32 comparisons with zext result."),
130 "Only i64 comparisons with zext result."),
133 "Only i32 comparisons with sext result."),
135 "Only i64 comparisons with sext result.")));
146 unsigned GlobalBaseReg = 0;
151 PPCDAGToDAGISel() =
delete;
161 if (Subtarget->hasROPProtect()) {
180 inline SDValue getI16Imm(
unsigned Imm,
const SDLoc &dl) {
181 return CurDAG->getTargetConstant(Imm, dl,
MVT::i16);
186 inline SDValue getI32Imm(
unsigned Imm,
const SDLoc &dl) {
187 return CurDAG->getTargetConstant(Imm, dl,
MVT::i32);
193 return CurDAG->getTargetConstant(Imm, dl,
MVT::i64);
198 return CurDAG->getTargetConstant(
199 Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout()));
204 static bool isRotateAndMask(
SDNode *
N,
unsigned Mask,
bool isShiftMask,
205 unsigned &SH,
unsigned &MB,
unsigned &ME);
209 SDNode *getGlobalBaseReg();
217 bool tryBitfieldInsert(
SDNode *
N);
218 bool tryBitPermutation(
SDNode *
N);
219 bool tryIntCompareInGPR(
SDNode *
N);
251 return PPCLowering->SelectOptimalAddrMode(Parent,
N, Disp,
Base, *CurDAG,
259 return PPCLowering->SelectOptimalAddrMode(Parent,
N, Disp,
Base, *CurDAG,
267 return PPCLowering->SelectOptimalAddrMode(Parent,
N, Disp,
Base, *CurDAG,
275 return PPCLowering->SelectOptimalAddrMode(Parent,
N, Disp,
Base, *CurDAG,
282 return PPCLowering->SelectOptimalAddrMode(Parent,
N, Disp,
Base, *CurDAG,
290 return PPCLowering->SelectOptimalAddrMode(Parent,
N, Disp,
Base, *CurDAG,
298 return PPCLowering->SelectForceXFormMode(
N, Disp,
Base, *CurDAG) ==
309 return PPCLowering->SelectAddressRegReg(
N,
Base,
Index, *CurDAG,
320 return PPCLowering->SelectAddressRegReg(
N,
Base,
Index, *CurDAG,
331 return PPCLowering->SelectAddressRegReg(
N,
Base,
Index, *CurDAG,
338 return PPCLowering->SelectAddressRegRegOnly(
N,
Base,
Index, *CurDAG);
347 return PPCLowering->SelectAddressRegImm(
N, Disp,
Base, *CurDAG,
355 return PPCLowering->SelectAddressRegImm(
N, Disp,
Base, *CurDAG,
Align(4));
362 return PPCLowering->SelectAddressRegImm(
N, Disp,
Base, *CurDAG,
370 return PPCLowering->SelectAddressRegImm34(
N, Disp,
Base, *CurDAG);
380 return PPCLowering->SelectAddressPCRel(
N,
Base);
389 unsigned ConstraintID,
390 std::vector<SDValue> &OutOps)
override {
391 switch(ConstraintID) {
393 errs() <<
"ConstraintID: " << ConstraintID <<
"\n";
408 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
409 dl,
Op.getValueType(),
412 OutOps.push_back(NewOp);
419#include "PPCGenDAGISel.inc"
423 bool tryFoldSWTestBRCC(
SDNode *
N);
424 bool trySelectLoopCountIntrinsic(
SDNode *
N);
425 bool tryAsSingleRLDICL(
SDNode *
N);
426 bool tryAsSingleRLDICR(
SDNode *
N);
427 bool tryAsSingleRLWINM(
SDNode *
N);
428 bool tryAsSingleRLWINM8(
SDNode *
N);
429 bool tryAsSingleRLWIMI(
SDNode *
N);
430 bool tryAsPairOfRLDICL(
SDNode *
N);
431 bool tryAsSingleRLDIMI(
SDNode *
N);
433 void PeepholePPC64();
434 void PeepholePPC64ZExt();
435 void PeepholeCROps();
440 bool AllUsersSelectZero(
SDNode *
N);
441 void SwapAllSelectUsers(
SDNode *
N);
443 bool isOffsetMultipleOf(
SDNode *
N,
unsigned Val)
const;
449char PPCDAGToDAGISel::ID = 0;
456SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
457 if (!GlobalBaseReg) {
465 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) ==
MVT::i32) {
468 if (!Subtarget->isSecurePlt() &&
478 TII.get(PPC::UpdateGBR), GlobalBaseReg)
484 RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass);
504 return CurDAG->getRegister(GlobalBaseReg,
505 PPCLowering->getPointerTy(CurDAG->getDataLayout()))
527 "GlobalVariables with an alignment requirement stricter than TOC entry "
528 "size not supported by the toc data transformation.");
532 assert(GVType->
isSized() &&
"A GlobalVariable's size must be known to be "
533 "supported by the toc data transformation.");
537 "supported by the toc data transformation.");
541 "supported by the toc data transformation.");
545 "supported by the toc data transformation.");
548 "A GlobalVariable with size larger than a TOC entry is not currently "
549 "supported by the toc data transformation.");
553 "currently supported by the toc data transformation.");
556 "Tentative definitions cannot have the mapping class XMC_TD.");
565 Imm = cast<ConstantSDNode>(
N)->getZExtValue();
575 Imm = cast<ConstantSDNode>(
N)->getZExtValue();
596 assert(isa<BasicBlockSDNode>(DestMBB));
626 if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb))
630 <<
"::" << BB->
getName() <<
"'\n"
631 <<
" -> " <<
TBB->
getName() <<
": " << TProb <<
"\n"
632 <<
" -> " << FBB->
getName() <<
": " << FProb <<
"\n");
648 return N->getOpcode() == Opc
654 int FI = cast<FrameIndexSDNode>(
N)->getIndex();
655 SDValue TFI = CurDAG->getTargetFrameIndex(FI,
N->getValueType(0));
656 unsigned Opc =
N->getValueType(0) ==
MVT::i32 ? PPC::ADDI : PPC::ADDI8;
658 CurDAG->SelectNodeTo(SN, Opc,
N->getValueType(0), TFI,
659 getSmallIPtrImm(
Offset, dl));
661 ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl,
N->getValueType(0), TFI,
662 getSmallIPtrImm(
Offset, dl)));
665bool PPCDAGToDAGISel::isRotateAndMask(
SDNode *
N,
unsigned Mask,
666 bool isShiftMask,
unsigned &SH,
667 unsigned &MB,
unsigned &ME) {
674 unsigned Indeterminant = ~0;
675 unsigned Opcode =
N->getOpcode();
676 if (
N->getNumOperands() != 2 ||
682 if (isShiftMask)
Mask =
Mask << Shift;
684 Indeterminant = ~(0xFFFFFFFFu << Shift);
687 if (isShiftMask)
Mask =
Mask >> Shift;
689 Indeterminant = ~(0xFFFFFFFFu >> Shift);
699 if (Mask && !(Mask & Indeterminant)) {
707bool PPCDAGToDAGISel::tryTLSXFormStore(
StoreSDNode *ST) {
718 EVT MemVT =
ST->getMemoryVT();
719 EVT RegVT =
ST->getValue().getValueType();
726 Opcode = (RegVT ==
MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS;
730 Opcode = (RegVT ==
MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS;
734 Opcode = (RegVT ==
MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS;
738 Opcode = PPC::STDXTLS;
746 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
747 transferMemOperands(ST, MN);
752bool PPCDAGToDAGISel::tryTLSXFormLoad(
LoadSDNode *LD) {
763 EVT MemVT =
LD->getMemoryVT();
764 EVT RegVT =
LD->getValueType(0);
770 Opcode = (RegVT ==
MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS;
774 Opcode = (RegVT ==
MVT::i32) ? PPC::LHZXTLS_32 : PPC::LHZXTLS;
778 Opcode = (RegVT ==
MVT::i32) ? PPC::LWZXTLS_32 : PPC::LWZXTLS;
782 Opcode = PPC::LDXTLS;
789 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
790 transferMemOperands(LD, MN);
797bool PPCDAGToDAGISel::tryBitfieldInsert(
SDNode *
N) {
802 KnownBits LKnown = CurDAG->computeKnownBits(Op0);
803 KnownBits RKnown = CurDAG->computeKnownBits(Op1);
808 if ((TargetMask | InsertMask) == 0xFFFFFFFF) {
811 unsigned Value, SH = 0;
812 TargetMask = ~TargetMask;
813 InsertMask = ~InsertMask;
861 SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl),
863 ReplaceNode(
N, CurDAG->getMachineNode(PPC::RLWIMI, dl,
MVT::i32, Ops));
871 unsigned MaxTruncation = 0;
878 Use->isMachineOpcode() ?
Use->getMachineOpcode() :
Use->getOpcode();
882 if (
Use->isMachineOpcode())
885 std::max(MaxTruncation, (
unsigned)
Use->getValueType(0).getSizeInBits());
888 if (
Use->isMachineOpcode())
894 MaxTruncation = std::max(MaxTruncation, MemVTSize);
903 MaxTruncation = std::max(MaxTruncation, 32u);
911 MaxTruncation = std::max(MaxTruncation, 16u);
919 MaxTruncation = std::max(MaxTruncation, 8u);
923 return MaxTruncation;
929 unsigned HiTZ = llvm::countr_zero<uint32_t>(
Hi_32(Imm));
930 unsigned LoLZ = llvm::countl_zero<uint32_t>(
Lo_32(Imm));
931 if ((HiTZ + LoLZ) >= Num)
939 unsigned TZ = llvm::countr_zero<uint64_t>(Imm);
940 unsigned LZ = llvm::countl_zero<uint64_t>(Imm);
941 unsigned TO = llvm::countr_one<uint64_t>(Imm);
942 unsigned LO = llvm::countl_one<uint64_t>(Imm);
943 unsigned Hi32 =
Hi_32(Imm);
944 unsigned Lo32 =
Lo_32(Imm);
948 auto getI32Imm = [CurDAG, dl](
unsigned Imm) {
956 if (isInt<16>(Imm)) {
962 if (TZ > 15 && (LZ > 32 || LO > 32))
964 getI32Imm((Imm >> 16) & 0xffff));
968 assert(LZ < 64 &&
"Unexpected leading zeros here.");
970 unsigned FO = llvm::countl_one<uint64_t>(Imm << LZ);
973 if (isInt<32>(Imm)) {
974 uint64_t ImmHi16 = (Imm >> 16) & 0xffff;
975 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
978 getI32Imm(Imm & 0xffff));
986 if ((LZ + FO + TZ) > 48) {
988 getI32Imm((Imm >> TZ) & 0xffff));
990 getI32Imm(TZ), getI32Imm(LZ));
1007 if ((LZ + TO) > 48) {
1011 assert(LZ <= 32 &&
"Unexpected shift value.");
1013 getI32Imm((Imm >> (48 - LZ) & 0xffff)));
1015 getI32Imm(48 - LZ), getI32Imm(LZ));
1033 if ((LZ + FO + TO) > 48) {
1035 getI32Imm((Imm >> TO) & 0xffff));
1037 getI32Imm(TO), getI32Imm(LZ));
1043 if (LZ == 32 && ((Lo32 & 0x8000) == 0)) {
1045 getI32Imm(Lo32 & 0xffff));
1047 getI32Imm(Lo32 >> 16));
1071 getI32Imm(RotImm & 0xffff));
1073 getI32Imm(Shift), getI32Imm(0));
1085 if ((LZ + FO + TZ) > 32) {
1086 uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff;
1087 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1090 getI32Imm((Imm >> TZ) & 0xffff));
1092 getI32Imm(TZ), getI32Imm(LZ));
1099 if ((LZ + TO) > 32) {
1103 assert(LZ <= 32 &&
"Unexpected shift value.");
1105 getI32Imm((Imm >> (48 - LZ)) & 0xffff));
1107 getI32Imm((Imm >> (32 - LZ)) & 0xffff));
1109 getI32Imm(32 - LZ), getI32Imm(LZ));
1117 if ((LZ + FO + TO) > 32) {
1119 getI32Imm((Imm >> (TO + 16)) & 0xffff));
1121 getI32Imm((Imm >> TO) & 0xffff));
1123 getI32Imm(TO), getI32Imm(LZ));
1128 uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff;
1129 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1132 getI32Imm(Lo32 & 0xffff));
1148 uint64_t ImmHi16 = (RotImm >> 16) & 0xffff;
1149 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1152 getI32Imm(RotImm & 0xffff));
1154 getI32Imm(Shift), getI32Imm(0));
1168 unsigned TZ = llvm::countr_zero<uint64_t>(Imm);
1169 unsigned LZ = llvm::countl_zero<uint64_t>(Imm);
1170 unsigned TO = llvm::countr_one<uint64_t>(Imm);
1171 unsigned FO = llvm::countl_one<uint64_t>(LZ == 64 ? 0 : (Imm << LZ));
1172 unsigned Hi32 =
Hi_32(Imm);
1173 unsigned Lo32 =
Lo_32(Imm);
1175 auto getI32Imm = [CurDAG, dl](
unsigned Imm) {
1179 auto getI64Imm = [CurDAG, dl](
uint64_t Imm) {
1194 SDNode *Result =
nullptr;
1201 if ((LZ + FO + TZ) > 30) {
1202 APInt SignedInt34 =
APInt(34, (Imm >> TZ) & 0x3ffffffff);
1205 getI64Imm(*Extended.getRawData()));
1207 getI32Imm(TZ), getI32Imm(LZ));
1223 if ((LZ + TO) > 30) {
1224 APInt SignedInt34 =
APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff);
1227 getI64Imm(*Extended.getRawData()));
1229 getI32Imm(30 - LZ), getI32Imm(LZ));
1236 if ((LZ + FO + TO) > 30) {
1237 APInt SignedInt34 =
APInt(34, (Imm >> TO) & 0x3ffffffff);
1240 getI64Imm(*Extended.getRawData()));
1242 getI32Imm(TO), getI32Imm(LZ));
1254 for (
unsigned Shift = 0; Shift < 63; ++Shift) {
1256 if (isInt<34>(RotImm)) {
1260 SDValue(Result, 0), getI32Imm(Shift),
1287 unsigned *InstCnt =
nullptr) {
1288 unsigned InstCntDirect = 0;
1299 if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) {
1300 unsigned InstCntDirectP = 0;
1307 if (ResultP && (!Result || InstCntDirectP < InstCntDirect)) {
1309 *InstCnt = InstCntDirectP;
1316 *InstCnt = InstCntDirect;
1319 auto getI32Imm = [CurDAG, dl](
unsigned Imm) {
1328 if (Hi16OfLo32 && Lo16OfLo32) {
1331 bool IsSelected =
false;
1337 SDValue(Result, 0), getI32Imm(Lo16));
1343 if (Hi16OfHi32 == Lo16OfHi32 && Lo16OfHi32 == Lo16OfLo32) {
1345 Result = getSplat(Hi16OfLo32, Lo16OfLo32);
1350 }
else if (Hi16OfHi32 == Hi16OfLo32 && Hi16OfLo32 == Lo16OfLo32) {
1352 Result = getSplat(Hi16OfHi32, Lo16OfHi32);
1355 getI32Imm(16), getI32Imm(31)};
1357 }
else if (Lo16OfHi32 == Lo16OfLo32 && Hi16OfLo32 == Lo16OfLo32) {
1359 Result = getSplat(Hi16OfHi32, Lo16OfHi32);
1362 getI32Imm(0), getI32Imm(15)};
1365 if (IsSelected ==
true) {
1378 SDValue(Result, 0), getI32Imm(Hi16OfLo32));
1383 getI32Imm(Lo16OfLo32));
1387 *InstCnt = InstCntDirect;
1396 int64_t Imm = cast<ConstantSDNode>(
N)->getZExtValue();
1400 if (isInt<16>(SextImm))
1408class BitPermutationSelector {
1423 VariableKnownToBeZero
1426 ValueBit(
SDValue V,
unsigned I, Kind K = Variable)
1428 ValueBit(Kind K = Variable) :
Idx(UINT32_MAX),
K(
K) {}
1431 return K == ConstZero ||
K == VariableKnownToBeZero;
1434 bool hasValue()
const {
1435 return K == Variable ||
K == VariableKnownToBeZero;
1439 assert(hasValue() &&
"Cannot get the value of a constant bit");
1443 unsigned getValueBitIndex()
const {
1444 assert(hasValue() &&
"Cannot get the value bit index of a constant bit");
1453 unsigned StartIdx, EndIdx;
1463 bool Repl32Coalesced;
1465 BitGroup(
SDValue V,
unsigned R,
unsigned S,
unsigned E)
1466 :
V(
V), RLAmt(
R), StartIdx(S), EndIdx(
E), Repl32(
false), Repl32CR(
false),
1467 Repl32Coalesced(
false) {
1468 LLVM_DEBUG(
dbgs() <<
"\tbit group for " <<
V.getNode() <<
" RLAmt = " << R
1469 <<
" [" << S <<
", " <<
E <<
"]\n");
1475 struct ValueRotInfo {
1477 unsigned RLAmt = std::numeric_limits<unsigned>::max();
1478 unsigned NumGroups = 0;
1479 unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max();
1480 bool Repl32 =
false;
1482 ValueRotInfo() =
default;
1490 if (Repl32 <
Other.Repl32)
1492 else if (Repl32 >
Other.Repl32)
1494 else if (NumGroups >
Other.NumGroups)
1496 else if (NumGroups <
Other.NumGroups)
1498 else if (RLAmt == 0 &&
Other.RLAmt != 0)
1500 else if (RLAmt != 0 &&
Other.RLAmt == 0)
1502 else if (FirstGroupStartIdx <
Other.FirstGroupStartIdx)
1508 using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>;
1509 using ValueBitsMemoizer =
1511 ValueBitsMemoizer Memoizer;
1517 std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(
SDValue V,
1519 auto &ValueEntry = Memoizer[
V];
1521 return std::make_pair(ValueEntry->first, &ValueEntry->second);
1522 ValueEntry.reset(
new ValueBitsMemoizedValue());
1523 bool &Interesting = ValueEntry->first;
1525 Bits.resize(NumBits);
1527 switch (
V.getOpcode()) {
1530 if (isa<ConstantSDNode>(
V.getOperand(1))) {
1531 unsigned RotAmt =
V.getConstantOperandVal(1);
1533 const auto &LHSBits = *getValueBits(
V.getOperand(0), NumBits).second;
1535 for (
unsigned i = 0; i < NumBits; ++i)
1536 Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt];
1538 return std::make_pair(Interesting =
true, &Bits);
1543 if (isa<ConstantSDNode>(
V.getOperand(1))) {
1544 unsigned ShiftAmt =
V.getConstantOperandVal(1);
1546 const auto &LHSBits = *getValueBits(
V.getOperand(0), NumBits).second;
1548 for (
unsigned i = ShiftAmt; i < NumBits; ++i)
1549 Bits[i] = LHSBits[i - ShiftAmt];
1551 for (
unsigned i = 0; i < ShiftAmt; ++i)
1552 Bits[i] = ValueBit(ValueBit::ConstZero);
1554 return std::make_pair(Interesting =
true, &Bits);
1559 if (isa<ConstantSDNode>(
V.getOperand(1))) {
1560 unsigned ShiftAmt =
V.getConstantOperandVal(1);
1562 const auto &LHSBits = *getValueBits(
V.getOperand(0), NumBits).second;
1564 for (
unsigned i = 0; i < NumBits - ShiftAmt; ++i)
1565 Bits[i] = LHSBits[i + ShiftAmt];
1567 for (
unsigned i = NumBits - ShiftAmt; i < NumBits; ++i)
1568 Bits[i] = ValueBit(ValueBit::ConstZero);
1570 return std::make_pair(Interesting =
true, &Bits);
1574 if (isa<ConstantSDNode>(
V.getOperand(1))) {
1582 std::tie(Interesting, LHSBits) = getValueBits(
V.getOperand(0), NumBits);
1584 for (
unsigned i = 0; i < NumBits; ++i)
1585 if (((Mask >> i) & 1) == 1)
1586 Bits[i] = (*LHSBits)[i];
1590 if ((*LHSBits)[i].
isZero())
1591 Bits[i] = (*LHSBits)[i];
1593 Bits[i] = ValueBit(ValueBit::ConstZero);
1596 return std::make_pair(Interesting, &Bits);
1600 const auto &LHSBits = *getValueBits(
V.getOperand(0), NumBits).second;
1601 const auto &RHSBits = *getValueBits(
V.getOperand(1), NumBits).second;
1603 bool AllDisjoint =
true;
1605 unsigned LastIdx = 0;
1606 for (
unsigned i = 0; i < NumBits; ++i) {
1614 if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal &&
1615 LHSBits[i].getValueBitIndex() == LastIdx + 1)
1616 Bits[i] = LHSBits[i];
1617 else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal &&
1618 RHSBits[i].getValueBitIndex() == LastIdx + 1)
1619 Bits[i] = RHSBits[i];
1621 Bits[i] = ValueBit(ValueBit::ConstZero);
1623 else if (LHSBits[i].
isZero())
1624 Bits[i] = RHSBits[i];
1625 else if (RHSBits[i].
isZero())
1626 Bits[i] = LHSBits[i];
1628 AllDisjoint =
false;
1632 if (Bits[i].hasValue()) {
1633 LastVal =
Bits[i].getValue();
1634 LastIdx =
Bits[i].getValueBitIndex();
1637 if (LastVal) LastVal =
SDValue();
1645 return std::make_pair(Interesting =
true, &Bits);
1650 V.getOperand(0).getValueType() !=
MVT::i32)
1654 const unsigned NumOperandBits = 32;
1655 std::tie(Interesting, LHSBits) = getValueBits(
V.getOperand(0),
1658 for (
unsigned i = 0; i < NumOperandBits; ++i)
1659 Bits[i] = (*LHSBits)[i];
1661 for (
unsigned i = NumOperandBits; i < NumBits; ++i)
1662 Bits[i] = ValueBit(ValueBit::ConstZero);
1664 return std::make_pair(Interesting, &Bits);
1668 EVT ToType =
V.getValueType();
1672 const unsigned NumAllBits =
FromType.getSizeInBits();
1674 std::tie(Interesting, InBits) = getValueBits(
V.getOperand(0),
1680 bool UseUpper32bit =
false;
1681 for (
unsigned i = 0; i < NumValidBits; ++i)
1682 if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) {
1683 UseUpper32bit =
true;
1689 for (
unsigned i = 0; i < NumValidBits; ++i)
1690 Bits[i] = (*InBits)[i];
1692 return std::make_pair(Interesting, &Bits);
1698 std::tie(Interesting, LHSBits) = getValueBits(
V.getOperand(0),
1701 EVT FromType = cast<VTSDNode>(
V.getOperand(1))->getVT();
1702 const unsigned NumValidBits =
FromType.getSizeInBits();
1703 for (
unsigned i = 0; i < NumValidBits; ++i)
1704 Bits[i] = (*LHSBits)[i];
1708 for (
unsigned i = NumValidBits; i < NumBits; ++i)
1709 Bits[i] = (*LHSBits)[i].hasValue()
1710 ? ValueBit((*LHSBits)[i].getValue(),
1711 (*LHSBits)[i].getValueBitIndex(),
1712 ValueBit::VariableKnownToBeZero)
1713 : ValueBit(ValueBit::ConstZero);
1715 return std::make_pair(Interesting, &Bits);
1720 EVT VT =
LD->getMemoryVT();
1723 for (
unsigned i = 0; i < NumValidBits; ++i)
1724 Bits[i] = ValueBit(V, i);
1727 for (
unsigned i = NumValidBits; i < NumBits; ++i)
1728 Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero);
1732 return std::make_pair(Interesting =
false, &Bits);
1737 for (
unsigned i = 0; i < NumBits; ++i)
1738 Bits[i] = ValueBit(V, i);
1740 return std::make_pair(Interesting =
false, &Bits);
1745 void computeRotationAmounts() {
1747 RLAmt.resize(
Bits.size());
1748 for (
unsigned i = 0; i <
Bits.size(); ++i)
1749 if (Bits[i].hasValue()) {
1750 unsigned VBI =
Bits[i].getValueBitIndex();
1754 RLAmt[i] =
Bits.size() - (VBI - i);
1755 }
else if (Bits[i].
isZero()) {
1757 RLAmt[i] = UINT32_MAX;
1766 void collectBitGroups(
bool LateMask) {
1769 unsigned LastRLAmt = RLAmt[0];
1771 unsigned LastGroupStartIdx = 0;
1772 bool IsGroupOfZeros = !
Bits[LastGroupStartIdx].hasValue();
1773 for (
unsigned i = 1; i <
Bits.size(); ++i) {
1774 unsigned ThisRLAmt = RLAmt[i];
1776 if (LateMask && !ThisValue) {
1777 ThisValue = LastValue;
1778 ThisRLAmt = LastRLAmt;
1781 if (BitGroups.empty())
1782 LastGroupStartIdx = 0;
1789 if (IsGroupOfZeros && Bits[i].
isZero())
1794 if (ThisRLAmt == LastRLAmt && ThisValue == LastValue)
1797 if (!(IsGroupOfZeros && ThisValue && !Bits[i].
isZero()))
1801 BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1803 LastRLAmt = ThisRLAmt;
1804 LastValue = ThisValue;
1805 LastGroupStartIdx = i;
1806 IsGroupOfZeros = !
Bits[LastGroupStartIdx].hasValue();
1809 BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1812 if (BitGroups.empty())
1816 if (BitGroups.size() > 1) {
1820 if (BitGroups[0].StartIdx == 0 &&
1821 BitGroups[BitGroups.size()-1].EndIdx ==
Bits.size()-1 &&
1822 BitGroups[0].V == BitGroups[BitGroups.size()-1].V &&
1823 BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) {
1824 LLVM_DEBUG(
dbgs() <<
"\tcombining final bit group with initial one\n");
1825 BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx;
1826 BitGroups.erase(BitGroups.begin());
1836 void collectValueRotInfo() {
1839 for (
auto &BG : BitGroups) {
1840 unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0);
1841 ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)];
1843 VRI.RLAmt = BG.RLAmt;
1844 VRI.Repl32 = BG.Repl32;
1846 VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx);
1851 ValueRotsVec.clear();
1852 for (
auto &
I : ValueRots) {
1853 ValueRotsVec.push_back(
I.second);
1866 void assignRepl32BitGroups() {
1877 auto IsAllLow32 = [
this](BitGroup & BG) {
1878 if (BG.StartIdx <= BG.EndIdx) {
1879 for (
unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) {
1880 if (!Bits[i].hasValue())
1882 if (Bits[i].getValueBitIndex() >= 32)
1886 for (
unsigned i = BG.StartIdx; i <
Bits.size(); ++i) {
1887 if (!Bits[i].hasValue())
1889 if (Bits[i].getValueBitIndex() >= 32)
1892 for (
unsigned i = 0; i <= BG.EndIdx; ++i) {
1893 if (!Bits[i].hasValue())
1895 if (Bits[i].getValueBitIndex() >= 32)
1903 for (
auto &BG : BitGroups) {
1907 if (BG.RLAmt == 0) {
1908 auto PotentiallyMerged = [
this](BitGroup & BG) {
1909 for (
auto &BG2 : BitGroups)
1910 if (&BG != &BG2 && BG.V == BG2.V &&
1911 (BG2.RLAmt == 0 || BG2.RLAmt == 32))
1915 if (!PotentiallyMerged(BG))
1918 if (BG.StartIdx < 32 && BG.EndIdx < 32) {
1919 if (IsAllLow32(BG)) {
1920 if (BG.RLAmt >= 32) {
1928 << BG.V.getNode() <<
" RLAmt = " << BG.RLAmt <<
" ["
1929 << BG.StartIdx <<
", " << BG.EndIdx <<
"]\n");
1935 for (
auto I = BitGroups.begin();
I != BitGroups.end();) {
1938 auto IP = (
I == BitGroups.begin()) ?
1939 std::prev(BitGroups.end()) : std::prev(
I);
1940 if (
I->Repl32 && IP->Repl32 &&
I->V == IP->V &&
I->RLAmt == IP->RLAmt &&
1941 I->StartIdx == (IP->EndIdx + 1) % 64 &&
I != IP) {
1943 LLVM_DEBUG(
dbgs() <<
"\tcombining 32-bit replicated bit group for "
1944 <<
I->V.getNode() <<
" RLAmt = " <<
I->RLAmt <<
" ["
1945 <<
I->StartIdx <<
", " <<
I->EndIdx
1946 <<
"] with group with range [" << IP->StartIdx <<
", "
1947 << IP->EndIdx <<
"]\n");
1949 IP->EndIdx =
I->EndIdx;
1950 IP->Repl32CR = IP->Repl32CR ||
I->Repl32CR;
1951 IP->Repl32Coalesced =
true;
1952 I = BitGroups.erase(
I);
1961 if (
I->StartIdx == 32 &&
I->EndIdx == 63) {
1962 assert(std::next(
I) == BitGroups.end() &&
1963 "bit group ends at index 63 but there is another?");
1964 auto IN = BitGroups.begin();
1966 if (IP->Repl32 && IN->Repl32 &&
I->V == IP->V &&
I->V == IN->V &&
1967 (
I->RLAmt % 32) == IP->RLAmt && (
I->RLAmt % 32) == IN->RLAmt &&
1968 IP->EndIdx == 31 && IN->StartIdx == 0 &&
I != IP &&
1972 <<
" RLAmt = " <<
I->RLAmt <<
" [" <<
I->StartIdx
1973 <<
", " <<
I->EndIdx
1974 <<
"] with 32-bit replicated groups with ranges ["
1975 << IP->StartIdx <<
", " << IP->EndIdx <<
"] and ["
1976 << IN->StartIdx <<
", " << IN->EndIdx <<
"]\n");
1984 IP->Repl32CR = IP->Repl32CR ||
I->RLAmt >= 32;
1985 IP->Repl32Coalesced =
true;
1986 I = BitGroups.erase(
I);
1991 IP->EndIdx = IN->EndIdx;
1992 IP->Repl32CR = IP->Repl32CR || IN->Repl32CR ||
I->RLAmt >= 32;
1993 IP->Repl32Coalesced =
true;
1994 I = BitGroups.erase(
I);
1995 BitGroups.erase(BitGroups.begin());
2010 return CurDAG->getTargetConstant(Imm, dl,
MVT::i32);
2015 for (
unsigned i = 0; i <
Bits.size(); ++i) {
2016 if (Bits[i].hasValue())
2018 Mask |= (UINT64_C(1) << i);
2029 if (
V.getValueSizeInBits() == 64)
2032 assert(
V.getValueSizeInBits() == 32);
2033 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl,
MVT::i32);
2034 SDValue ImDef =
SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl,
2036 SDValue ExtVal =
SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl,
2043 if (
V.getValueSizeInBits() == 32)
2046 assert(
V.getValueSizeInBits() == 64);
2047 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl,
MVT::i32);
2048 SDValue SubVal =
SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl,
2056 void SelectAndParts32(
const SDLoc &dl,
SDValue &Res,
unsigned *InstCnt) {
2060 for (ValueRotInfo &VRI : ValueRotsVec) {
2062 for (
unsigned i = 0; i <
Bits.size(); ++i) {
2063 if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V)
2065 if (RLAmt[i] != VRI.RLAmt)
2071 unsigned ANDIMask = (
Mask & UINT16_MAX), ANDISMask = Mask >> 16;
2072 assert((ANDIMask != 0 || ANDISMask != 0) &&
2073 "No set bits in mask for value bit groups");
2074 bool NeedsRotate = VRI.RLAmt != 0;
2090 unsigned NumAndInsts = (
unsigned) NeedsRotate +
2091 (
unsigned) (ANDIMask != 0) +
2092 (
unsigned) (ANDISMask != 0) +
2093 (
unsigned) (ANDIMask != 0 && ANDISMask != 0) +
2094 (
unsigned) (
bool) Res;
2096 LLVM_DEBUG(
dbgs() <<
"\t\trotation groups for " << VRI.V.getNode()
2097 <<
" RL: " << VRI.RLAmt <<
":"
2098 <<
"\n\t\t\tisel using masking: " << NumAndInsts
2099 <<
" using rotates: " << VRI.NumGroups <<
"\n");
2101 if (NumAndInsts >= VRI.NumGroups)
2106 if (InstCnt) *InstCnt += NumAndInsts;
2111 { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
2112 getI32Imm(0, dl), getI32Imm(31, dl) };
2116 VRot = TruncateToInt32(VRI.V, dl);
2122 VRot, getI32Imm(ANDIMask, dl)),
2127 getI32Imm(ANDISMask, dl)),
2132 TotalVal = ANDISVal;
2137 ANDIVal, ANDISVal), 0);
2147 eraseMatchingBitGroups([VRI](
const BitGroup &BG) {
2148 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
2154 SDNode *Select32(
SDNode *
N,
bool LateMask,
unsigned *InstCnt) {
2158 if (InstCnt) *InstCnt = 0;
2161 SelectAndParts32(dl, Res, InstCnt);
2166 if ((!NeedMask || LateMask) && !Res) {
2167 ValueRotInfo &VRI = ValueRotsVec[0];
2169 if (InstCnt) *InstCnt += 1;
2171 { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
2172 getI32Imm(0, dl), getI32Imm(31, dl) };
2176 Res = TruncateToInt32(VRI.V, dl);
2180 eraseMatchingBitGroups([VRI](
const BitGroup &BG) {
2181 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
2185 if (InstCnt) *InstCnt += BitGroups.size();
2188 for (
auto &BG : BitGroups) {
2191 { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
2192 getI32Imm(
Bits.size() - BG.EndIdx - 1, dl),
2193 getI32Imm(
Bits.size() - BG.StartIdx - 1, dl) };
2194 Res =
SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl,
MVT::i32, Ops), 0);
2197 { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
2198 getI32Imm(
Bits.size() - BG.EndIdx - 1, dl),
2199 getI32Imm(
Bits.size() - BG.StartIdx - 1, dl) };
2200 Res =
SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl,
MVT::i32, Ops), 0);
2207 unsigned ANDIMask = (
Mask & UINT16_MAX), ANDISMask = Mask >> 16;
2208 assert((ANDIMask != 0 || ANDISMask != 0) &&
2209 "No set bits in zeros mask?");
2211 if (InstCnt) *InstCnt += (
unsigned) (ANDIMask != 0) +
2213 (
unsigned) (ANDIMask != 0 && ANDISMask != 0);
2218 Res, getI32Imm(ANDIMask, dl)),
2223 getI32Imm(ANDISMask, dl)),
2232 ANDIVal, ANDISVal), 0);
2238 unsigned SelectRotMask64Count(
unsigned RLAmt,
bool Repl32,
2239 unsigned MaskStart,
unsigned MaskEnd,
2243 unsigned InstMaskStart = 64 - MaskEnd - 1,
2244 InstMaskEnd = 64 - MaskStart - 1;
2249 if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) ||
2250 InstMaskEnd == 63 - RLAmt)
2259 bool Repl32,
unsigned MaskStart,
unsigned MaskEnd,
2260 unsigned *InstCnt =
nullptr) {
2263 unsigned InstMaskStart = 64 - MaskEnd - 1,
2264 InstMaskEnd = 64 - MaskStart - 1;
2266 if (InstCnt) *InstCnt += 1;
2272 assert(InstMaskStart >= 32 &&
"Mask cannot start out of range");
2273 assert(InstMaskEnd >= 32 &&
"Mask cannot end out of range");
2275 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2276 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2281 if (InstMaskEnd == 63) {
2283 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2284 getI32Imm(InstMaskStart, dl) };
2285 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl,
MVT::i64, Ops), 0);
2288 if (InstMaskStart == 0) {
2290 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2291 getI32Imm(InstMaskEnd, dl) };
2292 return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl,
MVT::i64, Ops), 0);
2295 if (InstMaskEnd == 63 - RLAmt) {
2297 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2298 getI32Imm(InstMaskStart, dl) };
2299 return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl,
MVT::i64, Ops), 0);
2308 if (InstCnt) *InstCnt += 1;
2311 unsigned RLAmt2 = MaskStart;
2314 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2316 V = SelectRotMask64(V, dl, RLAmt1,
false, 0, 63);
2317 return SelectRotMask64(V, dl, RLAmt2,
false, MaskStart, MaskEnd);
2323 unsigned RLAmt,
bool Repl32,
unsigned MaskStart,
2324 unsigned MaskEnd,
unsigned *InstCnt =
nullptr) {
2327 unsigned InstMaskStart = 64 - MaskEnd - 1,
2328 InstMaskEnd = 64 - MaskStart - 1;
2330 if (InstCnt) *InstCnt += 1;
2336 assert(InstMaskStart >= 32 &&
"Mask cannot start out of range");
2337 assert(InstMaskEnd >= 32 &&
"Mask cannot end out of range");
2339 { ExtendToInt64(
Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2340 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2345 if (InstMaskEnd == 63 - RLAmt) {
2347 { ExtendToInt64(
Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2348 getI32Imm(InstMaskStart, dl) };
2349 return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl,
MVT::i64, Ops), 0);
2358 if (InstCnt) *InstCnt += 1;
2361 unsigned RLAmt2 = MaskStart;
2364 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2366 V = SelectRotMask64(V, dl, RLAmt1,
false, 0, 63);
2367 return SelectRotMaskIns64(
Base, V, dl, RLAmt2,
false, MaskStart, MaskEnd);
2370 void SelectAndParts64(
const SDLoc &dl,
SDValue &Res,
unsigned *InstCnt) {
2383 for (ValueRotInfo &VRI : ValueRotsVec) {
2391 auto MatchingBG = [VRI](
const BitGroup &BG) {
2395 unsigned EffRLAmt = BG.RLAmt;
2396 if (!VRI.Repl32 && BG.Repl32) {
2397 if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx &&
2398 !BG.Repl32Coalesced) {
2404 }
else if (VRI.Repl32 != BG.Repl32) {
2408 return VRI.RLAmt == EffRLAmt;
2411 for (
auto &BG : BitGroups) {
2412 if (!MatchingBG(BG))
2415 if (BG.StartIdx <= BG.EndIdx) {
2416 for (
unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i)
2417 Mask |= (UINT64_C(1) << i);
2419 for (
unsigned i = BG.StartIdx; i <
Bits.size(); ++i)
2420 Mask |= (UINT64_C(1) << i);
2421 for (
unsigned i = 0; i <= BG.EndIdx; ++i)
2422 Mask |= (UINT64_C(1) << i);
2429 bool Use32BitInsts = isUInt<32>(Mask);
2431 unsigned ANDIMask = (
Mask & UINT16_MAX),
2432 ANDISMask = (Mask >> 16) & UINT16_MAX;
2434 bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask));
2436 unsigned NumAndInsts = (
unsigned) NeedsRotate +
2437 (
unsigned) (
bool) Res;
2438 unsigned NumOfSelectInsts = 0;
2440 assert(NumOfSelectInsts > 0 &&
"Failed to select an i64 constant.");
2443 (
unsigned) (ANDIMask != 0 && ANDISMask != 0);
2445 NumAndInsts += NumOfSelectInsts + 1;
2447 unsigned NumRLInsts = 0;
2448 bool FirstBG =
true;
2449 bool MoreBG =
false;
2450 for (
auto &BG : BitGroups) {
2451 if (!MatchingBG(BG)) {
2456 SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx,
2461 LLVM_DEBUG(
dbgs() <<
"\t\trotation groups for " << VRI.V.getNode()
2462 <<
" RL: " << VRI.RLAmt << (VRI.Repl32 ?
" (32):" :
":")
2463 <<
"\n\t\t\tisel using masking: " << NumAndInsts
2464 <<
" using rotates: " << NumRLInsts <<
"\n");
2470 if (NumAndInsts > NumRLInsts)
2475 if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts)
2480 if (InstCnt) *InstCnt += NumAndInsts;
2487 if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)))
2488 VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2489 VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63);
2494 if (Use32BitInsts) {
2495 assert((ANDIMask != 0 || ANDISMask != 0) &&
2496 "No set bits in mask when using 32-bit ands for 64-bit value");
2500 ANDIVal =
SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl,
MVT::i64,
2501 ExtendToInt64(VRot, dl),
2502 getI32Imm(ANDIMask, dl)),
2507 ExtendToInt64(VRot, dl),
2508 getI32Imm(ANDISMask, dl)),
2512 TotalVal = ANDISVal;
2517 ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2522 ExtendToInt64(VRot, dl), TotalVal),
2530 ExtendToInt64(Res, dl), TotalVal),
2535 eraseMatchingBitGroups(MatchingBG);
2540 SDNode *Select64(
SDNode *
N,
bool LateMask,
unsigned *InstCnt) {
2544 if (InstCnt) *InstCnt = 0;
2547 SelectAndParts64(dl, Res, InstCnt);
2552 if ((!NeedMask || LateMask) && !Res) {
2556 unsigned MaxGroupsIdx = 0;
2557 if (!ValueRotsVec[0].Repl32) {
2558 for (
unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i)
2559 if (ValueRotsVec[i].Repl32) {
2560 if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups)
2566 ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx];
2567 bool NeedsRotate =
false;
2570 }
else if (VRI.Repl32) {
2571 for (
auto &BG : BitGroups) {
2572 if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt ||
2573 BG.Repl32 != VRI.Repl32)
2578 if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx)
2587 Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2588 VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63,
2595 eraseMatchingBitGroups([VRI](
const BitGroup &BG) {
2596 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt &&
2597 BG.Repl32 == VRI.Repl32;
2604 for (
auto I = BitGroups.begin(), IE = BitGroups.end();
I != IE; ++
I) {
2605 if (SelectRotMask64Count(
I->RLAmt,
I->Repl32,
I->StartIdx,
I->EndIdx,
2607 SelectRotMask64Count(
I->RLAmt,
I->Repl32,
I->StartIdx,
I->EndIdx,
2609 if (
I != BitGroups.begin()) {
2612 BitGroups.insert(BitGroups.begin(), BG);
2620 for (
auto &BG : BitGroups) {
2622 Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx,
2623 BG.EndIdx, InstCnt);
2625 Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32,
2626 BG.StartIdx, BG.EndIdx, InstCnt);
2635 bool Use32BitInsts = isUInt<32>(Mask);
2637 unsigned ANDIMask = (
Mask & UINT16_MAX),
2638 ANDISMask = (Mask >> 16) & UINT16_MAX;
2640 if (Use32BitInsts) {
2641 assert((ANDIMask != 0 || ANDISMask != 0) &&
2642 "No set bits in mask when using 32-bit ands for 64-bit value");
2644 if (InstCnt) *InstCnt += (
unsigned) (ANDIMask != 0) +
2646 (
unsigned) (ANDIMask != 0 && ANDISMask != 0);
2650 ANDIVal =
SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl,
MVT::i64,
2651 ExtendToInt64(Res, dl),
2652 getI32Imm(ANDIMask, dl)),
2657 ExtendToInt64(Res, dl),
2658 getI32Imm(ANDISMask, dl)),
2667 ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2669 unsigned NumOfSelectInsts = 0;
2673 ExtendToInt64(Res, dl), MaskVal),
2676 *InstCnt += NumOfSelectInsts + 1;
2685 collectBitGroups(LateMask);
2686 if (BitGroups.empty())
2690 if (
Bits.size() == 64)
2691 assignRepl32BitGroups();
2694 collectValueRotInfo();
2696 if (
Bits.size() == 32) {
2697 return Select32(
N, LateMask, InstCnt);
2699 assert(
Bits.size() == 64 &&
"Not 64 bits here?");
2700 return Select64(
N, LateMask, InstCnt);
2706 void eraseMatchingBitGroups(
function_ref<
bool(
const BitGroup &)>
F) {
2712 bool NeedMask =
false;
2732 getValueBits(
SDValue(
N, 0),
N->getValueType(0).getSizeInBits());
2737 LLVM_DEBUG(
dbgs() <<
"Considering bit-permutation-based instruction"
2738 " selection for: ");
2742 computeRotationAmounts();
2755 unsigned InstCnt = 0, InstCntLateMask = 0;
2758 LLVM_DEBUG(
dbgs() <<
"\t\tisel would use " << InstCnt <<
" instructions\n");
2763 <<
" instructions\n");
2765 if (InstCnt <= InstCntLateMask) {
2775class IntegerCompareEliminator {
2780 enum ExtOrTruncConversion {
Ext, Trunc };
2788 enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert };
2798 enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt };
2805 SDValue addExtOrTrunc(
SDValue NatWidthRes, ExtOrTruncConversion Conv);
2809 int64_t RHSValue,
SDLoc dl);
2811 int64_t RHSValue,
SDLoc dl);
2813 int64_t RHSValue,
SDLoc dl);
2815 int64_t RHSValue,
SDLoc dl);
2820 PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) {
2823 "Only expecting to use this on 64 bit targets.");
2828 switch (
N->getOpcode()) {
2839 return tryEXTEND(
N);
2843 return tryLogicOpOfCompares(
N);
2854 "Expecting a zero/sign extend node!");
2859 N->getOperand(0).getValueType() ==
MVT::i1 &&
2861 WideRes = computeLogicOpInGPR(
N->getOperand(0));
2862 else if (
N->getOperand(0).getOpcode() !=
ISD::SETCC)
2866 getSETCCInGPR(
N->getOperand(0),
2868 SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig);
2875 bool Output32Bit =
N->getValueType(0) ==
MVT::i32;
2881 if (Input32Bit != Output32Bit)
2882 ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext :
2883 ExtOrTruncConversion::Trunc);
2891SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(
SDNode *
N) {
2895 "Expected a logic operation on setcc results.");
2897 if (!LoweredLogical)
2902 unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt;
2911 if (IsBitwiseNegate &&
2914 else if (IsBitwiseNegate)
2916 OpToConvToRecForm = LoweredLogical.
getOperand(0);
2920 OpToConvToRecForm = LoweredLogical;
2930 if (NewOpc != -1 && IsBitwiseNegate) {
2933 "Expected a PPC::XORI8 only for bitwise negation.");
2935 std::vector<SDValue> Ops;
2936 for (
int i = 0, e = OpToConvToRecForm.
getNumOperands(); i < e; i++)
2937 Ops.push_back(OpToConvToRecForm.
getOperand(i));
2944 assert((NewOpc != -1 || !IsBitwiseNegate) &&
2945 "No record form available for AND8/OR8/XOR8?");
2973SDValue IntegerCompareEliminator::computeLogicOpInGPR(
SDValue LogicOp) {
2975 "Can only handle logic operations here.");
2977 "Can only handle logic operations on i1 values here.");
2989 unsigned OperandOpcode = Operand.
getOpcode();
2991 return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig);
2996 PPC::RLDICL, dl, InVT, InputOp,
2997 S->getI64Imm(0, dl),
2998 S->getI64Imm(63, dl)), 0);
3000 return computeLogicOpInGPR(Operand);
3009 if (!LHS || (!RHS && !IsBitwiseNegation))
3012 NumLogicOpsOnComparison++;
3016 LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext);
3017 if (!IsBitwiseNegation &&
RHS.getValueType() ==
MVT::i32)
3018 RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext);
3023 case ISD::AND: NewOpc = PPC::AND8;
break;
3024 case ISD::OR: NewOpc = PPC::OR8;
break;
3025 case ISD::XOR: NewOpc = PPC::XOR8;
break;
3028 if (IsBitwiseNegation) {
3029 RHS = S->getI64Imm(1, dl);
3030 NewOpc = PPC::XORI8;
3041SDValue IntegerCompareEliminator::signExtendInputIfNeeded(
SDValue Input) {
3043 "Can only sign-extend 32-bit values here.");
3051 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3053 LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
3057 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3062 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3065 SignExtensionsAdded++;
3074SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(
SDValue Input) {
3076 "Can only zero-extend 32-bit values here.");
3086 if (IsTruncateOfZExt)
3087 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3091 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3093 LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
3096 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3100 ZeroExtensionsAdded++;
3102 S->getI64Imm(0, dl),
3103 S->getI64Imm(32, dl)), 0);
3110SDValue IntegerCompareEliminator::addExtOrTrunc(
SDValue NatWidthRes,
3111 ExtOrTruncConversion Conv) {
3112 SDLoc dl(NatWidthRes);
3116 if (Conv == ExtOrTruncConversion::Ext) {
3121 ImDef, NatWidthRes, SubRegIdx), 0);
3124 assert(Conv == ExtOrTruncConversion::Trunc &&
3125 "Unknown convertion between 32 and 64 bit values.");
3131 NatWidthRes, SubRegIdx), 0);
3137IntegerCompareEliminator::getCompoundZeroComparisonInGPR(
SDValue LHS,
SDLoc dl,
3138 ZeroCompare CmpTy) {
3139 EVT InVT =
LHS.getValueType();
3145 case ZeroCompare::GEZExt:
3146 case ZeroCompare::GESExt:
3148 dl, InVT, LHS, LHS), 0);
3150 case ZeroCompare::LEZExt:
3151 case ZeroCompare::LESExt: {
3154 LHS = signExtendInputIfNeeded(LHS);
3159 Neg, S->getI64Imm(1, dl),
3160 S->getI64Imm(63, dl)), 0);
3164 S->getI64Imm(~0ULL, dl)), 0);
3174 (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt))
3176 ToExtend, S->getI64Imm(1, dl),
3177 S->getI64Imm(63, dl)), 0);
3179 (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt))
3181 S->getI64Imm(63, dl)), 0);
3183 assert(Is32Bit &&
"Should have handled the 32-bit sequences above.");
3186 case ZeroCompare::GEZExt: {
3187 SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3188 S->getI32Imm(31, dl) };
3192 case ZeroCompare::GESExt:
3194 S->getI32Imm(31, dl)), 0);
3195 case ZeroCompare::LEZExt:
3197 S->getI32Imm(1, dl)), 0);
3198 case ZeroCompare::LESExt:
3200 S->getI32Imm(-1, dl)), 0);
3211IntegerCompareEliminator::get32BitZExtCompare(
SDValue LHS,
SDValue RHS,
3213 int64_t RHSValue,
SDLoc dl) {
3217 bool IsRHSZero = RHSValue == 0;
3218 bool IsRHSOne = RHSValue == 1;
3219 bool IsRHSNegOne = RHSValue == -1LL;
3229 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
3230 S->getI32Imm(31, dl) };
3241 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
3242 S->getI32Imm(31, dl) };
3246 S->getI32Imm(1, dl)), 0);
3252 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3258 IsRHSZero = RHSConst && RHSConst->
isZero();
3269 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3273 LHS = signExtendInputIfNeeded(LHS);
3274 RHS = signExtendInputIfNeeded(RHS);
3279 S->getI64Imm(1, dl), S->getI64Imm(63, dl)),
3283 MVT::i64, Shift, S->getI32Imm(1, dl)), 0);
3291 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3297 LHS = signExtendInputIfNeeded(LHS);
3298 RHS = signExtendInputIfNeeded(RHS);
3302 Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0);
3308 IsRHSZero = RHSConst && RHSConst->
isZero();
3320 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3324 SDValue ShiftOps[] = {
LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3325 S->getI32Imm(31, dl) };
3333 LHS = signExtendInputIfNeeded(LHS);
3334 RHS = signExtendInputIfNeeded(RHS);
3338 SUBFNode, S->getI64Imm(1, dl),
3339 S->getI64Imm(63, dl)), 0);
3350 LHS = zeroExtendInputIfNeeded(LHS);
3351 RHS = zeroExtendInputIfNeeded(RHS);
3356 Subtract, S->getI64Imm(1, dl),
3357 S->getI64Imm(63, dl)), 0);
3359 S->getI32Imm(1, dl)), 0);
3370 LHS = zeroExtendInputIfNeeded(LHS);
3371 RHS = zeroExtendInputIfNeeded(RHS);
3375 Subtract, S->getI64Imm(1, dl),
3376 S->getI64Imm(63, dl)), 0);
3384IntegerCompareEliminator::get32BitSExtCompare(
SDValue LHS,
SDValue RHS,
3386 int64_t RHSValue,
SDLoc dl) {
3390 bool IsRHSZero = RHSValue == 0;
3391 bool IsRHSOne = RHSValue == 1;
3392 bool IsRHSNegOne = RHSValue == -1LL;
3405 SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl),
3406 S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3424 { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3429 S->getI32Imm(1, dl)), 0);
3436 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3442 IsRHSZero = RHSConst && RHSConst->
isZero();
3451 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3454 LHS = signExtendInputIfNeeded(LHS);
3455 RHS = signExtendInputIfNeeded(RHS);
3461 SUBFNode, S->getI64Imm(1, dl),
3462 S->getI64Imm(63, dl)), 0);
3464 S->getI32Imm(-1, dl)), 0);
3471 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3476 LHS = signExtendInputIfNeeded(LHS);
3477 RHS = signExtendInputIfNeeded(RHS);
3481 S->getI64Imm(63, dl)), 0);
3487 IsRHSZero = RHSConst && RHSConst->
isZero();
3498 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3502 S->getI32Imm(31, dl)), 0);
3507 LHS = signExtendInputIfNeeded(LHS);
3508 RHS = signExtendInputIfNeeded(RHS);
3512 SUBFNode, S->getI64Imm(63, dl)), 0);
3523 LHS = zeroExtendInputIfNeeded(LHS);
3524 RHS = zeroExtendInputIfNeeded(RHS);
3529 S->getI32Imm(1, dl), S->getI32Imm(63,dl)),
3532 S->getI32Imm(-1, dl)), 0);
3543 LHS = zeroExtendInputIfNeeded(LHS);
3544 RHS = zeroExtendInputIfNeeded(RHS);
3548 Subtract, S->getI64Imm(63, dl)), 0);
3556IntegerCompareEliminator::get64BitZExtCompare(
SDValue LHS,
SDValue RHS,
3558 int64_t RHSValue,
SDLoc dl) {
3562 bool IsRHSZero = RHSValue == 0;
3563 bool IsRHSOne = RHSValue == 1;
3564 bool IsRHSNegOne = RHSValue == -1LL;
3575 S->getI64Imm(58, dl),
3576 S->getI64Imm(63, dl)), 0);
3587 Xor, S->getI32Imm(~0U, dl)), 0);
3597 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3600 IsRHSZero = RHSConst && RHSConst->
isZero();
3609 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3612 S->getI64Imm(1, dl),
3613 S->getI64Imm(63, dl)), 0);
3616 S->getI64Imm(63, dl)), 0);
3621 ShiftR, ShiftL, SubtractCarry), 0);
3629 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3633 S->getI64Imm(~0ULL, dl)), 0);
3637 S->getI64Imm(1, dl),
3638 S->getI64Imm(63, dl)), 0);
3642 IsRHSZero = RHSConst && RHSConst->
isZero();
3652 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3655 S->getI64Imm(1, dl),
3656 S->getI64Imm(63, dl)), 0);
3659 LHS, S->getI64Imm(63, dl)), 0);
3662 RHS, S->getI64Imm(1, dl),
3663 S->getI64Imm(63, dl)), 0);
3669 SRDINode, SRADINode, SUBFC8Carry), 0);
3671 ADDE8Node, S->getI64Imm(1, dl)), 0);
3686 LHS, LHS, SUBFC8Carry), 0);
3688 SUBFE8Node, S->getI64Imm(1, dl)), 0);
3703 LHS, LHS, SubtractCarry), 0);
3713IntegerCompareEliminator::get64BitSExtCompare(
SDValue LHS,
SDValue RHS,
3715 int64_t RHSValue,
SDLoc dl) {
3719 bool IsRHSZero = RHSValue == 0;
3720 bool IsRHSOne = RHSValue == 1;
3721 bool IsRHSNegOne = RHSValue == -1LL;
3733 AddInput, S->getI32Imm(~0U, dl)), 0);
3746 Xor, S->getI32Imm(0, dl)), 0);
3748 SC,
SC.getValue(1)), 0);
3756 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3759 IsRHSZero = RHSConst && RHSConst->
isZero();
3768 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3771 S->getI64Imm(63, dl)), 0);
3774 S->getI64Imm(1, dl),
3775 S->getI64Imm(63, dl)), 0);
3781 ShiftR, ShiftL, SubtractCarry), 0);
3790 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3794 S->getI64Imm(-1, dl)), 0);
3798 S->getI64Imm(63, dl)), 0);
3802 IsRHSZero = RHSConst && RHSConst->
isZero();
3812 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3815 S->getI64Imm(63, dl)), 0);
3819 LHS, S->getI64Imm(63, dl)), 0);
3822 RHS, S->getI64Imm(1, dl),
3823 S->getI64Imm(63, dl)), 0);
3829 SRDINode, SRADINode, SUBFC8Carry), 0);
3832 ADDE8Node, S->getI64Imm(1, dl)), 0);
3849 LHS, SubtractCarry), 0);
3851 ExtSub, ExtSub), 0);
3865 LHS, LHS, SubCarry), 0);
3876 "An ISD::SETCC node required here.");
3884 for (
auto *CompareUse :
Compare.getNode()->uses())
3889 OmittedForNonExtendUses++;
3899 SetccInGPROpts ConvOpts) {
3902 "An ISD::SETCC node required here.");
3915 cast<CondCodeSDNode>(
Compare.getOperand(CCOpNum))->get();
3916 EVT InputVT =
LHS.getValueType();
3920 if (ConvOpts == SetccInGPROpts::ZExtInvert ||
3921 ConvOpts == SetccInGPROpts::SExtInvert)
3924 bool Inputs32Bit = InputVT ==
MVT::i32;
3929 bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig ||
3930 ConvOpts == SetccInGPROpts::SExtInvert;
3932 if (IsSext && Inputs32Bit)
3933 return get32BitSExtCompare(LHS, RHS,
CC, RHSValue, dl);
3934 else if (Inputs32Bit)
3935 return get32BitZExtCompare(LHS, RHS,
CC, RHSValue, dl);
3937 return get64BitSExtCompare(LHS, RHS,
CC, RHSValue, dl);
3938 return get64BitZExtCompare(LHS, RHS,
CC, RHSValue, dl);
3943bool PPCDAGToDAGISel::tryIntCompareInGPR(
SDNode *
N) {
3957 if (!(
CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1())
3960 switch (
N->getOpcode()) {
3967 IntegerCompareEliminator ICmpElim(CurDAG,
this);
3968 if (
SDNode *New = ICmpElim.Select(
N)) {
3969 ReplaceNode(
N, New);
3977bool PPCDAGToDAGISel::tryBitPermutation(
SDNode *
N) {
3985 switch (
N->getOpcode()) {
3990 if (Subtarget->isISA3_1() &&
N->getValueType(0) ==
MVT::i32 &&
3992 auto &OpRight =
N->getOperand(1);
4002 BitPermutationSelector BPS(CurDAG);
4003 if (
SDNode *New = BPS.Select(
N)) {
4004 ReplaceNode(
N, New);
4026 if (isUInt<16>(Imm))
4028 getI32Imm(Imm & 0xFFFF, dl)),
4031 if (isInt<16>((
int)Imm))
4033 getI32Imm(Imm & 0xFFFF, dl)),
4046 getI32Imm(Imm >> 16, dl)), 0);
4048 getI32Imm(Imm & 0xFFFF, dl)), 0);
4054 getI32Imm(Imm & 0xFFFF, dl)), 0);
4060 getI32Imm((
int)SImm & 0xFFFF,
4070 if (isUInt<16>(Imm))
4072 getI32Imm(Imm & 0xFFFF, dl)),
4077 getI32Imm(Imm & 0xFFFF, dl)),
4089 if (isUInt<32>(Imm)) {
4091 getI64Imm(Imm >> 16, dl)), 0);
4093 getI64Imm(Imm & 0xFFFF, dl)),
4101 getI64Imm(Imm & 0xFFFF, dl)), 0);
4107 getI64Imm(SImm & 0xFFFF, dl)),
4112 if (Subtarget->hasSPE()) {
4117 Opc = PPC::EFSCMPEQ;
4125 Opc = PPC::EFSCMPLT;
4133 Opc = PPC::EFSCMPGT;
4139 if (Subtarget->hasSPE()) {
4144 Opc = PPC::EFDCMPEQ;
4152 Opc = PPC::EFDCMPLT;
4160 Opc = PPC::EFDCMPGT;
4164 Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD;
4167 assert(Subtarget->hasP9Vector() &&
"XSCMPUQP requires Power9 Vector");
4168 Opc = PPC::XSCMPUQP;
4236 case ISD::SETO: Invert =
true;
return 3;
4253 bool HasVSX,
bool &Swap,
bool &Negate) {
4281 return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP;
4283 return PPC::XVCMPEQDP;
4288 return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP;
4290 return PPC::XVCMPGTDP;
4295 return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP;
4297 return PPC::XVCMPGEDP;
4325 return PPC::VCMPEQUB;
4327 return PPC::VCMPEQUH;
4329 return PPC::VCMPEQUW;
4331 return PPC::VCMPEQUD;
4333 return PPC::VCMPEQUQ;
4337 return PPC::VCMPGTSB;
4339 return PPC::VCMPGTSH;
4341 return PPC::VCMPGTSW;
4343 return PPC::VCMPGTSD;
4345 return PPC::VCMPGTSQ;
4349 return PPC::VCMPGTUB;
4351 return PPC::VCMPGTUH;
4353 return PPC::VCMPGTUW;
4355 return PPC::VCMPGTUD;
4357 return PPC::VCMPGTUQ;
4366bool PPCDAGToDAGISel::trySETCC(
SDNode *
N) {
4369 bool IsStrict =
N->isStrictFPOpcode();
4371 cast<CondCodeSDNode>(
N->getOperand(IsStrict ? 3 : 2))->get();
4374 bool isPPC64 = (PtrVT ==
MVT::i64);
4390 SDValue Ops[] = {
Op, getI32Imm(27, dl), getI32Imm(5, dl),
4391 getI32Imm(31, dl) };
4399 Op, getI32Imm(~0U, dl)), 0);
4404 SDValue Ops[] = {
Op, getI32Imm(1, dl), getI32Imm(31, dl),
4405 getI32Imm(31, dl) };
4413 SDValue Ops[] = {
T, getI32Imm(1, dl), getI32Imm(31, dl),
4414 getI32Imm(31, dl) };
4419 }
else if (Imm == ~0U) {
4426 Op, getI32Imm(1, dl)), 0);
4431 0),
Op.getValue(1));
4437 Op, getI32Imm(~0U, dl));
4444 getI32Imm(1, dl)), 0);
4447 SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl),
4448 getI32Imm(31, dl) };
4453 SDValue Ops[] = {
Op, getI32Imm(1, dl), getI32Imm(31, dl),
4454 getI32Imm(31, dl) };
4465 if (!IsStrict &&
LHS.getValueType().isVector()) {
4466 if (Subtarget->hasSPE())
4469 EVT VecVT =
LHS.getValueType();
4471 unsigned int VCmpInst =
4479 CurDAG->
SelectNodeTo(
N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR,
4488 if (Subtarget->useCRBits())
4493 SDValue CCReg = SelectCC(LHS, RHS,
CC, dl, Chain);
4500 if (Subtarget->hasSPE() &&
LHS.getValueType().isFloatingPoint()) {
4514 SDValue Ops[] = { IntCR, getI32Imm((32 - (3 -
Idx)) & 31, dl),
4515 getI32Imm(31, dl), getI32Imm(31, dl) };
4530bool PPCDAGToDAGISel::isOffsetMultipleOf(
SDNode *
N,
unsigned Val)
const {
4536 AddrOp =
N->getOperand(1);
4551 if ((SlotAlign % Val) != 0)
4566void PPCDAGToDAGISel::transferMemOperands(
SDNode *
N,
SDNode *Result) {
4573 bool &NeedSwapOps,
bool &IsUnCmp) {
4579 SDValue TrueRes =
N->getOperand(2);
4580 SDValue FalseRes =
N->getOperand(3);
4582 if (!TrueConst || (
N->getSimpleValueType(0) !=
MVT::i64 &&
4592 if ((TrueResVal < -1 || TrueResVal > 1) ||
4621 cast<CondCodeSDNode>(SetOrSelCC.
getOperand(InnerIsSel ? 4 : 2))->get();
4626 dyn_cast<ConstantSDNode>(SetOrSelCC.
getOperand(2));
4628 dyn_cast<ConstantSDNode>(SetOrSelCC.
getOperand(3));
4629 if (!SelCCTrueConst || !SelCCFalseConst)
4634 if (SelCCTVal == -1 && SelCCFVal == 1) {
4636 }
else if (SelCCTVal != 1 || SelCCFVal != -1)
4646 bool InnerSwapped =
false;
4647 if (
LHS == InnerRHS &&
RHS == InnerLHS)
4648 InnerSwapped =
true;
4649 else if (
LHS != InnerLHS ||
RHS != InnerRHS)
4660 NeedSwapOps = (InnerCC ==
ISD::SETGT) ? InnerSwapped : !InnerSwapped;
4677 NeedSwapOps = (TrueResVal == 1);
4696 NeedSwapOps = (TrueResVal == -1);
4705 LLVM_DEBUG(
dbgs() <<
"Found a node that can be lowered to a SETB: ");
4715 if (
N.getNumOperands() < 1 || !isa<ConstantSDNode>(
N.getOperand(0)) ||
4718 switch (
N.getConstantOperandVal(0)) {
4719 case Intrinsic::ppc_vsx_xvtdivdp:
4720 case Intrinsic::ppc_vsx_xvtdivsp:
4721 case Intrinsic::ppc_vsx_xvtsqrtdp:
4722 case Intrinsic::ppc_vsx_xvtsqrtsp:
4728bool PPCDAGToDAGISel::tryFoldSWTestBRCC(
SDNode *
N) {
4745 if (!isa<ConstantSDNode>(CmpRHS) ||
4746 cast<ConstantSDNode>(CmpRHS)->getSExtValue() != 0)