LLVM  14.0.0git
PPCISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines a pattern matching instruction selector for PowerPC,
10 // converting from a legalized dag to a PPC dag.
11 //
12 //===----------------------------------------------------------------------===//
13 
16 #include "PPC.h"
17 #include "PPCISelLowering.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCSubtarget.h"
20 #include "PPCTargetMachine.h"
21 #include "llvm/ADT/APInt.h"
22 #include "llvm/ADT/DenseMap.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallPtrSet.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/Statistic.h"
40 #include "llvm/IR/BasicBlock.h"
41 #include "llvm/IR/DebugLoc.h"
42 #include "llvm/IR/Function.h"
43 #include "llvm/IR/GlobalValue.h"
44 #include "llvm/IR/InlineAsm.h"
45 #include "llvm/IR/InstrTypes.h"
46 #include "llvm/IR/IntrinsicsPowerPC.h"
47 #include "llvm/IR/Module.h"
48 #include "llvm/Support/Casting.h"
49 #include "llvm/Support/CodeGen.h"
51 #include "llvm/Support/Compiler.h"
52 #include "llvm/Support/Debug.h"
54 #include "llvm/Support/KnownBits.h"
58 #include <algorithm>
59 #include <cassert>
60 #include <cstdint>
61 #include <iterator>
62 #include <limits>
63 #include <memory>
64 #include <new>
65 #include <tuple>
66 #include <utility>
67 
68 using namespace llvm;
69 
70 #define DEBUG_TYPE "ppc-codegen"
71 
72 STATISTIC(NumSextSetcc,
73  "Number of (sext(setcc)) nodes expanded into GPR sequence.");
74 STATISTIC(NumZextSetcc,
75  "Number of (zext(setcc)) nodes expanded into GPR sequence.");
76 STATISTIC(SignExtensionsAdded,
77  "Number of sign extensions for compare inputs added.");
78 STATISTIC(ZeroExtensionsAdded,
79  "Number of zero extensions for compare inputs added.");
80 STATISTIC(NumLogicOpsOnComparison,
81  "Number of logical ops on i1 values calculated in GPR.");
82 STATISTIC(OmittedForNonExtendUses,
83  "Number of compares not eliminated as they have non-extending uses.");
84 STATISTIC(NumP9Setb,
85  "Number of compares lowered to setb.");
86 
87 // FIXME: Remove this once the bug has been fixed!
88 cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug",
89 cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden);
90 
91 static cl::opt<bool>
92  UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true),
93  cl::desc("use aggressive ppc isel for bit permutations"),
94  cl::Hidden);
96  "ppc-bit-perm-rewriter-stress-rotates",
97  cl::desc("stress rotate selection in aggressive ppc isel for "
98  "bit permutations"),
99  cl::Hidden);
100 
102  "ppc-use-branch-hint", cl::init(true),
103  cl::desc("Enable static hinting of branches on ppc"),
104  cl::Hidden);
105 
107  "ppc-tls-opt", cl::init(true),
108  cl::desc("Enable tls optimization peephole"),
109  cl::Hidden);
110 
114 
116  "ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All),
117  cl::desc("Specify the types of comparisons to emit GPR-only code for."),
118  cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons."),
119  clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs."),
120  clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs."),
121  clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs."),
122  clEnumValN(ICGPR_NonExtIn, "nonextin",
123  "Only comparisons where inputs don't need [sz]ext."),
124  clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result."),
125  clEnumValN(ICGPR_ZextI32, "zexti32",
126  "Only i32 comparisons with zext result."),
127  clEnumValN(ICGPR_ZextI64, "zexti64",
128  "Only i64 comparisons with zext result."),
129  clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result."),
130  clEnumValN(ICGPR_SextI32, "sexti32",
131  "Only i32 comparisons with sext result."),
132  clEnumValN(ICGPR_SextI64, "sexti64",
133  "Only i64 comparisons with sext result.")));
134 namespace {
135 
136  //===--------------------------------------------------------------------===//
137  /// PPCDAGToDAGISel - PPC specific code to select PPC machine
138  /// instructions for SelectionDAG operations.
139  ///
140  class PPCDAGToDAGISel : public SelectionDAGISel {
141  const PPCTargetMachine &TM;
142  const PPCSubtarget *Subtarget = nullptr;
143  const PPCTargetLowering *PPCLowering = nullptr;
144  unsigned GlobalBaseReg = 0;
145 
146  public:
147  explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOpt::Level OptLevel)
148  : SelectionDAGISel(tm, OptLevel), TM(tm) {}
149 
150  bool runOnMachineFunction(MachineFunction &MF) override {
151  // Make sure we re-emit a set of the global base reg if necessary
152  GlobalBaseReg = 0;
153  Subtarget = &MF.getSubtarget<PPCSubtarget>();
154  PPCLowering = Subtarget->getTargetLowering();
155  if (Subtarget->hasROPProtect()) {
156  // Create a place on the stack for the ROP Protection Hash.
157  // The ROP Protection Hash will always be 8 bytes and aligned to 8
158  // bytes.
159  MachineFrameInfo &MFI = MF.getFrameInfo();
161  const int Result = MFI.CreateStackObject(8, Align(8), false);
162  FI->setROPProtectionHashSaveIndex(Result);
163  }
165 
166  return true;
167  }
168 
169  void PreprocessISelDAG() override;
170  void PostprocessISelDAG() override;
171 
172  /// getI16Imm - Return a target constant with the specified value, of type
173  /// i16.
174  inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) {
175  return CurDAG->getTargetConstant(Imm, dl, MVT::i16);
176  }
177 
178  /// getI32Imm - Return a target constant with the specified value, of type
179  /// i32.
180  inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
181  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
182  }
183 
184  /// getI64Imm - Return a target constant with the specified value, of type
185  /// i64.
186  inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) {
187  return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
188  }
189 
190  /// getSmallIPtrImm - Return a target constant of pointer type.
191  inline SDValue getSmallIPtrImm(unsigned Imm, const SDLoc &dl) {
192  return CurDAG->getTargetConstant(
193  Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout()));
194  }
195 
196  /// isRotateAndMask - Returns true if Mask and Shift can be folded into a
197  /// rotate and mask opcode and mask operation.
198  static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask,
199  unsigned &SH, unsigned &MB, unsigned &ME);
200 
201  /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC
202  /// base register. Return the virtual register that holds this value.
203  SDNode *getGlobalBaseReg();
204 
205  void selectFrameIndex(SDNode *SN, SDNode *N, unsigned Offset = 0);
206 
207  // Select - Convert the specified operand from a target-independent to a
208  // target-specific node if it hasn't already been changed.
209  void Select(SDNode *N) override;
210 
211  bool tryBitfieldInsert(SDNode *N);
212  bool tryBitPermutation(SDNode *N);
213  bool tryIntCompareInGPR(SDNode *N);
214 
215  // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into
216  // an X-Form load instruction with the offset being a relocation coming from
217  // the PPCISD::ADD_TLS.
218  bool tryTLSXFormLoad(LoadSDNode *N);
219  // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into
220  // an X-Form store instruction with the offset being a relocation coming from
221  // the PPCISD::ADD_TLS.
222  bool tryTLSXFormStore(StoreSDNode *N);
223  /// SelectCC - Select a comparison of the specified values with the
224  /// specified condition code, returning the CR# of the expression.
225  SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
226  const SDLoc &dl, SDValue Chain = SDValue());
227 
228  /// SelectAddrImmOffs - Return true if the operand is valid for a preinc
229  /// immediate field. Note that the operand at this point is already the
230  /// result of a prior SelectAddressRegImm call.
231  bool SelectAddrImmOffs(SDValue N, SDValue &Out) const {
232  if (N.getOpcode() == ISD::TargetConstant ||
233  N.getOpcode() == ISD::TargetGlobalAddress) {
234  Out = N;
235  return true;
236  }
237 
238  return false;
239  }
240 
241  /// SelectDSForm - Returns true if address N can be represented by the
242  /// addressing mode of DSForm instructions (a base register, plus a signed
243  /// 16-bit displacement that is a multiple of 4.
244  bool SelectDSForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
245  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
246  Align(4)) == PPC::AM_DSForm;
247  }
248 
249  /// SelectDQForm - Returns true if address N can be represented by the
250  /// addressing mode of DQForm instructions (a base register, plus a signed
251  /// 16-bit displacement that is a multiple of 16.
252  bool SelectDQForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
253  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
254  Align(16)) == PPC::AM_DQForm;
255  }
256 
257  /// SelectDForm - Returns true if address N can be represented by
258  /// the addressing mode of DForm instructions (a base register, plus a
259  /// signed 16-bit immediate.
260  bool SelectDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
261  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
262  None) == PPC::AM_DForm;
263  }
264 
265  /// SelectPCRelForm - Returns true if address N can be represented by
266  /// PC-Relative addressing mode.
267  bool SelectPCRelForm(SDNode *Parent, SDValue N, SDValue &Disp,
268  SDValue &Base) {
269  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
270  None) == PPC::AM_PCRel;
271  }
272 
273  /// SelectPDForm - Returns true if address N can be represented by Prefixed
274  /// DForm addressing mode (a base register, plus a signed 34-bit immediate.
275  bool SelectPDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
276  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
278  }
279 
280  /// SelectXForm - Returns true if address N can be represented by the
281  /// addressing mode of XForm instructions (an indexed [r+r] operation).
282  bool SelectXForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
283  return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
284  None) == PPC::AM_XForm;
285  }
286 
287  /// SelectForceXForm - Given the specified address, force it to be
288  /// represented as an indexed [r+r] operation (an XForm instruction).
289  bool SelectForceXForm(SDNode *Parent, SDValue N, SDValue &Disp,
290  SDValue &Base) {
291  return PPCLowering->SelectForceXFormMode(N, Disp, Base, *CurDAG) ==
293  }
294 
295  /// SelectAddrIdx - Given the specified address, check to see if it can be
296  /// represented as an indexed [r+r] operation.
297  /// This is for xform instructions whose associated displacement form is D.
298  /// The last parameter \p 0 means associated D form has no requirment for 16
299  /// bit signed displacement.
300  /// Returns false if it can be represented by [r+imm], which are preferred.
301  bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
302  return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG, None);
303  }
304 
305  /// SelectAddrIdx4 - Given the specified address, check to see if it can be
306  /// represented as an indexed [r+r] operation.
307  /// This is for xform instructions whose associated displacement form is DS.
308  /// The last parameter \p 4 means associated DS form 16 bit signed
309  /// displacement must be a multiple of 4.
310  /// Returns false if it can be represented by [r+imm], which are preferred.
311  bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) {
312  return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
313  Align(4));
314  }
315 
316  /// SelectAddrIdx16 - Given the specified address, check to see if it can be
317  /// represented as an indexed [r+r] operation.
318  /// This is for xform instructions whose associated displacement form is DQ.
319  /// The last parameter \p 16 means associated DQ form 16 bit signed
320  /// displacement must be a multiple of 16.
321  /// Returns false if it can be represented by [r+imm], which are preferred.
322  bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) {
323  return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
324  Align(16));
325  }
326 
327  /// SelectAddrIdxOnly - Given the specified address, force it to be
328  /// represented as an indexed [r+r] operation.
329  bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
330  return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG);
331  }
332 
333  /// SelectAddrImm - Returns true if the address N can be represented by
334  /// a base register plus a signed 16-bit displacement [r+imm].
335  /// The last parameter \p 0 means D form has no requirment for 16 bit signed
336  /// displacement.
337  bool SelectAddrImm(SDValue N, SDValue &Disp,
338  SDValue &Base) {
339  return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, None);
340  }
341 
342  /// SelectAddrImmX4 - Returns true if the address N can be represented by
343  /// a base register plus a signed 16-bit displacement that is a multiple of
344  /// 4 (last parameter). Suitable for use by STD and friends.
345  bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
346  return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4));
347  }
348 
349  /// SelectAddrImmX16 - Returns true if the address N can be represented by
350  /// a base register plus a signed 16-bit displacement that is a multiple of
351  /// 16(last parameter). Suitable for use by STXV and friends.
352  bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
353  return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG,
354  Align(16));
355  }
356 
357  /// SelectAddrImmX34 - Returns true if the address N can be represented by
358  /// a base register plus a signed 34-bit displacement. Suitable for use by
359  /// PSTXVP and friends.
360  bool SelectAddrImmX34(SDValue N, SDValue &Disp, SDValue &Base) {
361  return PPCLowering->SelectAddressRegImm34(N, Disp, Base, *CurDAG);
362  }
363 
364  // Select an address into a single register.
365  bool SelectAddr(SDValue N, SDValue &Base) {
366  Base = N;
367  return true;
368  }
369 
370  bool SelectAddrPCRel(SDValue N, SDValue &Base) {
371  return PPCLowering->SelectAddressPCRel(N, Base);
372  }
373 
374  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
375  /// inline asm expressions. It is always correct to compute the value into
376  /// a register. The case of adding a (possibly relocatable) constant to a
377  /// register can be improved, but it is wrong to substitute Reg+Reg for
378  /// Reg in an asm, because the load or store opcode would have to change.
379  bool SelectInlineAsmMemoryOperand(const SDValue &Op,
380  unsigned ConstraintID,
381  std::vector<SDValue> &OutOps) override {
382  switch(ConstraintID) {
383  default:
384  errs() << "ConstraintID: " << ConstraintID << "\n";
385  llvm_unreachable("Unexpected asm memory constraint");
392  // We need to make sure that this one operand does not end up in r0
393  // (because we might end up lowering this as 0(%op)).
394  const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
395  const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1);
396  SDLoc dl(Op);
397  SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
398  SDValue NewOp =
399  SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
400  dl, Op.getValueType(),
401  Op, RC), 0);
402 
403  OutOps.push_back(NewOp);
404  return false;
405  }
406  return true;
407  }
408 
409  StringRef getPassName() const override {
410  return "PowerPC DAG->DAG Pattern Instruction Selection";
411  }
412 
413 // Include the pieces autogenerated from the target description.
414 #include "PPCGenDAGISel.inc"
415 
416 private:
417  bool trySETCC(SDNode *N);
418  bool tryFoldSWTestBRCC(SDNode *N);
419  bool tryAsSingleRLDICL(SDNode *N);
420  bool tryAsSingleRLDICR(SDNode *N);
421  bool tryAsSingleRLWINM(SDNode *N);
422  bool tryAsSingleRLWINM8(SDNode *N);
423  bool tryAsSingleRLWIMI(SDNode *N);
424  bool tryAsPairOfRLDICL(SDNode *N);
425  bool tryAsSingleRLDIMI(SDNode *N);
426 
427  void PeepholePPC64();
428  void PeepholePPC64ZExt();
429  void PeepholeCROps();
430 
431  SDValue combineToCMPB(SDNode *N);
432  void foldBoolExts(SDValue &Res, SDNode *&N);
433 
434  bool AllUsersSelectZero(SDNode *N);
435  void SwapAllSelectUsers(SDNode *N);
436 
437  bool isOffsetMultipleOf(SDNode *N, unsigned Val) const;
438  void transferMemOperands(SDNode *N, SDNode *Result);
439  };
440 
441 } // end anonymous namespace
442 
443 /// getGlobalBaseReg - Output the instructions required to put the
444 /// base address to use for accessing globals into a register.
445 ///
446 SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
447  if (!GlobalBaseReg) {
448  const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
449  // Insert the set of GlobalBaseReg into the first MBB of the function
450  MachineBasicBlock &FirstMBB = MF->front();
452  const Module *M = MF->getFunction().getParent();
453  DebugLoc dl;
454 
455  if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) {
456  if (Subtarget->isTargetELF()) {
457  GlobalBaseReg = PPC::R30;
458  if (!Subtarget->isSecurePlt() &&
459  M->getPICLevel() == PICLevel::SmallPIC) {
460  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR));
461  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
462  MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
463  } else {
464  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
465  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
466  Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
467  BuildMI(FirstMBB, MBBI, dl,
468  TII.get(PPC::UpdateGBR), GlobalBaseReg)
470  MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
471  }
472  } else {
473  GlobalBaseReg =
474  RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass);
475  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
476  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
477  }
478  } else {
479  // We must ensure that this sequence is dominated by the prologue.
480  // FIXME: This is a bit of a big hammer since we don't get the benefits
481  // of shrink-wrapping whenever we emit this instruction. Considering
482  // this is used in any function where we emit a jump table, this may be
483  // a significant limitation. We should consider inserting this in the
484  // block where it is used and then commoning this sequence up if it
485  // appears in multiple places.
486  // Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of
487  // MovePCtoLR8.
488  MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true);
489  GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
490  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8));
491  BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg);
492  }
493  }
494  return CurDAG->getRegister(GlobalBaseReg,
495  PPCLowering->getPointerTy(CurDAG->getDataLayout()))
496  .getNode();
497 }
498 
499 // Check if a SDValue has the toc-data attribute.
500 static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) {
501  GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val);
502  if (!GA)
503  return false;
504 
505  const GlobalVariable *GV = dyn_cast_or_null<GlobalVariable>(GA->getGlobal());
506  if (!GV)
507  return false;
508 
509  if (!GV->hasAttribute("toc-data"))
510  return false;
511 
512  // TODO: These asserts should be updated as more support for the toc data
513  // transformation is added (64 bit, struct support, etc.).
514 
515  assert(PointerSize == 4 && "Only 32 Bit Codegen is currently supported by "
516  "the toc data transformation.");
517 
518  assert(PointerSize >= GV->getAlign().valueOrOne().value() &&
519  "GlobalVariables with an alignment requirement stricter then 4-bytes "
520  "not supported by the toc data transformation.");
521 
522  Type *GVType = GV->getValueType();
523 
524  assert(GVType->isSized() && "A GlobalVariable's size must be known to be "
525  "supported by the toc data transformation.");
526 
527  if (GVType->isVectorTy())
528  report_fatal_error("A GlobalVariable of Vector type is not currently "
529  "supported by the toc data transformation.");
530 
531  if (GVType->isArrayTy())
532  report_fatal_error("A GlobalVariable of Array type is not currently "
533  "supported by the toc data transformation.");
534 
535  if (GVType->isStructTy())
536  report_fatal_error("A GlobalVariable of Struct type is not currently "
537  "supported by the toc data transformation.");
538 
539  assert(GVType->getPrimitiveSizeInBits() <= PointerSize * 8 &&
540  "A GlobalVariable with size larger than 32 bits is not currently "
541  "supported by the toc data transformation.");
542 
543  if (GV->hasLocalLinkage() || GV->hasPrivateLinkage())
544  report_fatal_error("A GlobalVariable with private or local linkage is not "
545  "currently supported by the toc data transformation.");
546 
547  assert(!GV->hasCommonLinkage() &&
548  "Tentative definitions cannot have the mapping class XMC_TD.");
549 
550  return true;
551 }
552 
553 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
554 /// operand. If so Imm will receive the 32-bit value.
555 static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
556  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
557  Imm = cast<ConstantSDNode>(N)->getZExtValue();
558  return true;
559  }
560  return false;
561 }
562 
563 /// isInt64Immediate - This method tests to see if the node is a 64-bit constant
564 /// operand. If so Imm will receive the 64-bit value.
565 static bool isInt64Immediate(SDNode *N, uint64_t &Imm) {
566  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) {
567  Imm = cast<ConstantSDNode>(N)->getZExtValue();
568  return true;
569  }
570  return false;
571 }
572 
573 // isInt32Immediate - This method tests to see if a constant operand.
574 // If so Imm will receive the 32 bit value.
575 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
576  return isInt32Immediate(N.getNode(), Imm);
577 }
578 
579 /// isInt64Immediate - This method tests to see if the value is a 64-bit
580 /// constant operand. If so Imm will receive the 64-bit value.
581 static bool isInt64Immediate(SDValue N, uint64_t &Imm) {
582  return isInt64Immediate(N.getNode(), Imm);
583 }
584 
585 static unsigned getBranchHint(unsigned PCC,
586  const FunctionLoweringInfo &FuncInfo,
587  const SDValue &DestMBB) {
588  assert(isa<BasicBlockSDNode>(DestMBB));
589 
590  if (!FuncInfo.BPI) return PPC::BR_NO_HINT;
591 
592  const BasicBlock *BB = FuncInfo.MBB->getBasicBlock();
593  const Instruction *BBTerm = BB->getTerminator();
594 
595  if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT;
596 
597  const BasicBlock *TBB = BBTerm->getSuccessor(0);
598  const BasicBlock *FBB = BBTerm->getSuccessor(1);
599 
600  auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB);
601  auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB);
602 
603  // We only want to handle cases which are easy to predict at static time, e.g.
604  // C++ throw statement, that is very likely not taken, or calling never
605  // returned function, e.g. stdlib exit(). So we set Threshold to filter
606  // unwanted cases.
607  //
608  // Below is LLVM branch weight table, we only want to handle case 1, 2
609  //
610  // Case Taken:Nontaken Example
611  // 1. Unreachable 1048575:1 C++ throw, stdlib exit(),
612  // 2. Invoke-terminating 1:1048575
613  // 3. Coldblock 4:64 __builtin_expect
614  // 4. Loop Branch 124:4 For loop
615  // 5. PH/ZH/FPH 20:12
616  const uint32_t Threshold = 10000;
617 
618  if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb))
619  return PPC::BR_NO_HINT;
620 
621  LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName()
622  << "::" << BB->getName() << "'\n"
623  << " -> " << TBB->getName() << ": " << TProb << "\n"
624  << " -> " << FBB->getName() << ": " << FProb << "\n");
625 
626  const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB);
627 
628  // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities,
629  // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock
630  if (BBDN->getBasicBlock()->getBasicBlock() != TBB)
631  std::swap(TProb, FProb);
632 
633  return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT;
634 }
635 
636 // isOpcWithIntImmediate - This method tests to see if the node is a specific
637 // opcode and that it has a immediate integer right operand.
638 // If so Imm will receive the 32 bit value.
639 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
640  return N->getOpcode() == Opc
641  && isInt32Immediate(N->getOperand(1).getNode(), Imm);
642 }
643 
644 void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, unsigned Offset) {
645  SDLoc dl(SN);
646  int FI = cast<FrameIndexSDNode>(N)->getIndex();
647  SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0));
648  unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8;
649  if (SN->hasOneUse())
650  CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI,
651  getSmallIPtrImm(Offset, dl));
652  else
653  ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI,
654  getSmallIPtrImm(Offset, dl)));
655 }
656 
657 bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask,
658  bool isShiftMask, unsigned &SH,
659  unsigned &MB, unsigned &ME) {
660  // Don't even go down this path for i64, since different logic will be
661  // necessary for rldicl/rldicr/rldimi.
662  if (N->getValueType(0) != MVT::i32)
663  return false;
664 
665  unsigned Shift = 32;
666  unsigned Indeterminant = ~0; // bit mask marking indeterminant results
667  unsigned Opcode = N->getOpcode();
668  if (N->getNumOperands() != 2 ||
669  !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31))
670  return false;
671 
672  if (Opcode == ISD::SHL) {
673  // apply shift left to mask if it comes first
674  if (isShiftMask) Mask = Mask << Shift;
675  // determine which bits are made indeterminant by shift
676  Indeterminant = ~(0xFFFFFFFFu << Shift);
677  } else if (Opcode == ISD::SRL) {
678  // apply shift right to mask if it comes first
679  if (isShiftMask) Mask = Mask >> Shift;
680  // determine which bits are made indeterminant by shift
681  Indeterminant = ~(0xFFFFFFFFu >> Shift);
682  // adjust for the left rotate
683  Shift = 32 - Shift;
684  } else if (Opcode == ISD::ROTL) {
685  Indeterminant = 0;
686  } else {
687  return false;
688  }
689 
690  // if the mask doesn't intersect any Indeterminant bits
691  if (Mask && !(Mask & Indeterminant)) {
692  SH = Shift & 31;
693  // make sure the mask is still a mask (wrap arounds may not be)
694  return isRunOfOnes(Mask, MB, ME);
695  }
696  return false;
697 }
698 
699 bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) {
700  SDValue Base = ST->getBasePtr();
701  if (Base.getOpcode() != PPCISD::ADD_TLS)
702  return false;
703  SDValue Offset = ST->getOffset();
704  if (!Offset.isUndef())
705  return false;
706  if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR)
707  return false;
708 
709  SDLoc dl(ST);
710  EVT MemVT = ST->getMemoryVT();
711  EVT RegVT = ST->getValue().getValueType();
712 
713  unsigned Opcode;
714  switch (MemVT.getSimpleVT().SimpleTy) {
715  default:
716  return false;
717  case MVT::i8: {
718  Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS;
719  break;
720  }
721  case MVT::i16: {
722  Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS;
723  break;
724  }
725  case MVT::i32: {
726  Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS;
727  break;
728  }
729  case MVT::i64: {
730  Opcode = PPC::STDXTLS;
731  break;
732  }
733  }
734  SDValue Chain = ST->getChain();
735  SDVTList VTs = ST->getVTList();
736  SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1),
737  Chain};
738  SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
739  transferMemOperands(ST, MN);
740  ReplaceNode(ST, MN);
741  return true;
742 }
743 
744 bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) {
745  SDValue Base = LD->getBasePtr();
746  if (Base.getOpcode() != PPCISD::ADD_TLS)
747  return false;
748  SDValue Offset = LD->getOffset();
749  if (!Offset.isUndef())
750  return false;
751  if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR)
752  return false;
753 
754  SDLoc dl(LD);
755  EVT MemVT = LD->getMemoryVT();
756  EVT RegVT = LD->getValueType(0);
757  unsigned Opcode;
758  switch (MemVT.getSimpleVT().SimpleTy) {
759  default:
760  return false;
761  case MVT::i8: {
762  Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS;
763  break;
764  }
765  case MVT::i16: {
766  Opcode = (RegVT == MVT::i32) ? PPC::LHZXTLS_32 : PPC::LHZXTLS;
767  break;
768  }
769  case MVT::i32: {
770  Opcode = (RegVT == MVT::i32) ? PPC::LWZXTLS_32 : PPC::LWZXTLS;
771  break;
772  }
773  case MVT::i64: {
774  Opcode = PPC::LDXTLS;
775  break;
776  }
777  }
778  SDValue Chain = LD->getChain();
779  SDVTList VTs = LD->getVTList();
780  SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain};
781  SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
782  transferMemOperands(LD, MN);
783  ReplaceNode(LD, MN);
784  return true;
785 }
786 
787 /// Turn an or of two masked values into the rotate left word immediate then
788 /// mask insert (rlwimi) instruction.
789 bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) {
790  SDValue Op0 = N->getOperand(0);
791  SDValue Op1 = N->getOperand(1);
792  SDLoc dl(N);
793 
794  KnownBits LKnown = CurDAG->computeKnownBits(Op0);
795  KnownBits RKnown = CurDAG->computeKnownBits(Op1);
796 
797  unsigned TargetMask = LKnown.Zero.getZExtValue();
798  unsigned InsertMask = RKnown.Zero.getZExtValue();
799 
800  if ((TargetMask | InsertMask) == 0xFFFFFFFF) {
801  unsigned Op0Opc = Op0.getOpcode();
802  unsigned Op1Opc = Op1.getOpcode();
803  unsigned Value, SH = 0;
804  TargetMask = ~TargetMask;
805  InsertMask = ~InsertMask;
806 
807  // If the LHS has a foldable shift and the RHS does not, then swap it to the
808  // RHS so that we can fold the shift into the insert.
809  if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) {
810  if (Op0.getOperand(0).getOpcode() == ISD::SHL ||
811  Op0.getOperand(0).getOpcode() == ISD::SRL) {
812  if (Op1.getOperand(0).getOpcode() != ISD::SHL &&
813  Op1.getOperand(0).getOpcode() != ISD::SRL) {
814  std::swap(Op0, Op1);
815  std::swap(Op0Opc, Op1Opc);
816  std::swap(TargetMask, InsertMask);
817  }
818  }
819  } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) {
820  if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL &&
821  Op1.getOperand(0).getOpcode() != ISD::SRL) {
822  std::swap(Op0, Op1);
823  std::swap(Op0Opc, Op1Opc);
824  std::swap(TargetMask, InsertMask);
825  }
826  }
827 
828  unsigned MB, ME;
829  if (isRunOfOnes(InsertMask, MB, ME)) {
830  if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) &&
831  isInt32Immediate(Op1.getOperand(1), Value)) {
832  Op1 = Op1.getOperand(0);
833  SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value;
834  }
835  if (Op1Opc == ISD::AND) {
836  // The AND mask might not be a constant, and we need to make sure that
837  // if we're going to fold the masking with the insert, all bits not
838  // know to be zero in the mask are known to be one.
839  KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1));
840  bool CanFoldMask = InsertMask == MKnown.One.getZExtValue();
841 
842  unsigned SHOpc = Op1.getOperand(0).getOpcode();
843  if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask &&
845  // Note that Value must be in range here (less than 32) because
846  // otherwise there would not be any bits set in InsertMask.
847  Op1 = Op1.getOperand(0).getOperand(0);
848  SH = (SHOpc == ISD::SHL) ? Value : 32 - Value;
849  }
850  }
851 
852  SH &= 31;
853  SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl),
854  getI32Imm(ME, dl) };
855  ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops));
856  return true;
857  }
858  }
859  return false;
860 }
861 
862 static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) {
863  unsigned MaxTruncation = 0;
864  // Cannot use range-based for loop here as we need the actual use (i.e. we
865  // need the operand number corresponding to the use). A range-based for
866  // will unbox the use and provide an SDNode*.
867  for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end();
868  Use != UseEnd; ++Use) {
869  unsigned Opc =
870  Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode();
871  switch (Opc) {
872  default: return 0;
873  case ISD::TRUNCATE:
874  if (Use->isMachineOpcode())
875  return 0;
876  MaxTruncation =
877  std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits());
878  continue;
879  case ISD::STORE: {
880  if (Use->isMachineOpcode())
881  return 0;
882  StoreSDNode *STN = cast<StoreSDNode>(*Use);
883  unsigned MemVTSize = STN->getMemoryVT().getSizeInBits();
884  if (MemVTSize == 64 || Use.getOperandNo() != 0)
885  return 0;
886  MaxTruncation = std::max(MaxTruncation, MemVTSize);
887  continue;
888  }
889  case PPC::STW8:
890  case PPC::STWX8:
891  case PPC::STWU8:
892  case PPC::STWUX8:
893  if (Use.getOperandNo() != 0)
894  return 0;
895  MaxTruncation = std::max(MaxTruncation, 32u);
896  continue;
897  case PPC::STH8:
898  case PPC::STHX8:
899  case PPC::STHU8:
900  case PPC::STHUX8:
901  if (Use.getOperandNo() != 0)
902  return 0;
903  MaxTruncation = std::max(MaxTruncation, 16u);
904  continue;
905  case PPC::STB8:
906  case PPC::STBX8:
907  case PPC::STBU8:
908  case PPC::STBUX8:
909  if (Use.getOperandNo() != 0)
910  return 0;
911  MaxTruncation = std::max(MaxTruncation, 8u);
912  continue;
913  }
914  }
915  return MaxTruncation;
916 }
917 
918 // For any 32 < Num < 64, check if the Imm contains at least Num consecutive
919 // zeros and return the number of bits by the left of these consecutive zeros.
920 static int findContiguousZerosAtLeast(uint64_t Imm, unsigned Num) {
921  unsigned HiTZ = countTrailingZeros<uint32_t>(Hi_32(Imm));
922  unsigned LoLZ = countLeadingZeros<uint32_t>(Lo_32(Imm));
923  if ((HiTZ + LoLZ) >= Num)
924  return (32 + HiTZ);
925  return 0;
926 }
927 
928 // Direct materialization of 64-bit constants by enumerated patterns.
929 static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl,
930  uint64_t Imm, unsigned &InstCnt) {
931  unsigned TZ = countTrailingZeros<uint64_t>(Imm);
932  unsigned LZ = countLeadingZeros<uint64_t>(Imm);
933  unsigned TO = countTrailingOnes<uint64_t>(Imm);
934  unsigned LO = countLeadingOnes<uint64_t>(Imm);
935  unsigned Hi32 = Hi_32(Imm);
936  unsigned Lo32 = Lo_32(Imm);
937  SDNode *Result = nullptr;
938  unsigned Shift = 0;
939 
940  auto getI32Imm = [CurDAG, dl](unsigned Imm) {
941  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
942  };
943 
944  // Following patterns use 1 instructions to materialize the Imm.
945  InstCnt = 1;
946  // 1-1) Patterns : {zeros}{15-bit valve}
947  // {ones}{15-bit valve}
948  if (isInt<16>(Imm)) {
949  SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64);
950  return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
951  }
952  // 1-2) Patterns : {zeros}{15-bit valve}{16 zeros}
953  // {ones}{15-bit valve}{16 zeros}
954  if (TZ > 15 && (LZ > 32 || LO > 32))
955  return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
956  getI32Imm((Imm >> 16) & 0xffff));
957 
958  // Following patterns use 2 instructions to materialize the Imm.
959  InstCnt = 2;
960  assert(LZ < 64 && "Unexpected leading zeros here.");
961  // Count of ones follwing the leading zeros.
962  unsigned FO = countLeadingOnes<uint64_t>(Imm << LZ);
963  // 2-1) Patterns : {zeros}{31-bit value}
964  // {ones}{31-bit value}
965  if (isInt<32>(Imm)) {
966  uint64_t ImmHi16 = (Imm >> 16) & 0xffff;
967  unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
968  Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
969  return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
970  getI32Imm(Imm & 0xffff));
971  }
972  // 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros}
973  // {zeros}{15-bit value}{zeros}
974  // {zeros}{ones}{15-bit value}
975  // {ones}{15-bit value}{zeros}
976  // We can take advantage of LI's sign-extension semantics to generate leading
977  // ones, and then use RLDIC to mask off the ones in both sides after rotation.
978  if ((LZ + FO + TZ) > 48) {
979  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
980  getI32Imm((Imm >> TZ) & 0xffff));
981  return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
982  getI32Imm(TZ), getI32Imm(LZ));
983  }
984  // 2-3) Pattern : {zeros}{15-bit value}{ones}
985  // Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value,
986  // therefore we can take advantage of LI's sign-extension semantics, and then
987  // mask them off after rotation.
988  //
989  // +--LZ--||-15-bit-||--TO--+ +-------------|--16-bit--+
990  // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1|
991  // +------------------------+ +------------------------+
992  // 63 0 63 0
993  // Imm (Imm >> (48 - LZ) & 0xffff)
994  // +----sext-----|--16-bit--+ +clear-|-----------------+
995  // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111|
996  // +------------------------+ +------------------------+
997  // 63 0 63 0
998  // LI8: sext many leading zeros RLDICL: rotate left (48 - LZ), clear left LZ
999  if ((LZ + TO) > 48) {
1000  // Since the immediates with (LZ > 32) have been handled by previous
1001  // patterns, here we have (LZ <= 32) to make sure we will not shift right
1002  // the Imm by a negative value.
1003  assert(LZ <= 32 && "Unexpected shift value.");
1004  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1005  getI32Imm((Imm >> (48 - LZ) & 0xffff)));
1006  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1007  getI32Imm(48 - LZ), getI32Imm(LZ));
1008  }
1009  // 2-4) Patterns : {zeros}{ones}{15-bit value}{ones}
1010  // {ones}{15-bit value}{ones}
1011  // We can take advantage of LI's sign-extension semantics to generate leading
1012  // ones, and then use RLDICL to mask off the ones in left sides (if required)
1013  // after rotation.
1014  //
1015  // +-LZ-FO||-15-bit-||--TO--+ +-------------|--16-bit--+
1016  // |00011110bbbbbbbbb1111111| -> |000000000011110bbbbbbbbb|
1017  // +------------------------+ +------------------------+
1018  // 63 0 63 0
1019  // Imm (Imm >> TO) & 0xffff
1020  // +----sext-----|--16-bit--+ +LZ|---------------------+
1021  // |111111111111110bbbbbbbbb| -> |00011110bbbbbbbbb1111111|
1022  // +------------------------+ +------------------------+
1023  // 63 0 63 0
1024  // LI8: sext many leading zeros RLDICL: rotate left TO, clear left LZ
1025  if ((LZ + FO + TO) > 48) {
1026  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1027  getI32Imm((Imm >> TO) & 0xffff));
1028  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1029  getI32Imm(TO), getI32Imm(LZ));
1030  }
1031  // 2-5) Pattern : {32 zeros}{****}{0}{15-bit value}
1032  // If Hi32 is zero and the Lo16(in Lo32) can be presented as a positive 16 bit
1033  // value, we can use LI for Lo16 without generating leading ones then add the
1034  // Hi16(in Lo32).
1035  if (LZ == 32 && ((Lo32 & 0x8000) == 0)) {
1036  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1037  getI32Imm(Lo32 & 0xffff));
1038  return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0),
1039  getI32Imm(Lo32 >> 16));
1040  }
1041  // 2-6) Patterns : {******}{49 zeros}{******}
1042  // {******}{49 ones}{******}
1043  // If the Imm contains 49 consecutive zeros/ones, it means that a total of 15
1044  // bits remain on both sides. Rotate right the Imm to construct an int<16>
1045  // value, use LI for int<16> value and then use RLDICL without mask to rotate
1046  // it back.
1047  //
1048  // 1) findContiguousZerosAtLeast(Imm, 49)
1049  // +------|--zeros-|------+ +---ones--||---15 bit--+
1050  // |bbbbbb0000000000aaaaaa| -> |0000000000aaaaaabbbbbb|
1051  // +----------------------+ +----------------------+
1052  // 63 0 63 0
1053  //
1054  // 2) findContiguousZerosAtLeast(~Imm, 49)
1055  // +------|--ones--|------+ +---ones--||---15 bit--+
1056  // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb|
1057  // +----------------------+ +----------------------+
1058  // 63 0 63 0
1059  if ((Shift = findContiguousZerosAtLeast(Imm, 49)) ||
1060  (Shift = findContiguousZerosAtLeast(~Imm, 49))) {
1061  uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1062  Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1063  getI32Imm(RotImm & 0xffff));
1064  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1065  getI32Imm(Shift), getI32Imm(0));
1066  }
1067 
1068  // Following patterns use 3 instructions to materialize the Imm.
1069  InstCnt = 3;
1070  // 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros}
1071  // {zeros}{31-bit value}{zeros}
1072  // {zeros}{ones}{31-bit value}
1073  // {ones}{31-bit value}{zeros}
1074  // We can take advantage of LIS's sign-extension semantics to generate leading
1075  // ones, add the remaining bits with ORI, and then use RLDIC to mask off the
1076  // ones in both sides after rotation.
1077  if ((LZ + FO + TZ) > 32) {
1078  uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff;
1079  unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1080  Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1081  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1082  getI32Imm((Imm >> TZ) & 0xffff));
1083  return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
1084  getI32Imm(TZ), getI32Imm(LZ));
1085  }
1086  // 3-2) Pattern : {zeros}{31-bit value}{ones}
1087  // Shift right the Imm by (32 - LZ) bits to construct a negtive 32 bits value,
1088  // therefore we can take advantage of LIS's sign-extension semantics, add
1089  // the remaining bits with ORI, and then mask them off after rotation.
1090  // This is similar to Pattern 2-3, please refer to the diagram there.
1091  if ((LZ + TO) > 32) {
1092  // Since the immediates with (LZ > 32) have been handled by previous
1093  // patterns, here we have (LZ <= 32) to make sure we will not shift right
1094  // the Imm by a negative value.
1095  assert(LZ <= 32 && "Unexpected shift value.");
1096  Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
1097  getI32Imm((Imm >> (48 - LZ)) & 0xffff));
1098  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1099  getI32Imm((Imm >> (32 - LZ)) & 0xffff));
1100  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1101  getI32Imm(32 - LZ), getI32Imm(LZ));
1102  }
1103  // 3-3) Patterns : {zeros}{ones}{31-bit value}{ones}
1104  // {ones}{31-bit value}{ones}
1105  // We can take advantage of LIS's sign-extension semantics to generate leading
1106  // ones, add the remaining bits with ORI, and then use RLDICL to mask off the
1107  // ones in left sides (if required) after rotation.
1108  // This is similar to Pattern 2-4, please refer to the diagram there.
1109  if ((LZ + FO + TO) > 32) {
1110  Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
1111  getI32Imm((Imm >> (TO + 16)) & 0xffff));
1112  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1113  getI32Imm((Imm >> TO) & 0xffff));
1114  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1115  getI32Imm(TO), getI32Imm(LZ));
1116  }
1117  // 3-4) Patterns : High word == Low word
1118  if (Hi32 == Lo32) {
1119  // Handle the first 32 bits.
1120  uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff;
1121  unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1122  Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1123  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1124  getI32Imm(Lo32 & 0xffff));
1125  // Use rldimi to insert the Low word into High word.
1126  SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32),
1127  getI32Imm(0)};
1128  return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1129  }
1130  // 3-5) Patterns : {******}{33 zeros}{******}
1131  // {******}{33 ones}{******}
1132  // If the Imm contains 33 consecutive zeros/ones, it means that a total of 31
1133  // bits remain on both sides. Rotate right the Imm to construct an int<32>
1134  // value, use LIS + ORI for int<32> value and then use RLDICL without mask to
1135  // rotate it back.
1136  // This is similar to Pattern 2-6, please refer to the diagram there.
1137  if ((Shift = findContiguousZerosAtLeast(Imm, 33)) ||
1138  (Shift = findContiguousZerosAtLeast(~Imm, 33))) {
1139  uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1140  uint64_t ImmHi16 = (RotImm >> 16) & 0xffff;
1141  unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1142  Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1143  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1144  getI32Imm(RotImm & 0xffff));
1145  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1146  getI32Imm(Shift), getI32Imm(0));
1147  }
1148 
1149  InstCnt = 0;
1150  return nullptr;
1151 }
1152 
1153 // Try to select instructions to generate a 64 bit immediate using prefix as
1154 // well as non prefix instructions. The function will return the SDNode
1155 // to materialize that constant or it will return nullptr if it does not
1156 // find one. The variable InstCnt is set to the number of instructions that
1157 // were selected.
1159  uint64_t Imm, unsigned &InstCnt) {
1160  unsigned TZ = countTrailingZeros<uint64_t>(Imm);
1161  unsigned LZ = countLeadingZeros<uint64_t>(Imm);
1162  unsigned TO = countTrailingOnes<uint64_t>(Imm);
1163  unsigned FO = countLeadingOnes<uint64_t>(LZ == 64 ? 0 : (Imm << LZ));
1164  unsigned Hi32 = Hi_32(Imm);
1165  unsigned Lo32 = Lo_32(Imm);
1166 
1167  auto getI32Imm = [CurDAG, dl](unsigned Imm) {
1168  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1169  };
1170 
1171  auto getI64Imm = [CurDAG, dl](uint64_t Imm) {
1172  return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
1173  };
1174 
1175  // Following patterns use 1 instruction to materialize Imm.
1176  InstCnt = 1;
1177 
1178  // The pli instruction can materialize up to 34 bits directly.
1179  // If a constant fits within 34-bits, emit the pli instruction here directly.
1180  if (isInt<34>(Imm))
1181  return CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1182  CurDAG->getTargetConstant(Imm, dl, MVT::i64));
1183 
1184  // Require at least two instructions.
1185  InstCnt = 2;
1186  SDNode *Result = nullptr;
1187  // Patterns : {zeros}{ones}{33-bit value}{zeros}
1188  // {zeros}{33-bit value}{zeros}
1189  // {zeros}{ones}{33-bit value}
1190  // {ones}{33-bit value}{zeros}
1191  // We can take advantage of PLI's sign-extension semantics to generate leading
1192  // ones, and then use RLDIC to mask off the ones on both sides after rotation.
1193  if ((LZ + FO + TZ) > 30) {
1194  APInt SignedInt34 = APInt(34, (Imm >> TZ) & 0x3ffffffff);
1195  APInt Extended = SignedInt34.sext(64);
1196  Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1197  getI64Imm(*Extended.getRawData()));
1198  return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
1199  getI32Imm(TZ), getI32Imm(LZ));
1200  }
1201  // Pattern : {zeros}{33-bit value}{ones}
1202  // Shift right the Imm by (30 - LZ) bits to construct a negative 34 bit value,
1203  // therefore we can take advantage of PLI's sign-extension semantics, and then
1204  // mask them off after rotation.
1205  //
1206  // +--LZ--||-33-bit-||--TO--+ +-------------|--34-bit--+
1207  // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1|
1208  // +------------------------+ +------------------------+
1209  // 63 0 63 0
1210  //
1211  // +----sext-----|--34-bit--+ +clear-|-----------------+
1212  // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111|
1213  // +------------------------+ +------------------------+
1214  // 63 0 63 0
1215  if ((LZ + TO) > 30) {
1216  APInt SignedInt34 = APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff);
1217  APInt Extended = SignedInt34.sext(64);
1218  Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1219  getI64Imm(*Extended.getRawData()));
1220  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1221  getI32Imm(30 - LZ), getI32Imm(LZ));
1222  }
1223  // Patterns : {zeros}{ones}{33-bit value}{ones}
1224  // {ones}{33-bit value}{ones}
1225  // Similar to LI we can take advantage of PLI's sign-extension semantics to
1226  // generate leading ones, and then use RLDICL to mask off the ones in left
1227  // sides (if required) after rotation.
1228  if ((LZ + FO + TO) > 30) {
1229  APInt SignedInt34 = APInt(34, (Imm >> TO) & 0x3ffffffff);
1230  APInt Extended = SignedInt34.sext(64);
1231  Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1232  getI64Imm(*Extended.getRawData()));
1233  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1234  getI32Imm(TO), getI32Imm(LZ));
1235  }
1236  // Patterns : {******}{31 zeros}{******}
1237  // : {******}{31 ones}{******}
1238  // If Imm contains 31 consecutive zeros/ones then the remaining bit count
1239  // is 33. Rotate right the Imm to construct a int<33> value, we can use PLI
1240  // for the int<33> value and then use RLDICL without a mask to rotate it back.
1241  //
1242  // +------|--ones--|------+ +---ones--||---33 bit--+
1243  // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb|
1244  // +----------------------+ +----------------------+
1245  // 63 0 63 0
1246  for (unsigned Shift = 0; Shift < 63; ++Shift) {
1247  uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1248  if (isInt<34>(RotImm)) {
1249  Result =
1250  CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(RotImm));
1251  return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
1252  SDValue(Result, 0), getI32Imm(Shift),
1253  getI32Imm(0));
1254  }
1255  }
1256 
1257  // Patterns : High word == Low word
1258  // This is basically a splat of a 32 bit immediate.
1259  if (Hi32 == Lo32) {
1260  Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32));
1261  SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32),
1262  getI32Imm(0)};
1263  return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1264  }
1265 
1266  InstCnt = 3;
1267  // Catch-all
1268  // This pattern can form any 64 bit immediate in 3 instructions.
1269  SDNode *ResultHi =
1270  CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32));
1271  SDNode *ResultLo =
1272  CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Lo32));
1273  SDValue Ops[] = {SDValue(ResultLo, 0), SDValue(ResultHi, 0), getI32Imm(32),
1274  getI32Imm(0)};
1275  return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1276 }
1277 
1278 static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm,
1279  unsigned *InstCnt = nullptr) {
1280  unsigned InstCntDirect = 0;
1281  // No more than 3 instructions is used if we can select the i64 immediate
1282  // directly.
1283  SDNode *Result = selectI64ImmDirect(CurDAG, dl, Imm, InstCntDirect);
1284 
1285  const PPCSubtarget &Subtarget =
1287 
1288  // If we have prefixed instructions and there is a chance we can
1289  // materialize the constant with fewer prefixed instructions than
1290  // non-prefixed, try that.
1291  if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) {
1292  unsigned InstCntDirectP = 0;
1293  SDNode *ResultP = selectI64ImmDirectPrefix(CurDAG, dl, Imm, InstCntDirectP);
1294  // Use the prefix case in either of two cases:
1295  // 1) We have no result from the non-prefix case to use.
1296  // 2) The non-prefix case uses more instructions than the prefix case.
1297  // If the prefix and non-prefix cases use the same number of instructions
1298  // we will prefer the non-prefix case.
1299  if (ResultP && (!Result || InstCntDirectP < InstCntDirect)) {
1300  if (InstCnt)
1301  *InstCnt = InstCntDirectP;
1302  return ResultP;
1303  }
1304  }
1305 
1306  if (Result) {
1307  if (InstCnt)
1308  *InstCnt = InstCntDirect;
1309  return Result;
1310  }
1311  auto getI32Imm = [CurDAG, dl](unsigned Imm) {
1312  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1313  };
1314  // Handle the upper 32 bit value.
1315  Result =
1316  selectI64ImmDirect(CurDAG, dl, Imm & 0xffffffff00000000, InstCntDirect);
1317  // Add in the last bits as required.
1318  if (uint32_t Hi16 = (Lo_32(Imm) >> 16) & 0xffff) {
1319  Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64,
1320  SDValue(Result, 0), getI32Imm(Hi16));
1321  ++InstCntDirect;
1322  }
1323  if (uint32_t Lo16 = Lo_32(Imm) & 0xffff) {
1324  Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1325  getI32Imm(Lo16));
1326  ++InstCntDirect;
1327  }
1328  if (InstCnt)
1329  *InstCnt = InstCntDirect;
1330  return Result;
1331 }
1332 
1333 // Select a 64-bit constant.
1335  SDLoc dl(N);
1336 
1337  // Get 64 bit value.
1338  int64_t Imm = cast<ConstantSDNode>(N)->getZExtValue();
1339  if (unsigned MinSize = allUsesTruncate(CurDAG, N)) {
1340  uint64_t SextImm = SignExtend64(Imm, MinSize);
1341  SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
1342  if (isInt<16>(SextImm))
1343  return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
1344  }
1345  return selectI64Imm(CurDAG, dl, Imm);
1346 }
1347 
1348 namespace {
1349 
1350 class BitPermutationSelector {
1351  struct ValueBit {
1352  SDValue V;
1353 
1354  // The bit number in the value, using a convention where bit 0 is the
1355  // lowest-order bit.
1356  unsigned Idx;
1357 
1358  // ConstZero means a bit we need to mask off.
1359  // Variable is a bit comes from an input variable.
1360  // VariableKnownToBeZero is also a bit comes from an input variable,
1361  // but it is known to be already zero. So we do not need to mask them.
1362  enum Kind {
1363  ConstZero,
1364  Variable,
1365  VariableKnownToBeZero
1366  } K;
1367 
1368  ValueBit(SDValue V, unsigned I, Kind K = Variable)
1369  : V(V), Idx(I), K(K) {}
1370  ValueBit(Kind K = Variable)
1371  : V(SDValue(nullptr, 0)), Idx(UINT32_MAX), K(K) {}
1372 
1373  bool isZero() const {
1374  return K == ConstZero || K == VariableKnownToBeZero;
1375  }
1376 
1377  bool hasValue() const {
1378  return K == Variable || K == VariableKnownToBeZero;
1379  }
1380 
1381  SDValue getValue() const {
1382  assert(hasValue() && "Cannot get the value of a constant bit");
1383  return V;
1384  }
1385 
1386  unsigned getValueBitIndex() const {
1387  assert(hasValue() && "Cannot get the value bit index of a constant bit");
1388  return Idx;
1389  }
1390  };
1391 
1392  // A bit group has the same underlying value and the same rotate factor.
1393  struct BitGroup {
1394  SDValue V;
1395  unsigned RLAmt;
1396  unsigned StartIdx, EndIdx;
1397 
1398  // This rotation amount assumes that the lower 32 bits of the quantity are
1399  // replicated in the high 32 bits by the rotation operator (which is done
1400  // by rlwinm and friends in 64-bit mode).
1401  bool Repl32;
1402  // Did converting to Repl32 == true change the rotation factor? If it did,
1403  // it decreased it by 32.
1404  bool Repl32CR;
1405  // Was this group coalesced after setting Repl32 to true?
1406  bool Repl32Coalesced;
1407 
1408  BitGroup(SDValue V, unsigned R, unsigned S, unsigned E)
1409  : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false),
1410  Repl32Coalesced(false) {
1411  LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R
1412  << " [" << S << ", " << E << "]\n");
1413  }
1414  };
1415 
1416  // Information on each (Value, RLAmt) pair (like the number of groups
1417  // associated with each) used to choose the lowering method.
1418  struct ValueRotInfo {
1419  SDValue V;
1420  unsigned RLAmt = std::numeric_limits<unsigned>::max();
1421  unsigned NumGroups = 0;
1422  unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max();
1423  bool Repl32 = false;
1424 
1425  ValueRotInfo() = default;
1426 
1427  // For sorting (in reverse order) by NumGroups, and then by
1428  // FirstGroupStartIdx.
1429  bool operator < (const ValueRotInfo &Other) const {
1430  // We need to sort so that the non-Repl32 come first because, when we're
1431  // doing masking, the Repl32 bit groups might be subsumed into the 64-bit
1432  // masking operation.
1433  if (Repl32 < Other.Repl32)
1434  return true;
1435  else if (Repl32 > Other.Repl32)
1436  return false;
1437  else if (NumGroups > Other.NumGroups)
1438  return true;
1439  else if (NumGroups < Other.NumGroups)
1440  return false;
1441  else if (RLAmt == 0 && Other.RLAmt != 0)
1442  return true;
1443  else if (RLAmt != 0 && Other.RLAmt == 0)
1444  return false;
1445  else if (FirstGroupStartIdx < Other.FirstGroupStartIdx)
1446  return true;
1447  return false;
1448  }
1449  };
1450 
1451  using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>;
1452  using ValueBitsMemoizer =
1454  ValueBitsMemoizer Memoizer;
1455 
1456  // Return a pair of bool and a SmallVector pointer to a memoization entry.
1457  // The bool is true if something interesting was deduced, otherwise if we're
1458  // providing only a generic representation of V (or something else likewise
1459  // uninteresting for instruction selection) through the SmallVector.
1460  std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(SDValue V,
1461  unsigned NumBits) {
1462  auto &ValueEntry = Memoizer[V];
1463  if (ValueEntry)
1464  return std::make_pair(ValueEntry->first, &ValueEntry->second);
1465  ValueEntry.reset(new ValueBitsMemoizedValue());
1466  bool &Interesting = ValueEntry->first;
1467  SmallVector<ValueBit, 64> &Bits = ValueEntry->second;
1468  Bits.resize(NumBits);
1469 
1470  switch (V.getOpcode()) {
1471  default: break;
1472  case ISD::ROTL:
1473  if (isa<ConstantSDNode>(V.getOperand(1))) {
1474  unsigned RotAmt = V.getConstantOperandVal(1);
1475 
1476  const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1477 
1478  for (unsigned i = 0; i < NumBits; ++i)
1479  Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt];
1480 
1481  return std::make_pair(Interesting = true, &Bits);
1482  }
1483  break;
1484  case ISD::SHL:
1485  case PPCISD::SHL:
1486  if (isa<ConstantSDNode>(V.getOperand(1))) {
1487  unsigned ShiftAmt = V.getConstantOperandVal(1);
1488 
1489  const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1490 
1491  for (unsigned i = ShiftAmt; i < NumBits; ++i)
1492  Bits[i] = LHSBits[i - ShiftAmt];
1493 
1494  for (unsigned i = 0; i < ShiftAmt; ++i)
1495  Bits[i] = ValueBit(ValueBit::ConstZero);
1496 
1497  return std::make_pair(Interesting = true, &Bits);
1498  }
1499  break;
1500  case ISD::SRL:
1501  case PPCISD::SRL:
1502  if (isa<ConstantSDNode>(V.getOperand(1))) {
1503  unsigned ShiftAmt = V.getConstantOperandVal(1);
1504 
1505  const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1506 
1507  for (unsigned i = 0; i < NumBits - ShiftAmt; ++i)
1508  Bits[i] = LHSBits[i + ShiftAmt];
1509 
1510  for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i)
1511  Bits[i] = ValueBit(ValueBit::ConstZero);
1512 
1513  return std::make_pair(Interesting = true, &Bits);
1514  }
1515  break;
1516  case ISD::AND:
1517  if (isa<ConstantSDNode>(V.getOperand(1))) {
1519 
1520  const SmallVector<ValueBit, 64> *LHSBits;
1521  // Mark this as interesting, only if the LHS was also interesting. This
1522  // prevents the overall procedure from matching a single immediate 'and'
1523  // (which is non-optimal because such an and might be folded with other
1524  // things if we don't select it here).
1525  std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits);
1526 
1527  for (unsigned i = 0; i < NumBits; ++i)
1528  if (((Mask >> i) & 1) == 1)
1529  Bits[i] = (*LHSBits)[i];
1530  else {
1531  // AND instruction masks this bit. If the input is already zero,
1532  // we have nothing to do here. Otherwise, make the bit ConstZero.
1533  if ((*LHSBits)[i].isZero())
1534  Bits[i] = (*LHSBits)[i];
1535  else
1536  Bits[i] = ValueBit(ValueBit::ConstZero);
1537  }
1538 
1539  return std::make_pair(Interesting, &Bits);
1540  }
1541  break;
1542  case ISD::OR: {
1543  const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1544  const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second;
1545 
1546  bool AllDisjoint = true;
1547  SDValue LastVal = SDValue();
1548  unsigned LastIdx = 0;
1549  for (unsigned i = 0; i < NumBits; ++i) {
1550  if (LHSBits[i].isZero() && RHSBits[i].isZero()) {
1551  // If both inputs are known to be zero and one is ConstZero and
1552  // another is VariableKnownToBeZero, we can select whichever
1553  // we like. To minimize the number of bit groups, we select
1554  // VariableKnownToBeZero if this bit is the next bit of the same
1555  // input variable from the previous bit. Otherwise, we select
1556  // ConstZero.
1557  if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal &&
1558  LHSBits[i].getValueBitIndex() == LastIdx + 1)
1559  Bits[i] = LHSBits[i];
1560  else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal &&
1561  RHSBits[i].getValueBitIndex() == LastIdx + 1)
1562  Bits[i] = RHSBits[i];
1563  else
1564  Bits[i] = ValueBit(ValueBit::ConstZero);
1565  }
1566  else if (LHSBits[i].isZero())
1567  Bits[i] = RHSBits[i];
1568  else if (RHSBits[i].isZero())
1569  Bits[i] = LHSBits[i];
1570  else {
1571  AllDisjoint = false;
1572  break;
1573  }
1574  // We remember the value and bit index of this bit.
1575  if (Bits[i].hasValue()) {
1576  LastVal = Bits[i].getValue();
1577  LastIdx = Bits[i].getValueBitIndex();
1578  }
1579  else {
1580  if (LastVal) LastVal = SDValue();
1581  LastIdx = 0;
1582  }
1583  }
1584 
1585  if (!AllDisjoint)
1586  break;
1587 
1588  return std::make_pair(Interesting = true, &Bits);
1589  }
1590  case ISD::ZERO_EXTEND: {
1591  // We support only the case with zero extension from i32 to i64 so far.
1592  if (V.getValueType() != MVT::i64 ||
1593  V.getOperand(0).getValueType() != MVT::i32)
1594  break;
1595 
1596  const SmallVector<ValueBit, 64> *LHSBits;
1597  const unsigned NumOperandBits = 32;
1598  std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1599  NumOperandBits);
1600 
1601  for (unsigned i = 0; i < NumOperandBits; ++i)
1602  Bits[i] = (*LHSBits)[i];
1603 
1604  for (unsigned i = NumOperandBits; i < NumBits; ++i)
1605  Bits[i] = ValueBit(ValueBit::ConstZero);
1606 
1607  return std::make_pair(Interesting, &Bits);
1608  }
1609  case ISD::TRUNCATE: {
1611  EVT ToType = V.getValueType();
1612  // We support only the case with truncate from i64 to i32.
1613  if (FromType != MVT::i64 || ToType != MVT::i32)
1614  break;
1615  const unsigned NumAllBits = FromType.getSizeInBits();
1616  SmallVector<ValueBit, 64> *InBits;
1617  std::tie(Interesting, InBits) = getValueBits(V.getOperand(0),
1618  NumAllBits);
1619  const unsigned NumValidBits = ToType.getSizeInBits();
1620 
1621  // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value.
1622  // So, we cannot include this truncate.
1623  bool UseUpper32bit = false;
1624  for (unsigned i = 0; i < NumValidBits; ++i)
1625  if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) {
1626  UseUpper32bit = true;
1627  break;
1628  }
1629  if (UseUpper32bit)
1630  break;
1631 
1632  for (unsigned i = 0; i < NumValidBits; ++i)
1633  Bits[i] = (*InBits)[i];
1634 
1635  return std::make_pair(Interesting, &Bits);
1636  }
1637  case ISD::AssertZext: {
1638  // For AssertZext, we look through the operand and
1639  // mark the bits known to be zero.
1640  const SmallVector<ValueBit, 64> *LHSBits;
1641  std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1642  NumBits);
1643 
1644  EVT FromType = cast<VTSDNode>(V.getOperand(1))->getVT();
1645  const unsigned NumValidBits = FromType.getSizeInBits();
1646  for (unsigned i = 0; i < NumValidBits; ++i)
1647  Bits[i] = (*LHSBits)[i];
1648 
1649  // These bits are known to be zero but the AssertZext may be from a value
1650  // that already has some constant zero bits (i.e. from a masking and).
1651  for (unsigned i = NumValidBits; i < NumBits; ++i)
1652  Bits[i] = (*LHSBits)[i].hasValue()
1653  ? ValueBit((*LHSBits)[i].getValue(),
1654  (*LHSBits)[i].getValueBitIndex(),
1655  ValueBit::VariableKnownToBeZero)
1656  : ValueBit(ValueBit::ConstZero);
1657 
1658  return std::make_pair(Interesting, &Bits);
1659  }
1660  case ISD::LOAD:
1661  LoadSDNode *LD = cast<LoadSDNode>(V);
1662  if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) {
1663  EVT VT = LD->getMemoryVT();
1664  const unsigned NumValidBits = VT.getSizeInBits();
1665 
1666  for (unsigned i = 0; i < NumValidBits; ++i)
1667  Bits[i] = ValueBit(V, i);
1668 
1669  // These bits are known to be zero.
1670  for (unsigned i = NumValidBits; i < NumBits; ++i)
1671  Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero);
1672 
1673  // Zero-extending load itself cannot be optimized. So, it is not
1674  // interesting by itself though it gives useful information.
1675  return std::make_pair(Interesting = false, &Bits);
1676  }
1677  break;
1678  }
1679 
1680  for (unsigned i = 0; i < NumBits; ++i)
1681  Bits[i] = ValueBit(V, i);
1682 
1683  return std::make_pair(Interesting = false, &Bits);
1684  }
1685 
1686  // For each value (except the constant ones), compute the left-rotate amount
1687  // to get it from its original to final position.
1688  void computeRotationAmounts() {
1689  NeedMask = false;
1690  RLAmt.resize(Bits.size());
1691  for (unsigned i = 0; i < Bits.size(); ++i)
1692  if (Bits[i].hasValue()) {
1693  unsigned VBI = Bits[i].getValueBitIndex();
1694  if (i >= VBI)
1695  RLAmt[i] = i - VBI;
1696  else
1697  RLAmt[i] = Bits.size() - (VBI - i);
1698  } else if (Bits[i].isZero()) {
1699  NeedMask = true;
1700  RLAmt[i] = UINT32_MAX;
1701  } else {
1702  llvm_unreachable("Unknown value bit type");
1703  }
1704  }
1705 
1706  // Collect groups of consecutive bits with the same underlying value and
1707  // rotation factor. If we're doing late masking, we ignore zeros, otherwise
1708  // they break up groups.
1709  void collectBitGroups(bool LateMask) {
1710  BitGroups.clear();
1711 
1712  unsigned LastRLAmt = RLAmt[0];
1713  SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue();
1714  unsigned LastGroupStartIdx = 0;
1715  bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1716  for (unsigned i = 1; i < Bits.size(); ++i) {
1717  unsigned ThisRLAmt = RLAmt[i];
1718  SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue();
1719  if (LateMask && !ThisValue) {
1720  ThisValue = LastValue;
1721  ThisRLAmt = LastRLAmt;
1722  // If we're doing late masking, then the first bit group always starts
1723  // at zero (even if the first bits were zero).
1724  if (BitGroups.empty())
1725  LastGroupStartIdx = 0;
1726  }
1727 
1728  // If this bit is known to be zero and the current group is a bit group
1729  // of zeros, we do not need to terminate the current bit group even the
1730  // Value or RLAmt does not match here. Instead, we terminate this group
1731  // when the first non-zero bit appears later.
1732  if (IsGroupOfZeros && Bits[i].isZero())
1733  continue;
1734 
1735  // If this bit has the same underlying value and the same rotate factor as
1736  // the last one, then they're part of the same group.
1737  if (ThisRLAmt == LastRLAmt && ThisValue == LastValue)
1738  // We cannot continue the current group if this bits is not known to
1739  // be zero in a bit group of zeros.
1740  if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero()))
1741  continue;
1742 
1743  if (LastValue.getNode())
1744  BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1745  i-1));
1746  LastRLAmt = ThisRLAmt;
1747  LastValue = ThisValue;
1748  LastGroupStartIdx = i;
1749  IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1750  }
1751  if (LastValue.getNode())
1752  BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1753  Bits.size()-1));
1754 
1755  if (BitGroups.empty())
1756  return;
1757 
1758  // We might be able to combine the first and last groups.
1759  if (BitGroups.size() > 1) {
1760  // If the first and last groups are the same, then remove the first group
1761  // in favor of the last group, making the ending index of the last group
1762  // equal to the ending index of the to-be-removed first group.
1763  if (BitGroups[0].StartIdx == 0 &&
1764  BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 &&
1765  BitGroups[0].V == BitGroups[BitGroups.size()-1].V &&
1766  BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) {
1767  LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n");
1768  BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx;
1769  BitGroups.erase(BitGroups.begin());
1770  }
1771  }
1772  }
1773 
1774  // Take all (SDValue, RLAmt) pairs and sort them by the number of groups
1775  // associated with each. If the number of groups are same, we prefer a group
1776  // which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate
1777  // instruction. If there is a degeneracy, pick the one that occurs
1778  // first (in the final value).
1779  void collectValueRotInfo() {
1780  ValueRots.clear();
1781 
1782  for (auto &BG : BitGroups) {
1783  unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0);
1784  ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)];
1785  VRI.V = BG.V;
1786  VRI.RLAmt = BG.RLAmt;
1787  VRI.Repl32 = BG.Repl32;
1788  VRI.NumGroups += 1;
1789  VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx);
1790  }
1791 
1792  // Now that we've collected the various ValueRotInfo instances, we need to
1793  // sort them.
1794  ValueRotsVec.clear();
1795  for (auto &I : ValueRots) {
1796  ValueRotsVec.push_back(I.second);
1797  }
1798  llvm::sort(ValueRotsVec);
1799  }
1800 
1801  // In 64-bit mode, rlwinm and friends have a rotation operator that
1802  // replicates the low-order 32 bits into the high-order 32-bits. The mask
1803  // indices of these instructions can only be in the lower 32 bits, so they
1804  // can only represent some 64-bit bit groups. However, when they can be used,
1805  // the 32-bit replication can be used to represent, as a single bit group,
1806  // otherwise separate bit groups. We'll convert to replicated-32-bit bit
1807  // groups when possible. Returns true if any of the bit groups were
1808  // converted.
1809  void assignRepl32BitGroups() {
1810  // If we have bits like this:
1811  //
1812  // Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
1813  // V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24
1814  // Groups: | RLAmt = 8 | RLAmt = 40 |
1815  //
1816  // But, making use of a 32-bit operation that replicates the low-order 32
1817  // bits into the high-order 32 bits, this can be one bit group with a RLAmt
1818  // of 8.
1819 
1820  auto IsAllLow32 = [this](BitGroup & BG) {
1821  if (BG.StartIdx <= BG.EndIdx) {
1822  for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) {
1823  if (!Bits[i].hasValue())
1824  continue;
1825  if (Bits[i].getValueBitIndex() >= 32)
1826  return false;
1827  }
1828  } else {
1829  for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) {
1830  if (!Bits[i].hasValue())
1831  continue;
1832  if (Bits[i].getValueBitIndex() >= 32)
1833  return false;
1834  }
1835  for (unsigned i = 0; i <= BG.EndIdx; ++i) {
1836  if (!Bits[i].hasValue())
1837  continue;
1838  if (Bits[i].getValueBitIndex() >= 32)
1839  return false;
1840  }
1841  }
1842 
1843  return true;
1844  };
1845 
1846  for (auto &BG : BitGroups) {
1847  // If this bit group has RLAmt of 0 and will not be merged with
1848  // another bit group, we don't benefit from Repl32. We don't mark
1849  // such group to give more freedom for later instruction selection.
1850  if (BG.RLAmt == 0) {
1851  auto PotentiallyMerged = [this](BitGroup & BG) {
1852  for (auto &BG2 : BitGroups)
1853  if (&BG != &BG2 && BG.V == BG2.V &&
1854  (BG2.RLAmt == 0 || BG2.RLAmt == 32))
1855  return true;
1856  return false;
1857  };
1858  if (!PotentiallyMerged(BG))
1859  continue;
1860  }
1861  if (BG.StartIdx < 32 && BG.EndIdx < 32) {
1862  if (IsAllLow32(BG)) {
1863  if (BG.RLAmt >= 32) {
1864  BG.RLAmt -= 32;
1865  BG.Repl32CR = true;
1866  }
1867 
1868  BG.Repl32 = true;
1869 
1870  LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for "
1871  << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " ["
1872  << BG.StartIdx << ", " << BG.EndIdx << "]\n");
1873  }
1874  }
1875  }
1876 
1877  // Now walk through the bit groups, consolidating where possible.
1878  for (auto I = BitGroups.begin(); I != BitGroups.end();) {
1879  // We might want to remove this bit group by merging it with the previous
1880  // group (which might be the ending group).
1881  auto IP = (I == BitGroups.begin()) ?
1882  std::prev(BitGroups.end()) : std::prev(I);
1883  if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt &&
1884  I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) {
1885 
1886  LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for "
1887  << I->V.getNode() << " RLAmt = " << I->RLAmt << " ["
1888  << I->StartIdx << ", " << I->EndIdx
1889  << "] with group with range [" << IP->StartIdx << ", "
1890  << IP->EndIdx << "]\n");
1891 
1892  IP->EndIdx = I->EndIdx;
1893  IP->Repl32CR = IP->Repl32CR || I->Repl32CR;
1894  IP->Repl32Coalesced = true;
1895  I = BitGroups.erase(I);
1896  continue;
1897  } else {
1898  // There is a special case worth handling: If there is a single group
1899  // covering the entire upper 32 bits, and it can be merged with both
1900  // the next and previous groups (which might be the same group), then
1901  // do so. If it is the same group (so there will be only one group in
1902  // total), then we need to reverse the order of the range so that it
1903  // covers the entire 64 bits.
1904  if (I->StartIdx == 32 && I->EndIdx == 63) {
1905  assert(std::next(I) == BitGroups.end() &&
1906  "bit group ends at index 63 but there is another?");
1907  auto IN = BitGroups.begin();
1908 
1909  if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V &&
1910  (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt &&
1911  IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP &&
1912  IsAllLow32(*I)) {
1913 
1914  LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode()
1915  << " RLAmt = " << I->RLAmt << " [" << I->StartIdx
1916  << ", " << I->EndIdx
1917  << "] with 32-bit replicated groups with ranges ["
1918  << IP->StartIdx << ", " << IP->EndIdx << "] and ["
1919  << IN->StartIdx << ", " << IN->EndIdx << "]\n");
1920 
1921  if (IP == IN) {
1922  // There is only one other group; change it to cover the whole
1923  // range (backward, so that it can still be Repl32 but cover the
1924  // whole 64-bit range).
1925  IP->StartIdx = 31;
1926  IP->EndIdx = 30;
1927  IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32;
1928  IP->Repl32Coalesced = true;
1929  I = BitGroups.erase(I);
1930  } else {
1931  // There are two separate groups, one before this group and one
1932  // after us (at the beginning). We're going to remove this group,
1933  // but also the group at the very beginning.
1934  IP->EndIdx = IN->EndIdx;
1935  IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32;
1936  IP->Repl32Coalesced = true;
1937  I = BitGroups.erase(I);
1938  BitGroups.erase(BitGroups.begin());
1939  }
1940 
1941  // This must be the last group in the vector (and we might have
1942  // just invalidated the iterator above), so break here.
1943  break;
1944  }
1945  }
1946  }
1947 
1948  ++I;
1949  }
1950  }
1951 
1952  SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
1953  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1954  }
1955 
1956  uint64_t getZerosMask() {
1957  uint64_t Mask = 0;
1958  for (unsigned i = 0; i < Bits.size(); ++i) {
1959  if (Bits[i].hasValue())
1960  continue;
1961  Mask |= (UINT64_C(1) << i);
1962  }
1963 
1964  return ~Mask;
1965  }
1966 
1967  // This method extends an input value to 64 bit if input is 32-bit integer.
1968  // While selecting instructions in BitPermutationSelector in 64-bit mode,
1969  // an input value can be a 32-bit integer if a ZERO_EXTEND node is included.
1970  // In such case, we extend it to 64 bit to be consistent with other values.
1971  SDValue ExtendToInt64(SDValue V, const SDLoc &dl) {
1972  if (V.getValueSizeInBits() == 64)
1973  return V;
1974 
1975  assert(V.getValueSizeInBits() == 32);
1976  SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1977  SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl,
1978  MVT::i64), 0);
1979  SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl,
1980  MVT::i64, ImDef, V,
1981  SubRegIdx), 0);
1982  return ExtVal;
1983  }
1984 
1985  SDValue TruncateToInt32(SDValue V, const SDLoc &dl) {
1986  if (V.getValueSizeInBits() == 32)
1987  return V;
1988 
1989  assert(V.getValueSizeInBits() == 64);
1990  SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
1991  SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl,
1992  MVT::i32, V, SubRegIdx), 0);
1993  return SubVal;
1994  }
1995 
1996  // Depending on the number of groups for a particular value, it might be
1997  // better to rotate, mask explicitly (using andi/andis), and then or the
1998  // result. Select this part of the result first.
1999  void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2001  return;
2002 
2003  for (ValueRotInfo &VRI : ValueRotsVec) {
2004  unsigned Mask = 0;
2005  for (unsigned i = 0; i < Bits.size(); ++i) {
2006  if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V)
2007  continue;
2008  if (RLAmt[i] != VRI.RLAmt)
2009  continue;
2010  Mask |= (1u << i);
2011  }
2012 
2013  // Compute the masks for andi/andis that would be necessary.
2014  unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16;
2015  assert((ANDIMask != 0 || ANDISMask != 0) &&
2016  "No set bits in mask for value bit groups");
2017  bool NeedsRotate = VRI.RLAmt != 0;
2018 
2019  // We're trying to minimize the number of instructions. If we have one
2020  // group, using one of andi/andis can break even. If we have three
2021  // groups, we can use both andi and andis and break even (to use both
2022  // andi and andis we also need to or the results together). We need four
2023  // groups if we also need to rotate. To use andi/andis we need to do more
2024  // than break even because rotate-and-mask instructions tend to be easier
2025  // to schedule.
2026 
2027  // FIXME: We've biased here against using andi/andis, which is right for
2028  // POWER cores, but not optimal everywhere. For example, on the A2,
2029  // andi/andis have single-cycle latency whereas the rotate-and-mask
2030  // instructions take two cycles, and it would be better to bias toward
2031  // andi/andis in break-even cases.
2032 
2033  unsigned NumAndInsts = (unsigned) NeedsRotate +
2034  (unsigned) (ANDIMask != 0) +
2035  (unsigned) (ANDISMask != 0) +
2036  (unsigned) (ANDIMask != 0 && ANDISMask != 0) +
2037  (unsigned) (bool) Res;
2038 
2039  LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()
2040  << " RL: " << VRI.RLAmt << ":"
2041  << "\n\t\t\tisel using masking: " << NumAndInsts
2042  << " using rotates: " << VRI.NumGroups << "\n");
2043 
2044  if (NumAndInsts >= VRI.NumGroups)
2045  continue;
2046 
2047  LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n");
2048 
2049  if (InstCnt) *InstCnt += NumAndInsts;
2050 
2051  SDValue VRot;
2052  if (VRI.RLAmt) {
2053  SDValue Ops[] =
2054  { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
2055  getI32Imm(0, dl), getI32Imm(31, dl) };
2056  VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
2057  Ops), 0);
2058  } else {
2059  VRot = TruncateToInt32(VRI.V, dl);
2060  }
2061 
2062  SDValue ANDIVal, ANDISVal;
2063  if (ANDIMask != 0)
2064  ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32,
2065  VRot, getI32Imm(ANDIMask, dl)),
2066  0);
2067  if (ANDISMask != 0)
2068  ANDISVal =
2069  SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot,
2070  getI32Imm(ANDISMask, dl)),
2071  0);
2072 
2073  SDValue TotalVal;
2074  if (!ANDIVal)
2075  TotalVal = ANDISVal;
2076  else if (!ANDISVal)
2077  TotalVal = ANDIVal;
2078  else
2079  TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2080  ANDIVal, ANDISVal), 0);
2081 
2082  if (!Res)
2083  Res = TotalVal;
2084  else
2085  Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2086  Res, TotalVal), 0);
2087 
2088  // Now, remove all groups with this underlying value and rotation
2089  // factor.
2090  eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2091  return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
2092  });
2093  }
2094  }
2095 
2096  // Instruction selection for the 32-bit case.
2097  SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) {
2098  SDLoc dl(N);
2099  SDValue Res;
2100 
2101  if (InstCnt) *InstCnt = 0;
2102 
2103  // Take care of cases that should use andi/andis first.
2104  SelectAndParts32(dl, Res, InstCnt);
2105 
2106  // If we've not yet selected a 'starting' instruction, and we have no zeros
2107  // to fill in, select the (Value, RLAmt) with the highest priority (largest
2108  // number of groups), and start with this rotated value.
2109  if ((!NeedMask || LateMask) && !Res) {
2110  ValueRotInfo &VRI = ValueRotsVec[0];
2111  if (VRI.RLAmt) {
2112  if (InstCnt) *InstCnt += 1;
2113  SDValue Ops[] =
2114  { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
2115  getI32Imm(0, dl), getI32Imm(31, dl) };
2116  Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops),
2117  0);
2118  } else {
2119  Res = TruncateToInt32(VRI.V, dl);
2120  }
2121 
2122  // Now, remove all groups with this underlying value and rotation factor.
2123  eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2124  return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
2125  });
2126  }
2127 
2128  if (InstCnt) *InstCnt += BitGroups.size();
2129 
2130  // Insert the other groups (one at a time).
2131  for (auto &BG : BitGroups) {
2132  if (!Res) {
2133  SDValue Ops[] =
2134  { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
2135  getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
2136  getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
2137  Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
2138  } else {
2139  SDValue Ops[] =
2140  { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
2141  getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
2142  getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
2143  Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0);
2144  }
2145  }
2146 
2147  if (LateMask) {
2148  unsigned Mask = (unsigned) getZerosMask();
2149 
2150  unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16;
2151  assert((ANDIMask != 0 || ANDISMask != 0) &&
2152  "No set bits in zeros mask?");
2153 
2154  if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
2155  (unsigned) (ANDISMask != 0) +
2156  (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2157 
2158  SDValue ANDIVal, ANDISVal;
2159  if (ANDIMask != 0)
2160  ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32,
2161  Res, getI32Imm(ANDIMask, dl)),
2162  0);
2163  if (ANDISMask != 0)
2164  ANDISVal =
2165  SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res,
2166  getI32Imm(ANDISMask, dl)),
2167  0);
2168 
2169  if (!ANDIVal)
2170  Res = ANDISVal;
2171  else if (!ANDISVal)
2172  Res = ANDIVal;
2173  else
2174  Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2175  ANDIVal, ANDISVal), 0);
2176  }
2177 
2178  return Res.getNode();
2179  }
2180 
2181  unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32,
2182  unsigned MaskStart, unsigned MaskEnd,
2183  bool IsIns) {
2184  // In the notation used by the instructions, 'start' and 'end' are reversed
2185  // because bits are counted from high to low order.
2186  unsigned InstMaskStart = 64 - MaskEnd - 1,
2187  InstMaskEnd = 64 - MaskStart - 1;
2188 
2189  if (Repl32)
2190  return 1;
2191 
2192  if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) ||
2193  InstMaskEnd == 63 - RLAmt)
2194  return 1;
2195 
2196  return 2;
2197  }
2198 
2199  // For 64-bit values, not all combinations of rotates and masks are
2200  // available. Produce one if it is available.
2201  SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt,
2202  bool Repl32, unsigned MaskStart, unsigned MaskEnd,
2203  unsigned *InstCnt = nullptr) {
2204  // In the notation used by the instructions, 'start' and 'end' are reversed
2205  // because bits are counted from high to low order.
2206  unsigned InstMaskStart = 64 - MaskEnd - 1,
2207  InstMaskEnd = 64 - MaskStart - 1;
2208 
2209  if (InstCnt) *InstCnt += 1;
2210 
2211  if (Repl32) {
2212  // This rotation amount assumes that the lower 32 bits of the quantity
2213  // are replicated in the high 32 bits by the rotation operator (which is
2214  // done by rlwinm and friends).
2215  assert(InstMaskStart >= 32 && "Mask cannot start out of range");
2216  assert(InstMaskEnd >= 32 && "Mask cannot end out of range");
2217  SDValue Ops[] =
2218  { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2219  getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2220  return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64,
2221  Ops), 0);
2222  }
2223 
2224  if (InstMaskEnd == 63) {
2225  SDValue Ops[] =
2226  { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2227  getI32Imm(InstMaskStart, dl) };
2228  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0);
2229  }
2230 
2231  if (InstMaskStart == 0) {
2232  SDValue Ops[] =
2233  { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2234  getI32Imm(InstMaskEnd, dl) };
2235  return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0);
2236  }
2237 
2238  if (InstMaskEnd == 63 - RLAmt) {
2239  SDValue Ops[] =
2240  { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2241  getI32Imm(InstMaskStart, dl) };
2242  return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0);
2243  }
2244 
2245  // We cannot do this with a single instruction, so we'll use two. The
2246  // problem is that we're not free to choose both a rotation amount and mask
2247  // start and end independently. We can choose an arbitrary mask start and
2248  // end, but then the rotation amount is fixed. Rotation, however, can be
2249  // inverted, and so by applying an "inverse" rotation first, we can get the
2250  // desired result.
2251  if (InstCnt) *InstCnt += 1;
2252 
2253  // The rotation mask for the second instruction must be MaskStart.
2254  unsigned RLAmt2 = MaskStart;
2255  // The first instruction must rotate V so that the overall rotation amount
2256  // is RLAmt.
2257  unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2258  if (RLAmt1)
2259  V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
2260  return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd);
2261  }
2262 
2263  // For 64-bit values, not all combinations of rotates and masks are
2264  // available. Produce a rotate-mask-and-insert if one is available.
2265  SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl,
2266  unsigned RLAmt, bool Repl32, unsigned MaskStart,
2267  unsigned MaskEnd, unsigned *InstCnt = nullptr) {
2268  // In the notation used by the instructions, 'start' and 'end' are reversed
2269  // because bits are counted from high to low order.
2270  unsigned InstMaskStart = 64 - MaskEnd - 1,
2271  InstMaskEnd = 64 - MaskStart - 1;
2272 
2273  if (InstCnt) *InstCnt += 1;
2274 
2275  if (Repl32) {
2276  // This rotation amount assumes that the lower 32 bits of the quantity
2277  // are replicated in the high 32 bits by the rotation operator (which is
2278  // done by rlwinm and friends).
2279  assert(InstMaskStart >= 32 && "Mask cannot start out of range");
2280  assert(InstMaskEnd >= 32 && "Mask cannot end out of range");
2281  SDValue Ops[] =
2282  { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2283  getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2284  return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64,
2285  Ops), 0);
2286  }
2287 
2288  if (InstMaskEnd == 63 - RLAmt) {
2289  SDValue Ops[] =
2290  { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2291  getI32Imm(InstMaskStart, dl) };
2292  return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0);
2293  }
2294 
2295  // We cannot do this with a single instruction, so we'll use two. The
2296  // problem is that we're not free to choose both a rotation amount and mask
2297  // start and end independently. We can choose an arbitrary mask start and
2298  // end, but then the rotation amount is fixed. Rotation, however, can be
2299  // inverted, and so by applying an "inverse" rotation first, we can get the
2300  // desired result.
2301  if (InstCnt) *InstCnt += 1;
2302 
2303  // The rotation mask for the second instruction must be MaskStart.
2304  unsigned RLAmt2 = MaskStart;
2305  // The first instruction must rotate V so that the overall rotation amount
2306  // is RLAmt.
2307  unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2308  if (RLAmt1)
2309  V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
2310  return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd);
2311  }
2312 
2313  void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2315  return;
2316 
2317  // The idea here is the same as in the 32-bit version, but with additional
2318  // complications from the fact that Repl32 might be true. Because we
2319  // aggressively convert bit groups to Repl32 form (which, for small
2320  // rotation factors, involves no other change), and then coalesce, it might
2321  // be the case that a single 64-bit masking operation could handle both
2322  // some Repl32 groups and some non-Repl32 groups. If converting to Repl32
2323  // form allowed coalescing, then we must use a 32-bit rotaton in order to
2324  // completely capture the new combined bit group.
2325 
2326  for (ValueRotInfo &VRI : ValueRotsVec) {
2327  uint64_t Mask = 0;
2328 
2329  // We need to add to the mask all bits from the associated bit groups.
2330  // If Repl32 is false, we need to add bits from bit groups that have
2331  // Repl32 true, but are trivially convertable to Repl32 false. Such a
2332  // group is trivially convertable if it overlaps only with the lower 32
2333  // bits, and the group has not been coalesced.
2334  auto MatchingBG = [VRI](const BitGroup &BG) {
2335  if (VRI.V != BG.V)
2336  return false;
2337 
2338  unsigned EffRLAmt = BG.RLAmt;
2339  if (!VRI.Repl32 && BG.Repl32) {
2340  if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx &&
2341  !BG.Repl32Coalesced) {
2342  if (BG.Repl32CR)
2343  EffRLAmt += 32;
2344  } else {
2345  return false;
2346  }
2347  } else if (VRI.Repl32 != BG.Repl32) {
2348  return false;
2349  }
2350 
2351  return VRI.RLAmt == EffRLAmt;
2352  };
2353 
2354  for (auto &BG : BitGroups) {
2355  if (!MatchingBG(BG))
2356  continue;
2357 
2358  if (BG.StartIdx <= BG.EndIdx) {
2359  for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i)
2360  Mask |= (UINT64_C(1) << i);
2361  } else {
2362  for (unsigned i = BG.StartIdx; i < Bits.size(); ++i)
2363  Mask |= (UINT64_C(1) << i);
2364  for (unsigned i = 0; i <= BG.EndIdx; ++i)
2365  Mask |= (UINT64_C(1) << i);
2366  }
2367  }
2368 
2369  // We can use the 32-bit andi/andis technique if the mask does not
2370  // require any higher-order bits. This can save an instruction compared
2371  // to always using the general 64-bit technique.
2372  bool Use32BitInsts = isUInt<32>(Mask);
2373  // Compute the masks for andi/andis that would be necessary.
2374  unsigned ANDIMask = (Mask & UINT16_MAX),
2375  ANDISMask = (Mask >> 16) & UINT16_MAX;
2376 
2377  bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask));
2378 
2379  unsigned NumAndInsts = (unsigned) NeedsRotate +
2380  (unsigned) (bool) Res;
2381  unsigned NumOfSelectInsts = 0;
2382  selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts);
2383  assert(NumOfSelectInsts > 0 && "Failed to select an i64 constant.");
2384  if (Use32BitInsts)
2385  NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) +
2386  (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2387  else
2388  NumAndInsts += NumOfSelectInsts + /* and */ 1;
2389 
2390  unsigned NumRLInsts = 0;
2391  bool FirstBG = true;
2392  bool MoreBG = false;
2393  for (auto &BG : BitGroups) {
2394  if (!MatchingBG(BG)) {
2395  MoreBG = true;
2396  continue;
2397  }
2398  NumRLInsts +=
2399  SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx,
2400  !FirstBG);
2401  FirstBG = false;
2402  }
2403 
2404  LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()
2405  << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":")
2406  << "\n\t\t\tisel using masking: " << NumAndInsts
2407  << " using rotates: " << NumRLInsts << "\n");
2408 
2409  // When we'd use andi/andis, we bias toward using the rotates (andi only
2410  // has a record form, and is cracked on POWER cores). However, when using
2411  // general 64-bit constant formation, bias toward the constant form,
2412  // because that exposes more opportunities for CSE.
2413  if (NumAndInsts > NumRLInsts)
2414  continue;
2415  // When merging multiple bit groups, instruction or is used.
2416  // But when rotate is used, rldimi can inert the rotated value into any
2417  // register, so instruction or can be avoided.
2418  if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts)
2419  continue;
2420 
2421  LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n");
2422 
2423  if (InstCnt) *InstCnt += NumAndInsts;
2424 
2425  SDValue VRot;
2426  // We actually need to generate a rotation if we have a non-zero rotation
2427  // factor or, in the Repl32 case, if we care about any of the
2428  // higher-order replicated bits. In the latter case, we generate a mask
2429  // backward so that it actually includes the entire 64 bits.
2430  if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)))
2431  VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2432  VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63);
2433  else
2434  VRot = VRI.V;
2435 
2436  SDValue TotalVal;
2437  if (Use32BitInsts) {
2438  assert((ANDIMask != 0 || ANDISMask != 0) &&
2439  "No set bits in mask when using 32-bit ands for 64-bit value");
2440 
2441  SDValue ANDIVal, ANDISVal;
2442  if (ANDIMask != 0)
2443  ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64,
2444  ExtendToInt64(VRot, dl),
2445  getI32Imm(ANDIMask, dl)),
2446  0);
2447  if (ANDISMask != 0)
2448  ANDISVal =
2449  SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64,
2450  ExtendToInt64(VRot, dl),
2451  getI32Imm(ANDISMask, dl)),
2452  0);
2453 
2454  if (!ANDIVal)
2455  TotalVal = ANDISVal;
2456  else if (!ANDISVal)
2457  TotalVal = ANDIVal;
2458  else
2459  TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2460  ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2461  } else {
2462  TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0);
2463  TotalVal =
2464  SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2465  ExtendToInt64(VRot, dl), TotalVal),
2466  0);
2467  }
2468 
2469  if (!Res)
2470  Res = TotalVal;
2471  else
2472  Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2473  ExtendToInt64(Res, dl), TotalVal),
2474  0);
2475 
2476  // Now, remove all groups with this underlying value and rotation
2477  // factor.
2478  eraseMatchingBitGroups(MatchingBG);
2479  }
2480  }
2481 
2482  // Instruction selection for the 64-bit case.
2483  SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) {
2484  SDLoc dl(N);
2485  SDValue Res;
2486 
2487  if (InstCnt) *InstCnt = 0;
2488 
2489  // Take care of cases that should use andi/andis first.
2490  SelectAndParts64(dl, Res, InstCnt);
2491 
2492  // If we've not yet selected a 'starting' instruction, and we have no zeros
2493  // to fill in, select the (Value, RLAmt) with the highest priority (largest
2494  // number of groups), and start with this rotated value.
2495  if ((!NeedMask || LateMask) && !Res) {
2496  // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32
2497  // groups will come first, and so the VRI representing the largest number
2498  // of groups might not be first (it might be the first Repl32 groups).
2499  unsigned MaxGroupsIdx = 0;
2500  if (!ValueRotsVec[0].Repl32) {
2501  for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i)
2502  if (ValueRotsVec[i].Repl32) {
2503  if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups)
2504  MaxGroupsIdx = i;
2505  break;
2506  }
2507  }
2508 
2509  ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx];
2510  bool NeedsRotate = false;
2511  if (VRI.RLAmt) {
2512  NeedsRotate = true;
2513  } else if (VRI.Repl32) {
2514  for (auto &BG : BitGroups) {
2515  if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt ||
2516  BG.Repl32 != VRI.Repl32)
2517  continue;
2518 
2519  // We don't need a rotate if the bit group is confined to the lower
2520  // 32 bits.
2521  if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx)
2522  continue;
2523 
2524  NeedsRotate = true;
2525  break;
2526  }
2527  }
2528 
2529  if (NeedsRotate)
2530  Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2531  VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63,
2532  InstCnt);
2533  else
2534  Res = VRI.V;
2535 
2536  // Now, remove all groups with this underlying value and rotation factor.
2537  if (Res)
2538  eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2539  return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt &&
2540  BG.Repl32 == VRI.Repl32;
2541  });
2542  }
2543 
2544  // Because 64-bit rotates are more flexible than inserts, we might have a
2545  // preference regarding which one we do first (to save one instruction).
2546  if (!Res)
2547  for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) {
2548  if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2549  false) <
2550  SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2551  true)) {
2552  if (I != BitGroups.begin()) {
2553  BitGroup BG = *I;
2554  BitGroups.erase(I);
2555  BitGroups.insert(BitGroups.begin(), BG);
2556  }
2557 
2558  break;
2559  }
2560  }
2561 
2562  // Insert the other groups (one at a time).
2563  for (auto &BG : BitGroups) {
2564  if (!Res)
2565  Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx,
2566  BG.EndIdx, InstCnt);
2567  else
2568  Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32,
2569  BG.StartIdx, BG.EndIdx, InstCnt);
2570  }
2571 
2572  if (LateMask) {
2573  uint64_t Mask = getZerosMask();
2574 
2575  // We can use the 32-bit andi/andis technique if the mask does not
2576  // require any higher-order bits. This can save an instruction compared
2577  // to always using the general 64-bit technique.
2578  bool Use32BitInsts = isUInt<32>(Mask);
2579  // Compute the masks for andi/andis that would be necessary.
2580  unsigned ANDIMask = (Mask & UINT16_MAX),
2581  ANDISMask = (Mask >> 16) & UINT16_MAX;
2582 
2583  if (Use32BitInsts) {
2584  assert((ANDIMask != 0 || ANDISMask != 0) &&
2585  "No set bits in mask when using 32-bit ands for 64-bit value");
2586 
2587  if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
2588  (unsigned) (ANDISMask != 0) +
2589  (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2590 
2591  SDValue ANDIVal, ANDISVal;
2592  if (ANDIMask != 0)
2593  ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64,
2594  ExtendToInt64(Res, dl),
2595  getI32Imm(ANDIMask, dl)),
2596  0);
2597  if (ANDISMask != 0)
2598  ANDISVal =
2599  SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64,
2600  ExtendToInt64(Res, dl),
2601  getI32Imm(ANDISMask, dl)),
2602  0);
2603 
2604  if (!ANDIVal)
2605  Res = ANDISVal;
2606  else if (!ANDISVal)
2607  Res = ANDIVal;
2608  else
2609  Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2610  ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2611  } else {
2612  unsigned NumOfSelectInsts = 0;
2613  SDValue MaskVal =
2614  SDValue(selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts), 0);
2615  Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2616  ExtendToInt64(Res, dl), MaskVal),
2617  0);
2618  if (InstCnt)
2619  *InstCnt += NumOfSelectInsts + /* and */ 1;
2620  }
2621  }
2622 
2623  return Res.getNode();
2624  }
2625 
2626  SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) {
2627  // Fill in BitGroups.
2628  collectBitGroups(LateMask);
2629  if (BitGroups.empty())
2630  return nullptr;
2631 
2632  // For 64-bit values, figure out when we can use 32-bit instructions.
2633  if (Bits.size() == 64)
2634  assignRepl32BitGroups();
2635 
2636  // Fill in ValueRotsVec.
2637  collectValueRotInfo();
2638 
2639  if (Bits.size() == 32) {
2640  return Select32(N, LateMask, InstCnt);
2641  } else {
2642  assert(Bits.size() == 64 && "Not 64 bits here?");
2643  return Select64(N, LateMask, InstCnt);
2644  }
2645 
2646  return nullptr;
2647  }
2648 
2649  void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) {
2650  erase_if(BitGroups, F);
2651  }
2652 
2654 
2655  bool NeedMask = false;
2657 
2658  SmallVector<BitGroup, 16> BitGroups;
2659 
2660  DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots;
2661  SmallVector<ValueRotInfo, 16> ValueRotsVec;
2662 
2663  SelectionDAG *CurDAG = nullptr;
2664 
2665 public:
2666  BitPermutationSelector(SelectionDAG *DAG)
2667  : CurDAG(DAG) {}
2668 
2669  // Here we try to match complex bit permutations into a set of
2670  // rotate-and-shift/shift/and/or instructions, using a set of heuristics
2671  // known to produce optimal code for common cases (like i32 byte swapping).
2672  SDNode *Select(SDNode *N) {
2673  Memoizer.clear();
2674  auto Result =
2675  getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits());
2676  if (!Result.first)
2677  return nullptr;
2678  Bits = std::move(*Result.second);
2679 
2680  LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction"
2681  " selection for: ");
2682  LLVM_DEBUG(N->dump(CurDAG));
2683 
2684  // Fill it RLAmt and set NeedMask.
2685  computeRotationAmounts();
2686 
2687  if (!NeedMask)
2688  return Select(N, false);
2689 
2690  // We currently have two techniques for handling results with zeros: early
2691  // masking (the default) and late masking. Late masking is sometimes more
2692  // efficient, but because the structure of the bit groups is different, it
2693  // is hard to tell without generating both and comparing the results. With
2694  // late masking, we ignore zeros in the resulting value when inserting each
2695  // set of bit groups, and then mask in the zeros at the end. With early
2696  // masking, we only insert the non-zero parts of the result at every step.
2697 
2698  unsigned InstCnt = 0, InstCntLateMask = 0;
2699  LLVM_DEBUG(dbgs() << "\tEarly masking:\n");
2700  SDNode *RN = Select(N, false, &InstCnt);
2701  LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n");
2702 
2703  LLVM_DEBUG(dbgs() << "\tLate masking:\n");
2704  SDNode *RNLM = Select(N, true, &InstCntLateMask);
2705  LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask
2706  << " instructions\n");
2707 
2708  if (InstCnt <= InstCntLateMask) {
2709  LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n");
2710  return RN;
2711  }
2712 
2713  LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n");
2714  return RNLM;
2715  }
2716 };
2717 
2718 class IntegerCompareEliminator {
2719  SelectionDAG *CurDAG;
2720  PPCDAGToDAGISel *S;
2721  // Conversion type for interpreting results of a 32-bit instruction as
2722  // a 64-bit value or vice versa.
2723  enum ExtOrTruncConversion { Ext, Trunc };
2724 
2725  // Modifiers to guide how an ISD::SETCC node's result is to be computed
2726  // in a GPR.
2727  // ZExtOrig - use the original condition code, zero-extend value
2728  // ZExtInvert - invert the condition code, zero-extend value
2729  // SExtOrig - use the original condition code, sign-extend value
2730  // SExtInvert - invert the condition code, sign-extend value
2731  enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert };
2732 
2733  // Comparisons against zero to emit GPR code sequences for. Each of these
2734  // sequences may need to be emitted for two or more equivalent patterns.
2735  // For example (a >= 0) == (a > -1). The direction of the comparison (</>)
2736  // matters as well as the extension type: sext (-1/0), zext (1/0).
2737  // GEZExt - (zext (LHS >= 0))
2738  // GESExt - (sext (LHS >= 0))
2739  // LEZExt - (zext (LHS <= 0))
2740  // LESExt - (sext (LHS <= 0))
2741  enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt };
2742 
2743  SDNode *tryEXTEND(SDNode *N);
2744  SDNode *tryLogicOpOfCompares(SDNode *N);
2745  SDValue computeLogicOpInGPR(SDValue LogicOp);
2746  SDValue signExtendInputIfNeeded(SDValue Input);
2747  SDValue zeroExtendInputIfNeeded(SDValue Input);
2748  SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv);
2749  SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2750  ZeroCompare CmpTy);
2751  SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2752  int64_t RHSValue, SDLoc dl);
2753  SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2754  int64_t RHSValue, SDLoc dl);
2755  SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2756  int64_t RHSValue, SDLoc dl);
2757  SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2758  int64_t RHSValue, SDLoc dl);
2759  SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts);
2760 
2761 public:
2762  IntegerCompareEliminator(SelectionDAG *DAG,
2763  PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) {
2764  assert(CurDAG->getTargetLoweringInfo()
2765  .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 &&
2766  "Only expecting to use this on 64 bit targets.");
2767  }
2768  SDNode *Select(SDNode *N) {
2769  if (CmpInGPR == ICGPR_None)
2770  return nullptr;
2771  switch (N->getOpcode()) {
2772  default: break;
2773  case ISD::ZERO_EXTEND:
2774  if (CmpInGPR == ICGPR_Sext || CmpInGPR == ICGPR_SextI32 ||
2776  return nullptr;
2778  case ISD::SIGN_EXTEND:
2779  if (CmpInGPR == ICGPR_Zext || CmpInGPR == ICGPR_ZextI32 ||
2781  return nullptr;
2782  return tryEXTEND(N);
2783  case ISD::AND:
2784  case ISD::OR:
2785  case ISD::XOR:
2786  return tryLogicOpOfCompares(N);
2787  }
2788  return nullptr;
2789  }
2790 };
2791 
2792 static bool isLogicOp(unsigned Opc) {
2793  return Opc == ISD::AND || Opc == ISD::OR || Opc == ISD::XOR;
2794 }
2795 // The obvious case for wanting to keep the value in a GPR. Namely, the
2796 // result of the comparison is actually needed in a GPR.
2797 SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) {
2798  assert((N->getOpcode() == ISD::ZERO_EXTEND ||
2799  N->getOpcode() == ISD::SIGN_EXTEND) &&
2800  "Expecting a zero/sign extend node!");
2801  SDValue WideRes;
2802  // If we are zero-extending the result of a logical operation on i1
2803  // values, we can keep the values in GPRs.
2804  if (isLogicOp(N->getOperand(0).getOpcode()) &&
2805  N->getOperand(0).getValueType() == MVT::i1 &&
2806  N->getOpcode() == ISD::ZERO_EXTEND)
2807  WideRes = computeLogicOpInGPR(N->getOperand(0));
2808  else if (N->getOperand(0).getOpcode() != ISD::SETCC)
2809  return nullptr;
2810  else
2811  WideRes =
2812  getSETCCInGPR(N->getOperand(0),
2813  N->getOpcode() == ISD::SIGN_EXTEND ?
2814  SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig);
2815 
2816  if (!WideRes)
2817  return nullptr;
2818 
2819  SDLoc dl(N);
2820  bool Input32Bit = WideRes.getValueType() == MVT::i32;
2821  bool Output32Bit = N->getValueType(0) == MVT::i32;
2822 
2823  NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0;
2824  NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1;
2825 
2826  SDValue ConvOp = WideRes;
2827  if (Input32Bit != Output32Bit)
2828  ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext :
2829  ExtOrTruncConversion::Trunc);
2830  return ConvOp.getNode();
2831 }
2832 
2833 // Attempt to perform logical operations on the results of comparisons while
2834 // keeping the values in GPRs. Without doing so, these would end up being
2835 // lowered to CR-logical operations which suffer from significant latency and
2836 // low ILP.
2837 SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) {
2838  if (N->getValueType(0) != MVT::i1)
2839  return nullptr;
2840  assert(isLogicOp(N->getOpcode()) &&
2841  "Expected a logic operation on setcc results.");
2842  SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0));
2843  if (!LoweredLogical)
2844  return nullptr;
2845 
2846  SDLoc dl(N);
2847  bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8;
2848  unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt;
2849  SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
2850  SDValue LHS = LoweredLogical.getOperand(0);
2851  SDValue RHS = LoweredLogical.getOperand(1);
2852  SDValue WideOp;
2853  SDValue OpToConvToRecForm;
2854 
2855  // Look through any 32-bit to 64-bit implicit extend nodes to find the
2856  // opcode that is input to the XORI.
2857  if (IsBitwiseNegate &&
2858  LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG)
2859  OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1);
2860  else if (IsBitwiseNegate)
2861  // If the input to the XORI isn't an extension, that's what we're after.
2862  OpToConvToRecForm = LoweredLogical.getOperand(0);
2863  else
2864  // If this is not an XORI, it is a reg-reg logical op and we can convert
2865  // it to record-form.
2866  OpToConvToRecForm = LoweredLogical;
2867 
2868  // Get the record-form version of the node we're looking to use to get the
2869  // CR result from.
2870  uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode();
2871  int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc);
2872 
2873  // Convert the right node to record-form. This is either the logical we're
2874  // looking at or it is the input node to the negation (if we're looking at
2875  // a bitwise negation).
2876  if (NewOpc != -1 && IsBitwiseNegate) {
2877  // The input to the XORI has a record-form. Use it.
2878  assert(LoweredLogical.getConstantOperandVal(1) == 1 &&
2879  "Expected a PPC::XORI8 only for bitwise negation.");
2880  // Emit the record-form instruction.
2881  std::vector<SDValue> Ops;
2882  for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++)
2883  Ops.push_back(OpToConvToRecForm.getOperand(i));
2884 
2885  WideOp =
2886  SDValue(CurDAG->getMachineNode(NewOpc, dl,
2887  OpToConvToRecForm.getValueType(),
2888  MVT::Glue, Ops), 0);
2889  } else {
2890  assert((NewOpc != -1 || !IsBitwiseNegate) &&
2891  "No record form available for AND8/OR8/XOR8?");
2892  WideOp =
2893  SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc,
2894  dl, MVT::i64, MVT::Glue, LHS, RHS),
2895  0);
2896  }
2897 
2898  // Select this node to a single bit from CR0 set by the record-form node
2899  // just created. For bitwise negation, use the EQ bit which is the equivalent
2900  // of negating the result (i.e. it is a bit set when the result of the
2901  // operation is zero).
2902  SDValue SRIdxVal =
2903  CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32);
2904  SDValue CRBit =
2905  SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl,
2906  MVT::i1, CR0Reg, SRIdxVal,
2907  WideOp.getValue(1)), 0);
2908  return CRBit.getNode();
2909 }
2910 
2911 // Lower a logical operation on i1 values into a GPR sequence if possible.
2912 // The result can be kept in a GPR if requested.
2913 // Three types of inputs can be handled:
2914 // - SETCC
2915 // - TRUNCATE
2916 // - Logical operation (AND/OR/XOR)
2917 // There is also a special case that is handled (namely a complement operation
2918 // achieved with xor %a, -1).
2919 SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) {
2920  assert(isLogicOp(LogicOp.getOpcode()) &&
2921  "Can only handle logic operations here.");
2922  assert(LogicOp.getValueType() == MVT::i1 &&
2923  "Can only handle logic operations on i1 values here.");
2924  SDLoc dl(LogicOp);
2925  SDValue LHS, RHS;
2926 
2927  // Special case: xor %a, -1
2928  bool IsBitwiseNegation = isBitwiseNot(LogicOp);
2929 
2930  // Produces a GPR sequence for each operand of the binary logic operation.
2931  // For SETCC, it produces the respective comparison, for TRUNCATE it truncates
2932  // the value in a GPR and for logic operations, it will recursively produce
2933  // a GPR sequence for the operation.
2934  auto getLogicOperand = [&] (SDValue Operand) -> SDValue {
2935  unsigned OperandOpcode = Operand.getOpcode();
2936  if (OperandOpcode == ISD::SETCC)
2937  return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig);
2938  else if (OperandOpcode == ISD::TRUNCATE) {
2939  SDValue InputOp = Operand.getOperand(0);
2940  EVT InVT = InputOp.getValueType();
2941  return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 :
2942  PPC::RLDICL, dl, InVT, InputOp,
2943  S->getI64Imm(0, dl),
2944  S->getI64Imm(63, dl)), 0);
2945  } else if (isLogicOp(OperandOpcode))
2946  return computeLogicOpInGPR(Operand);
2947  return SDValue();
2948  };
2949  LHS = getLogicOperand(LogicOp.getOperand(0));
2950  RHS = getLogicOperand(LogicOp.getOperand(1));
2951 
2952  // If a GPR sequence can't be produced for the LHS we can't proceed.
2953  // Not producing a GPR sequence for the RHS is only a problem if this isn't
2954  // a bitwise negation operation.
2955  if (!LHS || (!RHS && !IsBitwiseNegation))
2956  return SDValue();
2957 
2958  NumLogicOpsOnComparison++;
2959 
2960  // We will use the inputs as 64-bit values.
2961  if (LHS.getValueType() == MVT::i32)
2962  LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext);
2963  if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32)
2964  RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext);
2965 
2966  unsigned NewOpc;
2967  switch (LogicOp.getOpcode()) {
2968  default: llvm_unreachable("Unknown logic operation.");
2969  case ISD::AND: NewOpc = PPC::AND8; break;
2970  case ISD::OR: NewOpc = PPC::OR8; break;
2971  case ISD::XOR: NewOpc = PPC::XOR8; break;
2972  }
2973 
2974  if (IsBitwiseNegation) {
2975  RHS = S->getI64Imm(1, dl);
2976  NewOpc = PPC::XORI8;
2977  }
2978 
2979  return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0);
2980 
2981 }
2982 
2983 /// If the value isn't guaranteed to be sign-extended to 64-bits, extend it.
2984 /// Otherwise just reinterpret it as a 64-bit value.
2985 /// Useful when emitting comparison code for 32-bit values without using
2986 /// the compare instruction (which only considers the lower 32-bits).
2987 SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) {
2988  assert(Input.getValueType() == MVT::i32 &&
2989  "Can only sign-extend 32-bit values here.");
2990  unsigned Opc = Input.getOpcode();
2991 
2992  // The value was sign extended and then truncated to 32-bits. No need to
2993  // sign extend it again.
2994  if (Opc == ISD::TRUNCATE &&
2995  (Input.getOperand(0).getOpcode() == ISD::AssertSext ||
2996  Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND))
2997  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
2998 
2999  LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
3000  // The input is a sign-extending load. All ppc sign-extending loads
3001  // sign-extend to the full 64-bits.
3002  if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD)
3003  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3004 
3005  ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
3006  // We don't sign-extend constants.
3007  if (InputConst)
3008  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3009 
3010  SDLoc dl(Input);
3011  SignExtensionsAdded++;
3012  return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl,
3013  MVT::i64, Input), 0);
3014 }
3015 
3016 /// If the value isn't guaranteed to be zero-extended to 64-bits, extend it.
3017 /// Otherwise just reinterpret it as a 64-bit value.
3018 /// Useful when emitting comparison code for 32-bit values without using
3019 /// the compare instruction (which only considers the lower 32-bits).
3020 SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) {
3021  assert(Input.getValueType() == MVT::i32 &&
3022  "Can only zero-extend 32-bit values here.");
3023  unsigned Opc = Input.getOpcode();
3024 
3025  // The only condition under which we can omit the actual extend instruction:
3026  // - The value is a positive constant
3027  // - The value comes from a load that isn't a sign-extending load
3028  // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext.
3029  bool IsTruncateOfZExt = Opc == ISD::TRUNCATE &&
3030  (Input.getOperand(0).getOpcode() == ISD::AssertZext ||
3031  Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND);
3032  if (IsTruncateOfZExt)
3033  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3034 
3035  ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
3036  if (InputConst && InputConst->getSExtValue() >= 0)
3037  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3038 
3039  LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
3040  // The input is a load that doesn't sign-extend (it will be zero-extended).
3041  if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD)
3042  return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3043 
3044  // None of the above, need to zero-extend.
3045  SDLoc dl(Input);
3046  ZeroExtensionsAdded++;
3047  return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input,
3048  S->getI64Imm(0, dl),
3049  S->getI64Imm(32, dl)), 0);
3050 }
3051 
3052 // Handle a 32-bit value in a 64-bit register and vice-versa. These are of
3053 // course not actual zero/sign extensions that will generate machine code,
3054 // they're just a way to reinterpret a 32 bit value in a register as a
3055 // 64 bit value and vice-versa.
3056 SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes,
3057  ExtOrTruncConversion Conv) {
3058  SDLoc dl(NatWidthRes);
3059 
3060  // For reinterpreting 32-bit values as 64 bit values, we generate
3061  // INSERT_SUBREG IMPLICIT_DEF:i64, <input>, TargetConstant:i32<1>
3062  if (Conv == ExtOrTruncConversion::Ext) {
3063  SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0);
3064  SDValue SubRegIdx =
3065  CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
3066  return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64,
3067  ImDef, NatWidthRes, SubRegIdx), 0);
3068  }
3069 
3070  assert(Conv == ExtOrTruncConversion::Trunc &&
3071  "Unknown convertion between 32 and 64 bit values.");
3072  // For reinterpreting 64-bit values as 32-bit values, we just need to
3073  // EXTRACT_SUBREG (i.e. extract the low word).
3074  SDValue SubRegIdx =
3075  CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
3076  return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32,
3077  NatWidthRes, SubRegIdx), 0);
3078 }
3079 
3080 // Produce a GPR sequence for compound comparisons (<=, >=) against zero.
3081 // Handle both zero-extensions and sign-extensions.
3082 SDValue
3083 IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
3084  ZeroCompare CmpTy) {
3085  EVT InVT = LHS.getValueType();
3086  bool Is32Bit = InVT == MVT::i32;
3087  SDValue ToExtend;
3088 
3089  // Produce the value that needs to be either zero or sign extended.
3090  switch (CmpTy) {
3091  case ZeroCompare::GEZExt:
3092  case ZeroCompare::GESExt:
3093  ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8,
3094  dl, InVT, LHS, LHS), 0);
3095  break;
3096  case ZeroCompare::LEZExt:
3097  case ZeroCompare::LESExt: {
3098  if (Is32Bit) {
3099  // Upper 32 bits cannot be undefined for this sequence.
3100  LHS = signExtendInputIfNeeded(LHS);
3101  SDValue Neg =
3102  SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3103  ToExtend =
3104  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3105  Neg, S->getI64Imm(1, dl),
3106  S->getI64Imm(63, dl)), 0);
3107  } else {
3108  SDValue Addi =
3109  SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3110  S->getI64Imm(~0ULL, dl)), 0);
3111  ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
3112  Addi, LHS), 0);
3113  }
3114  break;
3115  }
3116  }
3117 
3118  // For 64-bit sequences, the extensions are the same for the GE/LE cases.
3119  if (!Is32Bit &&
3120  (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt))
3121  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3122  ToExtend, S->getI64Imm(1, dl),
3123  S->getI64Imm(63, dl)), 0);
3124  if (!Is32Bit &&
3125  (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt))
3126  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend,
3127  S->getI64Imm(63, dl)), 0);
3128 
3129  assert(Is32Bit && "Should have handled the 32-bit sequences above.");
3130  // For 32-bit sequences, the extensions differ between GE/LE cases.
3131  switch (CmpTy) {
3132  case ZeroCompare::GEZExt: {
3133  SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3134  S->getI32Imm(31, dl) };
3135  return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3136  ShiftOps), 0);
3137  }
3138  case ZeroCompare::GESExt:
3139  return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend,
3140  S->getI32Imm(31, dl)), 0);
3141  case ZeroCompare::LEZExt:
3142  return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend,
3143  S->getI32Imm(1, dl)), 0);
3144  case ZeroCompare::LESExt:
3145  return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend,
3146  S->getI32Imm(-1, dl)), 0);
3147  }
3148 
3149  // The above case covers all the enumerators so it can't have a default clause
3150  // to avoid compiler warnings.
3151  llvm_unreachable("Unknown zero-comparison type.");
3152 }
3153 
3154 /// Produces a zero-extended result of comparing two 32-bit values according to
3155 /// the passed condition code.
3156 SDValue
3157 IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS,
3158  ISD::CondCode CC,
3159  int64_t RHSValue, SDLoc dl) {
3160  if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 ||
3162  return SDValue();
3163  bool IsRHSZero = RHSValue == 0;
3164  bool IsRHSOne = RHSValue == 1;
3165  bool IsRHSNegOne = RHSValue == -1LL;
3166  switch (CC) {
3167  default: return SDValue();
3168  case ISD::SETEQ: {
3169  // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5)
3170  // (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5)
3171  SDValue Xor = IsRHSZero ? LHS :
3172  SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3173  SDValue Clz =
3174  SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3175  SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
3176  S->getI32Imm(31, dl) };
3177  return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3178  ShiftOps), 0);
3179  }
3180  case ISD::SETNE: {
3181  // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1)
3182  // (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1)
3183  SDValue Xor = IsRHSZero ? LHS :
3184  SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3185  SDValue Clz =
3186  SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3187  SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
3188  S->getI32Imm(31, dl) };
3189  SDValue Shift =
3190  SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
3191  return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
3192  S->getI32Imm(1, dl)), 0);
3193  }
3194  case ISD::SETGE: {
3195  // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1)
3196  // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31)
3197  if(IsRHSZero)
3198  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3199 
3200  // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
3201  // by swapping inputs and falling through.
3202  std::swap(LHS, RHS);
3203  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3204  IsRHSZero = RHSConst && RHSConst->isZero();
3206  }
3207  case ISD::SETLE: {
3208  if (CmpInGPR == ICGPR_NonExtIn)
3209  return SDValue();
3210  // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1)
3211  // (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1)
3212  if(IsRHSZero) {
3213  if (CmpInGPR == ICGPR_NonExtIn)
3214  return SDValue();
3215  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3216  }
3217 
3218  // The upper 32-bits of the register can't be undefined for this sequence.
3219  LHS = signExtendInputIfNeeded(LHS);
3220  RHS = signExtendInputIfNeeded(RHS);
3221  SDValue Sub =
3222  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3223  SDValue Shift =
3224  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub,
3225  S->getI64Imm(1, dl), S->getI64Imm(63, dl)),
3226  0);
3227  return
3228  SDValue(CurDAG->getMachineNode(PPC::XORI8, dl,
3229  MVT::i64, Shift, S->getI32Imm(1, dl)), 0);
3230  }
3231  case ISD::SETGT: {
3232  // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63)
3233  // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31)
3234  // (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63)
3235  // Handle SETLT -1 (which is equivalent to SETGE 0).
3236  if (IsRHSNegOne)
3237  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3238 
3239  if (IsRHSZero) {
3240  if (CmpInGPR == ICGPR_NonExtIn)
3241  return SDValue();
3242  // The upper 32-bits of the register can't be undefined for this sequence.
3243  LHS = signExtendInputIfNeeded(LHS);
3244  RHS = signExtendInputIfNeeded(RHS);
3245  SDValue Neg =
3246  SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3247  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3248  Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0);
3249  }
3250  // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
3251  // (%b < %a) by swapping inputs and falling through.
3252  std::swap(LHS, RHS);
3253  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3254  IsRHSZero = RHSConst && RHSConst->isZero();
3255  IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3257  }
3258  case ISD::SETLT: {
3259  // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63)
3260  // (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1)
3261  // (zext (setcc %a, 0, setlt)) -> (lshr %a, 31)
3262  // Handle SETLT 1 (which is equivalent to SETLE 0).
3263  if (IsRHSOne) {
3264  if (CmpInGPR == ICGPR_NonExtIn)
3265  return SDValue();
3266  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3267  }
3268 
3269  if (IsRHSZero) {
3270  SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3271  S->getI32Imm(31, dl) };
3272  return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3273  ShiftOps), 0);
3274  }
3275 
3276  if (CmpInGPR == ICGPR_NonExtIn)
3277  return SDValue();
3278  // The upper 32-bits of the register can't be undefined for this sequence.
3279  LHS = signExtendInputIfNeeded(LHS);
3280  RHS = signExtendInputIfNeeded(RHS);
3281  SDValue SUBFNode =
3282  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3283  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3284  SUBFNode, S->getI64Imm(1, dl),
3285  S->getI64Imm(63, dl)), 0);
3286  }
3287  case ISD::SETUGE:
3288  // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1)
3289  // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1)
3290  std::swap(LHS, RHS);
3292  case ISD::SETULE: {
3293  if (CmpInGPR == ICGPR_NonExtIn)
3294  return SDValue();
3295  // The upper 32-bits of the register can't be undefined for this sequence.
3296  LHS = zeroExtendInputIfNeeded(LHS);
3297  RHS = zeroExtendInputIfNeeded(RHS);
3298  SDValue Subtract =
3299  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3300  SDValue SrdiNode =
3301  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3302  Subtract, S->getI64Imm(1, dl),
3303  S->getI64Imm(63, dl)), 0);
3304  return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode,
3305  S->getI32Imm(1, dl)), 0);
3306  }
3307  case ISD::SETUGT:
3308  // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63)
3309  // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63)
3310  std::swap(LHS, RHS);
3312  case ISD::SETULT: {
3313  if (CmpInGPR == ICGPR_NonExtIn)
3314  return SDValue();
3315  // The upper 32-bits of the register can't be undefined for this sequence.
3316  LHS = zeroExtendInputIfNeeded(LHS);
3317  RHS = zeroExtendInputIfNeeded(RHS);
3318  SDValue Subtract =
3319  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3320  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3321  Subtract, S->getI64Imm(1, dl),
3322  S->getI64Imm(63, dl)), 0);
3323  }
3324  }
3325 }
3326 
3327 /// Produces a sign-extended result of comparing two 32-bit values according to
3328 /// the passed condition code.
3329 SDValue
3330 IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS,
3331  ISD::CondCode CC,
3332  int64_t RHSValue, SDLoc dl) {
3333  if (CmpInGPR == ICGPR_I64 || CmpInGPR == ICGPR_SextI64 ||
3335  return SDValue();
3336  bool IsRHSZero = RHSValue == 0;
3337  bool IsRHSOne = RHSValue == 1;
3338  bool IsRHSNegOne = RHSValue == -1LL;
3339 
3340  switch (CC) {
3341  default: return SDValue();
3342  case ISD::SETEQ: {
3343  // (sext (setcc %a, %b, seteq)) ->
3344  // (ashr (shl (ctlz (xor %a, %b)), 58), 63)
3345  // (sext (setcc %a, 0, seteq)) ->
3346  // (ashr (shl (ctlz %a), 58), 63)
3347  SDValue CountInput = IsRHSZero ? LHS :
3348  SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3349  SDValue Cntlzw =
3350  SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0);
3351  SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl),
3352  S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3353  SDValue Slwi =
3354  SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0);
3355  return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0);
3356  }
3357  case ISD::SETNE: {
3358  // Bitwise xor the operands, count leading zeros, shift right by 5 bits and
3359  // flip the bit, finally take 2's complement.
3360  // (sext (setcc %a, %b, setne)) ->
3361  // (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1))
3362  // Same as above, but the first xor is not needed.
3363  // (sext (setcc %a, 0, setne)) ->
3364  // (neg (xor (lshr (ctlz %a), 5), 1))
3365  SDValue Xor = IsRHSZero ? LHS :
3366  SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3367  SDValue Clz =
3368  SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3369  SDValue ShiftOps[] =
3370  { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3371  SDValue Shift =
3372  SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
3373  SDValue Xori =
3374  SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
3375  S->getI32Imm(1, dl)), 0);
3376  return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0);
3377  }
3378  case ISD::SETGE: {
3379  // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1)
3380  // (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31)
3381  if (IsRHSZero)
3382  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3383 
3384  // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
3385  // by swapping inputs and falling through.
3386  std::swap(LHS, RHS);
3387  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3388  IsRHSZero = RHSConst && RHSConst->isZero();
3390  }
3391  case ISD::SETLE: {
3392  if (CmpInGPR == ICGPR_NonExtIn)
3393  return SDValue();
3394  // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1)
3395  // (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1)
3396  if (IsRHSZero)
3397  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3398 
3399  // The upper 32-bits of the register can't be undefined for this sequence.
3400  LHS = signExtendInputIfNeeded(LHS);
3401  RHS = signExtendInputIfNeeded(RHS);
3402  SDValue SUBFNode =
3403  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue,
3404  LHS, RHS), 0);
3405  SDValue Srdi =
3406  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3407  SUBFNode, S->getI64Imm(1, dl),
3408  S->getI64Imm(63, dl)), 0);
3409  return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi,
3410  S->getI32Imm(-1, dl)), 0);
3411  }
3412  case ISD::SETGT: {
3413  // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63)
3414  // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31)
3415  // (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63)
3416  if (IsRHSNegOne)
3417  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3418  if (IsRHSZero) {
3419  if (CmpInGPR == ICGPR_NonExtIn)
3420  return SDValue();
3421  // The upper 32-bits of the register can't be undefined for this sequence.
3422  LHS = signExtendInputIfNeeded(LHS);
3423  RHS = signExtendInputIfNeeded(RHS);
3424  SDValue Neg =
3425  SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3426  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg,
3427  S->getI64Imm(63, dl)), 0);
3428  }
3429  // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
3430  // (%b < %a) by swapping inputs and falling through.
3431  std::swap(LHS, RHS);
3432  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3433  IsRHSZero = RHSConst && RHSConst->isZero();
3434  IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3436  }
3437  case ISD::SETLT: {
3438  // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63)
3439  // (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1)
3440  // (sext (setcc %a, 0, setgt)) -> (ashr %a, 31)
3441  if (IsRHSOne) {
3442  if (CmpInGPR == ICGPR_NonExtIn)
3443  return SDValue();
3444  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3445  }
3446  if (IsRHSZero)
3447  return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS,
3448  S->getI32Imm(31, dl)), 0);
3449 
3450  if (CmpInGPR == ICGPR_NonExtIn)
3451  return SDValue();
3452  // The upper 32-bits of the register can't be undefined for this sequence.
3453  LHS = signExtendInputIfNeeded(LHS);
3454  RHS = signExtendInputIfNeeded(RHS);
3455  SDValue SUBFNode =
3456  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3457  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3458  SUBFNode, S->getI64Imm(63, dl)), 0);
3459  }
3460  case ISD::SETUGE:
3461  // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1)
3462  // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1)
3463  std::swap(LHS, RHS);
3465  case ISD::SETULE: {
3466  if (CmpInGPR == ICGPR_NonExtIn)
3467  return SDValue();
3468  // The upper 32-bits of the register can't be undefined for this sequence.
3469  LHS = zeroExtendInputIfNeeded(LHS);
3470  RHS = zeroExtendInputIfNeeded(RHS);
3471  SDValue Subtract =
3472  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3473  SDValue Shift =
3474  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract,
3475  S->getI32Imm(1, dl), S->getI32Imm(63,dl)),
3476  0);
3477  return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift,
3478  S->getI32Imm(-1, dl)), 0);
3479  }
3480  case ISD::SETUGT:
3481  // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63)
3482  // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63)
3483  std::swap(LHS, RHS);
3485  case ISD::SETULT: {
3486  if (CmpInGPR == ICGPR_NonExtIn)
3487  return SDValue();
3488  // The upper 32-bits of the register can't be undefined for this sequence.
3489  LHS = zeroExtendInputIfNeeded(LHS);
3490  RHS = zeroExtendInputIfNeeded(RHS);
3491  SDValue Subtract =
3492  SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3493  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3494  Subtract, S->getI64Imm(63, dl)), 0);
3495  }
3496  }
3497 }
3498 
3499 /// Produces a zero-extended result of comparing two 64-bit values according to
3500 /// the passed condition code.
3501 SDValue
3502 IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS,
3503  ISD::CondCode CC,
3504  int64_t RHSValue, SDLoc dl) {
3505  if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 ||
3507  return SDValue();
3508  bool IsRHSZero = RHSValue == 0;
3509  bool IsRHSOne = RHSValue == 1;
3510  bool IsRHSNegOne = RHSValue == -1LL;
3511  switch (CC) {
3512  default: return SDValue();
3513  case ISD::SETEQ: {
3514  // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6)
3515  // (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6)
3516  SDValue Xor = IsRHSZero ? LHS :
3517  SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3518  SDValue Clz =
3519  SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0);
3520  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz,
3521  S->getI64Imm(58, dl),
3522  S->getI64Imm(63, dl)), 0);
3523  }
3524  case ISD::SETNE: {
3525  // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3526  // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA)
3527  // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3528  // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3529  SDValue Xor = IsRHSZero ? LHS :
3530  SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3531  SDValue AC =
3532  SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3533  Xor, S->getI32Imm(~0U, dl)), 0);
3534  return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC,
3535  Xor, AC.getValue(1)), 0);
3536  }
3537  case ISD::SETGE: {
3538  // {subc.reg, subc.CA} = (subcarry %a, %b)
3539  // (zext (setcc %a, %b, setge)) ->
3540  // (adde (lshr %b, 63), (ashr %a, 63), subc.CA)
3541  // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63)
3542  if (IsRHSZero)
3543  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3544  std::swap(LHS, RHS);
3545  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3546  IsRHSZero = RHSConst && RHSConst->isZero();
3548  }
3549  case ISD::SETLE: {
3550  // {subc.reg, subc.CA} = (subcarry %b, %a)
3551  // (zext (setcc %a, %b, setge)) ->
3552  // (adde (lshr %a, 63), (ashr %b, 63), subc.CA)
3553  // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63)
3554  if (IsRHSZero)
3555  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3556  SDValue ShiftL =
3557  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3558  S->getI64Imm(1, dl),
3559  S->getI64Imm(63, dl)), 0);
3560  SDValue ShiftR =
3561  SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3562  S->getI64Imm(63, dl)), 0);
3563  SDValue SubtractCarry =
3564  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3565  LHS, RHS), 1);
3566  return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3567  ShiftR, ShiftL, SubtractCarry), 0);
3568  }
3569  case ISD::SETGT: {
3570  // {subc.reg, subc.CA} = (subcarry %b, %a)
3571  // (zext (setcc %a, %b, setgt)) ->
3572  // (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3573  // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63)
3574  if (IsRHSNegOne)
3575  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3576  if (IsRHSZero) {
3577  SDValue Addi =
3578  SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3579  S->getI64Imm(~0ULL, dl)), 0);
3580  SDValue Nor =
3581  SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0);
3582  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor,
3583  S->getI64Imm(1, dl),
3584  S->getI64Imm(63, dl)), 0);
3585  }
3586  std::swap(LHS, RHS);
3587  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3588  IsRHSZero = RHSConst && RHSConst->isZero();
3589  IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3591  }
3592  case ISD::SETLT: {
3593  // {subc.reg, subc.CA} = (subcarry %a, %b)
3594  // (zext (setcc %a, %b, setlt)) ->
3595  // (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3596  // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63)
3597  if (IsRHSOne)
3598  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3599  if (IsRHSZero)
3600  return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3601  S->getI64Imm(1, dl),
3602  S->getI64Imm(63, dl)), 0);
3603  SDValue SRADINode =
3604  SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3605  LHS, S->getI64Imm(63, dl)), 0);
3606  SDValue SRDINode =
3607  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3608  RHS, S->getI64Imm(1, dl),
3609  S->getI64Imm(63, dl)), 0);
3610  SDValue SUBFC8Carry =
3611  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3612  RHS, LHS), 1);
3613  SDValue ADDE8Node =
3614  SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3615  SRDINode, SRADINode, SUBFC8Carry), 0);
3616  return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3617  ADDE8Node, S->getI64Imm(1, dl)), 0);
3618  }
3619  case ISD::SETUGE:
3620  // {subc.reg, subc.CA} = (subcarry %a, %b)
3621  // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1)
3622  std::swap(LHS, RHS);
3624  case ISD::SETULE: {
3625  // {subc.reg, subc.CA} = (subcarry %b, %a)
3626  // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1)
3627  SDValue SUBFC8Carry =
3628  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3629  LHS, RHS), 1);
3630  SDValue SUBFE8Node =
3631  SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue,
3632  LHS, LHS, SUBFC8Carry), 0);
3633  return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64,
3634  SUBFE8Node, S->getI64Imm(1, dl)), 0);
3635  }
3636  case ISD::SETUGT:
3637  // {subc.reg, subc.CA} = (subcarry %b, %a)
3638  // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA)
3639  std::swap(LHS, RHS);
3641  case ISD::SETULT: {
3642  // {subc.reg, subc.CA} = (subcarry %a, %b)
3643  // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA)
3644  SDValue SubtractCarry =
3645  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3646  RHS, LHS), 1);
3647  SDValue ExtSub =
3648  SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3649  LHS, LHS, SubtractCarry), 0);
3650  return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3651  ExtSub), 0);
3652  }
3653  }
3654 }
3655 
3656 /// Produces a sign-extended result of comparing two 64-bit values according to
3657 /// the passed condition code.
3658 SDValue
3659 IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS,
3660  ISD::CondCode CC,
3661  int64_t RHSValue, SDLoc dl) {
3662  if (CmpInGPR == ICGPR_I32 || CmpInGPR == ICGPR_SextI32 ||
3664  return SDValue();
3665  bool IsRHSZero = RHSValue == 0;
3666  bool IsRHSOne = RHSValue == 1;
3667  bool IsRHSNegOne = RHSValue == -1LL;
3668  switch (CC) {
3669  default: return SDValue();
3670  case ISD::SETEQ: {
3671  // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3672  // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA)
3673  // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3674  // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3675  SDValue AddInput = IsRHSZero ? LHS :
3676  SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3677  SDValue Addic =
3678  SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3679  AddInput, S->getI32Imm(~0U, dl)), 0);
3680  return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic,
3681  Addic, Addic.getValue(1)), 0);
3682  }
3683  case ISD::SETNE: {
3684  // {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b))
3685  // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA)
3686  // {subfcz.reg, subfcz.CA} = (subcarry 0, %a)
3687  // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA)
3688  SDValue Xor = IsRHSZero ? LHS :
3689  SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3690  SDValue SC =
3691  SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue,
3692  Xor, S->getI32Imm(0, dl)), 0);
3693  return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC,
3694  SC, SC.getValue(1)), 0);
3695  }
3696  case ISD::SETGE: {
3697  // {subc.reg, subc.CA} = (subcarry %a, %b)
3698  // (zext (setcc %a, %b, setge)) ->
3699  // (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA))
3700  // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63))
3701  if (IsRHSZero)
3702  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3703  std::swap(LHS, RHS);
3704  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3705  IsRHSZero = RHSConst && RHSConst->isZero();
3707  }
3708  case ISD::SETLE: {
3709  // {subc.reg, subc.CA} = (subcarry %b, %a)
3710  // (zext (setcc %a, %b, setge)) ->
3711  // (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA))
3712  // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63)
3713  if (IsRHSZero)
3714  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3715  SDValue ShiftR =
3716  SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3717  S->getI64Imm(63, dl)), 0);
3718  SDValue ShiftL =
3719  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3720  S->getI64Imm(1, dl),
3721  S->getI64Imm(63, dl)), 0);
3722  SDValue SubtractCarry =
3723  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3724  LHS, RHS), 1);
3725  SDValue Adde =
3726  SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3727  ShiftR, ShiftL, SubtractCarry), 0);
3728  return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0);
3729  }
3730  case ISD::SETGT: {
3731  // {subc.reg, subc.CA} = (subcarry %b, %a)
3732  // (zext (setcc %a, %b, setgt)) ->
3733  // -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3734  // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63)
3735  if (IsRHSNegOne)
3736  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3737  if (IsRHSZero) {
3738  SDValue Add =
3739  SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3740  S->getI64Imm(-1, dl)), 0);
3741  SDValue Nor =
3742  SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0);
3743  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor,
3744  S->getI64Imm(63, dl)), 0);
3745  }
3746  std::swap(LHS, RHS);
3747  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3748  IsRHSZero = RHSConst && RHSConst->isZero();
3749  IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3751  }
3752  case ISD::SETLT: {
3753  // {subc.reg, subc.CA} = (subcarry %a, %b)
3754  // (zext (setcc %a, %b, setlt)) ->
3755  // -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3756  // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63)
3757  if (IsRHSOne)
3758  return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3759  if (IsRHSZero) {
3760  return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS,
3761  S->getI64Imm(63, dl)), 0);
3762  }
3763  SDValue SRADINode =
3764  SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3765  LHS, S->getI64Imm(63, dl)), 0);
3766  SDValue SRDINode =
3767  SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3768  RHS, S->getI64Imm(1, dl),
3769  S->getI64Imm(63, dl)), 0);
3770  SDValue SUBFC8Carry =
3771  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3772  RHS, LHS), 1);
3773  SDValue ADDE8Node =
3774  SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64,
3775  SRDINode, SRADINode, SUBFC8Carry), 0);
3776  SDValue XORI8Node =
3777  SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3778  ADDE8Node, S->getI64Imm(1, dl)), 0);
3779  return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3780  XORI8Node), 0);
3781  }
3782  case ISD::SETUGE:
3783  // {subc.reg, subc.CA} = (subcarry %a, %b)
3784  // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA)
3785  std::swap(LHS, RHS);
3787  case ISD::SETULE: {
3788  // {subc.reg, subc.CA} = (subcarry %b, %a)
3789  // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA)
3790  SDValue SubtractCarry =
3791  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3792  LHS, RHS), 1);
3793  SDValue ExtSub =
3794  SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS,
3795  LHS, SubtractCarry), 0);
3796  return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64,
3797  ExtSub, ExtSub), 0);
3798  }
3799  case ISD::SETUGT:
3800  // {subc.reg, subc.CA} = (subcarry %b, %a)
3801  // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA)
3802  std::swap(LHS, RHS);
3804  case ISD::SETULT: {
3805  // {subc.reg, subc.CA} = (subcarry %a, %b)
3806  // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA)
3807  SDValue SubCarry =
3808  SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3809  RHS, LHS), 1);
3810  return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3811  LHS, LHS, SubCarry), 0);
3812  }
3813  }
3814 }
3815 
3816 /// Do all uses of this SDValue need the result in a GPR?
3817 /// This is meant to be used on values that have type i1 since
3818 /// it is somewhat meaningless to ask if values of other types
3819 /// should be kept in GPR's.
3820 static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) {
3821  assert(Compare.getOpcode() == ISD::SETCC &&
3822  "An ISD::SETCC node required here.");
3823 
3824  // For values that have a single use, the caller should obviously already have
3825  // checked if that use is an extending use. We check the other uses here.
3826  if (Compare.hasOneUse())
3827  return true;
3828  // We want the value in a GPR if it is being extended, used for a select, or
3829  // used in logical operations.
3830  for (auto CompareUse : Compare.getNode()->uses())
3831  if (CompareUse->getOpcode() != ISD::SIGN_EXTEND &&
3832  CompareUse->getOpcode() != ISD::ZERO_EXTEND &&
3833  CompareUse->getOpcode() != ISD::SELECT &&
3834  !isLogicOp(CompareUse->getOpcode())) {
3835  OmittedForNonExtendUses++;
3836  return false;
3837  }
3838  return true;
3839 }
3840 
3841 /// Returns an equivalent of a SETCC node but with the result the same width as
3842 /// the inputs. This can also be used for SELECT_CC if either the true or false
3843 /// values is a power of two while the other is zero.
3844 SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare,
3845  SetccInGPROpts ConvOpts) {
3846  assert((Compare.getOpcode() == ISD::SETCC ||
3847  Compare.getOpcode() == ISD::SELECT_CC) &&
3848  "An ISD::SETCC node required here.");
3849 
3850  // Don't convert this comparison to a GPR sequence because there are uses
3851  // of the i1 result (i.e. uses that require the result in the CR).
3852  if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG))
3853  return SDValue();
3854 
3855  SDValue LHS = Compare.getOperand(0);
3856  SDValue RHS = Compare.getOperand(1);
3857 
3858  // The condition code is operand 2 for SETCC and operand 4 for SELECT_CC.
3859  int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2;
3860  ISD::CondCode CC =
3861  cast<CondCodeSDNode>(Compare.getOperand(CCOpNum))->get();
3862  EVT InputVT = LHS.getValueType();
3863  if (InputVT != MVT::i32 && InputVT != MVT::i64)
3864  return SDValue();
3865 
3866  if (ConvOpts == SetccInGPROpts::ZExtInvert ||
3867  ConvOpts == SetccInGPROpts::SExtInvert)
3868  CC = ISD::getSetCCInverse(CC, InputVT);
3869 
3870  bool Inputs32Bit = InputVT == MVT::i32;
3871 
3872  SDLoc dl(Compare);
3873  ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3874  int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX;
3875  bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig ||
3876  ConvOpts == SetccInGPROpts::SExtInvert;
3877 
3878  if (IsSext && Inputs32Bit)
3879  return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
3880  else if (Inputs32Bit)
3881  return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
3882  else if (IsSext)
3883  return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
3884  return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
3885 }
3886 
3887 } // end anonymous namespace
3888 
3889 bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) {
3890  if (N->getValueType(0) != MVT::i32 &&
3891  N->getValueType(0) != MVT::i64)
3892  return false;
3893 
3894  // This optimization will emit code that assumes 64-bit registers
3895  // so we don't want to run it in 32-bit mode. Also don't run it
3896  // on functions that are not to be optimized.
3897  if (TM.getOptLevel() == CodeGenOpt::None || !TM.isPPC64())
3898  return false;
3899 
3900  // For POWER10, it is more profitable to use the set boolean extension
3901  // instructions rather than the integer compare elimination codegen.
3902  // Users can override this via the command line option, `--ppc-gpr-icmps`.
3903  if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1())
3904  return false;
3905 
3906  switch (N->getOpcode()) {
3907  default: break;
3908  case ISD::ZERO_EXTEND:
3909  case ISD::SIGN_EXTEND:
3910  case ISD::AND:
3911  case ISD::OR:
3912  case ISD::XOR: {
3913  IntegerCompareEliminator ICmpElim(CurDAG, this);
3914  if (SDNode *New = ICmpElim.Select(N)) {
3915  ReplaceNode(N, New);
3916  return true;
3917  }
3918  }
3919  }
3920  return false;
3921 }
3922 
3923 bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) {
3924  if (N->getValueType(0) != MVT::i32 &&
3925  N->getValueType(0) != MVT::i64)
3926  return false;
3927 
3928  if (!UseBitPermRewriter)
3929  return false;
3930 
3931  switch (N->getOpcode()) {
3932  default: break;
3933  case ISD::ROTL:
3934  case ISD::SHL:
3935  case ISD::SRL:
3936  case ISD::AND:
3937  case ISD::OR: {
3938  BitPermutationSelector BPS(CurDAG);
3939  if (SDNode *New = BPS.Select(N)) {
3940  ReplaceNode(N, New);
3941  return true;
3942  }
3943  return false;
3944  }
3945  }
3946 
3947  return false;
3948 }
3949 
3950 /// SelectCC - Select a comparison of the specified values with the specified
3951 /// condition code, returning the CR# of the expression.
3952 SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
3953  const SDLoc &dl, SDValue Chain) {
3954  // Always select the LHS.
3955  unsigned Opc;
3956 
3957  if (LHS.getValueType() == MVT::i32) {
3958  unsigned Imm;
3959  if (CC == ISD::SETEQ || CC == ISD::SETNE) {
3960  if (isInt32Immediate(RHS, Imm)) {
3961  // SETEQ/SETNE comparison with 16-bit immediate, fold it.
3962  if (isUInt<16>(Imm))
3963  return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
3964  getI32Imm(Imm & 0xFFFF, dl)),
3965  0);
3966  // If this is a 16-bit signed immediate, fold it.
3967  if (isInt<16>((int)Imm))
3968  return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
3969  getI32Imm(Imm & 0xFFFF, dl)),
3970  0);
3971 
3972  // For non-equality comparisons, the default code would materialize the
3973  // constant, then compare against it, like this:
3974  // lis r2, 4660
3975  // ori r2, r2, 22136
3976  // cmpw cr0, r3, r2
3977  // Since we are just comparing for equality, we can emit this instead:
3978  // xoris r0,r3,0x1234
3979  // cmplwi cr0,r0,0x5678
3980  // beq cr0,L6
3981  SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS,
3982  getI32Imm(Imm >> 16, dl)), 0);
3983  return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor,
3984  getI32Imm(Imm & 0xFFFF, dl)), 0);
3985  }
3986  Opc = PPC::CMPLW;
3987  } else if (ISD::isUnsignedIntSetCC(CC)) {
3988  if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm))
3989  return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
3990  getI32Imm(Imm & 0xFFFF, dl)), 0);
3991  Opc = PPC::CMPLW;
3992  } else {
3993  int16_t SImm;
3994  if (isIntS16Immediate(RHS, SImm))
3995  return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
3996  getI32Imm((int)SImm & 0xFFFF,
3997  dl)),
3998  0);
3999  Opc = PPC::CMPW;
4000  }
4001  } else if (LHS.getValueType() == MVT::i64) {
4002  uint64_t Imm;
4003  if (CC == ISD::SETEQ || CC == ISD::SETNE) {
4004  if (isInt64Immediate(RHS.getNode(), Imm)) {
4005  // SETEQ/SETNE comparison with 16-bit immediate, fold it.
4006  if (isUInt<16>(Imm))
4007  return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
4008  getI32Imm(Imm & 0xFFFF, dl)),
4009  0);
4010  // If this is a 16-bit signed immediate, fold it.
4011  if (isInt<16>(Imm))
4012  return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
4013  getI32Imm(Imm & 0xFFFF, dl)),
4014  0);
4015 
4016  // For non-equality comparisons, the default code would materialize the
4017  // constant, then compare against it, like this:
4018  // lis r2, 4660
4019  // ori r2, r2, 22136
4020  // cmpd cr0, r3, r2
4021  // Since we are just comparing for equality, we can emit this instead:
4022  // xoris r0,r3,0x1234
4023  // cmpldi cr0,r0,0x5678
4024  // beq cr0,L6
4025  if (isUInt<32>(Imm)) {
4026  SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS,
4027  getI64Imm(Imm >> 16, dl)), 0);
4028  return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor,
4029  getI64Imm(Imm & 0xFFFF, dl)),
4030  0);
4031  }
4032  }
4033  Opc = PPC::CMPLD;
4034  } else if (ISD::isUnsignedIntSetCC(CC)) {
4035  if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm))
4036  return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
4037  getI64Imm(Imm & 0xFFFF, dl)), 0);
4038  Opc = PPC::CMPLD;
4039  } else {
4040  int16_t SImm;
4041  if (isIntS16Immediate(RHS, SImm))
4042  return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
4043  getI64Imm(SImm & 0xFFFF, dl)),
4044  0);
4045  Opc = PPC::CMPD;
4046  }
4047  } else if (LHS.getValueType() == MVT::f32) {
4048  if (Subtarget->hasSPE()) {
4049  switch (CC) {
4050  default:
4051  case ISD::SETEQ:
4052  case ISD::SETNE:
4053  Opc = PPC::EFSCMPEQ;
4054  break;
4055  case ISD::SETLT:
4056  case ISD::SETGE:
4057  case ISD::SETOLT:
4058  case ISD::SETOGE:
4059  case ISD::SETULT:
4060  case ISD::SETUGE:
4061  Opc = PPC::EFSCMPLT;
4062  break;
4063  case ISD::SETGT:
4064  case ISD::SETLE:
4065  case ISD::SETOGT:
4066  case ISD::SETOLE:
4067  case ISD::SETUGT:
4068  case ISD::SETULE:
4069  Opc = PPC::EFSCMPGT;
4070  break;
4071  }
4072  } else
4073  Opc = PPC::FCMPUS;
4074  } else if (LHS.getValueType() == MVT::f64) {
4075  if (Subtarget->hasSPE()) {
4076  switch (CC) {
4077  default:
4078  case ISD::SETEQ:
4079  case ISD::SETNE:
4080  Opc = PPC::EFDCMPEQ;
4081  break;
4082  case ISD::SETLT:
4083  case ISD::SETGE:
4084  case ISD::SETOLT:
4085  case ISD::SETOGE:
4086  case ISD::SETULT:
4087  case ISD::SETUGE:
4088  Opc = PPC::EFDCMPLT;
4089  break;
4090  case ISD::SETGT:
4091  case ISD::SETLE:
4092  case ISD::SETOGT:
4093  case ISD::SETOLE:
4094  case ISD::SETUGT:
4095  case ISD::SETULE:
4096  Opc = PPC::EFDCMPGT;
4097  break;
4098  }
4099  } else
4100  Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD;
4101  } else {
4102  assert(LHS.getValueType() == MVT::f128 && "Unknown vt!");
4103  assert(Subtarget->hasP9Vector() && "XSCMPUQP requires Power9 Vector");
4104  Opc = PPC::XSCMPUQP;
4105  }
4106  if (Chain)
4107  return SDValue(
4108  CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain),
4109  0);
4110  else
4111  return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0);
4112 }
4113 
4115  const PPCSubtarget *Subtarget) {
4116  // For SPE instructions, the result is in GT bit of the CR
4117  bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint();
4118 
4119  switch (CC) {
4120  case ISD::SETUEQ:
4121  case ISD::SETONE:
4122  case ISD::SETOLE:
4123  case ISD::SETOGE:
4124  llvm_unreachable("Should be lowered by legalize!");
4125  default: llvm_unreachable("Unknown condition!");
4126  case ISD::SETOEQ:
4127  case ISD::SETEQ:
4128  return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ;
4129  case ISD::SETUNE:
4130  case ISD::SETNE:
4131  return UseSPE ? PPC::PRED_LE : PPC::PRED_NE;
4132  case ISD::SETOLT:
4133  case ISD::SETLT:
4134  return UseSPE ? PPC::PRED_GT : PPC::PRED_LT;
4135  case ISD::SETULE:
4136  case ISD::SETLE:
4137  return PPC::PRED_LE;
4138  case ISD::SETOGT:
4139  case ISD::SETGT:
4140  return PPC::PRED_GT;
4141  case ISD::SETUGE:
4142  case ISD::SETGE:
4143  return UseSPE ? PPC::PRED_LE : PPC::PRED_GE;
4144  case ISD::SETO: return PPC::PRED_NU;
4145  case ISD::SETUO: return PPC::PRED_UN;
4146  // These two are invalid for floating point. Assume we have int.
4147  case ISD::SETULT: return PPC::PRED_LT;
4148  case ISD::SETUGT: return PPC::PRED_GT;
4149  }
4150 }
4151 
4152 /// getCRIdxForSetCC - Return the index of the condition register field
4153 /// associated with the SetCC condition, and whether or not the field is
4154 /// treated as inverted. That is, lt = 0; ge = 0 inverted.
4155 static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) {
4156  Invert = false;
4157  switch (CC) {
4158  default: llvm_unreachable("Unknown condition!");
4159  case ISD::SETOLT:
4160  case ISD::SETLT: return 0; // Bit #0 = SETOLT
4161  case ISD::SETOGT:
4162  case ISD::SETGT: return 1; // Bit #1 = SETOGT
4163  case ISD::SETOEQ:
4164  case ISD::SETEQ: return 2; // Bit #2 = SETOEQ
4165  case ISD::SETUO: return 3; // Bit #3 = SETUO
4166  case ISD::SETUGE:
4167  case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE
4168  case ISD::SETULE:
4169  case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE
4170  case ISD::SETUNE:
4171  case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE
4172  case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO
4173  case ISD::SETUEQ:
4174  case ISD::SETOGE:
4175  case ISD::SETOLE:
4176  case ISD::SETONE:
4177  llvm_unreachable("Invalid branch code: should be expanded by legalize");
4178  // These are invalid for floating point. Assume integer.
4179  case ISD::SETULT: return 0;
4180  case ISD::SETUGT: return 1;
4181  }
4182 }
4183 
4184 // getVCmpInst: return the vector compare instruction for the specified
4185 // vector type and condition code. Since this is for altivec specific code,
4186 // only support the altivec types (v16i8, v8i16, v4i32, v2i64, v1i128,
4187 // and v4f32).
4188 static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC,
4189  bool HasVSX, bool &Swap, bool &Negate) {
4190  Swap = false;
4191  Negate = false;
4192 
4193  if (VecVT.isFloatingPoint()) {
4194  /* Handle some cases by swapping input operands. */
4195  switch (CC) {
4196  case ISD::SETLE: CC = ISD::SETGE; Swap = true; break;
4197  case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
4198  case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break;
4199  case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break;
4200  case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
4201  case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break;
4202  default: break;
4203  }
4204  /* Handle some cases by negating the result. */
4205  switch (CC) {
4206  case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
4207  case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break;
4208  case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break;
4209  case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break;
4210  default: break;
4211  }
4212  /* We have instructions implementing the remaining cases. */
4213  switch (CC) {
4214  case ISD::SETEQ:
4215  case ISD::SETOEQ:
4216  if (VecVT == MVT::v4f32)
4217  return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP;
4218  else if (VecVT == MVT::v2f64)
4219  return PPC::XVCMPEQDP;
4220  break;
4221  case ISD::SETGT:
4222  case ISD::SETOGT:
4223  if (VecVT == MVT::v4f32)
4224  return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP;
4225  else if (VecVT == MVT::v2f64)
4226  return PPC::XVCMPGTDP;
4227  break;
4228  case ISD::SETGE:
4229  case ISD::SETOGE:
4230  if (VecVT == MVT::v4f32)
4231  return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP;
4232  else if (VecVT == MVT::v2f64)
4233  return PPC::XVCMPGEDP;
4234  break;
4235  default:
4236  break;
4237  }
4238  llvm_unreachable("Invalid floating-point vector compare condition");
4239  } else {
4240  /* Handle some cases by swapping input operands. */
4241  switch (CC) {
4242  case ISD::SETGE: CC = ISD::SETLE; Swap = true; break;
4243  case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
4244  case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
4245  case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break;
4246  default: break;
4247  }
4248  /* Handle some cases by negating the result. */
4249  switch (CC) {
4250  case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
4251  case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break;
4252  case ISD::SETLE: CC = ISD::SETGT; Negate = true; break;
4253  case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break;
4254  default: break;
4255  }
4256  /* We have instructions implementing the remaining cases. */
4257  switch (CC) {
4258  case ISD::SETEQ:
4259  case ISD::SETUEQ:
4260  if (VecVT == MVT::v16i8)
4261  return PPC::VCMPEQUB;
4262  else if (VecVT == MVT::v8i16)
4263  return PPC::VCMPEQUH;
4264  else if (VecVT == MVT::v4i32)
4265  return PPC::VCMPEQUW;
4266  else if (VecVT == MVT::v2i64)
4267  return PPC::VCMPEQUD;
4268  else if (VecVT == MVT::v1i128)
4269  return PPC::VCMPEQUQ;
4270  break;
4271  case ISD::SETGT:
4272  if (VecVT == MVT::v16i8)
4273  return PPC::VCMPGTSB;
4274  else if (VecVT == MVT::v8i16)
4275  return PPC::VCMPGTSH;
4276  else if (VecVT == MVT::v4i32)
4277  return PPC::VCMPGTSW;
4278  else if (VecVT == MVT::v2i64)
4279  return PPC::VCMPGTSD;
4280  else if (VecVT == MVT::v1i128)
4281  return PPC::VCMPGTSQ;
4282  break;
4283  case ISD::SETUGT:
4284  if (VecVT == MVT::v16i8)
4285  return PPC::VCMPGTUB;
4286  else if (VecVT == MVT::v8i16)
4287  return PPC::VCMPGTUH;
4288  else if (VecVT == MVT::v4i32)
4289  return PPC::VCMPGTUW;
4290  else if (VecVT == MVT::v2i64)
4291  return PPC::VCMPGTUD;
4292  else if (VecVT == MVT::v1i128)
4293  return PPC::VCMPGTUQ;
4294  break;
4295  default:
4296  break;
4297  }
4298  llvm_unreachable("Invalid integer vector compare condition");
4299  }
4300 }
4301 
4302 bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
4303  SDLoc dl(N);
4304  unsigned Imm;
4305  bool IsStrict = N->isStrictFPOpcode();
4306  ISD::CondCode CC =
4307  cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
4308  EVT PtrVT =
4309  CurDAG->getTargetLoweringInfo().getPointerTy(CurDAG->getDataLayout());
4310  bool isPPC64 = (PtrVT == MVT::i64);
4311  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4312 
4313  SDValue LHS = N->getOperand(IsStrict ? 1 : 0);
4314  SDValue RHS = N->getOperand(IsStrict ? 2 : 1);
4315 
4316  if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(RHS, Imm)) {
4317  // We can codegen setcc op, imm very efficiently compared to a brcond.
4318  // Check for those cases here.
4319  // setcc op, 0
4320  if (Imm == 0) {
4321  SDValue Op = LHS;
4322  switch (CC) {
4323  default: break;
4324  case ISD::SETEQ: {
4325  Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0);
4326  SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl),
4327  getI32Imm(31, dl) };
4328  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4329  return true;
4330  }
4331  case ISD::SETNE: {
4332  if (isPPC64) break;
4333  SDValue AD =
4334  SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4335  Op, getI32Imm(~0U, dl)), 0);
4336  CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1));
4337  return true;
4338  }
4339  case ISD::SETLT: {
4340  SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4341  getI32Imm(31, dl) };
4342  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4343  return true;
4344  }
4345  case ISD::SETGT: {
4346  SDValue T =
4347  SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0);
4348  T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0);
4349  SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl),
4350  getI32Imm(31, dl) };
4351  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4352  return true;
4353  }
4354  }
4355  } else if (Imm == ~0U) { // setcc op, -1
4356  SDValue Op = LHS;
4357  switch (CC) {
4358  default: break;
4359  case ISD::SETEQ:
4360  if (isPPC64) break;
4361  Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4362  Op, getI32Imm(1, dl)), 0);
4363  CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32,
4364  SDValue(CurDAG->getMachineNode(PPC::LI, dl,
4365  MVT::i32,
4366  getI32Imm(0, dl)),
4367  0), Op.getValue(1));
4368  return true;
4369  case ISD::SETNE: {
4370  if (isPPC64) break;
4371  Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0);
4372  SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4373  Op, getI32Imm(~0U, dl));
4374  CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op,
4375  SDValue(AD, 1));
4376  return true;
4377  }
4378  case ISD::SETLT: {
4379  SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op,
4380  getI32Imm(1, dl)), 0);
4381  SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD,
4382  Op), 0);
4383  SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl),
4384  getI32Imm(31, dl) };
4385  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4386  return true;
4387  }
4388  case ISD::SETGT: {
4389  SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4390  getI32Imm(31, dl) };
4391  Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4392  CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl));
4393  return true;
4394  }
4395  }
4396  }
4397  }
4398 
4399  // Altivec Vector compare instructions do not set any CR register by default and
4400  // vector compare operations return the same type as the operands.
4401  if (!IsStrict && LHS.getValueType().isVector()) {
4402  if (Subtarget->hasSPE())
4403  return false;
4404 
4405  EVT VecVT = LHS.getValueType();
4406  bool Swap, Negate;
4407  unsigned int VCmpInst =
4408  getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate);
4409  if (Swap)
4410  std::swap(LHS, RHS);
4411 
4412  EVT ResVT = VecVT.changeVectorElementTypeToInteger();
4413  if (Negate) {
4414  SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0);
4415  CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR,
4416  ResVT, VCmp, VCmp);
4417  return true;
4418  }
4419 
4420  CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS);
4421  return true;
4422  }
4423 
4424  if (Subtarget->useCRBits())
4425  return false;
4426 
4427  bool Inv;
4428  unsigned Idx = getCRIdxForSetCC(CC, Inv);
4429  SDValue CCReg = SelectCC(LHS, RHS, CC, dl, Chain);
4430  if (IsStrict)
4431  CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), CCReg.getValue(1));
4432  SDValue IntCR;
4433 
4434  // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that
4435  // The correct compare instruction is already set by SelectCC()
4436  if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) {
4437  Idx = 1;
4438  }
4439 
4440  // Force the ccreg into CR7.
4441  SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32);
4442 
4443  SDValue InFlag(nullptr, 0); // Null incoming flag value.
4444  CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg,
4445  InFlag).getValue(1);
4446 
4447  IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg,
4448  CCReg), 0);
4449 
4450  SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl),
4451  getI32Imm(31, dl), getI32Imm(31, dl) };
4452  if (!Inv) {
4453  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4454  return true;
4455  }
4456 
4457  // Get the specified bit.
4458  SDValue Tmp =
4459  SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4460  CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl));
4461  return true;
4462 }
4463 
4464 /// Does this node represent a load/store node whose address can be represented
4465 /// with a register plus an immediate that's a multiple of \p Val:
4466 bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const {
4467  LoadSDNode *LDN = dyn_cast<LoadSDNode>(N);
4468  StoreSDNode *STN = dyn_cast<StoreSDNode>(N);
4469  SDValue AddrOp;
4470  if (LDN)
4471  AddrOp = LDN->getOperand(1);
4472  else if (STN)
4473  AddrOp = STN->getOperand(2);
4474 
4475  // If the address points a frame object or a frame object with an offset,
4476  // we need to check the object alignment.
4477  short Imm = 0;
4478  if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(
4479  AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) :
4480  AddrOp)) {
4481  // If op0 is a frame index that is under aligned, we can't do it either,
4482  // because it is translated to r31 or r1 + slot + offset. We won't know the
4483  // slot number until the stack frame is finalized.
4484  const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo();
4485  unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value();
4486  if ((SlotAlign % Val) != 0)
4487  return false;
4488 
4489  // If we have an offset, we need further check on the offset.
4490  if (AddrOp.getOpcode() != ISD::ADD)
4491  return true;
4492  }
4493 
4494  if (AddrOp.getOpcode() == ISD::ADD)
4495  return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val);
4496 
4497  // If the address comes from the outside, the offset will be zero.
4498  return AddrOp.getOpcode() == ISD::CopyFromReg;
4499 }
4500 
4501 void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
4502  // Transfer memoperands.
4503  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
4504  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
4505 }
4506 
4507 static bool mayUseP9Setb(SDNode *N, const ISD::CondCode &CC, SelectionDAG *DAG,
4508  bool &NeedSwapOps, bool &IsUnCmp) {
4509 
4510  assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here.");
4511 
4512  SDValue LHS = N->getOperand(0);
4513  SDValue RHS = N->getOperand(1);
4514  SDValue TrueRes = N->getOperand(2);
4515  SDValue FalseRes = N->getOperand(3);
4516  ConstantSDNode *TrueConst = dyn_cast<ConstantSDNode>(TrueRes);
4517  if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 &&
4518  N->getSimpleValueType(0) != MVT::i32))
4519  return false;
4520 
4521  // We are looking for any of:
4522  // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4523  // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4524  // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq)
4525  // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq)
4526  int64_t TrueResVal = TrueConst->getSExtValue();
4527  if ((TrueResVal < -1 || TrueResVal > 1) ||
4528  (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) ||
4529  (TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) ||
4530  (TrueResVal == 0 &&
4531  (FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ)))
4532  return false;
4533 
4534  SDValue SetOrSelCC = FalseRes.getOpcode() == ISD::SELECT_CC
4535  ? FalseRes
4536  : FalseRes.getOperand(0);
4537  bool InnerIsSel = SetOrSelCC.getOpcode() == ISD::SELECT_CC;
4538  if (SetOrSelCC.getOpcode() != ISD::SETCC &&
4539  SetOrSelCC.getOpcode() != ISD::SELECT_CC)
4540  return false;
4541 
4542  // Without this setb optimization, the outer SELECT_CC will be manually
4543  // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass
4544  // transforms pseudo instruction to isel instruction. When there are more than
4545  // one use for result like zext/sext, with current optimization we only see
4546  // isel is replaced by setb but can't see any significant gain. Since
4547  // setb has longer latency than original isel, we should avoid this. Another
4548  // point is that setb requires comparison always kept, it can break the
4549  // opportunity to get the comparison away if we have in future.
4550  if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse()))
4551  return false;
4552 
4553  SDValue InnerLHS = SetOrSelCC.getOperand(0);
4554  SDValue InnerRHS = SetOrSelCC.getOperand(1);
4555  ISD::CondCode InnerCC =
4556  cast<CondCodeSDNode>(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get();
4557  // If the inner comparison is a select_cc, make sure the true/false values are
4558  // 1/-1 and canonicalize it if needed.
4559  if (InnerIsSel) {
4560  ConstantSDNode *SelCCTrueConst =
4561  dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(2));
4562  ConstantSDNode *SelCCFalseConst =
4563  dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(3));
4564  if (!SelCCTrueConst || !SelCCFalseConst)
4565  return false;
4566  int64_t SelCCTVal = SelCCTrueConst->getSExtValue();
4567  int64_t SelCCFVal = SelCCFalseConst->getSExtValue();
4568  // The values must be -1/1 (requiring a swap) or 1/-1.
4569  if (SelCCTVal == -1 && SelCCFVal == 1) {
4570  std::swap(InnerLHS, InnerRHS);
4571  } else if (SelCCTVal != 1 || SelCCFVal != -1)
4572  return false;
4573  }
4574 
4575  // Canonicalize unsigned case
4576  if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) {
4577  IsUnCmp = true;
4578  InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT;
4579  }
4580 
4581  bool InnerSwapped = false;
4582  if (LHS == InnerRHS && RHS == InnerLHS)
4583  InnerSwapped = true;
4584  else if (LHS != InnerLHS || RHS != InnerRHS)
4585  return false;
4586 
4587  switch (CC) {
4588  // (select_cc lhs, rhs, 0, \
4589  // (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq)
4590  case ISD::SETEQ:
4591  if (!InnerIsSel)
4592  return false;
4593  if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT)
4594  return false;
4595  NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped;
4596  break;
4597 
4598  // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4599  // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt)
4600  // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt)
4601  // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4602  // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt)
4603  // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt)
4604  case ISD::SETULT:
4605  if (!IsUnCmp && InnerCC != ISD::SETNE)
4606  return false;
4607  IsUnCmp = true;
4609  case ISD::SETLT:
4610  if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) ||
4611  (InnerCC == ISD::SETLT && InnerSwapped))
4612  NeedSwapOps = (TrueResVal == 1);
4613  else
4614  return false;
4615  break;
4616 
4617  // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4618  // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt)
4619  // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt)
4620  // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4621  // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt)
4622  // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt)
4623  case ISD::SETUGT:
4624  if (!IsUnCmp && InnerCC != ISD::SETNE)
4625  return false;
4626  IsUnCmp = true;
4628  case ISD::SETGT:
4629  if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) ||
4630  (InnerCC == ISD::SETGT && InnerSwapped))
4631  NeedSwapOps = (TrueResVal == -1);
4632  else
4633  return false;
4634  break;
4635 
4636  default:
4637  return false;
4638  }
4639 
4640  LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: ");
4641  LLVM_DEBUG(N->dump());
4642 
4643  return true;
4644 }
4645 
4646 // Return true if it's a software square-root/divide operand.
4647 static bool isSWTestOp(SDValue N) {
4648  if (N.getOpcode() == PPCISD::FTSQRT)
4649  return true;
4650  if (N.getNumOperands() < 1 || !isa<ConstantSDNode>(N.getOperand(0)))
4651  return false;
4652  switch (N.getConstantOperandVal(0)) {
4653  case Intrinsic::ppc_vsx_xvtdivdp:
4654  case Intrinsic::ppc_vsx_xvtdivsp:
4655  case Intrinsic::ppc_vsx_xvtsqrtdp:
4656  case Intrinsic::ppc_vsx_xvtsqrtsp:
4657  return true;
4658  }
4659  return false;
4660 }
4661 
4662 bool PPCDAGToDAGISel::tryFoldSWTestBRCC(SDNode *N) {
4663  assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected.");
4664  // We are looking for following patterns, where `truncate to i1` actually has
4665  // the same semantic with `and 1`.
4666  // (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp)
4667  // (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp)
4668  // (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp)
4669  // (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp)
4670  // (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp)
4671  // (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp)
4672  // (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp)
4673  // (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp)
4674  ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
4675  if (CC != ISD::SETEQ && CC != ISD::SETNE)
4676  return false;
4677 
4678  SDValue CmpRHS = N->getOperand(3);
4679  if (!isa<ConstantSDNode>(CmpRHS) ||
4680  cast<ConstantSDNode>(CmpRHS)->getSExtValue() != 0)
4681  return false;
4682 
4683  SDValue CmpLHS = N->getOperand(2);
4684  if (CmpLHS.getNumOperands() < 1 || !isSWTestOp(CmpLHS.getOperand(0)))
4685  return false;
4686 
4687  unsigned PCC = 0;
4688  bool IsCCNE = CC == ISD::SETNE;
4689  if (CmpLHS.getOpcode() == ISD::AND &&
4690  isa<ConstantSDNode>(CmpLHS.getOperand(1)))
4691  switch (CmpLHS.getConstantOperandVal(1)) {
4692  case 1:
4693  PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU;
4694  break;
4695  case 2:
4696  PCC = IsCCNE ? PPC::PRED_EQ : PPC::PRED_NE;
4697  break;
4698  case 4:
4699  PCC = IsCCNE ? PPC::PRED_GT : PPC::PRED_LE;
4700  break;
4701  case 8:
4702  PCC = IsCCNE ? PPC::PRED_LT : PPC::PRED_GE;
4703  break;
4704  default:
4705  return false;
4706  }
4707  else if (CmpLHS.getOpcode() == ISD::TRUNCATE &&
4708  CmpLHS.getValueType() == MVT::i1)
4709  PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU;
4710 
4711  if (PCC) {
4712  SDLoc dl(N);
4713  SDValue Ops[] = {getI32Imm(PCC, dl), CmpLHS.getOperand(0), N->getOperand(4),
4714  N->getOperand(0)};
4715  CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
4716  return true;
4717  }
4718  return false;
4719 }
4720 
4721 bool PPCDAGToDAGISel::tryAsSingleRLWINM(SDNode *N) {
4722  assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4723  unsigned Imm;
4724  if (!isInt32Immediate(N->getOperand(1), Imm))
4725  return false;
4726 
4727  SDLoc dl(N);
4728  SDValue Val = N->getOperand(0);
4729  unsigned SH, MB, ME;
4730  // If this is an and of a value rotated between 0 and 31 bits and then and'd
4731  // with a mask, emit rlwinm
4732  if (isRotateAndMask(Val.getNode(), Imm, false, SH, MB, ME)) {
4733  Val = Val.getOperand(0);
4734  SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl),
4735  getI32Imm(ME, dl)};
4736  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4737  return true;
4738  }
4739 
4740  // If this is just a masked value where the input is not handled, and
4741  // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm
4742  if (isRunOfOnes(Imm, MB, ME) && Val.getOpcode() != ISD::ROTL) {
4743  SDValue Ops[] = {Val, getI32Imm(0, dl), getI32Imm(MB, dl),
4744  getI32Imm(ME, dl)};
4745  CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4746  return true;
4747  }
4748 
4749  // AND X, 0 -> 0, not "rlwinm 32".
4750  if (Imm == 0) {
4751  ReplaceUses(SDValue(N, 0), N->getOperand(1));
4752  return true;
4753  }
4754 
4755  return false;
4756 }
4757 
4758 bool PPCDAGToDAGISel::tryAsSingleRLWINM8(SDNode *N) {
4759  assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4760  uint64_t Imm64;
4761  if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64))
4762  return false;
4763 
4764  unsigned MB, ME;
4765  if (isRunOfOnes64(Imm64, MB, ME) && MB >= 32 && MB <= ME) {
4766  // MB ME
4767  // +----------------------+
4768  // |xxxxxxxxxxx00011111000|
4769  // +----------------------+
4770  // 0 32 64
4771  // We can only do it if the MB is larger than 32 and MB <= ME
4772  // as RLWINM will replace the contents of [0 - 32) with [32 - 64) even
4773  // we didn't rotate it.
4774  SDLoc dl(N);
4775  SDValue Ops[] = {N->getOperand(0), getI64Imm(0, dl), getI64Imm(MB - 32, dl),
4776  getI64Imm(ME - 32, dl)};
4777  CurDAG->SelectNodeTo(N, PPC::RLWINM8, MVT::i64, Ops);
4778  return true;
4779  }
4780 
4781  return false;
4782 }
4783 
4784 bool PPCDAGToDAGISel::tryAsPairOfRLDICL(SDNode *N) {
4785  assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4786  uint64_t Imm64;
4787  if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64))
4788  return false;
4789 
4790  // Do nothing if it is 16-bit imm as the pattern in the .td file handle
4791  // it well with "andi.".
4792  if (isUInt<16>(Imm64))
4793  return false;
4794 
4795  SDLoc Loc(N);
4796  SDValue Val = N->getOperand(0);
4797 
4798  // Optimized with two rldicl's as follows:
4799  // Add missing bits on left to the mask and check that the mask is a
4800  // wrapped run of ones, i.e.
4801  // Change pattern |0001111100000011111111|
4802  // to |1111111100000011111111|.
4803  unsigned NumOfLeadingZeros = countLeadingZeros(Imm64);
4804  if (NumOfLeadingZeros != 0)
4805  Imm64 |= maskLeadingOnes<uint64_t>(NumOfLeadingZeros);
4806 
4807  unsigned MB, ME;
4808  if (!isRunOfOnes64(Imm64, MB, ME))
4809  return false;
4810 
4811  // ME MB MB-ME+63
4812  // +----------------------+ +----------------------+
4813  // |1111111100000011111111| -> |0000001111111111111111|
4814  // +----------------------+ +----------------------+
4815  // 0 63 0 63
4816  // There are ME + 1 ones on the left and (MB - ME + 63) & 63 zeros in between.
4817  unsigned OnesOnLeft = ME + 1;
4818  unsigned ZerosInBetween = (MB - ME + 63) & 63;
4819  // Rotate left by OnesOnLeft (so leading ones are now trailing ones) and clear
4820  // on the left the bits that are already zeros in the mask.
4821  Val = SDValue(CurDAG->getMachineNode(PPC::RLDICL, Loc, MVT::i64, Val,
4822  getI64Imm(OnesOnLeft, Loc),
4823  getI64Imm(ZerosInBetween, Loc)),
4824  0);
4825  // MB-ME+63 ME MB
4826  // +----------------------+ +----------------------+
4827  // |0000001111111111111111| -> |0001111100000011111111|
4828  // +----------------------+ +----------------------+
4829  // 0 63 0 63
4830  // Rotate back by 64 - OnesOnLeft to undo previous rotate. Then clear on the
4831  // left the number of ones we previously added.
4832  SDValue Ops[] = {Val, getI64Imm(64 - OnesOnLeft, Loc),
4833  getI64Imm(NumOfLeadingZeros, Loc)};
4834  CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops);
4835  return true;
4836 }
4837 
4838 bool PPCDAGToDAGISel::tryAsSingleRLWIMI(SDNode *N) {
4839  assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4840  unsigned Imm;
4841  if (!isInt32Immediate(N->getOperand(1), Imm))
4842  return false;
4843 
4844  SDValue Val = N->getOperand(0);
4845  unsigned Imm2;
4846  // ISD::OR doesn't get all the bitfield insertion fun.
4847  // (and (or x, c1), c2) where isRunOfOnes(~(c1^c2)) might be a
4848  // bitfield insert.
4849  if (Val.getOpcode() != ISD::OR || !isInt32Immediate(Val.getOperand(1), Imm2))
4850  return false;
4851 
4852  // The idea here is to check whether this is equivalent to:
4853  // (c1 & m) | (x & ~m)
4854  // where m is a run-of-ones mask. The logic here is that, for each bit in
4855  // c1 and c2:
4856  // - if both are 1, then the output will be 1.
4857  // - if both are 0, then the output will be 0.
4858  // - if the bit in c1 is 0, and the bit in c2 is 1, then the output will
4859  // come from x.
4860  // - if the bit in c1 is 1, and the bit in c2 is 0, then the output will
4861  // be 0.
4862  // If that last condition is never the case, then we can form m from the
4863  // bits that are the same between c1 and c2.
4864  unsigned MB, ME;
4865  if (isRunOfOnes(~(Imm ^ Imm2), MB, ME) && !(~Imm & Imm2)) {
4866  SDLoc dl(N);
4867  SDValue Ops[] = {Val.getOperand(0), Val.getOperand(1), getI32Imm(0, dl),
4868  getI32Imm(MB, dl), getI32Imm(ME, dl)};
4869  ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops));
4870  return true;
4871  }
4872 
4873  return false;
4874 }
4875 
4876 bool PPCDAGToDAGISel::tryAsSingleRLDICL(SDNode *N) {
4877  assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4878  uint64_t Imm64;
4879  if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) || !isMask_64(Imm64))
4880  return false;
4881 
4882  // If this is a 64-bit zero-extension mask, emit rldicl.
4883  unsigned MB = 64 - countTrailingOnes(Imm64);
4884  unsigned SH = 0;
4885  unsigned Imm;
4886  SDValue Val = N->getOperand(0);
4887  SDLoc dl(N);
4888 
4889  if (Val.getOpcode() == ISD::ANY_EXTEND) {
4890  auto Op0 = Val.getOperand(0);
4891  if (Op0.getOpcode() == ISD::SRL &&
4892  isInt32Immediate(Op0.getOperand(1).getNode(), Imm) && Imm <= MB) {
4893 
4894  auto ResultType = Val.getNode()->getValueType(0);
4895  auto ImDef = CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, ResultType);
4896  SDValue IDVal(ImDef, 0);
4897 
4898  Val = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, ResultType,
4899  IDVal, Op0.getOperand(0),
4900  getI32Imm(1, dl)),
4901  0);
4902  SH = 64 - Imm;
4903  }
4904  }
4905 
4906  // If the operand is a logical right shift, we can fold it into this
4907  // instruction: rldicl(rldicl(x, 64-n, n), 0, mb) -> rldicl(x, 64-n, mb)
4908  // for n <= mb. The right shift is really a left rotate followed by a
4909  // mask, and this mask is a more-restrictive sub-mask of the mask implied
4910  // by the shift.
4911  if (Val.getOpcode() == ISD::SRL &&
4912  isInt32Immediate(Val.getOperand(1).getNode(), Imm) && Imm <= MB) {
4913  assert(Imm < 64 && "Illegal shift amount");
4914  Val = Val.getOperand(0);
4915  SH = 64 - Imm;
4916  }
4917 
4918  SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl)};
4919  CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops);
4920  return true;
4921 }
4922 
4923 bool PPCDAGToDAGISel::tryAsSingleRLDICR(SDNode *N) {
4924  assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4925  uint64_t Imm64;
4926  if (!isInt64Immediate(N->getOperand(1).getNode(), Imm64) ||
4927  !isMask_64(~Imm64))
4928  return false;
4929 
4930  // If this is a negated 64-bit zero-extension mask,
4931  // i.e. the immediate is a sequence of ones from most significant side
4932  // and all zero for reminder, we should use rldicr.
4933  unsigned MB = 63 - countTrailingOnes(~Imm64);
4934  unsigned SH = 0;
4935  SDLoc dl(N);
4936  SDValue Ops[] = {N->getOperand(0), getI32Imm(SH, dl), getI32Imm(MB, dl)};
4937  CurDAG->SelectNodeTo(N, PPC::RLDICR, MVT::i64, Ops);
4938  return true;
4939 }