LLVM 19.0.0git
PPCISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- PPCISelDAGToDAG.cpp - PPC --pattern matching inst selector --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines a pattern matching instruction selector for PowerPC,
10// converting from a legalized dag to a PPC dag.
11//
12//===----------------------------------------------------------------------===//
13
16#include "PPC.h"
17#include "PPCISelLowering.h"
19#include "PPCSubtarget.h"
20#include "PPCTargetMachine.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/APSInt.h"
23#include "llvm/ADT/DenseMap.h"
24#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/Statistic.h"
43#include "llvm/IR/BasicBlock.h"
44#include "llvm/IR/DebugLoc.h"
45#include "llvm/IR/Function.h"
46#include "llvm/IR/GlobalValue.h"
47#include "llvm/IR/InlineAsm.h"
48#include "llvm/IR/InstrTypes.h"
49#include "llvm/IR/IntrinsicsPowerPC.h"
50#include "llvm/IR/Module.h"
55#include "llvm/Support/Debug.h"
60#include <algorithm>
61#include <cassert>
62#include <cstdint>
63#include <iterator>
64#include <limits>
65#include <memory>
66#include <new>
67#include <tuple>
68#include <utility>
69
70using namespace llvm;
71
72#define DEBUG_TYPE "ppc-isel"
73#define PASS_NAME "PowerPC DAG->DAG Pattern Instruction Selection"
74
75STATISTIC(NumSextSetcc,
76 "Number of (sext(setcc)) nodes expanded into GPR sequence.");
77STATISTIC(NumZextSetcc,
78 "Number of (zext(setcc)) nodes expanded into GPR sequence.");
79STATISTIC(SignExtensionsAdded,
80 "Number of sign extensions for compare inputs added.");
81STATISTIC(ZeroExtensionsAdded,
82 "Number of zero extensions for compare inputs added.");
83STATISTIC(NumLogicOpsOnComparison,
84 "Number of logical ops on i1 values calculated in GPR.");
85STATISTIC(OmittedForNonExtendUses,
86 "Number of compares not eliminated as they have non-extending uses.");
87STATISTIC(NumP9Setb,
88 "Number of compares lowered to setb.");
89
90// FIXME: Remove this once the bug has been fixed!
91cl::opt<bool> ANDIGlueBug("expose-ppc-andi-glue-bug",
92cl::desc("expose the ANDI glue bug on PPC"), cl::Hidden);
93
94static cl::opt<bool>
95 UseBitPermRewriter("ppc-use-bit-perm-rewriter", cl::init(true),
96 cl::desc("use aggressive ppc isel for bit permutations"),
99 "ppc-bit-perm-rewriter-stress-rotates",
100 cl::desc("stress rotate selection in aggressive ppc isel for "
101 "bit permutations"),
102 cl::Hidden);
103
105 "ppc-use-branch-hint", cl::init(true),
106 cl::desc("Enable static hinting of branches on ppc"),
107 cl::Hidden);
108
110 "ppc-tls-opt", cl::init(true),
111 cl::desc("Enable tls optimization peephole"),
112 cl::Hidden);
113
117
119 "ppc-gpr-icmps", cl::Hidden, cl::init(ICGPR_All),
120 cl::desc("Specify the types of comparisons to emit GPR-only code for."),
121 cl::values(clEnumValN(ICGPR_None, "none", "Do not modify integer comparisons."),
122 clEnumValN(ICGPR_All, "all", "All possible int comparisons in GPRs."),
123 clEnumValN(ICGPR_I32, "i32", "Only i32 comparisons in GPRs."),
124 clEnumValN(ICGPR_I64, "i64", "Only i64 comparisons in GPRs."),
125 clEnumValN(ICGPR_NonExtIn, "nonextin",
126 "Only comparisons where inputs don't need [sz]ext."),
127 clEnumValN(ICGPR_Zext, "zext", "Only comparisons with zext result."),
128 clEnumValN(ICGPR_ZextI32, "zexti32",
129 "Only i32 comparisons with zext result."),
130 clEnumValN(ICGPR_ZextI64, "zexti64",
131 "Only i64 comparisons with zext result."),
132 clEnumValN(ICGPR_Sext, "sext", "Only comparisons with sext result."),
133 clEnumValN(ICGPR_SextI32, "sexti32",
134 "Only i32 comparisons with sext result."),
135 clEnumValN(ICGPR_SextI64, "sexti64",
136 "Only i64 comparisons with sext result.")));
137namespace {
138
139 //===--------------------------------------------------------------------===//
140 /// PPCDAGToDAGISel - PPC specific code to select PPC machine
141 /// instructions for SelectionDAG operations.
142 ///
143 class PPCDAGToDAGISel : public SelectionDAGISel {
144 const PPCTargetMachine &TM;
145 const PPCSubtarget *Subtarget = nullptr;
146 const PPCTargetLowering *PPCLowering = nullptr;
147 unsigned GlobalBaseReg = 0;
148
149 public:
150 static char ID;
151
152 PPCDAGToDAGISel() = delete;
153
154 explicit PPCDAGToDAGISel(PPCTargetMachine &tm, CodeGenOptLevel OptLevel)
155 : SelectionDAGISel(ID, tm, OptLevel), TM(tm) {}
156
157 bool runOnMachineFunction(MachineFunction &MF) override {
158 // Make sure we re-emit a set of the global base reg if necessary
159 GlobalBaseReg = 0;
160 Subtarget = &MF.getSubtarget<PPCSubtarget>();
161 PPCLowering = Subtarget->getTargetLowering();
162 if (Subtarget->hasROPProtect()) {
163 // Create a place on the stack for the ROP Protection Hash.
164 // The ROP Protection Hash will always be 8 bytes and aligned to 8
165 // bytes.
166 MachineFrameInfo &MFI = MF.getFrameInfo();
168 const int Result = MFI.CreateStackObject(8, Align(8), false);
170 }
172
173 return true;
174 }
175
176 void PreprocessISelDAG() override;
177 void PostprocessISelDAG() override;
178
179 /// getI16Imm - Return a target constant with the specified value, of type
180 /// i16.
181 inline SDValue getI16Imm(unsigned Imm, const SDLoc &dl) {
182 return CurDAG->getTargetConstant(Imm, dl, MVT::i16);
183 }
184
185 /// getI32Imm - Return a target constant with the specified value, of type
186 /// i32.
187 inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
188 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
189 }
190
191 /// getI64Imm - Return a target constant with the specified value, of type
192 /// i64.
193 inline SDValue getI64Imm(uint64_t Imm, const SDLoc &dl) {
194 return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
195 }
196
197 /// getSmallIPtrImm - Return a target constant of pointer type.
198 inline SDValue getSmallIPtrImm(uint64_t Imm, const SDLoc &dl) {
199 return CurDAG->getTargetConstant(
200 Imm, dl, PPCLowering->getPointerTy(CurDAG->getDataLayout()));
201 }
202
203 /// isRotateAndMask - Returns true if Mask and Shift can be folded into a
204 /// rotate and mask opcode and mask operation.
205 static bool isRotateAndMask(SDNode *N, unsigned Mask, bool isShiftMask,
206 unsigned &SH, unsigned &MB, unsigned &ME);
207
208 /// getGlobalBaseReg - insert code into the entry mbb to materialize the PIC
209 /// base register. Return the virtual register that holds this value.
210 SDNode *getGlobalBaseReg();
211
212 void selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset = 0);
213
214 // Select - Convert the specified operand from a target-independent to a
215 // target-specific node if it hasn't already been changed.
216 void Select(SDNode *N) override;
217
218 bool tryBitfieldInsert(SDNode *N);
219 bool tryBitPermutation(SDNode *N);
220 bool tryIntCompareInGPR(SDNode *N);
221
222 // tryTLSXFormLoad - Convert an ISD::LOAD fed by a PPCISD::ADD_TLS into
223 // an X-Form load instruction with the offset being a relocation coming from
224 // the PPCISD::ADD_TLS.
225 bool tryTLSXFormLoad(LoadSDNode *N);
226 // tryTLSXFormStore - Convert an ISD::STORE fed by a PPCISD::ADD_TLS into
227 // an X-Form store instruction with the offset being a relocation coming from
228 // the PPCISD::ADD_TLS.
229 bool tryTLSXFormStore(StoreSDNode *N);
230 /// SelectCC - Select a comparison of the specified values with the
231 /// specified condition code, returning the CR# of the expression.
232 SDValue SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
233 const SDLoc &dl, SDValue Chain = SDValue());
234
235 /// SelectAddrImmOffs - Return true if the operand is valid for a preinc
236 /// immediate field. Note that the operand at this point is already the
237 /// result of a prior SelectAddressRegImm call.
238 bool SelectAddrImmOffs(SDValue N, SDValue &Out) const {
239 if (N.getOpcode() == ISD::TargetConstant ||
240 N.getOpcode() == ISD::TargetGlobalAddress) {
241 Out = N;
242 return true;
243 }
244
245 return false;
246 }
247
248 /// SelectDSForm - Returns true if address N can be represented by the
249 /// addressing mode of DSForm instructions (a base register, plus a signed
250 /// 16-bit displacement that is a multiple of 4.
251 bool SelectDSForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
252 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
253 Align(4)) == PPC::AM_DSForm;
254 }
255
256 /// SelectDQForm - Returns true if address N can be represented by the
257 /// addressing mode of DQForm instructions (a base register, plus a signed
258 /// 16-bit displacement that is a multiple of 16.
259 bool SelectDQForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
260 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
261 Align(16)) == PPC::AM_DQForm;
262 }
263
264 /// SelectDForm - Returns true if address N can be represented by
265 /// the addressing mode of DForm instructions (a base register, plus a
266 /// signed 16-bit immediate.
267 bool SelectDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
268 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
269 std::nullopt) == PPC::AM_DForm;
270 }
271
272 /// SelectPCRelForm - Returns true if address N can be represented by
273 /// PC-Relative addressing mode.
274 bool SelectPCRelForm(SDNode *Parent, SDValue N, SDValue &Disp,
275 SDValue &Base) {
276 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
277 std::nullopt) == PPC::AM_PCRel;
278 }
279
280 /// SelectPDForm - Returns true if address N can be represented by Prefixed
281 /// DForm addressing mode (a base register, plus a signed 34-bit immediate.
282 bool SelectPDForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
283 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
284 std::nullopt) ==
286 }
287
288 /// SelectXForm - Returns true if address N can be represented by the
289 /// addressing mode of XForm instructions (an indexed [r+r] operation).
290 bool SelectXForm(SDNode *Parent, SDValue N, SDValue &Disp, SDValue &Base) {
291 return PPCLowering->SelectOptimalAddrMode(Parent, N, Disp, Base, *CurDAG,
292 std::nullopt) == PPC::AM_XForm;
293 }
294
295 /// SelectForceXForm - Given the specified address, force it to be
296 /// represented as an indexed [r+r] operation (an XForm instruction).
297 bool SelectForceXForm(SDNode *Parent, SDValue N, SDValue &Disp,
298 SDValue &Base) {
299 return PPCLowering->SelectForceXFormMode(N, Disp, Base, *CurDAG) ==
301 }
302
303 /// SelectAddrIdx - Given the specified address, check to see if it can be
304 /// represented as an indexed [r+r] operation.
305 /// This is for xform instructions whose associated displacement form is D.
306 /// The last parameter \p 0 means associated D form has no requirment for 16
307 /// bit signed displacement.
308 /// Returns false if it can be represented by [r+imm], which are preferred.
309 bool SelectAddrIdx(SDValue N, SDValue &Base, SDValue &Index) {
310 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
311 std::nullopt);
312 }
313
314 /// SelectAddrIdx4 - Given the specified address, check to see if it can be
315 /// represented as an indexed [r+r] operation.
316 /// This is for xform instructions whose associated displacement form is DS.
317 /// The last parameter \p 4 means associated DS form 16 bit signed
318 /// displacement must be a multiple of 4.
319 /// Returns false if it can be represented by [r+imm], which are preferred.
320 bool SelectAddrIdxX4(SDValue N, SDValue &Base, SDValue &Index) {
321 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
322 Align(4));
323 }
324
325 /// SelectAddrIdx16 - Given the specified address, check to see if it can be
326 /// represented as an indexed [r+r] operation.
327 /// This is for xform instructions whose associated displacement form is DQ.
328 /// The last parameter \p 16 means associated DQ form 16 bit signed
329 /// displacement must be a multiple of 16.
330 /// Returns false if it can be represented by [r+imm], which are preferred.
331 bool SelectAddrIdxX16(SDValue N, SDValue &Base, SDValue &Index) {
332 return PPCLowering->SelectAddressRegReg(N, Base, Index, *CurDAG,
333 Align(16));
334 }
335
336 /// SelectAddrIdxOnly - Given the specified address, force it to be
337 /// represented as an indexed [r+r] operation.
338 bool SelectAddrIdxOnly(SDValue N, SDValue &Base, SDValue &Index) {
339 return PPCLowering->SelectAddressRegRegOnly(N, Base, Index, *CurDAG);
340 }
341
342 /// SelectAddrImm - Returns true if the address N can be represented by
343 /// a base register plus a signed 16-bit displacement [r+imm].
344 /// The last parameter \p 0 means D form has no requirment for 16 bit signed
345 /// displacement.
346 bool SelectAddrImm(SDValue N, SDValue &Disp,
347 SDValue &Base) {
348 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG,
349 std::nullopt);
350 }
351
352 /// SelectAddrImmX4 - Returns true if the address N can be represented by
353 /// a base register plus a signed 16-bit displacement that is a multiple of
354 /// 4 (last parameter). Suitable for use by STD and friends.
355 bool SelectAddrImmX4(SDValue N, SDValue &Disp, SDValue &Base) {
356 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG, Align(4));
357 }
358
359 /// SelectAddrImmX16 - Returns true if the address N can be represented by
360 /// a base register plus a signed 16-bit displacement that is a multiple of
361 /// 16(last parameter). Suitable for use by STXV and friends.
362 bool SelectAddrImmX16(SDValue N, SDValue &Disp, SDValue &Base) {
363 return PPCLowering->SelectAddressRegImm(N, Disp, Base, *CurDAG,
364 Align(16));
365 }
366
367 /// SelectAddrImmX34 - Returns true if the address N can be represented by
368 /// a base register plus a signed 34-bit displacement. Suitable for use by
369 /// PSTXVP and friends.
370 bool SelectAddrImmX34(SDValue N, SDValue &Disp, SDValue &Base) {
371 return PPCLowering->SelectAddressRegImm34(N, Disp, Base, *CurDAG);
372 }
373
374 // Select an address into a single register.
375 bool SelectAddr(SDValue N, SDValue &Base) {
376 Base = N;
377 return true;
378 }
379
380 bool SelectAddrPCRel(SDValue N, SDValue &Base) {
381 return PPCLowering->SelectAddressPCRel(N, Base);
382 }
383
384 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
385 /// inline asm expressions. It is always correct to compute the value into
386 /// a register. The case of adding a (possibly relocatable) constant to a
387 /// register can be improved, but it is wrong to substitute Reg+Reg for
388 /// Reg in an asm, because the load or store opcode would have to change.
390 InlineAsm::ConstraintCode ConstraintID,
391 std::vector<SDValue> &OutOps) override {
392 switch(ConstraintID) {
393 default:
394 errs() << "ConstraintID: "
395 << InlineAsm::getMemConstraintName(ConstraintID) << "\n";
396 llvm_unreachable("Unexpected asm memory constraint");
403 // We need to make sure that this one operand does not end up in r0
404 // (because we might end up lowering this as 0(%op)).
405 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
406 const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF, /*Kind=*/1);
407 SDLoc dl(Op);
408 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i32);
409 SDValue NewOp =
410 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
411 dl, Op.getValueType(),
412 Op, RC), 0);
413
414 OutOps.push_back(NewOp);
415 return false;
416 }
417 return true;
418 }
419
420// Include the pieces autogenerated from the target description.
421#include "PPCGenDAGISel.inc"
422
423private:
424 bool trySETCC(SDNode *N);
425 bool tryFoldSWTestBRCC(SDNode *N);
426 bool trySelectLoopCountIntrinsic(SDNode *N);
427 bool tryAsSingleRLDICL(SDNode *N);
428 bool tryAsSingleRLDCL(SDNode *N);
429 bool tryAsSingleRLDICR(SDNode *N);
430 bool tryAsSingleRLWINM(SDNode *N);
431 bool tryAsSingleRLWINM8(SDNode *N);
432 bool tryAsSingleRLWIMI(SDNode *N);
433 bool tryAsPairOfRLDICL(SDNode *N);
434 bool tryAsSingleRLDIMI(SDNode *N);
435
436 void PeepholePPC64();
437 void PeepholePPC64ZExt();
438 void PeepholeCROps();
439
440 SDValue combineToCMPB(SDNode *N);
441 void foldBoolExts(SDValue &Res, SDNode *&N);
442
443 bool AllUsersSelectZero(SDNode *N);
444 void SwapAllSelectUsers(SDNode *N);
445
446 bool isOffsetMultipleOf(SDNode *N, unsigned Val) const;
447 void transferMemOperands(SDNode *N, SDNode *Result);
448 };
449
450} // end anonymous namespace
451
452char PPCDAGToDAGISel::ID = 0;
453
454INITIALIZE_PASS(PPCDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false)
455
456/// getGlobalBaseReg - Output the instructions required to put the
457/// base address to use for accessing globals into a register.
458///
459SDNode *PPCDAGToDAGISel::getGlobalBaseReg() {
460 if (!GlobalBaseReg) {
461 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
462 // Insert the set of GlobalBaseReg into the first MBB of the function
463 MachineBasicBlock &FirstMBB = MF->front();
465 const Module *M = MF->getFunction().getParent();
466 DebugLoc dl;
467
468 if (PPCLowering->getPointerTy(CurDAG->getDataLayout()) == MVT::i32) {
469 if (Subtarget->isTargetELF()) {
470 GlobalBaseReg = PPC::R30;
471 if (!Subtarget->isSecurePlt() &&
472 M->getPICLevel() == PICLevel::SmallPIC) {
473 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MoveGOTtoLR));
474 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
475 MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
476 } else {
477 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
478 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
479 Register TempReg = RegInfo->createVirtualRegister(&PPC::GPRCRegClass);
480 BuildMI(FirstMBB, MBBI, dl,
481 TII.get(PPC::UpdateGBR), GlobalBaseReg)
482 .addReg(TempReg, RegState::Define).addReg(GlobalBaseReg);
483 MF->getInfo<PPCFunctionInfo>()->setUsesPICBase(true);
484 }
485 } else {
487 RegInfo->createVirtualRegister(&PPC::GPRC_and_GPRC_NOR0RegClass);
488 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR));
489 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR), GlobalBaseReg);
490 }
491 } else {
492 // We must ensure that this sequence is dominated by the prologue.
493 // FIXME: This is a bit of a big hammer since we don't get the benefits
494 // of shrink-wrapping whenever we emit this instruction. Considering
495 // this is used in any function where we emit a jump table, this may be
496 // a significant limitation. We should consider inserting this in the
497 // block where it is used and then commoning this sequence up if it
498 // appears in multiple places.
499 // Note: on ISA 3.0 cores, we can use lnia (addpcis) instead of
500 // MovePCtoLR8.
501 MF->getInfo<PPCFunctionInfo>()->setShrinkWrapDisabled(true);
502 GlobalBaseReg = RegInfo->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
503 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MovePCtoLR8));
504 BuildMI(FirstMBB, MBBI, dl, TII.get(PPC::MFLR8), GlobalBaseReg);
505 }
506 }
507 return CurDAG->getRegister(GlobalBaseReg,
508 PPCLowering->getPointerTy(CurDAG->getDataLayout()))
509 .getNode();
510}
511
512// Check if a SDValue has the toc-data attribute.
513static bool hasTocDataAttr(SDValue Val, unsigned PointerSize) {
514 GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Val);
515 if (!GA)
516 return false;
517
518 const GlobalVariable *GV = dyn_cast_or_null<GlobalVariable>(GA->getGlobal());
519 if (!GV)
520 return false;
521
522 if (!GV->hasAttribute("toc-data"))
523 return false;
524
525 // TODO: These asserts should be updated as more support for the toc data
526 // transformation is added (struct support, etc.).
527
528 assert(
529 PointerSize >= GV->getAlign().valueOrOne().value() &&
530 "GlobalVariables with an alignment requirement stricter than TOC entry "
531 "size not supported by the toc data transformation.");
532
533 Type *GVType = GV->getValueType();
534
535 assert(GVType->isSized() && "A GlobalVariable's size must be known to be "
536 "supported by the toc data transformation.");
537
538 if (GVType->isVectorTy())
539 report_fatal_error("A GlobalVariable of Vector type is not currently "
540 "supported by the toc data transformation.");
541
542 if (GVType->isArrayTy())
543 report_fatal_error("A GlobalVariable of Array type is not currently "
544 "supported by the toc data transformation.");
545
546 if (GVType->isStructTy())
547 report_fatal_error("A GlobalVariable of Struct type is not currently "
548 "supported by the toc data transformation.");
549
550 assert(GVType->getPrimitiveSizeInBits() <= PointerSize * 8 &&
551 "A GlobalVariable with size larger than a TOC entry is not currently "
552 "supported by the toc data transformation.");
553
554 if (GV->hasPrivateLinkage())
555 report_fatal_error("A GlobalVariable with private linkage is not "
556 "currently supported by the toc data transformation.");
557
558 return true;
559}
560
561/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
562/// operand. If so Imm will receive the 32-bit value.
563static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
564 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
565 Imm = N->getAsZExtVal();
566 return true;
567 }
568 return false;
569}
570
571/// isInt64Immediate - This method tests to see if the node is a 64-bit constant
572/// operand. If so Imm will receive the 64-bit value.
573static bool isInt64Immediate(SDNode *N, uint64_t &Imm) {
574 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i64) {
575 Imm = N->getAsZExtVal();
576 return true;
577 }
578 return false;
579}
580
581// isInt32Immediate - This method tests to see if a constant operand.
582// If so Imm will receive the 32 bit value.
583static bool isInt32Immediate(SDValue N, unsigned &Imm) {
584 return isInt32Immediate(N.getNode(), Imm);
585}
586
587/// isInt64Immediate - This method tests to see if the value is a 64-bit
588/// constant operand. If so Imm will receive the 64-bit value.
589static bool isInt64Immediate(SDValue N, uint64_t &Imm) {
590 return isInt64Immediate(N.getNode(), Imm);
591}
592
593static unsigned getBranchHint(unsigned PCC,
594 const FunctionLoweringInfo &FuncInfo,
595 const SDValue &DestMBB) {
596 assert(isa<BasicBlockSDNode>(DestMBB));
597
598 if (!FuncInfo.BPI) return PPC::BR_NO_HINT;
599
600 const BasicBlock *BB = FuncInfo.MBB->getBasicBlock();
601 const Instruction *BBTerm = BB->getTerminator();
602
603 if (BBTerm->getNumSuccessors() != 2) return PPC::BR_NO_HINT;
604
605 const BasicBlock *TBB = BBTerm->getSuccessor(0);
606 const BasicBlock *FBB = BBTerm->getSuccessor(1);
607
608 auto TProb = FuncInfo.BPI->getEdgeProbability(BB, TBB);
609 auto FProb = FuncInfo.BPI->getEdgeProbability(BB, FBB);
610
611 // We only want to handle cases which are easy to predict at static time, e.g.
612 // C++ throw statement, that is very likely not taken, or calling never
613 // returned function, e.g. stdlib exit(). So we set Threshold to filter
614 // unwanted cases.
615 //
616 // Below is LLVM branch weight table, we only want to handle case 1, 2
617 //
618 // Case Taken:Nontaken Example
619 // 1. Unreachable 1048575:1 C++ throw, stdlib exit(),
620 // 2. Invoke-terminating 1:1048575
621 // 3. Coldblock 4:64 __builtin_expect
622 // 4. Loop Branch 124:4 For loop
623 // 5. PH/ZH/FPH 20:12
624 const uint32_t Threshold = 10000;
625
626 if (std::max(TProb, FProb) / Threshold < std::min(TProb, FProb))
627 return PPC::BR_NO_HINT;
628
629 LLVM_DEBUG(dbgs() << "Use branch hint for '" << FuncInfo.Fn->getName()
630 << "::" << BB->getName() << "'\n"
631 << " -> " << TBB->getName() << ": " << TProb << "\n"
632 << " -> " << FBB->getName() << ": " << FProb << "\n");
633
634 const BasicBlockSDNode *BBDN = cast<BasicBlockSDNode>(DestMBB);
635
636 // If Dest BasicBlock is False-BasicBlock (FBB), swap branch probabilities,
637 // because we want 'TProb' stands for 'branch probability' to Dest BasicBlock
638 if (BBDN->getBasicBlock()->getBasicBlock() != TBB)
639 std::swap(TProb, FProb);
640
641 return (TProb > FProb) ? PPC::BR_TAKEN_HINT : PPC::BR_NONTAKEN_HINT;
642}
643
644// isOpcWithIntImmediate - This method tests to see if the node is a specific
645// opcode and that it has a immediate integer right operand.
646// If so Imm will receive the 32 bit value.
647static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
648 return N->getOpcode() == Opc
649 && isInt32Immediate(N->getOperand(1).getNode(), Imm);
650}
651
652void PPCDAGToDAGISel::selectFrameIndex(SDNode *SN, SDNode *N, uint64_t Offset) {
653 SDLoc dl(SN);
654 int FI = cast<FrameIndexSDNode>(N)->getIndex();
655 SDValue TFI = CurDAG->getTargetFrameIndex(FI, N->getValueType(0));
656 unsigned Opc = N->getValueType(0) == MVT::i32 ? PPC::ADDI : PPC::ADDI8;
657 if (SN->hasOneUse())
658 CurDAG->SelectNodeTo(SN, Opc, N->getValueType(0), TFI,
659 getSmallIPtrImm(Offset, dl));
660 else
661 ReplaceNode(SN, CurDAG->getMachineNode(Opc, dl, N->getValueType(0), TFI,
662 getSmallIPtrImm(Offset, dl)));
663}
664
665bool PPCDAGToDAGISel::isRotateAndMask(SDNode *N, unsigned Mask,
666 bool isShiftMask, unsigned &SH,
667 unsigned &MB, unsigned &ME) {
668 // Don't even go down this path for i64, since different logic will be
669 // necessary for rldicl/rldicr/rldimi.
670 if (N->getValueType(0) != MVT::i32)
671 return false;
672
673 unsigned Shift = 32;
674 unsigned Indeterminant = ~0; // bit mask marking indeterminant results
675 unsigned Opcode = N->getOpcode();
676 if (N->getNumOperands() != 2 ||
677 !isInt32Immediate(N->getOperand(1).getNode(), Shift) || (Shift > 31))
678 return false;
679
680 if (Opcode == ISD::SHL) {
681 // apply shift left to mask if it comes first
682 if (isShiftMask) Mask = Mask << Shift;
683 // determine which bits are made indeterminant by shift
684 Indeterminant = ~(0xFFFFFFFFu << Shift);
685 } else if (Opcode == ISD::SRL) {
686 // apply shift right to mask if it comes first
687 if (isShiftMask) Mask = Mask >> Shift;
688 // determine which bits are made indeterminant by shift
689 Indeterminant = ~(0xFFFFFFFFu >> Shift);
690 // adjust for the left rotate
691 Shift = 32 - Shift;
692 } else if (Opcode == ISD::ROTL) {
693 Indeterminant = 0;
694 } else {
695 return false;
696 }
697
698 // if the mask doesn't intersect any Indeterminant bits
699 if (Mask && !(Mask & Indeterminant)) {
700 SH = Shift & 31;
701 // make sure the mask is still a mask (wrap arounds may not be)
702 return isRunOfOnes(Mask, MB, ME);
703 }
704 return false;
705}
706
707// isThreadPointerAcquisitionNode - Check if the operands of an ADD_TLS
708// instruction use the thread pointer.
710 assert(
711 Base.getOpcode() == PPCISD::ADD_TLS &&
712 "Only expecting the ADD_TLS instruction to acquire the thread pointer!");
713 const PPCSubtarget &Subtarget =
715 SDValue ADDTLSOp1 = Base.getOperand(0);
716 unsigned ADDTLSOp1Opcode = ADDTLSOp1.getOpcode();
717
718 // Account for when ADD_TLS is used for the initial-exec TLS model on Linux.
719 //
720 // Although ADD_TLS does not explicitly use the thread pointer
721 // register when LD_GOT_TPREL_L is one of it's operands, the LD_GOT_TPREL_L
722 // instruction will have a relocation specifier, @got@tprel, that is used to
723 // generate a GOT entry. The linker replaces this entry with an offset for a
724 // for a thread local variable, which will be relative to the thread pointer.
725 if (ADDTLSOp1Opcode == PPCISD::LD_GOT_TPREL_L)
726 return true;
727 // When using PC-Relative instructions for initial-exec, a MAT_PCREL_ADDR
728 // node is produced instead to represent the aforementioned situation.
729 LoadSDNode *LD = dyn_cast<LoadSDNode>(ADDTLSOp1);
730 if (LD && LD->getBasePtr().getOpcode() == PPCISD::MAT_PCREL_ADDR)
731 return true;
732
733 // A GET_TPOINTER PPCISD node (only produced on AIX 32-bit mode) as an operand
734 // to ADD_TLS represents a call to .__get_tpointer to get the thread pointer,
735 // later returning it into R3.
736 if (ADDTLSOp1Opcode == PPCISD::GET_TPOINTER)
737 return true;
738
739 // The ADD_TLS note is explicitly acquiring the thread pointer (X13/R13).
740 RegisterSDNode *AddFirstOpReg =
741 dyn_cast_or_null<RegisterSDNode>(ADDTLSOp1.getNode());
742 if (AddFirstOpReg &&
743 AddFirstOpReg->getReg() == Subtarget.getThreadPointerRegister())
744 return true;
745
746 return false;
747}
748
749// canOptimizeTLSDFormToXForm - Optimize TLS accesses when an ADD_TLS
750// instruction is present. An ADD_TLS instruction, followed by a D-Form memory
751// operation, can be optimized to use an X-Form load or store, allowing the
752// ADD_TLS node to be removed completely.
754
755 // Do not do this transformation at -O0.
756 if (CurDAG->getTarget().getOptLevel() == CodeGenOptLevel::None)
757 return false;
758
759 // In order to perform this optimization inside tryTLSXForm[Load|Store],
760 // Base is expected to be an ADD_TLS node.
761 if (Base.getOpcode() != PPCISD::ADD_TLS)
762 return false;
763 for (auto *ADDTLSUse : Base.getNode()->uses()) {
764 // The optimization to convert the D-Form load/store into its X-Form
765 // counterpart should only occur if the source value offset of the load/
766 // store is 0. This also means that The offset should always be undefined.
767 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(ADDTLSUse)) {
768 if (LD->getSrcValueOffset() != 0 || !LD->getOffset().isUndef())
769 return false;
770 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(ADDTLSUse)) {
771 if (ST->getSrcValueOffset() != 0 || !ST->getOffset().isUndef())
772 return false;
773 } else // Don't optimize if there are ADD_TLS users that aren't load/stores.
774 return false;
775 }
776
777 if (Base.getOperand(1).getOpcode() == PPCISD::TLS_LOCAL_EXEC_MAT_ADDR)
778 return false;
779
780 // Does the ADD_TLS node of the load/store use the thread pointer?
781 // If the thread pointer is not used as one of the operands of ADD_TLS,
782 // then this optimization is not valid.
783 return isThreadPointerAcquisitionNode(Base, CurDAG);
784}
785
786bool PPCDAGToDAGISel::tryTLSXFormStore(StoreSDNode *ST) {
787 SDValue Base = ST->getBasePtr();
788 if (!canOptimizeTLSDFormToXForm(CurDAG, Base))
789 return false;
790
791 SDLoc dl(ST);
792 EVT MemVT = ST->getMemoryVT();
793 EVT RegVT = ST->getValue().getValueType();
794
795 unsigned Opcode;
796 switch (MemVT.getSimpleVT().SimpleTy) {
797 default:
798 return false;
799 case MVT::i8: {
800 Opcode = (RegVT == MVT::i32) ? PPC::STBXTLS_32 : PPC::STBXTLS;
801 break;
802 }
803 case MVT::i16: {
804 Opcode = (RegVT == MVT::i32) ? PPC::STHXTLS_32 : PPC::STHXTLS;
805 break;
806 }
807 case MVT::i32: {
808 Opcode = (RegVT == MVT::i32) ? PPC::STWXTLS_32 : PPC::STWXTLS;
809 break;
810 }
811 case MVT::i64: {
812 Opcode = PPC::STDXTLS;
813 break;
814 }
815 case MVT::f32: {
816 Opcode = PPC::STFSXTLS;
817 break;
818 }
819 case MVT::f64: {
820 Opcode = PPC::STFDXTLS;
821 break;
822 }
823 }
824 SDValue Chain = ST->getChain();
825 SDVTList VTs = ST->getVTList();
826 SDValue Ops[] = {ST->getValue(), Base.getOperand(0), Base.getOperand(1),
827 Chain};
828 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
829 transferMemOperands(ST, MN);
830 ReplaceNode(ST, MN);
831 return true;
832}
833
834bool PPCDAGToDAGISel::tryTLSXFormLoad(LoadSDNode *LD) {
835 SDValue Base = LD->getBasePtr();
836 if (!canOptimizeTLSDFormToXForm(CurDAG, Base))
837 return false;
838
839 SDLoc dl(LD);
840 EVT MemVT = LD->getMemoryVT();
841 EVT RegVT = LD->getValueType(0);
842 bool isSExt = LD->getExtensionType() == ISD::SEXTLOAD;
843 unsigned Opcode;
844 switch (MemVT.getSimpleVT().SimpleTy) {
845 default:
846 return false;
847 case MVT::i8: {
848 Opcode = (RegVT == MVT::i32) ? PPC::LBZXTLS_32 : PPC::LBZXTLS;
849 break;
850 }
851 case MVT::i16: {
852 if (RegVT == MVT::i32)
853 Opcode = isSExt ? PPC::LHAXTLS_32 : PPC::LHZXTLS_32;
854 else
855 Opcode = isSExt ? PPC::LHAXTLS : PPC::LHZXTLS;
856 break;
857 }
858 case MVT::i32: {
859 if (RegVT == MVT::i32)
860 Opcode = isSExt ? PPC::LWAXTLS_32 : PPC::LWZXTLS_32;
861 else
862 Opcode = isSExt ? PPC::LWAXTLS : PPC::LWZXTLS;
863 break;
864 }
865 case MVT::i64: {
866 Opcode = PPC::LDXTLS;
867 break;
868 }
869 case MVT::f32: {
870 Opcode = PPC::LFSXTLS;
871 break;
872 }
873 case MVT::f64: {
874 Opcode = PPC::LFDXTLS;
875 break;
876 }
877 }
878 SDValue Chain = LD->getChain();
879 SDVTList VTs = LD->getVTList();
880 SDValue Ops[] = {Base.getOperand(0), Base.getOperand(1), Chain};
881 SDNode *MN = CurDAG->getMachineNode(Opcode, dl, VTs, Ops);
882 transferMemOperands(LD, MN);
883 ReplaceNode(LD, MN);
884 return true;
885}
886
887/// Turn an or of two masked values into the rotate left word immediate then
888/// mask insert (rlwimi) instruction.
889bool PPCDAGToDAGISel::tryBitfieldInsert(SDNode *N) {
890 SDValue Op0 = N->getOperand(0);
891 SDValue Op1 = N->getOperand(1);
892 SDLoc dl(N);
893
894 KnownBits LKnown = CurDAG->computeKnownBits(Op0);
895 KnownBits RKnown = CurDAG->computeKnownBits(Op1);
896
897 unsigned TargetMask = LKnown.Zero.getZExtValue();
898 unsigned InsertMask = RKnown.Zero.getZExtValue();
899
900 if ((TargetMask | InsertMask) == 0xFFFFFFFF) {
901 unsigned Op0Opc = Op0.getOpcode();
902 unsigned Op1Opc = Op1.getOpcode();
903 unsigned Value, SH = 0;
904 TargetMask = ~TargetMask;
905 InsertMask = ~InsertMask;
906
907 // If the LHS has a foldable shift and the RHS does not, then swap it to the
908 // RHS so that we can fold the shift into the insert.
909 if (Op0Opc == ISD::AND && Op1Opc == ISD::AND) {
910 if (Op0.getOperand(0).getOpcode() == ISD::SHL ||
911 Op0.getOperand(0).getOpcode() == ISD::SRL) {
912 if (Op1.getOperand(0).getOpcode() != ISD::SHL &&
913 Op1.getOperand(0).getOpcode() != ISD::SRL) {
914 std::swap(Op0, Op1);
915 std::swap(Op0Opc, Op1Opc);
916 std::swap(TargetMask, InsertMask);
917 }
918 }
919 } else if (Op0Opc == ISD::SHL || Op0Opc == ISD::SRL) {
920 if (Op1Opc == ISD::AND && Op1.getOperand(0).getOpcode() != ISD::SHL &&
921 Op1.getOperand(0).getOpcode() != ISD::SRL) {
922 std::swap(Op0, Op1);
923 std::swap(Op0Opc, Op1Opc);
924 std::swap(TargetMask, InsertMask);
925 }
926 }
927
928 unsigned MB, ME;
929 if (isRunOfOnes(InsertMask, MB, ME)) {
930 if ((Op1Opc == ISD::SHL || Op1Opc == ISD::SRL) &&
932 Op1 = Op1.getOperand(0);
933 SH = (Op1Opc == ISD::SHL) ? Value : 32 - Value;
934 }
935 if (Op1Opc == ISD::AND) {
936 // The AND mask might not be a constant, and we need to make sure that
937 // if we're going to fold the masking with the insert, all bits not
938 // know to be zero in the mask are known to be one.
939 KnownBits MKnown = CurDAG->computeKnownBits(Op1.getOperand(1));
940 bool CanFoldMask = InsertMask == MKnown.One.getZExtValue();
941
942 unsigned SHOpc = Op1.getOperand(0).getOpcode();
943 if ((SHOpc == ISD::SHL || SHOpc == ISD::SRL) && CanFoldMask &&
945 // Note that Value must be in range here (less than 32) because
946 // otherwise there would not be any bits set in InsertMask.
947 Op1 = Op1.getOperand(0).getOperand(0);
948 SH = (SHOpc == ISD::SHL) ? Value : 32 - Value;
949 }
950 }
951
952 SH &= 31;
953 SDValue Ops[] = { Op0, Op1, getI32Imm(SH, dl), getI32Imm(MB, dl),
954 getI32Imm(ME, dl) };
955 ReplaceNode(N, CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops));
956 return true;
957 }
958 }
959 return false;
960}
961
962static unsigned allUsesTruncate(SelectionDAG *CurDAG, SDNode *N) {
963 unsigned MaxTruncation = 0;
964 // Cannot use range-based for loop here as we need the actual use (i.e. we
965 // need the operand number corresponding to the use). A range-based for
966 // will unbox the use and provide an SDNode*.
967 for (SDNode::use_iterator Use = N->use_begin(), UseEnd = N->use_end();
968 Use != UseEnd; ++Use) {
969 unsigned Opc =
970 Use->isMachineOpcode() ? Use->getMachineOpcode() : Use->getOpcode();
971 switch (Opc) {
972 default: return 0;
973 case ISD::TRUNCATE:
974 if (Use->isMachineOpcode())
975 return 0;
976 MaxTruncation =
977 std::max(MaxTruncation, (unsigned)Use->getValueType(0).getSizeInBits());
978 continue;
979 case ISD::STORE: {
980 if (Use->isMachineOpcode())
981 return 0;
982 StoreSDNode *STN = cast<StoreSDNode>(*Use);
983 unsigned MemVTSize = STN->getMemoryVT().getSizeInBits();
984 if (MemVTSize == 64 || Use.getOperandNo() != 0)
985 return 0;
986 MaxTruncation = std::max(MaxTruncation, MemVTSize);
987 continue;
988 }
989 case PPC::STW8:
990 case PPC::STWX8:
991 case PPC::STWU8:
992 case PPC::STWUX8:
993 if (Use.getOperandNo() != 0)
994 return 0;
995 MaxTruncation = std::max(MaxTruncation, 32u);
996 continue;
997 case PPC::STH8:
998 case PPC::STHX8:
999 case PPC::STHU8:
1000 case PPC::STHUX8:
1001 if (Use.getOperandNo() != 0)
1002 return 0;
1003 MaxTruncation = std::max(MaxTruncation, 16u);
1004 continue;
1005 case PPC::STB8:
1006 case PPC::STBX8:
1007 case PPC::STBU8:
1008 case PPC::STBUX8:
1009 if (Use.getOperandNo() != 0)
1010 return 0;
1011 MaxTruncation = std::max(MaxTruncation, 8u);
1012 continue;
1013 }
1014 }
1015 return MaxTruncation;
1016}
1017
1018// For any 32 < Num < 64, check if the Imm contains at least Num consecutive
1019// zeros and return the number of bits by the left of these consecutive zeros.
1020static int findContiguousZerosAtLeast(uint64_t Imm, unsigned Num) {
1021 unsigned HiTZ = llvm::countr_zero<uint32_t>(Hi_32(Imm));
1022 unsigned LoLZ = llvm::countl_zero<uint32_t>(Lo_32(Imm));
1023 if ((HiTZ + LoLZ) >= Num)
1024 return (32 + HiTZ);
1025 return 0;
1026}
1027
1028// Direct materialization of 64-bit constants by enumerated patterns.
1029static SDNode *selectI64ImmDirect(SelectionDAG *CurDAG, const SDLoc &dl,
1030 uint64_t Imm, unsigned &InstCnt) {
1031 unsigned TZ = llvm::countr_zero<uint64_t>(Imm);
1032 unsigned LZ = llvm::countl_zero<uint64_t>(Imm);
1033 unsigned TO = llvm::countr_one<uint64_t>(Imm);
1034 unsigned LO = llvm::countl_one<uint64_t>(Imm);
1035 unsigned Hi32 = Hi_32(Imm);
1036 unsigned Lo32 = Lo_32(Imm);
1037 SDNode *Result = nullptr;
1038 unsigned Shift = 0;
1039
1040 auto getI32Imm = [CurDAG, dl](unsigned Imm) {
1041 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1042 };
1043
1044 // Following patterns use 1 instructions to materialize the Imm.
1045 InstCnt = 1;
1046 // 1-1) Patterns : {zeros}{15-bit valve}
1047 // {ones}{15-bit valve}
1048 if (isInt<16>(Imm)) {
1049 SDValue SDImm = CurDAG->getTargetConstant(Imm, dl, MVT::i64);
1050 return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
1051 }
1052 // 1-2) Patterns : {zeros}{15-bit valve}{16 zeros}
1053 // {ones}{15-bit valve}{16 zeros}
1054 if (TZ > 15 && (LZ > 32 || LO > 32))
1055 return CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
1056 getI32Imm((Imm >> 16) & 0xffff));
1057
1058 // Following patterns use 2 instructions to materialize the Imm.
1059 InstCnt = 2;
1060 assert(LZ < 64 && "Unexpected leading zeros here.");
1061 // Count of ones follwing the leading zeros.
1062 unsigned FO = llvm::countl_one<uint64_t>(Imm << LZ);
1063 // 2-1) Patterns : {zeros}{31-bit value}
1064 // {ones}{31-bit value}
1065 if (isInt<32>(Imm)) {
1066 uint64_t ImmHi16 = (Imm >> 16) & 0xffff;
1067 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1068 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1069 return CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1070 getI32Imm(Imm & 0xffff));
1071 }
1072 // 2-2) Patterns : {zeros}{ones}{15-bit value}{zeros}
1073 // {zeros}{15-bit value}{zeros}
1074 // {zeros}{ones}{15-bit value}
1075 // {ones}{15-bit value}{zeros}
1076 // We can take advantage of LI's sign-extension semantics to generate leading
1077 // ones, and then use RLDIC to mask off the ones in both sides after rotation.
1078 if ((LZ + FO + TZ) > 48) {
1079 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1080 getI32Imm((Imm >> TZ) & 0xffff));
1081 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
1082 getI32Imm(TZ), getI32Imm(LZ));
1083 }
1084 // 2-3) Pattern : {zeros}{15-bit value}{ones}
1085 // Shift right the Imm by (48 - LZ) bits to construct a negtive 16 bits value,
1086 // therefore we can take advantage of LI's sign-extension semantics, and then
1087 // mask them off after rotation.
1088 //
1089 // +--LZ--||-15-bit-||--TO--+ +-------------|--16-bit--+
1090 // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1|
1091 // +------------------------+ +------------------------+
1092 // 63 0 63 0
1093 // Imm (Imm >> (48 - LZ) & 0xffff)
1094 // +----sext-----|--16-bit--+ +clear-|-----------------+
1095 // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111|
1096 // +------------------------+ +------------------------+
1097 // 63 0 63 0
1098 // LI8: sext many leading zeros RLDICL: rotate left (48 - LZ), clear left LZ
1099 if ((LZ + TO) > 48) {
1100 // Since the immediates with (LZ > 32) have been handled by previous
1101 // patterns, here we have (LZ <= 32) to make sure we will not shift right
1102 // the Imm by a negative value.
1103 assert(LZ <= 32 && "Unexpected shift value.");
1104 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1105 getI32Imm((Imm >> (48 - LZ) & 0xffff)));
1106 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1107 getI32Imm(48 - LZ), getI32Imm(LZ));
1108 }
1109 // 2-4) Patterns : {zeros}{ones}{15-bit value}{ones}
1110 // {ones}{15-bit value}{ones}
1111 // We can take advantage of LI's sign-extension semantics to generate leading
1112 // ones, and then use RLDICL to mask off the ones in left sides (if required)
1113 // after rotation.
1114 //
1115 // +-LZ-FO||-15-bit-||--TO--+ +-------------|--16-bit--+
1116 // |00011110bbbbbbbbb1111111| -> |000000000011110bbbbbbbbb|
1117 // +------------------------+ +------------------------+
1118 // 63 0 63 0
1119 // Imm (Imm >> TO) & 0xffff
1120 // +----sext-----|--16-bit--+ +LZ|---------------------+
1121 // |111111111111110bbbbbbbbb| -> |00011110bbbbbbbbb1111111|
1122 // +------------------------+ +------------------------+
1123 // 63 0 63 0
1124 // LI8: sext many leading zeros RLDICL: rotate left TO, clear left LZ
1125 if ((LZ + FO + TO) > 48) {
1126 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1127 getI32Imm((Imm >> TO) & 0xffff));
1128 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1129 getI32Imm(TO), getI32Imm(LZ));
1130 }
1131 // 2-5) Pattern : {32 zeros}{****}{0}{15-bit value}
1132 // If Hi32 is zero and the Lo16(in Lo32) can be presented as a positive 16 bit
1133 // value, we can use LI for Lo16 without generating leading ones then add the
1134 // Hi16(in Lo32).
1135 if (LZ == 32 && ((Lo32 & 0x8000) == 0)) {
1136 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1137 getI32Imm(Lo32 & 0xffff));
1138 return CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64, SDValue(Result, 0),
1139 getI32Imm(Lo32 >> 16));
1140 }
1141 // 2-6) Patterns : {******}{49 zeros}{******}
1142 // {******}{49 ones}{******}
1143 // If the Imm contains 49 consecutive zeros/ones, it means that a total of 15
1144 // bits remain on both sides. Rotate right the Imm to construct an int<16>
1145 // value, use LI for int<16> value and then use RLDICL without mask to rotate
1146 // it back.
1147 //
1148 // 1) findContiguousZerosAtLeast(Imm, 49)
1149 // +------|--zeros-|------+ +---ones--||---15 bit--+
1150 // |bbbbbb0000000000aaaaaa| -> |0000000000aaaaaabbbbbb|
1151 // +----------------------+ +----------------------+
1152 // 63 0 63 0
1153 //
1154 // 2) findContiguousZerosAtLeast(~Imm, 49)
1155 // +------|--ones--|------+ +---ones--||---15 bit--+
1156 // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb|
1157 // +----------------------+ +----------------------+
1158 // 63 0 63 0
1159 if ((Shift = findContiguousZerosAtLeast(Imm, 49)) ||
1160 (Shift = findContiguousZerosAtLeast(~Imm, 49))) {
1161 uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1162 Result = CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64,
1163 getI32Imm(RotImm & 0xffff));
1164 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1165 getI32Imm(Shift), getI32Imm(0));
1166 }
1167 // 2-7) Patterns : High word == Low word
1168 // This may require 2 to 3 instructions, depending on whether Lo32 can be
1169 // materialized in 1 instruction.
1170 if (Hi32 == Lo32) {
1171 // Handle the first 32 bits.
1172 uint64_t ImmHi16 = (Lo32 >> 16) & 0xffff;
1173 uint64_t ImmLo16 = Lo32 & 0xffff;
1174 if (isInt<16>(Lo32))
1175 Result =
1176 CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, getI32Imm(ImmLo16));
1177 else if (!ImmLo16)
1178 Result =
1179 CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(ImmHi16));
1180 else {
1181 InstCnt = 3;
1182 Result =
1183 CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(ImmHi16));
1184 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64,
1185 SDValue(Result, 0), getI32Imm(ImmLo16));
1186 }
1187 // Use rldimi to insert the Low word into High word.
1188 SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32),
1189 getI32Imm(0)};
1190 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1191 }
1192
1193 // Following patterns use 3 instructions to materialize the Imm.
1194 InstCnt = 3;
1195 // 3-1) Patterns : {zeros}{ones}{31-bit value}{zeros}
1196 // {zeros}{31-bit value}{zeros}
1197 // {zeros}{ones}{31-bit value}
1198 // {ones}{31-bit value}{zeros}
1199 // We can take advantage of LIS's sign-extension semantics to generate leading
1200 // ones, add the remaining bits with ORI, and then use RLDIC to mask off the
1201 // ones in both sides after rotation.
1202 if ((LZ + FO + TZ) > 32) {
1203 uint64_t ImmHi16 = (Imm >> (TZ + 16)) & 0xffff;
1204 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1205 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1206 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1207 getI32Imm((Imm >> TZ) & 0xffff));
1208 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
1209 getI32Imm(TZ), getI32Imm(LZ));
1210 }
1211 // 3-2) Pattern : {zeros}{31-bit value}{ones}
1212 // Shift right the Imm by (32 - LZ) bits to construct a negative 32 bits
1213 // value, therefore we can take advantage of LIS's sign-extension semantics,
1214 // add the remaining bits with ORI, and then mask them off after rotation.
1215 // This is similar to Pattern 2-3, please refer to the diagram there.
1216 if ((LZ + TO) > 32) {
1217 // Since the immediates with (LZ > 32) have been handled by previous
1218 // patterns, here we have (LZ <= 32) to make sure we will not shift right
1219 // the Imm by a negative value.
1220 assert(LZ <= 32 && "Unexpected shift value.");
1221 Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
1222 getI32Imm((Imm >> (48 - LZ)) & 0xffff));
1223 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1224 getI32Imm((Imm >> (32 - LZ)) & 0xffff));
1225 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1226 getI32Imm(32 - LZ), getI32Imm(LZ));
1227 }
1228 // 3-3) Patterns : {zeros}{ones}{31-bit value}{ones}
1229 // {ones}{31-bit value}{ones}
1230 // We can take advantage of LIS's sign-extension semantics to generate leading
1231 // ones, add the remaining bits with ORI, and then use RLDICL to mask off the
1232 // ones in left sides (if required) after rotation.
1233 // This is similar to Pattern 2-4, please refer to the diagram there.
1234 if ((LZ + FO + TO) > 32) {
1235 Result = CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64,
1236 getI32Imm((Imm >> (TO + 16)) & 0xffff));
1237 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1238 getI32Imm((Imm >> TO) & 0xffff));
1239 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1240 getI32Imm(TO), getI32Imm(LZ));
1241 }
1242 // 3-4) Patterns : {******}{33 zeros}{******}
1243 // {******}{33 ones}{******}
1244 // If the Imm contains 33 consecutive zeros/ones, it means that a total of 31
1245 // bits remain on both sides. Rotate right the Imm to construct an int<32>
1246 // value, use LIS + ORI for int<32> value and then use RLDICL without mask to
1247 // rotate it back.
1248 // This is similar to Pattern 2-6, please refer to the diagram there.
1249 if ((Shift = findContiguousZerosAtLeast(Imm, 33)) ||
1250 (Shift = findContiguousZerosAtLeast(~Imm, 33))) {
1251 uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1252 uint64_t ImmHi16 = (RotImm >> 16) & 0xffff;
1253 unsigned Opcode = ImmHi16 ? PPC::LIS8 : PPC::LI8;
1254 Result = CurDAG->getMachineNode(Opcode, dl, MVT::i64, getI32Imm(ImmHi16));
1255 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1256 getI32Imm(RotImm & 0xffff));
1257 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1258 getI32Imm(Shift), getI32Imm(0));
1259 }
1260
1261 InstCnt = 0;
1262 return nullptr;
1263}
1264
1265// Try to select instructions to generate a 64 bit immediate using prefix as
1266// well as non prefix instructions. The function will return the SDNode
1267// to materialize that constant or it will return nullptr if it does not
1268// find one. The variable InstCnt is set to the number of instructions that
1269// were selected.
1271 uint64_t Imm, unsigned &InstCnt) {
1272 unsigned TZ = llvm::countr_zero<uint64_t>(Imm);
1273 unsigned LZ = llvm::countl_zero<uint64_t>(Imm);
1274 unsigned TO = llvm::countr_one<uint64_t>(Imm);
1275 unsigned FO = llvm::countl_one<uint64_t>(LZ == 64 ? 0 : (Imm << LZ));
1276 unsigned Hi32 = Hi_32(Imm);
1277 unsigned Lo32 = Lo_32(Imm);
1278
1279 auto getI32Imm = [CurDAG, dl](unsigned Imm) {
1280 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1281 };
1282
1283 auto getI64Imm = [CurDAG, dl](uint64_t Imm) {
1284 return CurDAG->getTargetConstant(Imm, dl, MVT::i64);
1285 };
1286
1287 // Following patterns use 1 instruction to materialize Imm.
1288 InstCnt = 1;
1289
1290 // The pli instruction can materialize up to 34 bits directly.
1291 // If a constant fits within 34-bits, emit the pli instruction here directly.
1292 if (isInt<34>(Imm))
1293 return CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1294 CurDAG->getTargetConstant(Imm, dl, MVT::i64));
1295
1296 // Require at least two instructions.
1297 InstCnt = 2;
1298 SDNode *Result = nullptr;
1299 // Patterns : {zeros}{ones}{33-bit value}{zeros}
1300 // {zeros}{33-bit value}{zeros}
1301 // {zeros}{ones}{33-bit value}
1302 // {ones}{33-bit value}{zeros}
1303 // We can take advantage of PLI's sign-extension semantics to generate leading
1304 // ones, and then use RLDIC to mask off the ones on both sides after rotation.
1305 if ((LZ + FO + TZ) > 30) {
1306 APInt SignedInt34 = APInt(34, (Imm >> TZ) & 0x3ffffffff);
1307 APInt Extended = SignedInt34.sext(64);
1308 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1309 getI64Imm(*Extended.getRawData()));
1310 return CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, SDValue(Result, 0),
1311 getI32Imm(TZ), getI32Imm(LZ));
1312 }
1313 // Pattern : {zeros}{33-bit value}{ones}
1314 // Shift right the Imm by (30 - LZ) bits to construct a negative 34 bit value,
1315 // therefore we can take advantage of PLI's sign-extension semantics, and then
1316 // mask them off after rotation.
1317 //
1318 // +--LZ--||-33-bit-||--TO--+ +-------------|--34-bit--+
1319 // |00000001bbbbbbbbb1111111| -> |00000000000001bbbbbbbbb1|
1320 // +------------------------+ +------------------------+
1321 // 63 0 63 0
1322 //
1323 // +----sext-----|--34-bit--+ +clear-|-----------------+
1324 // |11111111111111bbbbbbbbb1| -> |00000001bbbbbbbbb1111111|
1325 // +------------------------+ +------------------------+
1326 // 63 0 63 0
1327 if ((LZ + TO) > 30) {
1328 APInt SignedInt34 = APInt(34, (Imm >> (30 - LZ)) & 0x3ffffffff);
1329 APInt Extended = SignedInt34.sext(64);
1330 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1331 getI64Imm(*Extended.getRawData()));
1332 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1333 getI32Imm(30 - LZ), getI32Imm(LZ));
1334 }
1335 // Patterns : {zeros}{ones}{33-bit value}{ones}
1336 // {ones}{33-bit value}{ones}
1337 // Similar to LI we can take advantage of PLI's sign-extension semantics to
1338 // generate leading ones, and then use RLDICL to mask off the ones in left
1339 // sides (if required) after rotation.
1340 if ((LZ + FO + TO) > 30) {
1341 APInt SignedInt34 = APInt(34, (Imm >> TO) & 0x3ffffffff);
1342 APInt Extended = SignedInt34.sext(64);
1343 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64,
1344 getI64Imm(*Extended.getRawData()));
1345 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, SDValue(Result, 0),
1346 getI32Imm(TO), getI32Imm(LZ));
1347 }
1348 // Patterns : {******}{31 zeros}{******}
1349 // : {******}{31 ones}{******}
1350 // If Imm contains 31 consecutive zeros/ones then the remaining bit count
1351 // is 33. Rotate right the Imm to construct a int<33> value, we can use PLI
1352 // for the int<33> value and then use RLDICL without a mask to rotate it back.
1353 //
1354 // +------|--ones--|------+ +---ones--||---33 bit--+
1355 // |bbbbbb1111111111aaaaaa| -> |1111111111aaaaaabbbbbb|
1356 // +----------------------+ +----------------------+
1357 // 63 0 63 0
1358 for (unsigned Shift = 0; Shift < 63; ++Shift) {
1359 uint64_t RotImm = APInt(64, Imm).rotr(Shift).getZExtValue();
1360 if (isInt<34>(RotImm)) {
1361 Result =
1362 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(RotImm));
1363 return CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
1364 SDValue(Result, 0), getI32Imm(Shift),
1365 getI32Imm(0));
1366 }
1367 }
1368
1369 // Patterns : High word == Low word
1370 // This is basically a splat of a 32 bit immediate.
1371 if (Hi32 == Lo32) {
1372 Result = CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32));
1373 SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32),
1374 getI32Imm(0)};
1375 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1376 }
1377
1378 InstCnt = 3;
1379 // Catch-all
1380 // This pattern can form any 64 bit immediate in 3 instructions.
1381 SDNode *ResultHi =
1382 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Hi32));
1383 SDNode *ResultLo =
1384 CurDAG->getMachineNode(PPC::PLI8, dl, MVT::i64, getI64Imm(Lo32));
1385 SDValue Ops[] = {SDValue(ResultLo, 0), SDValue(ResultHi, 0), getI32Imm(32),
1386 getI32Imm(0)};
1387 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1388}
1389
1390static SDNode *selectI64Imm(SelectionDAG *CurDAG, const SDLoc &dl, uint64_t Imm,
1391 unsigned *InstCnt = nullptr) {
1392 unsigned InstCntDirect = 0;
1393 // No more than 3 instructions are used if we can select the i64 immediate
1394 // directly.
1395 SDNode *Result = selectI64ImmDirect(CurDAG, dl, Imm, InstCntDirect);
1396
1397 const PPCSubtarget &Subtarget =
1399
1400 // If we have prefixed instructions and there is a chance we can
1401 // materialize the constant with fewer prefixed instructions than
1402 // non-prefixed, try that.
1403 if (Subtarget.hasPrefixInstrs() && InstCntDirect != 1) {
1404 unsigned InstCntDirectP = 0;
1405 SDNode *ResultP = selectI64ImmDirectPrefix(CurDAG, dl, Imm, InstCntDirectP);
1406 // Use the prefix case in either of two cases:
1407 // 1) We have no result from the non-prefix case to use.
1408 // 2) The non-prefix case uses more instructions than the prefix case.
1409 // If the prefix and non-prefix cases use the same number of instructions
1410 // we will prefer the non-prefix case.
1411 if (ResultP && (!Result || InstCntDirectP < InstCntDirect)) {
1412 if (InstCnt)
1413 *InstCnt = InstCntDirectP;
1414 return ResultP;
1415 }
1416 }
1417
1418 if (Result) {
1419 if (InstCnt)
1420 *InstCnt = InstCntDirect;
1421 return Result;
1422 }
1423 auto getI32Imm = [CurDAG, dl](unsigned Imm) {
1424 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
1425 };
1426
1427 uint32_t Hi16OfLo32 = (Lo_32(Imm) >> 16) & 0xffff;
1428 uint32_t Lo16OfLo32 = Lo_32(Imm) & 0xffff;
1429
1430 // Try to use 4 instructions to materialize the immediate which is "almost" a
1431 // splat of a 32 bit immediate.
1432 if (Hi16OfLo32 && Lo16OfLo32) {
1433 uint32_t Hi16OfHi32 = (Hi_32(Imm) >> 16) & 0xffff;
1434 uint32_t Lo16OfHi32 = Hi_32(Imm) & 0xffff;
1435 bool IsSelected = false;
1436
1437 auto getSplat = [CurDAG, dl, getI32Imm](uint32_t Hi16, uint32_t Lo16) {
1438 SDNode *Result =
1439 CurDAG->getMachineNode(PPC::LIS8, dl, MVT::i64, getI32Imm(Hi16));
1440 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64,
1441 SDValue(Result, 0), getI32Imm(Lo16));
1442 SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(32),
1443 getI32Imm(0)};
1444 return CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1445 };
1446
1447 if (Hi16OfHi32 == Lo16OfHi32 && Lo16OfHi32 == Lo16OfLo32) {
1448 IsSelected = true;
1449 Result = getSplat(Hi16OfLo32, Lo16OfLo32);
1450 // Modify Hi16OfHi32.
1451 SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(48),
1452 getI32Imm(0)};
1453 Result = CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops);
1454 } else if (Hi16OfHi32 == Hi16OfLo32 && Hi16OfLo32 == Lo16OfLo32) {
1455 IsSelected = true;
1456 Result = getSplat(Hi16OfHi32, Lo16OfHi32);
1457 // Modify Lo16OfLo32.
1458 SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(16),
1459 getI32Imm(16), getI32Imm(31)};
1460 Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops);
1461 } else if (Lo16OfHi32 == Lo16OfLo32 && Hi16OfLo32 == Lo16OfLo32) {
1462 IsSelected = true;
1463 Result = getSplat(Hi16OfHi32, Lo16OfHi32);
1464 // Modify Hi16OfLo32.
1465 SDValue Ops[] = {SDValue(Result, 0), SDValue(Result, 0), getI32Imm(16),
1466 getI32Imm(0), getI32Imm(15)};
1467 Result = CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64, Ops);
1468 }
1469 if (IsSelected == true) {
1470 if (InstCnt)
1471 *InstCnt = 4;
1472 return Result;
1473 }
1474 }
1475
1476 // Handle the upper 32 bit value.
1477 Result =
1478 selectI64ImmDirect(CurDAG, dl, Imm & 0xffffffff00000000, InstCntDirect);
1479 // Add in the last bits as required.
1480 if (Hi16OfLo32) {
1481 Result = CurDAG->getMachineNode(PPC::ORIS8, dl, MVT::i64,
1482 SDValue(Result, 0), getI32Imm(Hi16OfLo32));
1483 ++InstCntDirect;
1484 }
1485 if (Lo16OfLo32) {
1486 Result = CurDAG->getMachineNode(PPC::ORI8, dl, MVT::i64, SDValue(Result, 0),
1487 getI32Imm(Lo16OfLo32));
1488 ++InstCntDirect;
1489 }
1490 if (InstCnt)
1491 *InstCnt = InstCntDirect;
1492 return Result;
1493}
1494
1495// Select a 64-bit constant.
1497 SDLoc dl(N);
1498
1499 // Get 64 bit value.
1500 int64_t Imm = N->getAsZExtVal();
1501 if (unsigned MinSize = allUsesTruncate(CurDAG, N)) {
1502 uint64_t SextImm = SignExtend64(Imm, MinSize);
1503 SDValue SDImm = CurDAG->getTargetConstant(SextImm, dl, MVT::i64);
1504 if (isInt<16>(SextImm))
1505 return CurDAG->getMachineNode(PPC::LI8, dl, MVT::i64, SDImm);
1506 }
1507 return selectI64Imm(CurDAG, dl, Imm);
1508}
1509
1510namespace {
1511
1512class BitPermutationSelector {
1513 struct ValueBit {
1514 SDValue V;
1515
1516 // The bit number in the value, using a convention where bit 0 is the
1517 // lowest-order bit.
1518 unsigned Idx;
1519
1520 // ConstZero means a bit we need to mask off.
1521 // Variable is a bit comes from an input variable.
1522 // VariableKnownToBeZero is also a bit comes from an input variable,
1523 // but it is known to be already zero. So we do not need to mask them.
1524 enum Kind {
1525 ConstZero,
1526 Variable,
1527 VariableKnownToBeZero
1528 } K;
1529
1530 ValueBit(SDValue V, unsigned I, Kind K = Variable)
1531 : V(V), Idx(I), K(K) {}
1532 ValueBit(Kind K = Variable) : Idx(UINT32_MAX), K(K) {}
1533
1534 bool isZero() const {
1535 return K == ConstZero || K == VariableKnownToBeZero;
1536 }
1537
1538 bool hasValue() const {
1539 return K == Variable || K == VariableKnownToBeZero;
1540 }
1541
1542 SDValue getValue() const {
1543 assert(hasValue() && "Cannot get the value of a constant bit");
1544 return V;
1545 }
1546
1547 unsigned getValueBitIndex() const {
1548 assert(hasValue() && "Cannot get the value bit index of a constant bit");
1549 return Idx;
1550 }
1551 };
1552
1553 // A bit group has the same underlying value and the same rotate factor.
1554 struct BitGroup {
1555 SDValue V;
1556 unsigned RLAmt;
1557 unsigned StartIdx, EndIdx;
1558
1559 // This rotation amount assumes that the lower 32 bits of the quantity are
1560 // replicated in the high 32 bits by the rotation operator (which is done
1561 // by rlwinm and friends in 64-bit mode).
1562 bool Repl32;
1563 // Did converting to Repl32 == true change the rotation factor? If it did,
1564 // it decreased it by 32.
1565 bool Repl32CR;
1566 // Was this group coalesced after setting Repl32 to true?
1567 bool Repl32Coalesced;
1568
1569 BitGroup(SDValue V, unsigned R, unsigned S, unsigned E)
1570 : V(V), RLAmt(R), StartIdx(S), EndIdx(E), Repl32(false), Repl32CR(false),
1571 Repl32Coalesced(false) {
1572 LLVM_DEBUG(dbgs() << "\tbit group for " << V.getNode() << " RLAmt = " << R
1573 << " [" << S << ", " << E << "]\n");
1574 }
1575 };
1576
1577 // Information on each (Value, RLAmt) pair (like the number of groups
1578 // associated with each) used to choose the lowering method.
1579 struct ValueRotInfo {
1580 SDValue V;
1581 unsigned RLAmt = std::numeric_limits<unsigned>::max();
1582 unsigned NumGroups = 0;
1583 unsigned FirstGroupStartIdx = std::numeric_limits<unsigned>::max();
1584 bool Repl32 = false;
1585
1586 ValueRotInfo() = default;
1587
1588 // For sorting (in reverse order) by NumGroups, and then by
1589 // FirstGroupStartIdx.
1590 bool operator < (const ValueRotInfo &Other) const {
1591 // We need to sort so that the non-Repl32 come first because, when we're
1592 // doing masking, the Repl32 bit groups might be subsumed into the 64-bit
1593 // masking operation.
1594 if (Repl32 < Other.Repl32)
1595 return true;
1596 else if (Repl32 > Other.Repl32)
1597 return false;
1598 else if (NumGroups > Other.NumGroups)
1599 return true;
1600 else if (NumGroups < Other.NumGroups)
1601 return false;
1602 else if (RLAmt == 0 && Other.RLAmt != 0)
1603 return true;
1604 else if (RLAmt != 0 && Other.RLAmt == 0)
1605 return false;
1606 else if (FirstGroupStartIdx < Other.FirstGroupStartIdx)
1607 return true;
1608 return false;
1609 }
1610 };
1611
1612 using ValueBitsMemoizedValue = std::pair<bool, SmallVector<ValueBit, 64>>;
1613 using ValueBitsMemoizer =
1615 ValueBitsMemoizer Memoizer;
1616
1617 // Return a pair of bool and a SmallVector pointer to a memoization entry.
1618 // The bool is true if something interesting was deduced, otherwise if we're
1619 // providing only a generic representation of V (or something else likewise
1620 // uninteresting for instruction selection) through the SmallVector.
1621 std::pair<bool, SmallVector<ValueBit, 64> *> getValueBits(SDValue V,
1622 unsigned NumBits) {
1623 auto &ValueEntry = Memoizer[V];
1624 if (ValueEntry)
1625 return std::make_pair(ValueEntry->first, &ValueEntry->second);
1626 ValueEntry.reset(new ValueBitsMemoizedValue());
1627 bool &Interesting = ValueEntry->first;
1628 SmallVector<ValueBit, 64> &Bits = ValueEntry->second;
1629 Bits.resize(NumBits);
1630
1631 switch (V.getOpcode()) {
1632 default: break;
1633 case ISD::ROTL:
1634 if (isa<ConstantSDNode>(V.getOperand(1))) {
1635 assert(isPowerOf2_32(NumBits) && "rotl bits should be power of 2!");
1636 unsigned RotAmt = V.getConstantOperandVal(1) & (NumBits - 1);
1637
1638 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1639
1640 for (unsigned i = 0; i < NumBits; ++i)
1641 Bits[i] = LHSBits[i < RotAmt ? i + (NumBits - RotAmt) : i - RotAmt];
1642
1643 return std::make_pair(Interesting = true, &Bits);
1644 }
1645 break;
1646 case ISD::SHL:
1647 case PPCISD::SHL:
1648 if (isa<ConstantSDNode>(V.getOperand(1))) {
1649 // sld takes 7 bits, slw takes 6.
1650 unsigned ShiftAmt = V.getConstantOperandVal(1) & ((NumBits << 1) - 1);
1651
1652 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1653
1654 if (ShiftAmt >= NumBits) {
1655 for (unsigned i = 0; i < NumBits; ++i)
1656 Bits[i] = ValueBit(ValueBit::ConstZero);
1657 } else {
1658 for (unsigned i = ShiftAmt; i < NumBits; ++i)
1659 Bits[i] = LHSBits[i - ShiftAmt];
1660 for (unsigned i = 0; i < ShiftAmt; ++i)
1661 Bits[i] = ValueBit(ValueBit::ConstZero);
1662 }
1663
1664 return std::make_pair(Interesting = true, &Bits);
1665 }
1666 break;
1667 case ISD::SRL:
1668 case PPCISD::SRL:
1669 if (isa<ConstantSDNode>(V.getOperand(1))) {
1670 // srd takes lowest 7 bits, srw takes 6.
1671 unsigned ShiftAmt = V.getConstantOperandVal(1) & ((NumBits << 1) - 1);
1672
1673 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1674
1675 if (ShiftAmt >= NumBits) {
1676 for (unsigned i = 0; i < NumBits; ++i)
1677 Bits[i] = ValueBit(ValueBit::ConstZero);
1678 } else {
1679 for (unsigned i = 0; i < NumBits - ShiftAmt; ++i)
1680 Bits[i] = LHSBits[i + ShiftAmt];
1681 for (unsigned i = NumBits - ShiftAmt; i < NumBits; ++i)
1682 Bits[i] = ValueBit(ValueBit::ConstZero);
1683 }
1684
1685 return std::make_pair(Interesting = true, &Bits);
1686 }
1687 break;
1688 case ISD::AND:
1689 if (isa<ConstantSDNode>(V.getOperand(1))) {
1690 uint64_t Mask = V.getConstantOperandVal(1);
1691
1692 const SmallVector<ValueBit, 64> *LHSBits;
1693 // Mark this as interesting, only if the LHS was also interesting. This
1694 // prevents the overall procedure from matching a single immediate 'and'
1695 // (which is non-optimal because such an and might be folded with other
1696 // things if we don't select it here).
1697 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0), NumBits);
1698
1699 for (unsigned i = 0; i < NumBits; ++i)
1700 if (((Mask >> i) & 1) == 1)
1701 Bits[i] = (*LHSBits)[i];
1702 else {
1703 // AND instruction masks this bit. If the input is already zero,
1704 // we have nothing to do here. Otherwise, make the bit ConstZero.
1705 if ((*LHSBits)[i].isZero())
1706 Bits[i] = (*LHSBits)[i];
1707 else
1708 Bits[i] = ValueBit(ValueBit::ConstZero);
1709 }
1710
1711 return std::make_pair(Interesting, &Bits);
1712 }
1713 break;
1714 case ISD::OR: {
1715 const auto &LHSBits = *getValueBits(V.getOperand(0), NumBits).second;
1716 const auto &RHSBits = *getValueBits(V.getOperand(1), NumBits).second;
1717
1718 bool AllDisjoint = true;
1719 SDValue LastVal = SDValue();
1720 unsigned LastIdx = 0;
1721 for (unsigned i = 0; i < NumBits; ++i) {
1722 if (LHSBits[i].isZero() && RHSBits[i].isZero()) {
1723 // If both inputs are known to be zero and one is ConstZero and
1724 // another is VariableKnownToBeZero, we can select whichever
1725 // we like. To minimize the number of bit groups, we select
1726 // VariableKnownToBeZero if this bit is the next bit of the same
1727 // input variable from the previous bit. Otherwise, we select
1728 // ConstZero.
1729 if (LHSBits[i].hasValue() && LHSBits[i].getValue() == LastVal &&
1730 LHSBits[i].getValueBitIndex() == LastIdx + 1)
1731 Bits[i] = LHSBits[i];
1732 else if (RHSBits[i].hasValue() && RHSBits[i].getValue() == LastVal &&
1733 RHSBits[i].getValueBitIndex() == LastIdx + 1)
1734 Bits[i] = RHSBits[i];
1735 else
1736 Bits[i] = ValueBit(ValueBit::ConstZero);
1737 }
1738 else if (LHSBits[i].isZero())
1739 Bits[i] = RHSBits[i];
1740 else if (RHSBits[i].isZero())
1741 Bits[i] = LHSBits[i];
1742 else {
1743 AllDisjoint = false;
1744 break;
1745 }
1746 // We remember the value and bit index of this bit.
1747 if (Bits[i].hasValue()) {
1748 LastVal = Bits[i].getValue();
1749 LastIdx = Bits[i].getValueBitIndex();
1750 }
1751 else {
1752 if (LastVal) LastVal = SDValue();
1753 LastIdx = 0;
1754 }
1755 }
1756
1757 if (!AllDisjoint)
1758 break;
1759
1760 return std::make_pair(Interesting = true, &Bits);
1761 }
1762 case ISD::ZERO_EXTEND: {
1763 // We support only the case with zero extension from i32 to i64 so far.
1764 if (V.getValueType() != MVT::i64 ||
1765 V.getOperand(0).getValueType() != MVT::i32)
1766 break;
1767
1768 const SmallVector<ValueBit, 64> *LHSBits;
1769 const unsigned NumOperandBits = 32;
1770 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1771 NumOperandBits);
1772
1773 for (unsigned i = 0; i < NumOperandBits; ++i)
1774 Bits[i] = (*LHSBits)[i];
1775
1776 for (unsigned i = NumOperandBits; i < NumBits; ++i)
1777 Bits[i] = ValueBit(ValueBit::ConstZero);
1778
1779 return std::make_pair(Interesting, &Bits);
1780 }
1781 case ISD::TRUNCATE: {
1782 EVT FromType = V.getOperand(0).getValueType();
1783 EVT ToType = V.getValueType();
1784 // We support only the case with truncate from i64 to i32.
1785 if (FromType != MVT::i64 || ToType != MVT::i32)
1786 break;
1787 const unsigned NumAllBits = FromType.getSizeInBits();
1789 std::tie(Interesting, InBits) = getValueBits(V.getOperand(0),
1790 NumAllBits);
1791 const unsigned NumValidBits = ToType.getSizeInBits();
1792
1793 // A 32-bit instruction cannot touch upper 32-bit part of 64-bit value.
1794 // So, we cannot include this truncate.
1795 bool UseUpper32bit = false;
1796 for (unsigned i = 0; i < NumValidBits; ++i)
1797 if ((*InBits)[i].hasValue() && (*InBits)[i].getValueBitIndex() >= 32) {
1798 UseUpper32bit = true;
1799 break;
1800 }
1801 if (UseUpper32bit)
1802 break;
1803
1804 for (unsigned i = 0; i < NumValidBits; ++i)
1805 Bits[i] = (*InBits)[i];
1806
1807 return std::make_pair(Interesting, &Bits);
1808 }
1809 case ISD::AssertZext: {
1810 // For AssertZext, we look through the operand and
1811 // mark the bits known to be zero.
1812 const SmallVector<ValueBit, 64> *LHSBits;
1813 std::tie(Interesting, LHSBits) = getValueBits(V.getOperand(0),
1814 NumBits);
1815
1816 EVT FromType = cast<VTSDNode>(V.getOperand(1))->getVT();
1817 const unsigned NumValidBits = FromType.getSizeInBits();
1818 for (unsigned i = 0; i < NumValidBits; ++i)
1819 Bits[i] = (*LHSBits)[i];
1820
1821 // These bits are known to be zero but the AssertZext may be from a value
1822 // that already has some constant zero bits (i.e. from a masking and).
1823 for (unsigned i = NumValidBits; i < NumBits; ++i)
1824 Bits[i] = (*LHSBits)[i].hasValue()
1825 ? ValueBit((*LHSBits)[i].getValue(),
1826 (*LHSBits)[i].getValueBitIndex(),
1827 ValueBit::VariableKnownToBeZero)
1828 : ValueBit(ValueBit::ConstZero);
1829
1830 return std::make_pair(Interesting, &Bits);
1831 }
1832 case ISD::LOAD:
1833 LoadSDNode *LD = cast<LoadSDNode>(V);
1834 if (ISD::isZEXTLoad(V.getNode()) && V.getResNo() == 0) {
1835 EVT VT = LD->getMemoryVT();
1836 const unsigned NumValidBits = VT.getSizeInBits();
1837
1838 for (unsigned i = 0; i < NumValidBits; ++i)
1839 Bits[i] = ValueBit(V, i);
1840
1841 // These bits are known to be zero.
1842 for (unsigned i = NumValidBits; i < NumBits; ++i)
1843 Bits[i] = ValueBit(V, i, ValueBit::VariableKnownToBeZero);
1844
1845 // Zero-extending load itself cannot be optimized. So, it is not
1846 // interesting by itself though it gives useful information.
1847 return std::make_pair(Interesting = false, &Bits);
1848 }
1849 break;
1850 }
1851
1852 for (unsigned i = 0; i < NumBits; ++i)
1853 Bits[i] = ValueBit(V, i);
1854
1855 return std::make_pair(Interesting = false, &Bits);
1856 }
1857
1858 // For each value (except the constant ones), compute the left-rotate amount
1859 // to get it from its original to final position.
1860 void computeRotationAmounts() {
1861 NeedMask = false;
1862 RLAmt.resize(Bits.size());
1863 for (unsigned i = 0; i < Bits.size(); ++i)
1864 if (Bits[i].hasValue()) {
1865 unsigned VBI = Bits[i].getValueBitIndex();
1866 if (i >= VBI)
1867 RLAmt[i] = i - VBI;
1868 else
1869 RLAmt[i] = Bits.size() - (VBI - i);
1870 } else if (Bits[i].isZero()) {
1871 NeedMask = true;
1872 RLAmt[i] = UINT32_MAX;
1873 } else {
1874 llvm_unreachable("Unknown value bit type");
1875 }
1876 }
1877
1878 // Collect groups of consecutive bits with the same underlying value and
1879 // rotation factor. If we're doing late masking, we ignore zeros, otherwise
1880 // they break up groups.
1881 void collectBitGroups(bool LateMask) {
1882 BitGroups.clear();
1883
1884 unsigned LastRLAmt = RLAmt[0];
1885 SDValue LastValue = Bits[0].hasValue() ? Bits[0].getValue() : SDValue();
1886 unsigned LastGroupStartIdx = 0;
1887 bool IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1888 for (unsigned i = 1; i < Bits.size(); ++i) {
1889 unsigned ThisRLAmt = RLAmt[i];
1890 SDValue ThisValue = Bits[i].hasValue() ? Bits[i].getValue() : SDValue();
1891 if (LateMask && !ThisValue) {
1892 ThisValue = LastValue;
1893 ThisRLAmt = LastRLAmt;
1894 // If we're doing late masking, then the first bit group always starts
1895 // at zero (even if the first bits were zero).
1896 if (BitGroups.empty())
1897 LastGroupStartIdx = 0;
1898 }
1899
1900 // If this bit is known to be zero and the current group is a bit group
1901 // of zeros, we do not need to terminate the current bit group even the
1902 // Value or RLAmt does not match here. Instead, we terminate this group
1903 // when the first non-zero bit appears later.
1904 if (IsGroupOfZeros && Bits[i].isZero())
1905 continue;
1906
1907 // If this bit has the same underlying value and the same rotate factor as
1908 // the last one, then they're part of the same group.
1909 if (ThisRLAmt == LastRLAmt && ThisValue == LastValue)
1910 // We cannot continue the current group if this bits is not known to
1911 // be zero in a bit group of zeros.
1912 if (!(IsGroupOfZeros && ThisValue && !Bits[i].isZero()))
1913 continue;
1914
1915 if (LastValue.getNode())
1916 BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1917 i-1));
1918 LastRLAmt = ThisRLAmt;
1919 LastValue = ThisValue;
1920 LastGroupStartIdx = i;
1921 IsGroupOfZeros = !Bits[LastGroupStartIdx].hasValue();
1922 }
1923 if (LastValue.getNode())
1924 BitGroups.push_back(BitGroup(LastValue, LastRLAmt, LastGroupStartIdx,
1925 Bits.size()-1));
1926
1927 if (BitGroups.empty())
1928 return;
1929
1930 // We might be able to combine the first and last groups.
1931 if (BitGroups.size() > 1) {
1932 // If the first and last groups are the same, then remove the first group
1933 // in favor of the last group, making the ending index of the last group
1934 // equal to the ending index of the to-be-removed first group.
1935 if (BitGroups[0].StartIdx == 0 &&
1936 BitGroups[BitGroups.size()-1].EndIdx == Bits.size()-1 &&
1937 BitGroups[0].V == BitGroups[BitGroups.size()-1].V &&
1938 BitGroups[0].RLAmt == BitGroups[BitGroups.size()-1].RLAmt) {
1939 LLVM_DEBUG(dbgs() << "\tcombining final bit group with initial one\n");
1940 BitGroups[BitGroups.size()-1].EndIdx = BitGroups[0].EndIdx;
1941 BitGroups.erase(BitGroups.begin());
1942 }
1943 }
1944 }
1945
1946 // Take all (SDValue, RLAmt) pairs and sort them by the number of groups
1947 // associated with each. If the number of groups are same, we prefer a group
1948 // which does not require rotate, i.e. RLAmt is 0, to avoid the first rotate
1949 // instruction. If there is a degeneracy, pick the one that occurs
1950 // first (in the final value).
1951 void collectValueRotInfo() {
1952 ValueRots.clear();
1953
1954 for (auto &BG : BitGroups) {
1955 unsigned RLAmtKey = BG.RLAmt + (BG.Repl32 ? 64 : 0);
1956 ValueRotInfo &VRI = ValueRots[std::make_pair(BG.V, RLAmtKey)];
1957 VRI.V = BG.V;
1958 VRI.RLAmt = BG.RLAmt;
1959 VRI.Repl32 = BG.Repl32;
1960 VRI.NumGroups += 1;
1961 VRI.FirstGroupStartIdx = std::min(VRI.FirstGroupStartIdx, BG.StartIdx);
1962 }
1963
1964 // Now that we've collected the various ValueRotInfo instances, we need to
1965 // sort them.
1966 ValueRotsVec.clear();
1967 for (auto &I : ValueRots) {
1968 ValueRotsVec.push_back(I.second);
1969 }
1970 llvm::sort(ValueRotsVec);
1971 }
1972
1973 // In 64-bit mode, rlwinm and friends have a rotation operator that
1974 // replicates the low-order 32 bits into the high-order 32-bits. The mask
1975 // indices of these instructions can only be in the lower 32 bits, so they
1976 // can only represent some 64-bit bit groups. However, when they can be used,
1977 // the 32-bit replication can be used to represent, as a single bit group,
1978 // otherwise separate bit groups. We'll convert to replicated-32-bit bit
1979 // groups when possible. Returns true if any of the bit groups were
1980 // converted.
1981 void assignRepl32BitGroups() {
1982 // If we have bits like this:
1983 //
1984 // Indices: 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
1985 // V bits: ... 7 6 5 4 3 2 1 0 31 30 29 28 27 26 25 24
1986 // Groups: | RLAmt = 8 | RLAmt = 40 |
1987 //
1988 // But, making use of a 32-bit operation that replicates the low-order 32
1989 // bits into the high-order 32 bits, this can be one bit group with a RLAmt
1990 // of 8.
1991
1992 auto IsAllLow32 = [this](BitGroup & BG) {
1993 if (BG.StartIdx <= BG.EndIdx) {
1994 for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i) {
1995 if (!Bits[i].hasValue())
1996 continue;
1997 if (Bits[i].getValueBitIndex() >= 32)
1998 return false;
1999 }
2000 } else {
2001 for (unsigned i = BG.StartIdx; i < Bits.size(); ++i) {
2002 if (!Bits[i].hasValue())
2003 continue;
2004 if (Bits[i].getValueBitIndex() >= 32)
2005 return false;
2006 }
2007 for (unsigned i = 0; i <= BG.EndIdx; ++i) {
2008 if (!Bits[i].hasValue())
2009 continue;
2010 if (Bits[i].getValueBitIndex() >= 32)
2011 return false;
2012 }
2013 }
2014
2015 return true;
2016 };
2017
2018 for (auto &BG : BitGroups) {
2019 // If this bit group has RLAmt of 0 and will not be merged with
2020 // another bit group, we don't benefit from Repl32. We don't mark
2021 // such group to give more freedom for later instruction selection.
2022 if (BG.RLAmt == 0) {
2023 auto PotentiallyMerged = [this](BitGroup & BG) {
2024 for (auto &BG2 : BitGroups)
2025 if (&BG != &BG2 && BG.V == BG2.V &&
2026 (BG2.RLAmt == 0 || BG2.RLAmt == 32))
2027 return true;
2028 return false;
2029 };
2030 if (!PotentiallyMerged(BG))
2031 continue;
2032 }
2033 if (BG.StartIdx < 32 && BG.EndIdx < 32) {
2034 if (IsAllLow32(BG)) {
2035 if (BG.RLAmt >= 32) {
2036 BG.RLAmt -= 32;
2037 BG.Repl32CR = true;
2038 }
2039
2040 BG.Repl32 = true;
2041
2042 LLVM_DEBUG(dbgs() << "\t32-bit replicated bit group for "
2043 << BG.V.getNode() << " RLAmt = " << BG.RLAmt << " ["
2044 << BG.StartIdx << ", " << BG.EndIdx << "]\n");
2045 }
2046 }
2047 }
2048
2049 // Now walk through the bit groups, consolidating where possible.
2050 for (auto I = BitGroups.begin(); I != BitGroups.end();) {
2051 // We might want to remove this bit group by merging it with the previous
2052 // group (which might be the ending group).
2053 auto IP = (I == BitGroups.begin()) ?
2054 std::prev(BitGroups.end()) : std::prev(I);
2055 if (I->Repl32 && IP->Repl32 && I->V == IP->V && I->RLAmt == IP->RLAmt &&
2056 I->StartIdx == (IP->EndIdx + 1) % 64 && I != IP) {
2057
2058 LLVM_DEBUG(dbgs() << "\tcombining 32-bit replicated bit group for "
2059 << I->V.getNode() << " RLAmt = " << I->RLAmt << " ["
2060 << I->StartIdx << ", " << I->EndIdx
2061 << "] with group with range [" << IP->StartIdx << ", "
2062 << IP->EndIdx << "]\n");
2063
2064 IP->EndIdx = I->EndIdx;
2065 IP->Repl32CR = IP->Repl32CR || I->Repl32CR;
2066 IP->Repl32Coalesced = true;
2067 I = BitGroups.erase(I);
2068 continue;
2069 } else {
2070 // There is a special case worth handling: If there is a single group
2071 // covering the entire upper 32 bits, and it can be merged with both
2072 // the next and previous groups (which might be the same group), then
2073 // do so. If it is the same group (so there will be only one group in
2074 // total), then we need to reverse the order of the range so that it
2075 // covers the entire 64 bits.
2076 if (I->StartIdx == 32 && I->EndIdx == 63) {
2077 assert(std::next(I) == BitGroups.end() &&
2078 "bit group ends at index 63 but there is another?");
2079 auto IN = BitGroups.begin();
2080
2081 if (IP->Repl32 && IN->Repl32 && I->V == IP->V && I->V == IN->V &&
2082 (I->RLAmt % 32) == IP->RLAmt && (I->RLAmt % 32) == IN->RLAmt &&
2083 IP->EndIdx == 31 && IN->StartIdx == 0 && I != IP &&
2084 IsAllLow32(*I)) {
2085
2086 LLVM_DEBUG(dbgs() << "\tcombining bit group for " << I->V.getNode()
2087 << " RLAmt = " << I->RLAmt << " [" << I->StartIdx
2088 << ", " << I->EndIdx
2089 << "] with 32-bit replicated groups with ranges ["
2090 << IP->StartIdx << ", " << IP->EndIdx << "] and ["
2091 << IN->StartIdx << ", " << IN->EndIdx << "]\n");
2092
2093 if (IP == IN) {
2094 // There is only one other group; change it to cover the whole
2095 // range (backward, so that it can still be Repl32 but cover the
2096 // whole 64-bit range).
2097 IP->StartIdx = 31;
2098 IP->EndIdx = 30;
2099 IP->Repl32CR = IP->Repl32CR || I->RLAmt >= 32;
2100 IP->Repl32Coalesced = true;
2101 I = BitGroups.erase(I);
2102 } else {
2103 // There are two separate groups, one before this group and one
2104 // after us (at the beginning). We're going to remove this group,
2105 // but also the group at the very beginning.
2106 IP->EndIdx = IN->EndIdx;
2107 IP->Repl32CR = IP->Repl32CR || IN->Repl32CR || I->RLAmt >= 32;
2108 IP->Repl32Coalesced = true;
2109 I = BitGroups.erase(I);
2110 BitGroups.erase(BitGroups.begin());
2111 }
2112
2113 // This must be the last group in the vector (and we might have
2114 // just invalidated the iterator above), so break here.
2115 break;
2116 }
2117 }
2118 }
2119
2120 ++I;
2121 }
2122 }
2123
2124 SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
2125 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
2126 }
2127
2128 uint64_t getZerosMask() {
2129 uint64_t Mask = 0;
2130 for (unsigned i = 0; i < Bits.size(); ++i) {
2131 if (Bits[i].hasValue())
2132 continue;
2133 Mask |= (UINT64_C(1) << i);
2134 }
2135
2136 return ~Mask;
2137 }
2138
2139 // This method extends an input value to 64 bit if input is 32-bit integer.
2140 // While selecting instructions in BitPermutationSelector in 64-bit mode,
2141 // an input value can be a 32-bit integer if a ZERO_EXTEND node is included.
2142 // In such case, we extend it to 64 bit to be consistent with other values.
2143 SDValue ExtendToInt64(SDValue V, const SDLoc &dl) {
2144 if (V.getValueSizeInBits() == 64)
2145 return V;
2146
2147 assert(V.getValueSizeInBits() == 32);
2148 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
2149 SDValue ImDef = SDValue(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl,
2150 MVT::i64), 0);
2151 SDValue ExtVal = SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl,
2152 MVT::i64, ImDef, V,
2153 SubRegIdx), 0);
2154 return ExtVal;
2155 }
2156
2157 SDValue TruncateToInt32(SDValue V, const SDLoc &dl) {
2158 if (V.getValueSizeInBits() == 32)
2159 return V;
2160
2161 assert(V.getValueSizeInBits() == 64);
2162 SDValue SubRegIdx = CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
2163 SDValue SubVal = SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl,
2164 MVT::i32, V, SubRegIdx), 0);
2165 return SubVal;
2166 }
2167
2168 // Depending on the number of groups for a particular value, it might be
2169 // better to rotate, mask explicitly (using andi/andis), and then or the
2170 // result. Select this part of the result first.
2171 void SelectAndParts32(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2173 return;
2174
2175 for (ValueRotInfo &VRI : ValueRotsVec) {
2176 unsigned Mask = 0;
2177 for (unsigned i = 0; i < Bits.size(); ++i) {
2178 if (!Bits[i].hasValue() || Bits[i].getValue() != VRI.V)
2179 continue;
2180 if (RLAmt[i] != VRI.RLAmt)
2181 continue;
2182 Mask |= (1u << i);
2183 }
2184
2185 // Compute the masks for andi/andis that would be necessary.
2186 unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16;
2187 assert((ANDIMask != 0 || ANDISMask != 0) &&
2188 "No set bits in mask for value bit groups");
2189 bool NeedsRotate = VRI.RLAmt != 0;
2190
2191 // We're trying to minimize the number of instructions. If we have one
2192 // group, using one of andi/andis can break even. If we have three
2193 // groups, we can use both andi and andis and break even (to use both
2194 // andi and andis we also need to or the results together). We need four
2195 // groups if we also need to rotate. To use andi/andis we need to do more
2196 // than break even because rotate-and-mask instructions tend to be easier
2197 // to schedule.
2198
2199 // FIXME: We've biased here against using andi/andis, which is right for
2200 // POWER cores, but not optimal everywhere. For example, on the A2,
2201 // andi/andis have single-cycle latency whereas the rotate-and-mask
2202 // instructions take two cycles, and it would be better to bias toward
2203 // andi/andis in break-even cases.
2204
2205 unsigned NumAndInsts = (unsigned) NeedsRotate +
2206 (unsigned) (ANDIMask != 0) +
2207 (unsigned) (ANDISMask != 0) +
2208 (unsigned) (ANDIMask != 0 && ANDISMask != 0) +
2209 (unsigned) (bool) Res;
2210
2211 LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()
2212 << " RL: " << VRI.RLAmt << ":"
2213 << "\n\t\t\tisel using masking: " << NumAndInsts
2214 << " using rotates: " << VRI.NumGroups << "\n");
2215
2216 if (NumAndInsts >= VRI.NumGroups)
2217 continue;
2218
2219 LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n");
2220
2221 if (InstCnt) *InstCnt += NumAndInsts;
2222
2223 SDValue VRot;
2224 if (VRI.RLAmt) {
2225 SDValue Ops[] =
2226 { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
2227 getI32Imm(0, dl), getI32Imm(31, dl) };
2228 VRot = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
2229 Ops), 0);
2230 } else {
2231 VRot = TruncateToInt32(VRI.V, dl);
2232 }
2233
2234 SDValue ANDIVal, ANDISVal;
2235 if (ANDIMask != 0)
2236 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32,
2237 VRot, getI32Imm(ANDIMask, dl)),
2238 0);
2239 if (ANDISMask != 0)
2240 ANDISVal =
2241 SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, VRot,
2242 getI32Imm(ANDISMask, dl)),
2243 0);
2244
2245 SDValue TotalVal;
2246 if (!ANDIVal)
2247 TotalVal = ANDISVal;
2248 else if (!ANDISVal)
2249 TotalVal = ANDIVal;
2250 else
2251 TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2252 ANDIVal, ANDISVal), 0);
2253
2254 if (!Res)
2255 Res = TotalVal;
2256 else
2257 Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2258 Res, TotalVal), 0);
2259
2260 // Now, remove all groups with this underlying value and rotation
2261 // factor.
2262 eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2263 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
2264 });
2265 }
2266 }
2267
2268 // Instruction selection for the 32-bit case.
2269 SDNode *Select32(SDNode *N, bool LateMask, unsigned *InstCnt) {
2270 SDLoc dl(N);
2271 SDValue Res;
2272
2273 if (InstCnt) *InstCnt = 0;
2274
2275 // Take care of cases that should use andi/andis first.
2276 SelectAndParts32(dl, Res, InstCnt);
2277
2278 // If we've not yet selected a 'starting' instruction, and we have no zeros
2279 // to fill in, select the (Value, RLAmt) with the highest priority (largest
2280 // number of groups), and start with this rotated value.
2281 if ((!NeedMask || LateMask) && !Res) {
2282 ValueRotInfo &VRI = ValueRotsVec[0];
2283 if (VRI.RLAmt) {
2284 if (InstCnt) *InstCnt += 1;
2285 SDValue Ops[] =
2286 { TruncateToInt32(VRI.V, dl), getI32Imm(VRI.RLAmt, dl),
2287 getI32Imm(0, dl), getI32Imm(31, dl) };
2288 Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops),
2289 0);
2290 } else {
2291 Res = TruncateToInt32(VRI.V, dl);
2292 }
2293
2294 // Now, remove all groups with this underlying value and rotation factor.
2295 eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2296 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt;
2297 });
2298 }
2299
2300 if (InstCnt) *InstCnt += BitGroups.size();
2301
2302 // Insert the other groups (one at a time).
2303 for (auto &BG : BitGroups) {
2304 if (!Res) {
2305 SDValue Ops[] =
2306 { TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
2307 getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
2308 getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
2309 Res = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
2310 } else {
2311 SDValue Ops[] =
2312 { Res, TruncateToInt32(BG.V, dl), getI32Imm(BG.RLAmt, dl),
2313 getI32Imm(Bits.size() - BG.EndIdx - 1, dl),
2314 getI32Imm(Bits.size() - BG.StartIdx - 1, dl) };
2315 Res = SDValue(CurDAG->getMachineNode(PPC::RLWIMI, dl, MVT::i32, Ops), 0);
2316 }
2317 }
2318
2319 if (LateMask) {
2320 unsigned Mask = (unsigned) getZerosMask();
2321
2322 unsigned ANDIMask = (Mask & UINT16_MAX), ANDISMask = Mask >> 16;
2323 assert((ANDIMask != 0 || ANDISMask != 0) &&
2324 "No set bits in zeros mask?");
2325
2326 if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
2327 (unsigned) (ANDISMask != 0) +
2328 (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2329
2330 SDValue ANDIVal, ANDISVal;
2331 if (ANDIMask != 0)
2332 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI_rec, dl, MVT::i32,
2333 Res, getI32Imm(ANDIMask, dl)),
2334 0);
2335 if (ANDISMask != 0)
2336 ANDISVal =
2337 SDValue(CurDAG->getMachineNode(PPC::ANDIS_rec, dl, MVT::i32, Res,
2338 getI32Imm(ANDISMask, dl)),
2339 0);
2340
2341 if (!ANDIVal)
2342 Res = ANDISVal;
2343 else if (!ANDISVal)
2344 Res = ANDIVal;
2345 else
2346 Res = SDValue(CurDAG->getMachineNode(PPC::OR, dl, MVT::i32,
2347 ANDIVal, ANDISVal), 0);
2348 }
2349
2350 return Res.getNode();
2351 }
2352
2353 unsigned SelectRotMask64Count(unsigned RLAmt, bool Repl32,
2354 unsigned MaskStart, unsigned MaskEnd,
2355 bool IsIns) {
2356 // In the notation used by the instructions, 'start' and 'end' are reversed
2357 // because bits are counted from high to low order.
2358 unsigned InstMaskStart = 64 - MaskEnd - 1,
2359 InstMaskEnd = 64 - MaskStart - 1;
2360
2361 if (Repl32)
2362 return 1;
2363
2364 if ((!IsIns && (InstMaskEnd == 63 || InstMaskStart == 0)) ||
2365 InstMaskEnd == 63 - RLAmt)
2366 return 1;
2367
2368 return 2;
2369 }
2370
2371 // For 64-bit values, not all combinations of rotates and masks are
2372 // available. Produce one if it is available.
2373 SDValue SelectRotMask64(SDValue V, const SDLoc &dl, unsigned RLAmt,
2374 bool Repl32, unsigned MaskStart, unsigned MaskEnd,
2375 unsigned *InstCnt = nullptr) {
2376 // In the notation used by the instructions, 'start' and 'end' are reversed
2377 // because bits are counted from high to low order.
2378 unsigned InstMaskStart = 64 - MaskEnd - 1,
2379 InstMaskEnd = 64 - MaskStart - 1;
2380
2381 if (InstCnt) *InstCnt += 1;
2382
2383 if (Repl32) {
2384 // This rotation amount assumes that the lower 32 bits of the quantity
2385 // are replicated in the high 32 bits by the rotation operator (which is
2386 // done by rlwinm and friends).
2387 assert(InstMaskStart >= 32 && "Mask cannot start out of range");
2388 assert(InstMaskEnd >= 32 && "Mask cannot end out of range");
2389 SDValue Ops[] =
2390 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2391 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2392 return SDValue(CurDAG->getMachineNode(PPC::RLWINM8, dl, MVT::i64,
2393 Ops), 0);
2394 }
2395
2396 if (InstMaskEnd == 63) {
2397 SDValue Ops[] =
2398 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2399 getI32Imm(InstMaskStart, dl) };
2400 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Ops), 0);
2401 }
2402
2403 if (InstMaskStart == 0) {
2404 SDValue Ops[] =
2405 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2406 getI32Imm(InstMaskEnd, dl) };
2407 return SDValue(CurDAG->getMachineNode(PPC::RLDICR, dl, MVT::i64, Ops), 0);
2408 }
2409
2410 if (InstMaskEnd == 63 - RLAmt) {
2411 SDValue Ops[] =
2412 { ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2413 getI32Imm(InstMaskStart, dl) };
2414 return SDValue(CurDAG->getMachineNode(PPC::RLDIC, dl, MVT::i64, Ops), 0);
2415 }
2416
2417 // We cannot do this with a single instruction, so we'll use two. The
2418 // problem is that we're not free to choose both a rotation amount and mask
2419 // start and end independently. We can choose an arbitrary mask start and
2420 // end, but then the rotation amount is fixed. Rotation, however, can be
2421 // inverted, and so by applying an "inverse" rotation first, we can get the
2422 // desired result.
2423 if (InstCnt) *InstCnt += 1;
2424
2425 // The rotation mask for the second instruction must be MaskStart.
2426 unsigned RLAmt2 = MaskStart;
2427 // The first instruction must rotate V so that the overall rotation amount
2428 // is RLAmt.
2429 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2430 if (RLAmt1)
2431 V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
2432 return SelectRotMask64(V, dl, RLAmt2, false, MaskStart, MaskEnd);
2433 }
2434
2435 // For 64-bit values, not all combinations of rotates and masks are
2436 // available. Produce a rotate-mask-and-insert if one is available.
2437 SDValue SelectRotMaskIns64(SDValue Base, SDValue V, const SDLoc &dl,
2438 unsigned RLAmt, bool Repl32, unsigned MaskStart,
2439 unsigned MaskEnd, unsigned *InstCnt = nullptr) {
2440 // In the notation used by the instructions, 'start' and 'end' are reversed
2441 // because bits are counted from high to low order.
2442 unsigned InstMaskStart = 64 - MaskEnd - 1,
2443 InstMaskEnd = 64 - MaskStart - 1;
2444
2445 if (InstCnt) *InstCnt += 1;
2446
2447 if (Repl32) {
2448 // This rotation amount assumes that the lower 32 bits of the quantity
2449 // are replicated in the high 32 bits by the rotation operator (which is
2450 // done by rlwinm and friends).
2451 assert(InstMaskStart >= 32 && "Mask cannot start out of range");
2452 assert(InstMaskEnd >= 32 && "Mask cannot end out of range");
2453 SDValue Ops[] =
2454 { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2455 getI32Imm(InstMaskStart - 32, dl), getI32Imm(InstMaskEnd - 32, dl) };
2456 return SDValue(CurDAG->getMachineNode(PPC::RLWIMI8, dl, MVT::i64,
2457 Ops), 0);
2458 }
2459
2460 if (InstMaskEnd == 63 - RLAmt) {
2461 SDValue Ops[] =
2462 { ExtendToInt64(Base, dl), ExtendToInt64(V, dl), getI32Imm(RLAmt, dl),
2463 getI32Imm(InstMaskStart, dl) };
2464 return SDValue(CurDAG->getMachineNode(PPC::RLDIMI, dl, MVT::i64, Ops), 0);
2465 }
2466
2467 // We cannot do this with a single instruction, so we'll use two. The
2468 // problem is that we're not free to choose both a rotation amount and mask
2469 // start and end independently. We can choose an arbitrary mask start and
2470 // end, but then the rotation amount is fixed. Rotation, however, can be
2471 // inverted, and so by applying an "inverse" rotation first, we can get the
2472 // desired result.
2473 if (InstCnt) *InstCnt += 1;
2474
2475 // The rotation mask for the second instruction must be MaskStart.
2476 unsigned RLAmt2 = MaskStart;
2477 // The first instruction must rotate V so that the overall rotation amount
2478 // is RLAmt.
2479 unsigned RLAmt1 = (64 + RLAmt - RLAmt2) % 64;
2480 if (RLAmt1)
2481 V = SelectRotMask64(V, dl, RLAmt1, false, 0, 63);
2482 return SelectRotMaskIns64(Base, V, dl, RLAmt2, false, MaskStart, MaskEnd);
2483 }
2484
2485 void SelectAndParts64(const SDLoc &dl, SDValue &Res, unsigned *InstCnt) {
2487 return;
2488
2489 // The idea here is the same as in the 32-bit version, but with additional
2490 // complications from the fact that Repl32 might be true. Because we
2491 // aggressively convert bit groups to Repl32 form (which, for small
2492 // rotation factors, involves no other change), and then coalesce, it might
2493 // be the case that a single 64-bit masking operation could handle both
2494 // some Repl32 groups and some non-Repl32 groups. If converting to Repl32
2495 // form allowed coalescing, then we must use a 32-bit rotaton in order to
2496 // completely capture the new combined bit group.
2497
2498 for (ValueRotInfo &VRI : ValueRotsVec) {
2499 uint64_t Mask = 0;
2500
2501 // We need to add to the mask all bits from the associated bit groups.
2502 // If Repl32 is false, we need to add bits from bit groups that have
2503 // Repl32 true, but are trivially convertable to Repl32 false. Such a
2504 // group is trivially convertable if it overlaps only with the lower 32
2505 // bits, and the group has not been coalesced.
2506 auto MatchingBG = [VRI](const BitGroup &BG) {
2507 if (VRI.V != BG.V)
2508 return false;
2509
2510 unsigned EffRLAmt = BG.RLAmt;
2511 if (!VRI.Repl32 && BG.Repl32) {
2512 if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx <= BG.EndIdx &&
2513 !BG.Repl32Coalesced) {
2514 if (BG.Repl32CR)
2515 EffRLAmt += 32;
2516 } else {
2517 return false;
2518 }
2519 } else if (VRI.Repl32 != BG.Repl32) {
2520 return false;
2521 }
2522
2523 return VRI.RLAmt == EffRLAmt;
2524 };
2525
2526 for (auto &BG : BitGroups) {
2527 if (!MatchingBG(BG))
2528 continue;
2529
2530 if (BG.StartIdx <= BG.EndIdx) {
2531 for (unsigned i = BG.StartIdx; i <= BG.EndIdx; ++i)
2532 Mask |= (UINT64_C(1) << i);
2533 } else {
2534 for (unsigned i = BG.StartIdx; i < Bits.size(); ++i)
2535 Mask |= (UINT64_C(1) << i);
2536 for (unsigned i = 0; i <= BG.EndIdx; ++i)
2537 Mask |= (UINT64_C(1) << i);
2538 }
2539 }
2540
2541 // We can use the 32-bit andi/andis technique if the mask does not
2542 // require any higher-order bits. This can save an instruction compared
2543 // to always using the general 64-bit technique.
2544 bool Use32BitInsts = isUInt<32>(Mask);
2545 // Compute the masks for andi/andis that would be necessary.
2546 unsigned ANDIMask = (Mask & UINT16_MAX),
2547 ANDISMask = (Mask >> 16) & UINT16_MAX;
2548
2549 bool NeedsRotate = VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask));
2550
2551 unsigned NumAndInsts = (unsigned) NeedsRotate +
2552 (unsigned) (bool) Res;
2553 unsigned NumOfSelectInsts = 0;
2554 selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts);
2555 assert(NumOfSelectInsts > 0 && "Failed to select an i64 constant.");
2556 if (Use32BitInsts)
2557 NumAndInsts += (unsigned) (ANDIMask != 0) + (unsigned) (ANDISMask != 0) +
2558 (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2559 else
2560 NumAndInsts += NumOfSelectInsts + /* and */ 1;
2561
2562 unsigned NumRLInsts = 0;
2563 bool FirstBG = true;
2564 bool MoreBG = false;
2565 for (auto &BG : BitGroups) {
2566 if (!MatchingBG(BG)) {
2567 MoreBG = true;
2568 continue;
2569 }
2570 NumRLInsts +=
2571 SelectRotMask64Count(BG.RLAmt, BG.Repl32, BG.StartIdx, BG.EndIdx,
2572 !FirstBG);
2573 FirstBG = false;
2574 }
2575
2576 LLVM_DEBUG(dbgs() << "\t\trotation groups for " << VRI.V.getNode()
2577 << " RL: " << VRI.RLAmt << (VRI.Repl32 ? " (32):" : ":")
2578 << "\n\t\t\tisel using masking: " << NumAndInsts
2579 << " using rotates: " << NumRLInsts << "\n");
2580
2581 // When we'd use andi/andis, we bias toward using the rotates (andi only
2582 // has a record form, and is cracked on POWER cores). However, when using
2583 // general 64-bit constant formation, bias toward the constant form,
2584 // because that exposes more opportunities for CSE.
2585 if (NumAndInsts > NumRLInsts)
2586 continue;
2587 // When merging multiple bit groups, instruction or is used.
2588 // But when rotate is used, rldimi can inert the rotated value into any
2589 // register, so instruction or can be avoided.
2590 if ((Use32BitInsts || MoreBG) && NumAndInsts == NumRLInsts)
2591 continue;
2592
2593 LLVM_DEBUG(dbgs() << "\t\t\t\tusing masking\n");
2594
2595 if (InstCnt) *InstCnt += NumAndInsts;
2596
2597 SDValue VRot;
2598 // We actually need to generate a rotation if we have a non-zero rotation
2599 // factor or, in the Repl32 case, if we care about any of the
2600 // higher-order replicated bits. In the latter case, we generate a mask
2601 // backward so that it actually includes the entire 64 bits.
2602 if (VRI.RLAmt || (VRI.Repl32 && !isUInt<32>(Mask)))
2603 VRot = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2604 VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63);
2605 else
2606 VRot = VRI.V;
2607
2608 SDValue TotalVal;
2609 if (Use32BitInsts) {
2610 assert((ANDIMask != 0 || ANDISMask != 0) &&
2611 "No set bits in mask when using 32-bit ands for 64-bit value");
2612
2613 SDValue ANDIVal, ANDISVal;
2614 if (ANDIMask != 0)
2615 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64,
2616 ExtendToInt64(VRot, dl),
2617 getI32Imm(ANDIMask, dl)),
2618 0);
2619 if (ANDISMask != 0)
2620 ANDISVal =
2621 SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64,
2622 ExtendToInt64(VRot, dl),
2623 getI32Imm(ANDISMask, dl)),
2624 0);
2625
2626 if (!ANDIVal)
2627 TotalVal = ANDISVal;
2628 else if (!ANDISVal)
2629 TotalVal = ANDIVal;
2630 else
2631 TotalVal = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2632 ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2633 } else {
2634 TotalVal = SDValue(selectI64Imm(CurDAG, dl, Mask), 0);
2635 TotalVal =
2636 SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2637 ExtendToInt64(VRot, dl), TotalVal),
2638 0);
2639 }
2640
2641 if (!Res)
2642 Res = TotalVal;
2643 else
2644 Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2645 ExtendToInt64(Res, dl), TotalVal),
2646 0);
2647
2648 // Now, remove all groups with this underlying value and rotation
2649 // factor.
2650 eraseMatchingBitGroups(MatchingBG);
2651 }
2652 }
2653
2654 // Instruction selection for the 64-bit case.
2655 SDNode *Select64(SDNode *N, bool LateMask, unsigned *InstCnt) {
2656 SDLoc dl(N);
2657 SDValue Res;
2658
2659 if (InstCnt) *InstCnt = 0;
2660
2661 // Take care of cases that should use andi/andis first.
2662 SelectAndParts64(dl, Res, InstCnt);
2663
2664 // If we've not yet selected a 'starting' instruction, and we have no zeros
2665 // to fill in, select the (Value, RLAmt) with the highest priority (largest
2666 // number of groups), and start with this rotated value.
2667 if ((!NeedMask || LateMask) && !Res) {
2668 // If we have both Repl32 groups and non-Repl32 groups, the non-Repl32
2669 // groups will come first, and so the VRI representing the largest number
2670 // of groups might not be first (it might be the first Repl32 groups).
2671 unsigned MaxGroupsIdx = 0;
2672 if (!ValueRotsVec[0].Repl32) {
2673 for (unsigned i = 0, ie = ValueRotsVec.size(); i < ie; ++i)
2674 if (ValueRotsVec[i].Repl32) {
2675 if (ValueRotsVec[i].NumGroups > ValueRotsVec[0].NumGroups)
2676 MaxGroupsIdx = i;
2677 break;
2678 }
2679 }
2680
2681 ValueRotInfo &VRI = ValueRotsVec[MaxGroupsIdx];
2682 bool NeedsRotate = false;
2683 if (VRI.RLAmt) {
2684 NeedsRotate = true;
2685 } else if (VRI.Repl32) {
2686 for (auto &BG : BitGroups) {
2687 if (BG.V != VRI.V || BG.RLAmt != VRI.RLAmt ||
2688 BG.Repl32 != VRI.Repl32)
2689 continue;
2690
2691 // We don't need a rotate if the bit group is confined to the lower
2692 // 32 bits.
2693 if (BG.StartIdx < 32 && BG.EndIdx < 32 && BG.StartIdx < BG.EndIdx)
2694 continue;
2695
2696 NeedsRotate = true;
2697 break;
2698 }
2699 }
2700
2701 if (NeedsRotate)
2702 Res = SelectRotMask64(VRI.V, dl, VRI.RLAmt, VRI.Repl32,
2703 VRI.Repl32 ? 31 : 0, VRI.Repl32 ? 30 : 63,
2704 InstCnt);
2705 else
2706 Res = VRI.V;
2707
2708 // Now, remove all groups with this underlying value and rotation factor.
2709 if (Res)
2710 eraseMatchingBitGroups([VRI](const BitGroup &BG) {
2711 return BG.V == VRI.V && BG.RLAmt == VRI.RLAmt &&
2712 BG.Repl32 == VRI.Repl32;
2713 });
2714 }
2715
2716 // Because 64-bit rotates are more flexible than inserts, we might have a
2717 // preference regarding which one we do first (to save one instruction).
2718 if (!Res)
2719 for (auto I = BitGroups.begin(), IE = BitGroups.end(); I != IE; ++I) {
2720 if (SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2721 false) <
2722 SelectRotMask64Count(I->RLAmt, I->Repl32, I->StartIdx, I->EndIdx,
2723 true)) {
2724 if (I != BitGroups.begin()) {
2725 BitGroup BG = *I;
2726 BitGroups.erase(I);
2727 BitGroups.insert(BitGroups.begin(), BG);
2728 }
2729
2730 break;
2731 }
2732 }
2733
2734 // Insert the other groups (one at a time).
2735 for (auto &BG : BitGroups) {
2736 if (!Res)
2737 Res = SelectRotMask64(BG.V, dl, BG.RLAmt, BG.Repl32, BG.StartIdx,
2738 BG.EndIdx, InstCnt);
2739 else
2740 Res = SelectRotMaskIns64(Res, BG.V, dl, BG.RLAmt, BG.Repl32,
2741 BG.StartIdx, BG.EndIdx, InstCnt);
2742 }
2743
2744 if (LateMask) {
2745 uint64_t Mask = getZerosMask();
2746
2747 // We can use the 32-bit andi/andis technique if the mask does not
2748 // require any higher-order bits. This can save an instruction compared
2749 // to always using the general 64-bit technique.
2750 bool Use32BitInsts = isUInt<32>(Mask);
2751 // Compute the masks for andi/andis that would be necessary.
2752 unsigned ANDIMask = (Mask & UINT16_MAX),
2753 ANDISMask = (Mask >> 16) & UINT16_MAX;
2754
2755 if (Use32BitInsts) {
2756 assert((ANDIMask != 0 || ANDISMask != 0) &&
2757 "No set bits in mask when using 32-bit ands for 64-bit value");
2758
2759 if (InstCnt) *InstCnt += (unsigned) (ANDIMask != 0) +
2760 (unsigned) (ANDISMask != 0) +
2761 (unsigned) (ANDIMask != 0 && ANDISMask != 0);
2762
2763 SDValue ANDIVal, ANDISVal;
2764 if (ANDIMask != 0)
2765 ANDIVal = SDValue(CurDAG->getMachineNode(PPC::ANDI8_rec, dl, MVT::i64,
2766 ExtendToInt64(Res, dl),
2767 getI32Imm(ANDIMask, dl)),
2768 0);
2769 if (ANDISMask != 0)
2770 ANDISVal =
2771 SDValue(CurDAG->getMachineNode(PPC::ANDIS8_rec, dl, MVT::i64,
2772 ExtendToInt64(Res, dl),
2773 getI32Imm(ANDISMask, dl)),
2774 0);
2775
2776 if (!ANDIVal)
2777 Res = ANDISVal;
2778 else if (!ANDISVal)
2779 Res = ANDIVal;
2780 else
2781 Res = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
2782 ExtendToInt64(ANDIVal, dl), ANDISVal), 0);
2783 } else {
2784 unsigned NumOfSelectInsts = 0;
2785 SDValue MaskVal =
2786 SDValue(selectI64Imm(CurDAG, dl, Mask, &NumOfSelectInsts), 0);
2787 Res = SDValue(CurDAG->getMachineNode(PPC::AND8, dl, MVT::i64,
2788 ExtendToInt64(Res, dl), MaskVal),
2789 0);
2790 if (InstCnt)
2791 *InstCnt += NumOfSelectInsts + /* and */ 1;
2792 }
2793 }
2794
2795 return Res.getNode();
2796 }
2797
2798 SDNode *Select(SDNode *N, bool LateMask, unsigned *InstCnt = nullptr) {
2799 // Fill in BitGroups.
2800 collectBitGroups(LateMask);
2801 if (BitGroups.empty())
2802 return nullptr;
2803
2804 // For 64-bit values, figure out when we can use 32-bit instructions.
2805 if (Bits.size() == 64)
2806 assignRepl32BitGroups();
2807
2808 // Fill in ValueRotsVec.
2809 collectValueRotInfo();
2810
2811 if (Bits.size() == 32) {
2812 return Select32(N, LateMask, InstCnt);
2813 } else {
2814 assert(Bits.size() == 64 && "Not 64 bits here?");
2815 return Select64(N, LateMask, InstCnt);
2816 }
2817
2818 return nullptr;
2819 }
2820
2821 void eraseMatchingBitGroups(function_ref<bool(const BitGroup &)> F) {
2822 erase_if(BitGroups, F);
2823 }
2824
2826
2827 bool NeedMask = false;
2829
2830 SmallVector<BitGroup, 16> BitGroups;
2831
2832 DenseMap<std::pair<SDValue, unsigned>, ValueRotInfo> ValueRots;
2833 SmallVector<ValueRotInfo, 16> ValueRotsVec;
2834
2835 SelectionDAG *CurDAG = nullptr;
2836
2837public:
2838 BitPermutationSelector(SelectionDAG *DAG)
2839 : CurDAG(DAG) {}
2840
2841 // Here we try to match complex bit permutations into a set of
2842 // rotate-and-shift/shift/and/or instructions, using a set of heuristics
2843 // known to produce optimal code for common cases (like i32 byte swapping).
2844 SDNode *Select(SDNode *N) {
2845 Memoizer.clear();
2846 auto Result =
2847 getValueBits(SDValue(N, 0), N->getValueType(0).getSizeInBits());
2848 if (!Result.first)
2849 return nullptr;
2850 Bits = std::move(*Result.second);
2851
2852 LLVM_DEBUG(dbgs() << "Considering bit-permutation-based instruction"
2853 " selection for: ");
2854 LLVM_DEBUG(N->dump(CurDAG));
2855
2856 // Fill it RLAmt and set NeedMask.
2857 computeRotationAmounts();
2858
2859 if (!NeedMask)
2860 return Select(N, false);
2861
2862 // We currently have two techniques for handling results with zeros: early
2863 // masking (the default) and late masking. Late masking is sometimes more
2864 // efficient, but because the structure of the bit groups is different, it
2865 // is hard to tell without generating both and comparing the results. With
2866 // late masking, we ignore zeros in the resulting value when inserting each
2867 // set of bit groups, and then mask in the zeros at the end. With early
2868 // masking, we only insert the non-zero parts of the result at every step.
2869
2870 unsigned InstCnt = 0, InstCntLateMask = 0;
2871 LLVM_DEBUG(dbgs() << "\tEarly masking:\n");
2872 SDNode *RN = Select(N, false, &InstCnt);
2873 LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCnt << " instructions\n");
2874
2875 LLVM_DEBUG(dbgs() << "\tLate masking:\n");
2876 SDNode *RNLM = Select(N, true, &InstCntLateMask);
2877 LLVM_DEBUG(dbgs() << "\t\tisel would use " << InstCntLateMask
2878 << " instructions\n");
2879
2880 if (InstCnt <= InstCntLateMask) {
2881 LLVM_DEBUG(dbgs() << "\tUsing early-masking for isel\n");
2882 return RN;
2883 }
2884
2885 LLVM_DEBUG(dbgs() << "\tUsing late-masking for isel\n");
2886 return RNLM;
2887 }
2888};
2889
2890class IntegerCompareEliminator {
2891 SelectionDAG *CurDAG;
2892 PPCDAGToDAGISel *S;
2893 // Conversion type for interpreting results of a 32-bit instruction as
2894 // a 64-bit value or vice versa.
2895 enum ExtOrTruncConversion { Ext, Trunc };
2896
2897 // Modifiers to guide how an ISD::SETCC node's result is to be computed
2898 // in a GPR.
2899 // ZExtOrig - use the original condition code, zero-extend value
2900 // ZExtInvert - invert the condition code, zero-extend value
2901 // SExtOrig - use the original condition code, sign-extend value
2902 // SExtInvert - invert the condition code, sign-extend value
2903 enum SetccInGPROpts { ZExtOrig, ZExtInvert, SExtOrig, SExtInvert };
2904
2905 // Comparisons against zero to emit GPR code sequences for. Each of these
2906 // sequences may need to be emitted for two or more equivalent patterns.
2907 // For example (a >= 0) == (a > -1). The direction of the comparison (</>)
2908 // matters as well as the extension type: sext (-1/0), zext (1/0).
2909 // GEZExt - (zext (LHS >= 0))
2910 // GESExt - (sext (LHS >= 0))
2911 // LEZExt - (zext (LHS <= 0))
2912 // LESExt - (sext (LHS <= 0))
2913 enum ZeroCompare { GEZExt, GESExt, LEZExt, LESExt };
2914
2915 SDNode *tryEXTEND(SDNode *N);
2916 SDNode *tryLogicOpOfCompares(SDNode *N);
2917 SDValue computeLogicOpInGPR(SDValue LogicOp);
2918 SDValue signExtendInputIfNeeded(SDValue Input);
2919 SDValue zeroExtendInputIfNeeded(SDValue Input);
2920 SDValue addExtOrTrunc(SDValue NatWidthRes, ExtOrTruncConversion Conv);
2921 SDValue getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
2922 ZeroCompare CmpTy);
2923 SDValue get32BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2924 int64_t RHSValue, SDLoc dl);
2925 SDValue get32BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2926 int64_t RHSValue, SDLoc dl);
2927 SDValue get64BitZExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2928 int64_t RHSValue, SDLoc dl);
2929 SDValue get64BitSExtCompare(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2930 int64_t RHSValue, SDLoc dl);
2931 SDValue getSETCCInGPR(SDValue Compare, SetccInGPROpts ConvOpts);
2932
2933public:
2934 IntegerCompareEliminator(SelectionDAG *DAG,
2935 PPCDAGToDAGISel *Sel) : CurDAG(DAG), S(Sel) {
2937 .getPointerTy(CurDAG->getDataLayout()).getSizeInBits() == 64 &&
2938 "Only expecting to use this on 64 bit targets.");
2939 }
2940 SDNode *Select(SDNode *N) {
2941 if (CmpInGPR == ICGPR_None)
2942 return nullptr;
2943 switch (N->getOpcode()) {
2944 default: break;
2945 case ISD::ZERO_EXTEND:
2948 return nullptr;
2949 [[fallthrough]];
2950 case ISD::SIGN_EXTEND:
2953 return nullptr;
2954 return tryEXTEND(N);
2955 case ISD::AND:
2956 case ISD::OR:
2957 case ISD::XOR:
2958 return tryLogicOpOfCompares(N);
2959 }
2960 return nullptr;
2961 }
2962};
2963
2964// The obvious case for wanting to keep the value in a GPR. Namely, the
2965// result of the comparison is actually needed in a GPR.
2966SDNode *IntegerCompareEliminator::tryEXTEND(SDNode *N) {
2967 assert((N->getOpcode() == ISD::ZERO_EXTEND ||
2968 N->getOpcode() == ISD::SIGN_EXTEND) &&
2969 "Expecting a zero/sign extend node!");
2970 SDValue WideRes;
2971 // If we are zero-extending the result of a logical operation on i1
2972 // values, we can keep the values in GPRs.
2973 if (ISD::isBitwiseLogicOp(N->getOperand(0).getOpcode()) &&
2974 N->getOperand(0).getValueType() == MVT::i1 &&
2975 N->getOpcode() == ISD::ZERO_EXTEND)
2976 WideRes = computeLogicOpInGPR(N->getOperand(0));
2977 else if (N->getOperand(0).getOpcode() != ISD::SETCC)
2978 return nullptr;
2979 else
2980 WideRes =
2981 getSETCCInGPR(N->getOperand(0),
2982 N->getOpcode() == ISD::SIGN_EXTEND ?
2983 SetccInGPROpts::SExtOrig : SetccInGPROpts::ZExtOrig);
2984
2985 if (!WideRes)
2986 return nullptr;
2987
2988 SDLoc dl(N);
2989 bool Input32Bit = WideRes.getValueType() == MVT::i32;
2990 bool Output32Bit = N->getValueType(0) == MVT::i32;
2991
2992 NumSextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 1 : 0;
2993 NumZextSetcc += N->getOpcode() == ISD::SIGN_EXTEND ? 0 : 1;
2994
2995 SDValue ConvOp = WideRes;
2996 if (Input32Bit != Output32Bit)
2997 ConvOp = addExtOrTrunc(WideRes, Input32Bit ? ExtOrTruncConversion::Ext :
2998 ExtOrTruncConversion::Trunc);
2999 return ConvOp.getNode();
3000}
3001
3002// Attempt to perform logical operations on the results of comparisons while
3003// keeping the values in GPRs. Without doing so, these would end up being
3004// lowered to CR-logical operations which suffer from significant latency and
3005// low ILP.
3006SDNode *IntegerCompareEliminator::tryLogicOpOfCompares(SDNode *N) {
3007 if (N->getValueType(0) != MVT::i1)
3008 return nullptr;
3009 assert(ISD::isBitwiseLogicOp(N->getOpcode()) &&
3010 "Expected a logic operation on setcc results.");
3011 SDValue LoweredLogical = computeLogicOpInGPR(SDValue(N, 0));
3012 if (!LoweredLogical)
3013 return nullptr;
3014
3015 SDLoc dl(N);
3016 bool IsBitwiseNegate = LoweredLogical.getMachineOpcode() == PPC::XORI8;
3017 unsigned SubRegToExtract = IsBitwiseNegate ? PPC::sub_eq : PPC::sub_gt;
3018 SDValue CR0Reg = CurDAG->getRegister(PPC::CR0, MVT::i32);
3019 SDValue LHS = LoweredLogical.getOperand(0);
3020 SDValue RHS = LoweredLogical.getOperand(1);
3021 SDValue WideOp;
3022 SDValue OpToConvToRecForm;
3023
3024 // Look through any 32-bit to 64-bit implicit extend nodes to find the
3025 // opcode that is input to the XORI.
3026 if (IsBitwiseNegate &&
3027 LoweredLogical.getOperand(0).getMachineOpcode() == PPC::INSERT_SUBREG)
3028 OpToConvToRecForm = LoweredLogical.getOperand(0).getOperand(1);
3029 else if (IsBitwiseNegate)
3030 // If the input to the XORI isn't an extension, that's what we're after.
3031 OpToConvToRecForm = LoweredLogical.getOperand(0);
3032 else
3033 // If this is not an XORI, it is a reg-reg logical op and we can convert
3034 // it to record-form.
3035 OpToConvToRecForm = LoweredLogical;
3036
3037 // Get the record-form version of the node we're looking to use to get the
3038 // CR result from.
3039 uint16_t NonRecOpc = OpToConvToRecForm.getMachineOpcode();
3040 int NewOpc = PPCInstrInfo::getRecordFormOpcode(NonRecOpc);
3041
3042 // Convert the right node to record-form. This is either the logical we're
3043 // looking at or it is the input node to the negation (if we're looking at
3044 // a bitwise negation).
3045 if (NewOpc != -1 && IsBitwiseNegate) {
3046 // The input to the XORI has a record-form. Use it.
3047 assert(LoweredLogical.getConstantOperandVal(1) == 1 &&
3048 "Expected a PPC::XORI8 only for bitwise negation.");
3049 // Emit the record-form instruction.
3050 std::vector<SDValue> Ops;
3051 for (int i = 0, e = OpToConvToRecForm.getNumOperands(); i < e; i++)
3052 Ops.push_back(OpToConvToRecForm.getOperand(i));
3053
3054 WideOp =
3055 SDValue(CurDAG->getMachineNode(NewOpc, dl,
3056 OpToConvToRecForm.getValueType(),
3057 MVT::Glue, Ops), 0);
3058 } else {
3059 assert((NewOpc != -1 || !IsBitwiseNegate) &&
3060 "No record form available for AND8/OR8/XOR8?");
3061 WideOp =
3062 SDValue(CurDAG->getMachineNode(NewOpc == -1 ? PPC::ANDI8_rec : NewOpc,
3063 dl, MVT::i64, MVT::Glue, LHS, RHS),
3064 0);
3065 }
3066
3067 // Select this node to a single bit from CR0 set by the record-form node
3068 // just created. For bitwise negation, use the EQ bit which is the equivalent
3069 // of negating the result (i.e. it is a bit set when the result of the
3070 // operation is zero).
3071 SDValue SRIdxVal =
3072 CurDAG->getTargetConstant(SubRegToExtract, dl, MVT::i32);
3073 SDValue CRBit =
3074 SDValue(CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl,
3075 MVT::i1, CR0Reg, SRIdxVal,
3076 WideOp.getValue(1)), 0);
3077 return CRBit.getNode();
3078}
3079
3080// Lower a logical operation on i1 values into a GPR sequence if possible.
3081// The result can be kept in a GPR if requested.
3082// Three types of inputs can be handled:
3083// - SETCC
3084// - TRUNCATE
3085// - Logical operation (AND/OR/XOR)
3086// There is also a special case that is handled (namely a complement operation
3087// achieved with xor %a, -1).
3088SDValue IntegerCompareEliminator::computeLogicOpInGPR(SDValue LogicOp) {
3090 "Can only handle logic operations here.");
3091 assert(LogicOp.getValueType() == MVT::i1 &&
3092 "Can only handle logic operations on i1 values here.");
3093 SDLoc dl(LogicOp);
3094 SDValue LHS, RHS;
3095
3096 // Special case: xor %a, -1
3097 bool IsBitwiseNegation = isBitwiseNot(LogicOp);
3098
3099 // Produces a GPR sequence for each operand of the binary logic operation.
3100 // For SETCC, it produces the respective comparison, for TRUNCATE it truncates
3101 // the value in a GPR and for logic operations, it will recursively produce
3102 // a GPR sequence for the operation.
3103 auto getLogicOperand = [&] (SDValue Operand) -> SDValue {
3104 unsigned OperandOpcode = Operand.getOpcode();
3105 if (OperandOpcode == ISD::SETCC)
3106 return getSETCCInGPR(Operand, SetccInGPROpts::ZExtOrig);
3107 else if (OperandOpcode == ISD::TRUNCATE) {
3108 SDValue InputOp = Operand.getOperand(0);
3109 EVT InVT = InputOp.getValueType();
3110 return SDValue(CurDAG->getMachineNode(InVT == MVT::i32 ? PPC::RLDICL_32 :
3111 PPC::RLDICL, dl, InVT, InputOp,
3112 S->getI64Imm(0, dl),
3113 S->getI64Imm(63, dl)), 0);
3114 } else if (ISD::isBitwiseLogicOp(OperandOpcode))
3115 return computeLogicOpInGPR(Operand);
3116 return SDValue();
3117 };
3118 LHS = getLogicOperand(LogicOp.getOperand(0));
3119 RHS = getLogicOperand(LogicOp.getOperand(1));
3120
3121 // If a GPR sequence can't be produced for the LHS we can't proceed.
3122 // Not producing a GPR sequence for the RHS is only a problem if this isn't
3123 // a bitwise negation operation.
3124 if (!LHS || (!RHS && !IsBitwiseNegation))
3125 return SDValue();
3126
3127 NumLogicOpsOnComparison++;
3128
3129 // We will use the inputs as 64-bit values.
3130 if (LHS.getValueType() == MVT::i32)
3131 LHS = addExtOrTrunc(LHS, ExtOrTruncConversion::Ext);
3132 if (!IsBitwiseNegation && RHS.getValueType() == MVT::i32)
3133 RHS = addExtOrTrunc(RHS, ExtOrTruncConversion::Ext);
3134
3135 unsigned NewOpc;
3136 switch (LogicOp.getOpcode()) {
3137 default: llvm_unreachable("Unknown logic operation.");
3138 case ISD::AND: NewOpc = PPC::AND8; break;
3139 case ISD::OR: NewOpc = PPC::OR8; break;
3140 case ISD::XOR: NewOpc = PPC::XOR8; break;
3141 }
3142
3143 if (IsBitwiseNegation) {
3144 RHS = S->getI64Imm(1, dl);
3145 NewOpc = PPC::XORI8;
3146 }
3147
3148 return SDValue(CurDAG->getMachineNode(NewOpc, dl, MVT::i64, LHS, RHS), 0);
3149
3150}
3151
3152/// If the value isn't guaranteed to be sign-extended to 64-bits, extend it.
3153/// Otherwise just reinterpret it as a 64-bit value.
3154/// Useful when emitting comparison code for 32-bit values without using
3155/// the compare instruction (which only considers the lower 32-bits).
3156SDValue IntegerCompareEliminator::signExtendInputIfNeeded(SDValue Input) {
3157 assert(Input.getValueType() == MVT::i32 &&
3158 "Can only sign-extend 32-bit values here.");
3159 unsigned Opc = Input.getOpcode();
3160
3161 // The value was sign extended and then truncated to 32-bits. No need to
3162 // sign extend it again.
3163 if (Opc == ISD::TRUNCATE &&
3164 (Input.getOperand(0).getOpcode() == ISD::AssertSext ||
3165 Input.getOperand(0).getOpcode() == ISD::SIGN_EXTEND))
3166 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3167
3168 LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
3169 // The input is a sign-extending load. All ppc sign-extending loads
3170 // sign-extend to the full 64-bits.
3171 if (InputLoad && InputLoad->getExtensionType() == ISD::SEXTLOAD)
3172 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3173
3174 ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
3175 // We don't sign-extend constants.
3176 if (InputConst)
3177 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3178
3179 SDLoc dl(Input);
3180 SignExtensionsAdded++;
3181 return SDValue(CurDAG->getMachineNode(PPC::EXTSW_32_64, dl,
3182 MVT::i64, Input), 0);
3183}
3184
3185/// If the value isn't guaranteed to be zero-extended to 64-bits, extend it.
3186/// Otherwise just reinterpret it as a 64-bit value.
3187/// Useful when emitting comparison code for 32-bit values without using
3188/// the compare instruction (which only considers the lower 32-bits).
3189SDValue IntegerCompareEliminator::zeroExtendInputIfNeeded(SDValue Input) {
3190 assert(Input.getValueType() == MVT::i32 &&
3191 "Can only zero-extend 32-bit values here.");
3192 unsigned Opc = Input.getOpcode();
3193
3194 // The only condition under which we can omit the actual extend instruction:
3195 // - The value is a positive constant
3196 // - The value comes from a load that isn't a sign-extending load
3197 // An ISD::TRUNCATE needs to be zero-extended unless it is fed by a zext.
3198 bool IsTruncateOfZExt = Opc == ISD::TRUNCATE &&
3199 (Input.getOperand(0).getOpcode() == ISD::AssertZext ||
3200 Input.getOperand(0).getOpcode() == ISD::ZERO_EXTEND);
3201 if (IsTruncateOfZExt)
3202 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3203
3204 ConstantSDNode *InputConst = dyn_cast<ConstantSDNode>(Input);
3205 if (InputConst && InputConst->getSExtValue() >= 0)
3206 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3207
3208 LoadSDNode *InputLoad = dyn_cast<LoadSDNode>(Input);
3209 // The input is a load that doesn't sign-extend (it will be zero-extended).
3210 if (InputLoad && InputLoad->getExtensionType() != ISD::SEXTLOAD)
3211 return addExtOrTrunc(Input, ExtOrTruncConversion::Ext);
3212
3213 // None of the above, need to zero-extend.
3214 SDLoc dl(Input);
3215 ZeroExtensionsAdded++;
3216 return SDValue(CurDAG->getMachineNode(PPC::RLDICL_32_64, dl, MVT::i64, Input,
3217 S->getI64Imm(0, dl),
3218 S->getI64Imm(32, dl)), 0);
3219}
3220
3221// Handle a 32-bit value in a 64-bit register and vice-versa. These are of
3222// course not actual zero/sign extensions that will generate machine code,
3223// they're just a way to reinterpret a 32 bit value in a register as a
3224// 64 bit value and vice-versa.
3225SDValue IntegerCompareEliminator::addExtOrTrunc(SDValue NatWidthRes,
3226 ExtOrTruncConversion Conv) {
3227 SDLoc dl(NatWidthRes);
3228
3229 // For reinterpreting 32-bit values as 64 bit values, we generate
3230 // INSERT_SUBREG IMPLICIT_DEF:i64, <input>, TargetConstant:i32<1>
3231 if (Conv == ExtOrTruncConversion::Ext) {
3232 SDValue ImDef(CurDAG->getMachineNode(PPC::IMPLICIT_DEF, dl, MVT::i64), 0);
3233 SDValue SubRegIdx =
3234 CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
3235 return SDValue(CurDAG->getMachineNode(PPC::INSERT_SUBREG, dl, MVT::i64,
3236 ImDef, NatWidthRes, SubRegIdx), 0);
3237 }
3238
3239 assert(Conv == ExtOrTruncConversion::Trunc &&
3240 "Unknown convertion between 32 and 64 bit values.");
3241 // For reinterpreting 64-bit values as 32-bit values, we just need to
3242 // EXTRACT_SUBREG (i.e. extract the low word).
3243 SDValue SubRegIdx =
3244 CurDAG->getTargetConstant(PPC::sub_32, dl, MVT::i32);
3245 return SDValue(CurDAG->getMachineNode(PPC::EXTRACT_SUBREG, dl, MVT::i32,
3246 NatWidthRes, SubRegIdx), 0);
3247}
3248
3249// Produce a GPR sequence for compound comparisons (<=, >=) against zero.
3250// Handle both zero-extensions and sign-extensions.
3251SDValue
3252IntegerCompareEliminator::getCompoundZeroComparisonInGPR(SDValue LHS, SDLoc dl,
3253 ZeroCompare CmpTy) {
3254 EVT InVT = LHS.getValueType();
3255 bool Is32Bit = InVT == MVT::i32;
3256 SDValue ToExtend;
3257
3258 // Produce the value that needs to be either zero or sign extended.
3259 switch (CmpTy) {
3260 case ZeroCompare::GEZExt:
3261 case ZeroCompare::GESExt:
3262 ToExtend = SDValue(CurDAG->getMachineNode(Is32Bit ? PPC::NOR : PPC::NOR8,
3263 dl, InVT, LHS, LHS), 0);
3264 break;
3265 case ZeroCompare::LEZExt:
3266 case ZeroCompare::LESExt: {
3267 if (Is32Bit) {
3268 // Upper 32 bits cannot be undefined for this sequence.
3269 LHS = signExtendInputIfNeeded(LHS);
3270 SDValue Neg =
3271 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3272 ToExtend =
3273 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3274 Neg, S->getI64Imm(1, dl),
3275 S->getI64Imm(63, dl)), 0);
3276 } else {
3277 SDValue Addi =
3278 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3279 S->getI64Imm(~0ULL, dl)), 0);
3280 ToExtend = SDValue(CurDAG->getMachineNode(PPC::OR8, dl, MVT::i64,
3281 Addi, LHS), 0);
3282 }
3283 break;
3284 }
3285 }
3286
3287 // For 64-bit sequences, the extensions are the same for the GE/LE cases.
3288 if (!Is32Bit &&
3289 (CmpTy == ZeroCompare::GEZExt || CmpTy == ZeroCompare::LEZExt))
3290 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3291 ToExtend, S->getI64Imm(1, dl),
3292 S->getI64Imm(63, dl)), 0);
3293 if (!Is32Bit &&
3294 (CmpTy == ZeroCompare::GESExt || CmpTy == ZeroCompare::LESExt))
3295 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, ToExtend,
3296 S->getI64Imm(63, dl)), 0);
3297
3298 assert(Is32Bit && "Should have handled the 32-bit sequences above.");
3299 // For 32-bit sequences, the extensions differ between GE/LE cases.
3300 switch (CmpTy) {
3301 case ZeroCompare::GEZExt: {
3302 SDValue ShiftOps[] = { ToExtend, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3303 S->getI32Imm(31, dl) };
3304 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3305 ShiftOps), 0);
3306 }
3307 case ZeroCompare::GESExt:
3308 return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, ToExtend,
3309 S->getI32Imm(31, dl)), 0);
3310 case ZeroCompare::LEZExt:
3311 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, ToExtend,
3312 S->getI32Imm(1, dl)), 0);
3313 case ZeroCompare::LESExt:
3314 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, ToExtend,
3315 S->getI32Imm(-1, dl)), 0);
3316 }
3317
3318 // The above case covers all the enumerators so it can't have a default clause
3319 // to avoid compiler warnings.
3320 llvm_unreachable("Unknown zero-comparison type.");
3321}
3322
3323/// Produces a zero-extended result of comparing two 32-bit values according to
3324/// the passed condition code.
3325SDValue
3326IntegerCompareEliminator::get32BitZExtCompare(SDValue LHS, SDValue RHS,
3328 int64_t RHSValue, SDLoc dl) {
3331 return SDValue();
3332 bool IsRHSZero = RHSValue == 0;
3333 bool IsRHSOne = RHSValue == 1;
3334 bool IsRHSNegOne = RHSValue == -1LL;
3335 switch (CC) {
3336 default: return SDValue();
3337 case ISD::SETEQ: {
3338 // (zext (setcc %a, %b, seteq)) -> (lshr (cntlzw (xor %a, %b)), 5)
3339 // (zext (setcc %a, 0, seteq)) -> (lshr (cntlzw %a), 5)
3340 SDValue Xor = IsRHSZero ? LHS :
3341 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3342 SDValue Clz =
3343 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3344 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
3345 S->getI32Imm(31, dl) };
3346 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3347 ShiftOps), 0);
3348 }
3349 case ISD::SETNE: {
3350 // (zext (setcc %a, %b, setne)) -> (xor (lshr (cntlzw (xor %a, %b)), 5), 1)
3351 // (zext (setcc %a, 0, setne)) -> (xor (lshr (cntlzw %a), 5), 1)
3352 SDValue Xor = IsRHSZero ? LHS :
3353 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3354 SDValue Clz =
3355 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3356 SDValue ShiftOps[] = { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl),
3357 S->getI32Imm(31, dl) };
3358 SDValue Shift =
3359 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
3360 return SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
3361 S->getI32Imm(1, dl)), 0);
3362 }
3363 case ISD::SETGE: {
3364 // (zext (setcc %a, %b, setge)) -> (xor (lshr (sub %a, %b), 63), 1)
3365 // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 31)
3366 if(IsRHSZero)
3367 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3368
3369 // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
3370 // by swapping inputs and falling through.
3371 std::swap(LHS, RHS);
3372 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3373 IsRHSZero = RHSConst && RHSConst->isZero();
3374 [[fallthrough]];
3375 }
3376 case ISD::SETLE: {
3377 if (CmpInGPR == ICGPR_NonExtIn)
3378 return SDValue();
3379 // (zext (setcc %a, %b, setle)) -> (xor (lshr (sub %b, %a), 63), 1)
3380 // (zext (setcc %a, 0, setle)) -> (xor (lshr (- %a), 63), 1)
3381 if(IsRHSZero) {
3382 if (CmpInGPR == ICGPR_NonExtIn)
3383 return SDValue();
3384 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3385 }
3386
3387 // The upper 32-bits of the register can't be undefined for this sequence.
3388 LHS = signExtendInputIfNeeded(LHS);
3389 RHS = signExtendInputIfNeeded(RHS);
3390 SDValue Sub =
3391 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3392 SDValue Shift =
3393 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Sub,
3394 S->getI64Imm(1, dl), S->getI64Imm(63, dl)),
3395 0);
3396 return
3397 SDValue(CurDAG->getMachineNode(PPC::XORI8, dl,
3398 MVT::i64, Shift, S->getI32Imm(1, dl)), 0);
3399 }
3400 case ISD::SETGT: {
3401 // (zext (setcc %a, %b, setgt)) -> (lshr (sub %b, %a), 63)
3402 // (zext (setcc %a, -1, setgt)) -> (lshr (~ %a), 31)
3403 // (zext (setcc %a, 0, setgt)) -> (lshr (- %a), 63)
3404 // Handle SETLT -1 (which is equivalent to SETGE 0).
3405 if (IsRHSNegOne)
3406 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3407
3408 if (IsRHSZero) {
3409 if (CmpInGPR == ICGPR_NonExtIn)
3410 return SDValue();
3411 // The upper 32-bits of the register can't be undefined for this sequence.
3412 LHS = signExtendInputIfNeeded(LHS);
3413 RHS = signExtendInputIfNeeded(RHS);
3414 SDValue Neg =
3415 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3416 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3417 Neg, S->getI32Imm(1, dl), S->getI32Imm(63, dl)), 0);
3418 }
3419 // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
3420 // (%b < %a) by swapping inputs and falling through.
3421 std::swap(LHS, RHS);
3422 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3423 IsRHSZero = RHSConst && RHSConst->isZero();
3424 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3425 [[fallthrough]];
3426 }
3427 case ISD::SETLT: {
3428 // (zext (setcc %a, %b, setlt)) -> (lshr (sub %a, %b), 63)
3429 // (zext (setcc %a, 1, setlt)) -> (xor (lshr (- %a), 63), 1)
3430 // (zext (setcc %a, 0, setlt)) -> (lshr %a, 31)
3431 // Handle SETLT 1 (which is equivalent to SETLE 0).
3432 if (IsRHSOne) {
3433 if (CmpInGPR == ICGPR_NonExtIn)
3434 return SDValue();
3435 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3436 }
3437
3438 if (IsRHSZero) {
3439 SDValue ShiftOps[] = { LHS, S->getI32Imm(1, dl), S->getI32Imm(31, dl),
3440 S->getI32Imm(31, dl) };
3441 return SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32,
3442 ShiftOps), 0);
3443 }
3444
3445 if (CmpInGPR == ICGPR_NonExtIn)
3446 return SDValue();
3447 // The upper 32-bits of the register can't be undefined for this sequence.
3448 LHS = signExtendInputIfNeeded(LHS);
3449 RHS = signExtendInputIfNeeded(RHS);
3450 SDValue SUBFNode =
3451 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3452 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3453 SUBFNode, S->getI64Imm(1, dl),
3454 S->getI64Imm(63, dl)), 0);
3455 }
3456 case ISD::SETUGE:
3457 // (zext (setcc %a, %b, setuge)) -> (xor (lshr (sub %b, %a), 63), 1)
3458 // (zext (setcc %a, %b, setule)) -> (xor (lshr (sub %a, %b), 63), 1)
3459 std::swap(LHS, RHS);
3460 [[fallthrough]];
3461 case ISD::SETULE: {
3462 if (CmpInGPR == ICGPR_NonExtIn)
3463 return SDValue();
3464 // The upper 32-bits of the register can't be undefined for this sequence.
3465 LHS = zeroExtendInputIfNeeded(LHS);
3466 RHS = zeroExtendInputIfNeeded(RHS);
3467 SDValue Subtract =
3468 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3469 SDValue SrdiNode =
3470 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3471 Subtract, S->getI64Imm(1, dl),
3472 S->getI64Imm(63, dl)), 0);
3473 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64, SrdiNode,
3474 S->getI32Imm(1, dl)), 0);
3475 }
3476 case ISD::SETUGT:
3477 // (zext (setcc %a, %b, setugt)) -> (lshr (sub %b, %a), 63)
3478 // (zext (setcc %a, %b, setult)) -> (lshr (sub %a, %b), 63)
3479 std::swap(LHS, RHS);
3480 [[fallthrough]];
3481 case ISD::SETULT: {
3482 if (CmpInGPR == ICGPR_NonExtIn)
3483 return SDValue();
3484 // The upper 32-bits of the register can't be undefined for this sequence.
3485 LHS = zeroExtendInputIfNeeded(LHS);
3486 RHS = zeroExtendInputIfNeeded(RHS);
3487 SDValue Subtract =
3488 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3489 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3490 Subtract, S->getI64Imm(1, dl),
3491 S->getI64Imm(63, dl)), 0);
3492 }
3493 }
3494}
3495
3496/// Produces a sign-extended result of comparing two 32-bit values according to
3497/// the passed condition code.
3498SDValue
3499IntegerCompareEliminator::get32BitSExtCompare(SDValue LHS, SDValue RHS,
3501 int64_t RHSValue, SDLoc dl) {
3504 return SDValue();
3505 bool IsRHSZero = RHSValue == 0;
3506 bool IsRHSOne = RHSValue == 1;
3507 bool IsRHSNegOne = RHSValue == -1LL;
3508
3509 switch (CC) {
3510 default: return SDValue();
3511 case ISD::SETEQ: {
3512 // (sext (setcc %a, %b, seteq)) ->
3513 // (ashr (shl (ctlz (xor %a, %b)), 58), 63)
3514 // (sext (setcc %a, 0, seteq)) ->
3515 // (ashr (shl (ctlz %a), 58), 63)
3516 SDValue CountInput = IsRHSZero ? LHS :
3517 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3518 SDValue Cntlzw =
3519 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, CountInput), 0);
3520 SDValue SHLOps[] = { Cntlzw, S->getI32Imm(27, dl),
3521 S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3522 SDValue Slwi =
3523 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, SHLOps), 0);
3524 return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Slwi), 0);
3525 }
3526 case ISD::SETNE: {
3527 // Bitwise xor the operands, count leading zeros, shift right by 5 bits and
3528 // flip the bit, finally take 2's complement.
3529 // (sext (setcc %a, %b, setne)) ->
3530 // (neg (xor (lshr (ctlz (xor %a, %b)), 5), 1))
3531 // Same as above, but the first xor is not needed.
3532 // (sext (setcc %a, 0, setne)) ->
3533 // (neg (xor (lshr (ctlz %a), 5), 1))
3534 SDValue Xor = IsRHSZero ? LHS :
3535 SDValue(CurDAG->getMachineNode(PPC::XOR, dl, MVT::i32, LHS, RHS), 0);
3536 SDValue Clz =
3537 SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Xor), 0);
3538 SDValue ShiftOps[] =
3539 { Clz, S->getI32Imm(27, dl), S->getI32Imm(5, dl), S->getI32Imm(31, dl) };
3540 SDValue Shift =
3541 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, ShiftOps), 0);
3542 SDValue Xori =
3543 SDValue(CurDAG->getMachineNode(PPC::XORI, dl, MVT::i32, Shift,
3544 S->getI32Imm(1, dl)), 0);
3545 return SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Xori), 0);
3546 }
3547 case ISD::SETGE: {
3548 // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %a, %b), 63), -1)
3549 // (sext (setcc %a, 0, setge)) -> (ashr (~ %a), 31)
3550 if (IsRHSZero)
3551 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3552
3553 // Not a special case (i.e. RHS == 0). Handle (%a >= %b) as (%b <= %a)
3554 // by swapping inputs and falling through.
3555 std::swap(LHS, RHS);
3556 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3557 IsRHSZero = RHSConst && RHSConst->isZero();
3558 [[fallthrough]];
3559 }
3560 case ISD::SETLE: {
3561 if (CmpInGPR == ICGPR_NonExtIn)
3562 return SDValue();
3563 // (sext (setcc %a, %b, setge)) -> (add (lshr (sub %b, %a), 63), -1)
3564 // (sext (setcc %a, 0, setle)) -> (add (lshr (- %a), 63), -1)
3565 if (IsRHSZero)
3566 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3567
3568 // The upper 32-bits of the register can't be undefined for this sequence.
3569 LHS = signExtendInputIfNeeded(LHS);
3570 RHS = signExtendInputIfNeeded(RHS);
3571 SDValue SUBFNode =
3572 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, MVT::Glue,
3573 LHS, RHS), 0);
3574 SDValue Srdi =
3575 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3576 SUBFNode, S->getI64Imm(1, dl),
3577 S->getI64Imm(63, dl)), 0);
3578 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Srdi,
3579 S->getI32Imm(-1, dl)), 0);
3580 }
3581 case ISD::SETGT: {
3582 // (sext (setcc %a, %b, setgt)) -> (ashr (sub %b, %a), 63)
3583 // (sext (setcc %a, -1, setgt)) -> (ashr (~ %a), 31)
3584 // (sext (setcc %a, 0, setgt)) -> (ashr (- %a), 63)
3585 if (IsRHSNegOne)
3586 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3587 if (IsRHSZero) {
3588 if (CmpInGPR == ICGPR_NonExtIn)
3589 return SDValue();
3590 // The upper 32-bits of the register can't be undefined for this sequence.
3591 LHS = signExtendInputIfNeeded(LHS);
3592 RHS = signExtendInputIfNeeded(RHS);
3593 SDValue Neg =
3594 SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, LHS), 0);
3595 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Neg,
3596 S->getI64Imm(63, dl)), 0);
3597 }
3598 // Not a special case (i.e. RHS == 0 or RHS == -1). Handle (%a > %b) as
3599 // (%b < %a) by swapping inputs and falling through.
3600 std::swap(LHS, RHS);
3601 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3602 IsRHSZero = RHSConst && RHSConst->isZero();
3603 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3604 [[fallthrough]];
3605 }
3606 case ISD::SETLT: {
3607 // (sext (setcc %a, %b, setgt)) -> (ashr (sub %a, %b), 63)
3608 // (sext (setcc %a, 1, setgt)) -> (add (lshr (- %a), 63), -1)
3609 // (sext (setcc %a, 0, setgt)) -> (ashr %a, 31)
3610 if (IsRHSOne) {
3611 if (CmpInGPR == ICGPR_NonExtIn)
3612 return SDValue();
3613 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3614 }
3615 if (IsRHSZero)
3616 return SDValue(CurDAG->getMachineNode(PPC::SRAWI, dl, MVT::i32, LHS,
3617 S->getI32Imm(31, dl)), 0);
3618
3619 if (CmpInGPR == ICGPR_NonExtIn)
3620 return SDValue();
3621 // The upper 32-bits of the register can't be undefined for this sequence.
3622 LHS = signExtendInputIfNeeded(LHS);
3623 RHS = signExtendInputIfNeeded(RHS);
3624 SDValue SUBFNode =
3625 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3626 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3627 SUBFNode, S->getI64Imm(63, dl)), 0);
3628 }
3629 case ISD::SETUGE:
3630 // (sext (setcc %a, %b, setuge)) -> (add (lshr (sub %a, %b), 63), -1)
3631 // (sext (setcc %a, %b, setule)) -> (add (lshr (sub %b, %a), 63), -1)
3632 std::swap(LHS, RHS);
3633 [[fallthrough]];
3634 case ISD::SETULE: {
3635 if (CmpInGPR == ICGPR_NonExtIn)
3636 return SDValue();
3637 // The upper 32-bits of the register can't be undefined for this sequence.
3638 LHS = zeroExtendInputIfNeeded(LHS);
3639 RHS = zeroExtendInputIfNeeded(RHS);
3640 SDValue Subtract =
3641 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, LHS, RHS), 0);
3642 SDValue Shift =
3643 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Subtract,
3644 S->getI32Imm(1, dl), S->getI32Imm(63,dl)),
3645 0);
3646 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, Shift,
3647 S->getI32Imm(-1, dl)), 0);
3648 }
3649 case ISD::SETUGT:
3650 // (sext (setcc %a, %b, setugt)) -> (ashr (sub %b, %a), 63)
3651 // (sext (setcc %a, %b, setugt)) -> (ashr (sub %a, %b), 63)
3652 std::swap(LHS, RHS);
3653 [[fallthrough]];
3654 case ISD::SETULT: {
3655 if (CmpInGPR == ICGPR_NonExtIn)
3656 return SDValue();
3657 // The upper 32-bits of the register can't be undefined for this sequence.
3658 LHS = zeroExtendInputIfNeeded(LHS);
3659 RHS = zeroExtendInputIfNeeded(RHS);
3660 SDValue Subtract =
3661 SDValue(CurDAG->getMachineNode(PPC::SUBF8, dl, MVT::i64, RHS, LHS), 0);
3662 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3663 Subtract, S->getI64Imm(63, dl)), 0);
3664 }
3665 }
3666}
3667
3668/// Produces a zero-extended result of comparing two 64-bit values according to
3669/// the passed condition code.
3670SDValue
3671IntegerCompareEliminator::get64BitZExtCompare(SDValue LHS, SDValue RHS,
3673 int64_t RHSValue, SDLoc dl) {
3676 return SDValue();
3677 bool IsRHSZero = RHSValue == 0;
3678 bool IsRHSOne = RHSValue == 1;
3679 bool IsRHSNegOne = RHSValue == -1LL;
3680 switch (CC) {
3681 default: return SDValue();
3682 case ISD::SETEQ: {
3683 // (zext (setcc %a, %b, seteq)) -> (lshr (ctlz (xor %a, %b)), 6)
3684 // (zext (setcc %a, 0, seteq)) -> (lshr (ctlz %a), 6)
3685 SDValue Xor = IsRHSZero ? LHS :
3686 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3687 SDValue Clz =
3688 SDValue(CurDAG->getMachineNode(PPC::CNTLZD, dl, MVT::i64, Xor), 0);
3689 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Clz,
3690 S->getI64Imm(58, dl),
3691 S->getI64Imm(63, dl)), 0);
3692 }
3693 case ISD::SETNE: {
3694 // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3695 // (zext (setcc %a, %b, setne)) -> (sube addc.reg, addc.reg, addc.CA)
3696 // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3697 // (zext (setcc %a, 0, setne)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3698 SDValue Xor = IsRHSZero ? LHS :
3699 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3700 SDValue AC =
3701 SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3702 Xor, S->getI32Imm(~0U, dl)), 0);
3703 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, AC,
3704 Xor, AC.getValue(1)), 0);
3705 }
3706 case ISD::SETGE: {
3707 // {subc.reg, subc.CA} = (subcarry %a, %b)
3708 // (zext (setcc %a, %b, setge)) ->
3709 // (adde (lshr %b, 63), (ashr %a, 63), subc.CA)
3710 // (zext (setcc %a, 0, setge)) -> (lshr (~ %a), 63)
3711 if (IsRHSZero)
3712 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3713 std::swap(LHS, RHS);
3714 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3715 IsRHSZero = RHSConst && RHSConst->isZero();
3716 [[fallthrough]];
3717 }
3718 case ISD::SETLE: {
3719 // {subc.reg, subc.CA} = (subcarry %b, %a)
3720 // (zext (setcc %a, %b, setge)) ->
3721 // (adde (lshr %a, 63), (ashr %b, 63), subc.CA)
3722 // (zext (setcc %a, 0, setge)) -> (lshr (or %a, (add %a, -1)), 63)
3723 if (IsRHSZero)
3724 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3725 SDValue ShiftL =
3726 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3727 S->getI64Imm(1, dl),
3728 S->getI64Imm(63, dl)), 0);
3729 SDValue ShiftR =
3730 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3731 S->getI64Imm(63, dl)), 0);
3732 SDValue SubtractCarry =
3733 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3734 LHS, RHS), 1);
3735 return SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3736 ShiftR, ShiftL, SubtractCarry), 0);
3737 }
3738 case ISD::SETGT: {
3739 // {subc.reg, subc.CA} = (subcarry %b, %a)
3740 // (zext (setcc %a, %b, setgt)) ->
3741 // (xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3742 // (zext (setcc %a, 0, setgt)) -> (lshr (nor (add %a, -1), %a), 63)
3743 if (IsRHSNegOne)
3744 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GEZExt);
3745 if (IsRHSZero) {
3746 SDValue Addi =
3747 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3748 S->getI64Imm(~0ULL, dl)), 0);
3749 SDValue Nor =
3750 SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Addi, LHS), 0);
3751 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, Nor,
3752 S->getI64Imm(1, dl),
3753 S->getI64Imm(63, dl)), 0);
3754 }
3755 std::swap(LHS, RHS);
3756 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3757 IsRHSZero = RHSConst && RHSConst->isZero();
3758 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3759 [[fallthrough]];
3760 }
3761 case ISD::SETLT: {
3762 // {subc.reg, subc.CA} = (subcarry %a, %b)
3763 // (zext (setcc %a, %b, setlt)) ->
3764 // (xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3765 // (zext (setcc %a, 0, setlt)) -> (lshr %a, 63)
3766 if (IsRHSOne)
3767 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LEZExt);
3768 if (IsRHSZero)
3769 return SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3770 S->getI64Imm(1, dl),
3771 S->getI64Imm(63, dl)), 0);
3772 SDValue SRADINode =
3773 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3774 LHS, S->getI64Imm(63, dl)), 0);
3775 SDValue SRDINode =
3776 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3777 RHS, S->getI64Imm(1, dl),
3778 S->getI64Imm(63, dl)), 0);
3779 SDValue SUBFC8Carry =
3780 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3781 RHS, LHS), 1);
3782 SDValue ADDE8Node =
3783 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3784 SRDINode, SRADINode, SUBFC8Carry), 0);
3785 return SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3786 ADDE8Node, S->getI64Imm(1, dl)), 0);
3787 }
3788 case ISD::SETUGE:
3789 // {subc.reg, subc.CA} = (subcarry %a, %b)
3790 // (zext (setcc %a, %b, setuge)) -> (add (sube %b, %b, subc.CA), 1)
3791 std::swap(LHS, RHS);
3792 [[fallthrough]];
3793 case ISD::SETULE: {
3794 // {subc.reg, subc.CA} = (subcarry %b, %a)
3795 // (zext (setcc %a, %b, setule)) -> (add (sube %a, %a, subc.CA), 1)
3796 SDValue SUBFC8Carry =
3797 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3798 LHS, RHS), 1);
3799 SDValue SUBFE8Node =
3800 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue,
3801 LHS, LHS, SUBFC8Carry), 0);
3802 return SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64,
3803 SUBFE8Node, S->getI64Imm(1, dl)), 0);
3804 }
3805 case ISD::SETUGT:
3806 // {subc.reg, subc.CA} = (subcarry %b, %a)
3807 // (zext (setcc %a, %b, setugt)) -> -(sube %b, %b, subc.CA)
3808 std::swap(LHS, RHS);
3809 [[fallthrough]];
3810 case ISD::SETULT: {
3811 // {subc.reg, subc.CA} = (subcarry %a, %b)
3812 // (zext (setcc %a, %b, setult)) -> -(sube %a, %a, subc.CA)
3813 SDValue SubtractCarry =
3814 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3815 RHS, LHS), 1);
3816 SDValue ExtSub =
3817 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3818 LHS, LHS, SubtractCarry), 0);
3819 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3820 ExtSub), 0);
3821 }
3822 }
3823}
3824
3825/// Produces a sign-extended result of comparing two 64-bit values according to
3826/// the passed condition code.
3827SDValue
3828IntegerCompareEliminator::get64BitSExtCompare(SDValue LHS, SDValue RHS,
3830 int64_t RHSValue, SDLoc dl) {
3833 return SDValue();
3834 bool IsRHSZero = RHSValue == 0;
3835 bool IsRHSOne = RHSValue == 1;
3836 bool IsRHSNegOne = RHSValue == -1LL;
3837 switch (CC) {
3838 default: return SDValue();
3839 case ISD::SETEQ: {
3840 // {addc.reg, addc.CA} = (addcarry (xor %a, %b), -1)
3841 // (sext (setcc %a, %b, seteq)) -> (sube addc.reg, addc.reg, addc.CA)
3842 // {addcz.reg, addcz.CA} = (addcarry %a, -1)
3843 // (sext (setcc %a, 0, seteq)) -> (sube addcz.reg, addcz.reg, addcz.CA)
3844 SDValue AddInput = IsRHSZero ? LHS :
3845 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3846 SDValue Addic =
3847 SDValue(CurDAG->getMachineNode(PPC::ADDIC8, dl, MVT::i64, MVT::Glue,
3848 AddInput, S->getI32Imm(~0U, dl)), 0);
3849 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, Addic,
3850 Addic, Addic.getValue(1)), 0);
3851 }
3852 case ISD::SETNE: {
3853 // {subfc.reg, subfc.CA} = (subcarry 0, (xor %a, %b))
3854 // (sext (setcc %a, %b, setne)) -> (sube subfc.reg, subfc.reg, subfc.CA)
3855 // {subfcz.reg, subfcz.CA} = (subcarry 0, %a)
3856 // (sext (setcc %a, 0, setne)) -> (sube subfcz.reg, subfcz.reg, subfcz.CA)
3857 SDValue Xor = IsRHSZero ? LHS :
3858 SDValue(CurDAG->getMachineNode(PPC::XOR8, dl, MVT::i64, LHS, RHS), 0);
3859 SDValue SC =
3860 SDValue(CurDAG->getMachineNode(PPC::SUBFIC8, dl, MVT::i64, MVT::Glue,
3861 Xor, S->getI32Imm(0, dl)), 0);
3862 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, SC,
3863 SC, SC.getValue(1)), 0);
3864 }
3865 case ISD::SETGE: {
3866 // {subc.reg, subc.CA} = (subcarry %a, %b)
3867 // (zext (setcc %a, %b, setge)) ->
3868 // (- (adde (lshr %b, 63), (ashr %a, 63), subc.CA))
3869 // (zext (setcc %a, 0, setge)) -> (~ (ashr %a, 63))
3870 if (IsRHSZero)
3871 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3872 std::swap(LHS, RHS);
3873 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3874 IsRHSZero = RHSConst && RHSConst->isZero();
3875 [[fallthrough]];
3876 }
3877 case ISD::SETLE: {
3878 // {subc.reg, subc.CA} = (subcarry %b, %a)
3879 // (zext (setcc %a, %b, setge)) ->
3880 // (- (adde (lshr %a, 63), (ashr %b, 63), subc.CA))
3881 // (zext (setcc %a, 0, setge)) -> (ashr (or %a, (add %a, -1)), 63)
3882 if (IsRHSZero)
3883 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3884 SDValue ShiftR =
3885 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, RHS,
3886 S->getI64Imm(63, dl)), 0);
3887 SDValue ShiftL =
3888 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64, LHS,
3889 S->getI64Imm(1, dl),
3890 S->getI64Imm(63, dl)), 0);
3891 SDValue SubtractCarry =
3892 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3893 LHS, RHS), 1);
3894 SDValue Adde =
3895 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64, MVT::Glue,
3896 ShiftR, ShiftL, SubtractCarry), 0);
3897 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64, Adde), 0);
3898 }
3899 case ISD::SETGT: {
3900 // {subc.reg, subc.CA} = (subcarry %b, %a)
3901 // (zext (setcc %a, %b, setgt)) ->
3902 // -(xor (adde (lshr %a, 63), (ashr %b, 63), subc.CA), 1)
3903 // (zext (setcc %a, 0, setgt)) -> (ashr (nor (add %a, -1), %a), 63)
3904 if (IsRHSNegOne)
3905 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::GESExt);
3906 if (IsRHSZero) {
3907 SDValue Add =
3908 SDValue(CurDAG->getMachineNode(PPC::ADDI8, dl, MVT::i64, LHS,
3909 S->getI64Imm(-1, dl)), 0);
3910 SDValue Nor =
3911 SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64, Add, LHS), 0);
3912 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, Nor,
3913 S->getI64Imm(63, dl)), 0);
3914 }
3915 std::swap(LHS, RHS);
3916 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
3917 IsRHSZero = RHSConst && RHSConst->isZero();
3918 IsRHSOne = RHSConst && RHSConst->getSExtValue() == 1;
3919 [[fallthrough]];
3920 }
3921 case ISD::SETLT: {
3922 // {subc.reg, subc.CA} = (subcarry %a, %b)
3923 // (zext (setcc %a, %b, setlt)) ->
3924 // -(xor (adde (lshr %b, 63), (ashr %a, 63), subc.CA), 1)
3925 // (zext (setcc %a, 0, setlt)) -> (ashr %a, 63)
3926 if (IsRHSOne)
3927 return getCompoundZeroComparisonInGPR(LHS, dl, ZeroCompare::LESExt);
3928 if (IsRHSZero) {
3929 return SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64, LHS,
3930 S->getI64Imm(63, dl)), 0);
3931 }
3932 SDValue SRADINode =
3933 SDValue(CurDAG->getMachineNode(PPC::SRADI, dl, MVT::i64,
3934 LHS, S->getI64Imm(63, dl)), 0);
3935 SDValue SRDINode =
3936 SDValue(CurDAG->getMachineNode(PPC::RLDICL, dl, MVT::i64,
3937 RHS, S->getI64Imm(1, dl),
3938 S->getI64Imm(63, dl)), 0);
3939 SDValue SUBFC8Carry =
3940 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3941 RHS, LHS), 1);
3942 SDValue ADDE8Node =
3943 SDValue(CurDAG->getMachineNode(PPC::ADDE8, dl, MVT::i64,
3944 SRDINode, SRADINode, SUBFC8Carry), 0);
3945 SDValue XORI8Node =
3946 SDValue(CurDAG->getMachineNode(PPC::XORI8, dl, MVT::i64,
3947 ADDE8Node, S->getI64Imm(1, dl)), 0);
3948 return SDValue(CurDAG->getMachineNode(PPC::NEG8, dl, MVT::i64,
3949 XORI8Node), 0);
3950 }
3951 case ISD::SETUGE:
3952 // {subc.reg, subc.CA} = (subcarry %a, %b)
3953 // (sext (setcc %a, %b, setuge)) -> ~(sube %b, %b, subc.CA)
3954 std::swap(LHS, RHS);
3955 [[fallthrough]];
3956 case ISD::SETULE: {
3957 // {subc.reg, subc.CA} = (subcarry %b, %a)
3958 // (sext (setcc %a, %b, setule)) -> ~(sube %a, %a, subc.CA)
3959 SDValue SubtractCarry =
3960 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3961 LHS, RHS), 1);
3962 SDValue ExtSub =
3963 SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64, MVT::Glue, LHS,
3964 LHS, SubtractCarry), 0);
3965 return SDValue(CurDAG->getMachineNode(PPC::NOR8, dl, MVT::i64,
3966 ExtSub, ExtSub), 0);
3967 }
3968 case ISD::SETUGT:
3969 // {subc.reg, subc.CA} = (subcarry %b, %a)
3970 // (sext (setcc %a, %b, setugt)) -> (sube %b, %b, subc.CA)
3971 std::swap(LHS, RHS);
3972 [[fallthrough]];
3973 case ISD::SETULT: {
3974 // {subc.reg, subc.CA} = (subcarry %a, %b)
3975 // (sext (setcc %a, %b, setult)) -> (sube %a, %a, subc.CA)
3976 SDValue SubCarry =
3977 SDValue(CurDAG->getMachineNode(PPC::SUBFC8, dl, MVT::i64, MVT::Glue,
3978 RHS, LHS), 1);
3979 return SDValue(CurDAG->getMachineNode(PPC::SUBFE8, dl, MVT::i64,
3980 LHS, LHS, SubCarry), 0);
3981 }
3982 }
3983}
3984
3985/// Do all uses of this SDValue need the result in a GPR?
3986/// This is meant to be used on values that have type i1 since
3987/// it is somewhat meaningless to ask if values of other types
3988/// should be kept in GPR's.
3989static bool allUsesExtend(SDValue Compare, SelectionDAG *CurDAG) {
3990 assert(Compare.getOpcode() == ISD::SETCC &&
3991 "An ISD::SETCC node required here.");
3992
3993 // For values that have a single use, the caller should obviously already have
3994 // checked if that use is an extending use. We check the other uses here.
3995 if (Compare.hasOneUse())
3996 return true;
3997 // We want the value in a GPR if it is being extended, used for a select, or
3998 // used in logical operations.
3999 for (auto *CompareUse : Compare.getNode()->uses())
4000 if (CompareUse->getOpcode() != ISD::SIGN_EXTEND &&
4001 CompareUse->getOpcode() != ISD::ZERO_EXTEND &&
4002 CompareUse->getOpcode() != ISD::SELECT &&
4003 !ISD::isBitwiseLogicOp(CompareUse->getOpcode())) {
4004 OmittedForNonExtendUses++;
4005 return false;
4006 }
4007 return true;
4008}
4009
4010/// Returns an equivalent of a SETCC node but with the result the same width as
4011/// the inputs. This can also be used for SELECT_CC if either the true or false
4012/// values is a power of two while the other is zero.
4013SDValue IntegerCompareEliminator::getSETCCInGPR(SDValue Compare,
4014 SetccInGPROpts ConvOpts) {
4015 assert((Compare.getOpcode() == ISD::SETCC ||
4016 Compare.getOpcode() == ISD::SELECT_CC) &&
4017 "An ISD::SETCC node required here.");
4018
4019 // Don't convert this comparison to a GPR sequence because there are uses
4020 // of the i1 result (i.e. uses that require the result in the CR).
4021 if ((Compare.getOpcode() == ISD::SETCC) && !allUsesExtend(Compare, CurDAG))
4022 return SDValue();
4023
4024 SDValue LHS = Compare.getOperand(0);
4025 SDValue RHS = Compare.getOperand(1);
4026
4027 // The condition code is operand 2 for SETCC and operand 4 for SELECT_CC.
4028 int CCOpNum = Compare.getOpcode() == ISD::SELECT_CC ? 4 : 2;
4030 cast<CondCodeSDNode>(Compare.getOperand(CCOpNum))->get();
4031 EVT InputVT = LHS.getValueType();
4032 if (InputVT != MVT::i32 && InputVT != MVT::i64)
4033 return SDValue();
4034
4035 if (ConvOpts == SetccInGPROpts::ZExtInvert ||
4036 ConvOpts == SetccInGPROpts::SExtInvert)
4037 CC = ISD::getSetCCInverse(CC, InputVT);
4038
4039 bool Inputs32Bit = InputVT == MVT::i32;
4040
4041 SDLoc dl(Compare);
4042 ConstantSDNode *RHSConst = dyn_cast<ConstantSDNode>(RHS);
4043 int64_t RHSValue = RHSConst ? RHSConst->getSExtValue() : INT64_MAX;
4044 bool IsSext = ConvOpts == SetccInGPROpts::SExtOrig ||
4045 ConvOpts == SetccInGPROpts::SExtInvert;
4046
4047 if (IsSext && Inputs32Bit)
4048 return get32BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
4049 else if (Inputs32Bit)
4050 return get32BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
4051 else if (IsSext)
4052 return get64BitSExtCompare(LHS, RHS, CC, RHSValue, dl);
4053 return get64BitZExtCompare(LHS, RHS, CC, RHSValue, dl);
4054}
4055
4056} // end anonymous namespace
4057
4058bool PPCDAGToDAGISel::tryIntCompareInGPR(SDNode *N) {
4059 if (N->getValueType(0) != MVT::i32 &&
4060 N->getValueType(0) != MVT::i64)
4061 return false;
4062
4063 // This optimization will emit code that assumes 64-bit registers
4064 // so we don't want to run it in 32-bit mode. Also don't run it
4065 // on functions that are not to be optimized.
4066 if (TM.getOptLevel() == CodeGenOptLevel::None || !TM.isPPC64())
4067 return false;
4068
4069 // For POWER10, it is more profitable to use the set boolean extension
4070 // instructions rather than the integer compare elimination codegen.
4071 // Users can override this via the command line option, `--ppc-gpr-icmps`.
4072 if (!(CmpInGPR.getNumOccurrences() > 0) && Subtarget->isISA3_1())
4073 return false;
4074
4075 switch (N->getOpcode()) {
4076 default: break;
4077 case ISD::ZERO_EXTEND:
4078 case ISD::SIGN_EXTEND:
4079 case ISD::AND:
4080 case ISD::OR:
4081 case ISD::XOR: {
4082 IntegerCompareEliminator ICmpElim(CurDAG, this);
4083 if (SDNode *New = ICmpElim.Select(N)) {
4084 ReplaceNode(N, New);
4085 return true;
4086 }
4087 }
4088 }
4089 return false;
4090}
4091
4092bool PPCDAGToDAGISel::tryBitPermutation(SDNode *N) {
4093 if (N->getValueType(0) != MVT::i32 &&
4094 N->getValueType(0) != MVT::i64)
4095 return false;
4096
4097 if (!UseBitPermRewriter)
4098 return false;
4099
4100 switch (N->getOpcode()) {
4101 default: break;
4102 case ISD::SRL:
4103 // If we are on P10, we have a pattern for 32-bit (srl (bswap r), 16) that
4104 // uses the BRH instruction.
4105 if (Subtarget->isISA3_1() && N->getValueType(0) == MVT::i32 &&
4106 N->getOperand(0).getOpcode() == ISD::BSWAP) {
4107 auto &OpRight = N->getOperand(1);
4108 ConstantSDNode *SRLConst = dyn_cast<ConstantSDNode>(OpRight);
4109 if (SRLConst && SRLConst->getSExtValue() == 16)
4110 return false;
4111 }
4112 [[fallthrough]];
4113 case ISD::ROTL:
4114 case ISD::SHL:
4115 case ISD::AND:
4116 case ISD::OR: {
4117 BitPermutationSelector BPS(CurDAG);
4118 if (SDNode *New = BPS.Select(N)) {
4119 ReplaceNode(N, New);
4120 return true;
4121 }
4122 return false;
4123 }
4124 }
4125
4126 return false;
4127}
4128
4129/// SelectCC - Select a comparison of the specified values with the specified
4130/// condition code, returning the CR# of the expression.
4131SDValue PPCDAGToDAGISel::SelectCC(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4132 const SDLoc &dl, SDValue Chain) {
4133 // Always select the LHS.
4134 unsigned Opc;
4135
4136 if (LHS.getValueType() == MVT::i32) {
4137 unsigned Imm;
4138 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
4139 if (isInt32Immediate(RHS, Imm)) {
4140 // SETEQ/SETNE comparison with 16-bit immediate, fold it.
4141 if (isUInt<16>(Imm))
4142 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
4143 getI32Imm(Imm & 0xFFFF, dl)),
4144 0);
4145 // If this is a 16-bit signed immediate, fold it.
4146 if (isInt<16>((int)Imm))
4147 return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
4148 getI32Imm(Imm & 0xFFFF, dl)),
4149 0);
4150
4151 // For non-equality comparisons, the default code would materialize the
4152 // constant, then compare against it, like this:
4153 // lis r2, 4660
4154 // ori r2, r2, 22136
4155 // cmpw cr0, r3, r2
4156 // Since we are just comparing for equality, we can emit this instead:
4157 // xoris r0,r3,0x1234
4158 // cmplwi cr0,r0,0x5678
4159 // beq cr0,L6
4160 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS, dl, MVT::i32, LHS,
4161 getI32Imm(Imm >> 16, dl)), 0);
4162 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, Xor,
4163 getI32Imm(Imm & 0xFFFF, dl)), 0);
4164 }
4165 Opc = PPC::CMPLW;
4166 } else if (ISD::isUnsignedIntSetCC(CC)) {
4167 if (isInt32Immediate(RHS, Imm) && isUInt<16>(Imm))
4168 return SDValue(CurDAG->getMachineNode(PPC::CMPLWI, dl, MVT::i32, LHS,
4169 getI32Imm(Imm & 0xFFFF, dl)), 0);
4170 Opc = PPC::CMPLW;
4171 } else {
4172 int16_t SImm;
4173 if (isIntS16Immediate(RHS, SImm))
4174 return SDValue(CurDAG->getMachineNode(PPC::CMPWI, dl, MVT::i32, LHS,
4175 getI32Imm((int)SImm & 0xFFFF,
4176 dl)),
4177 0);
4178 Opc = PPC::CMPW;
4179 }
4180 } else if (LHS.getValueType() == MVT::i64) {
4181 uint64_t Imm;
4182 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
4183 if (isInt64Immediate(RHS.getNode(), Imm)) {
4184 // SETEQ/SETNE comparison with 16-bit immediate, fold it.
4185 if (isUInt<16>(Imm))
4186 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
4187 getI32Imm(Imm & 0xFFFF, dl)),
4188 0);
4189 // If this is a 16-bit signed immediate, fold it.
4190 if (isInt<16>(Imm))
4191 return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
4192 getI32Imm(Imm & 0xFFFF, dl)),
4193 0);
4194
4195 // For non-equality comparisons, the default code would materialize the
4196 // constant, then compare against it, like this:
4197 // lis r2, 4660
4198 // ori r2, r2, 22136
4199 // cmpd cr0, r3, r2
4200 // Since we are just comparing for equality, we can emit this instead:
4201 // xoris r0,r3,0x1234
4202 // cmpldi cr0,r0,0x5678
4203 // beq cr0,L6
4204 if (isUInt<32>(Imm)) {
4205 SDValue Xor(CurDAG->getMachineNode(PPC::XORIS8, dl, MVT::i64, LHS,
4206 getI64Imm(Imm >> 16, dl)), 0);
4207 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, Xor,
4208 getI64Imm(Imm & 0xFFFF, dl)),
4209 0);
4210 }
4211 }
4212 Opc = PPC::CMPLD;
4213 } else if (ISD::isUnsignedIntSetCC(CC)) {
4214 if (isInt64Immediate(RHS.getNode(), Imm) && isUInt<16>(Imm))
4215 return SDValue(CurDAG->getMachineNode(PPC::CMPLDI, dl, MVT::i64, LHS,
4216 getI64Imm(Imm & 0xFFFF, dl)), 0);
4217 Opc = PPC::CMPLD;
4218 } else {
4219 int16_t SImm;
4220 if (isIntS16Immediate(RHS, SImm))
4221 return SDValue(CurDAG->getMachineNode(PPC::CMPDI, dl, MVT::i64, LHS,
4222 getI64Imm(SImm & 0xFFFF, dl)),
4223 0);
4224 Opc = PPC::CMPD;
4225 }
4226 } else if (LHS.getValueType() == MVT::f32) {
4227 if (Subtarget->hasSPE()) {
4228 switch (CC) {
4229 default:
4230 case ISD::SETEQ:
4231 case ISD::SETNE:
4232 Opc = PPC::EFSCMPEQ;
4233 break;
4234 case ISD::SETLT:
4235 case ISD::SETGE:
4236 case ISD::SETOLT:
4237 case ISD::SETOGE:
4238 case ISD::SETULT:
4239 case ISD::SETUGE:
4240 Opc = PPC::EFSCMPLT;
4241 break;
4242 case ISD::SETGT:
4243 case ISD::SETLE:
4244 case ISD::SETOGT:
4245 case ISD::SETOLE:
4246 case ISD::SETUGT:
4247 case ISD::SETULE:
4248 Opc = PPC::EFSCMPGT;
4249 break;
4250 }
4251 } else
4252 Opc = PPC::FCMPUS;
4253 } else if (LHS.getValueType() == MVT::f64) {
4254 if (Subtarget->hasSPE()) {
4255 switch (CC) {
4256 default:
4257 case ISD::SETEQ:
4258 case ISD::SETNE:
4259 Opc = PPC::EFDCMPEQ;
4260 break;
4261 case ISD::SETLT:
4262 case ISD::SETGE:
4263 case ISD::SETOLT:
4264 case ISD::SETOGE:
4265 case ISD::SETULT:
4266 case ISD::SETUGE:
4267 Opc = PPC::EFDCMPLT;
4268 break;
4269 case ISD::SETGT:
4270 case ISD::SETLE:
4271 case ISD::SETOGT:
4272 case ISD::SETOLE:
4273 case ISD::SETUGT:
4274 case ISD::SETULE:
4275 Opc = PPC::EFDCMPGT;
4276 break;
4277 }
4278 } else
4279 Opc = Subtarget->hasVSX() ? PPC::XSCMPUDP : PPC::FCMPUD;
4280 } else {
4281 assert(LHS.getValueType() == MVT::f128 && "Unknown vt!");
4282 assert(Subtarget->hasP9Vector() && "XSCMPUQP requires Power9 Vector");
4283 Opc = PPC::XSCMPUQP;
4284 }
4285 if (Chain)
4286 return SDValue(
4287 CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::Other, LHS, RHS, Chain),
4288 0);
4289 else
4290 return SDValue(CurDAG->getMachineNode(Opc, dl, MVT::i32, LHS, RHS), 0);
4291}
4292
4294 const PPCSubtarget *Subtarget) {
4295 // For SPE instructions, the result is in GT bit of the CR
4296 bool UseSPE = Subtarget->hasSPE() && VT.isFloatingPoint();
4297
4298 switch (CC) {
4299 case ISD::SETUEQ:
4300 case ISD::SETONE:
4301 case ISD::SETOLE:
4302 case ISD::SETOGE:
4303 llvm_unreachable("Should be lowered by legalize!");
4304 default: llvm_unreachable("Unknown condition!");
4305 case ISD::SETOEQ:
4306 case ISD::SETEQ:
4307 return UseSPE ? PPC::PRED_GT : PPC::PRED_EQ;
4308 case ISD::SETUNE:
4309 case ISD::SETNE:
4310 return UseSPE ? PPC::PRED_LE : PPC::PRED_NE;
4311 case ISD::SETOLT:
4312 case ISD::SETLT:
4313 return UseSPE ? PPC::PRED_GT : PPC::PRED_LT;
4314 case ISD::SETULE:
4315 case ISD::SETLE:
4316 return PPC::PRED_LE;
4317 case ISD::SETOGT:
4318 case ISD::SETGT:
4319 return PPC::PRED_GT;
4320 case ISD::SETUGE:
4321 case ISD::SETGE:
4322 return UseSPE ? PPC::PRED_LE : PPC::PRED_GE;
4323 case ISD::SETO: return PPC::PRED_NU;
4324 case ISD::SETUO: return PPC::PRED_UN;
4325 // These two are invalid for floating point. Assume we have int.
4326 case ISD::SETULT: return PPC::PRED_LT;
4327 case ISD::SETUGT: return PPC::PRED_GT;
4328 }
4329}
4330
4331/// getCRIdxForSetCC - Return the index of the condition register field
4332/// associated with the SetCC condition, and whether or not the field is
4333/// treated as inverted. That is, lt = 0; ge = 0 inverted.
4334static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert) {
4335 Invert = false;
4336 switch (CC) {
4337 default: llvm_unreachable("Unknown condition!");
4338 case ISD::SETOLT:
4339 case ISD::SETLT: return 0; // Bit #0 = SETOLT
4340 case ISD::SETOGT:
4341 case ISD::SETGT: return 1; // Bit #1 = SETOGT
4342 case ISD::SETOEQ:
4343 case ISD::SETEQ: return 2; // Bit #2 = SETOEQ
4344 case ISD::SETUO: return 3; // Bit #3 = SETUO
4345 case ISD::SETUGE:
4346 case ISD::SETGE: Invert = true; return 0; // !Bit #0 = SETUGE
4347 case ISD::SETULE:
4348 case ISD::SETLE: Invert = true; return 1; // !Bit #1 = SETULE
4349 case ISD::SETUNE:
4350 case ISD::SETNE: Invert = true; return 2; // !Bit #2 = SETUNE
4351 case ISD::SETO: Invert = true; return 3; // !Bit #3 = SETO
4352 case ISD::SETUEQ:
4353 case ISD::SETOGE:
4354 case ISD::SETOLE:
4355 case ISD::SETONE:
4356 llvm_unreachable("Invalid branch code: should be expanded by legalize");
4357 // These are invalid for floating point. Assume integer.
4358 case ISD::SETULT: return 0;
4359 case ISD::SETUGT: return 1;
4360 }
4361}
4362
4363// getVCmpInst: return the vector compare instruction for the specified
4364// vector type and condition code. Since this is for altivec specific code,
4365// only support the altivec types (v16i8, v8i16, v4i32, v2i64, v1i128,
4366// and v4f32).
4367static unsigned int getVCmpInst(MVT VecVT, ISD::CondCode CC,
4368 bool HasVSX, bool &Swap, bool &Negate) {
4369 Swap = false;
4370 Negate = false;
4371
4372 if (VecVT.isFloatingPoint()) {
4373 /* Handle some cases by swapping input operands. */
4374 switch (CC) {
4375 case ISD::SETLE: CC = ISD::SETGE; Swap = true; break;
4376 case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
4377 case ISD::SETOLE: CC = ISD::SETOGE; Swap = true; break;
4378 case ISD::SETOLT: CC = ISD::SETOGT; Swap = true; break;
4379 case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
4380 case ISD::SETUGT: CC = ISD::SETULT; Swap = true; break;
4381 default: break;
4382 }
4383 /* Handle some cases by negating the result. */
4384 switch (CC) {
4385 case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
4386 case ISD::SETUNE: CC = ISD::SETOEQ; Negate = true; break;
4387 case ISD::SETULE: CC = ISD::SETOGT; Negate = true; break;
4388 case ISD::SETULT: CC = ISD::SETOGE; Negate = true; break;
4389 default: break;
4390 }
4391 /* We have instructions implementing the remaining cases. */
4392 switch (CC) {
4393 case ISD::SETEQ:
4394 case ISD::SETOEQ:
4395 if (VecVT == MVT::v4f32)
4396 return HasVSX ? PPC::XVCMPEQSP : PPC::VCMPEQFP;
4397 else if (VecVT == MVT::v2f64)
4398 return PPC::XVCMPEQDP;
4399 break;
4400 case ISD::SETGT:
4401 case ISD::SETOGT:
4402 if (VecVT == MVT::v4f32)
4403 return HasVSX ? PPC::XVCMPGTSP : PPC::VCMPGTFP;
4404 else if (VecVT == MVT::v2f64)
4405 return PPC::XVCMPGTDP;
4406 break;
4407 case ISD::SETGE:
4408 case ISD::SETOGE:
4409 if (VecVT == MVT::v4f32)
4410 return HasVSX ? PPC::XVCMPGESP : PPC::VCMPGEFP;
4411 else if (VecVT == MVT::v2f64)
4412 return PPC::XVCMPGEDP;
4413 break;
4414 default:
4415 break;
4416 }
4417 llvm_unreachable("Invalid floating-point vector compare condition");
4418 } else {
4419 /* Handle some cases by swapping input operands. */
4420 switch (CC) {
4421 case ISD::SETGE: CC = ISD::SETLE; Swap = true; break;
4422 case ISD::SETLT: CC = ISD::SETGT; Swap = true; break;
4423 case ISD::SETUGE: CC = ISD::SETULE; Swap = true; break;
4424 case ISD::SETULT: CC = ISD::SETUGT; Swap = true; break;
4425 default: break;
4426 }
4427 /* Handle some cases by negating the result. */
4428 switch (CC) {
4429 case ISD::SETNE: CC = ISD::SETEQ; Negate = true; break;
4430 case ISD::SETUNE: CC = ISD::SETUEQ; Negate = true; break;
4431 case ISD::SETLE: CC = ISD::SETGT; Negate = true; break;
4432 case ISD::SETULE: CC = ISD::SETUGT; Negate = true; break;
4433 default: break;
4434 }
4435 /* We have instructions implementing the remaining cases. */
4436 switch (CC) {
4437 case ISD::SETEQ:
4438 case ISD::SETUEQ:
4439 if (VecVT == MVT::v16i8)
4440 return PPC::VCMPEQUB;
4441 else if (VecVT == MVT::v8i16)
4442 return PPC::VCMPEQUH;
4443 else if (VecVT == MVT::v4i32)
4444 return PPC::VCMPEQUW;
4445 else if (VecVT == MVT::v2i64)
4446 return PPC::VCMPEQUD;
4447 else if (VecVT == MVT::v1i128)
4448 return PPC::VCMPEQUQ;
4449 break;
4450 case ISD::SETGT:
4451 if (VecVT == MVT::v16i8)
4452 return PPC::VCMPGTSB;
4453 else if (VecVT == MVT::v8i16)
4454 return PPC::VCMPGTSH;
4455 else if (VecVT == MVT::v4i32)
4456 return PPC::VCMPGTSW;
4457 else if (VecVT == MVT::v2i64)
4458 return PPC::VCMPGTSD;
4459 else if (VecVT == MVT::v1i128)
4460 return PPC::VCMPGTSQ;
4461 break;
4462 case ISD::SETUGT:
4463 if (VecVT == MVT::v16i8)
4464 return PPC::VCMPGTUB;
4465 else if (VecVT == MVT::v8i16)
4466 return PPC::VCMPGTUH;
4467 else if (VecVT == MVT::v4i32)
4468 return PPC::VCMPGTUW;
4469 else if (VecVT == MVT::v2i64)
4470 return PPC::VCMPGTUD;
4471 else if (VecVT == MVT::v1i128)
4472 return PPC::VCMPGTUQ;
4473 break;
4474 default:
4475 break;
4476 }
4477 llvm_unreachable("Invalid integer vector compare condition");
4478 }
4479}
4480
4481bool PPCDAGToDAGISel::trySETCC(SDNode *N) {
4482 SDLoc dl(N);
4483 unsigned Imm;
4484 bool IsStrict = N->isStrictFPOpcode();
4486 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
4487 EVT PtrVT =
4489 bool isPPC64 = (PtrVT == MVT::i64);
4490 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4491
4492 SDValue LHS = N->getOperand(IsStrict ? 1 : 0);
4493 SDValue RHS = N->getOperand(IsStrict ? 2 : 1);
4494
4495 if (!IsStrict && !Subtarget->useCRBits() && isInt32Immediate(RHS, Imm)) {
4496 // We can codegen setcc op, imm very efficiently compared to a brcond.
4497 // Check for those cases here.
4498 // setcc op, 0
4499 if (Imm == 0) {
4500 SDValue Op = LHS;
4501 switch (CC) {
4502 default: break;
4503 case ISD::SETEQ: {
4504 Op = SDValue(CurDAG->getMachineNode(PPC::CNTLZW, dl, MVT::i32, Op), 0);
4505 SDValue Ops[] = { Op, getI32Imm(27, dl), getI32Imm(5, dl),
4506 getI32Imm(31, dl) };
4507 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4508 return true;
4509 }
4510 case ISD::SETNE: {
4511 if (isPPC64) break;
4512 SDValue AD =
4513 SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4514 Op, getI32Imm(~0U, dl)), 0);
4515 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, AD, Op, AD.getValue(1));
4516 return true;
4517 }
4518 case ISD::SETLT: {
4519 SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4520 getI32Imm(31, dl) };
4521 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4522 return true;
4523 }
4524 case ISD::SETGT: {
4525 SDValue T =
4526 SDValue(CurDAG->getMachineNode(PPC::NEG, dl, MVT::i32, Op), 0);
4527 T = SDValue(CurDAG->getMachineNode(PPC::ANDC, dl, MVT::i32, T, Op), 0);
4528 SDValue Ops[] = { T, getI32Imm(1, dl), getI32Imm(31, dl),
4529 getI32Imm(31, dl) };
4530 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4531 return true;
4532 }
4533 }
4534 } else if (Imm == ~0U) { // setcc op, -1
4535 SDValue Op = LHS;
4536 switch (CC) {
4537 default: break;
4538 case ISD::SETEQ:
4539 if (isPPC64) break;
4540 Op = SDValue(CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4541 Op, getI32Imm(1, dl)), 0);
4542 CurDAG->SelectNodeTo(N, PPC::ADDZE, MVT::i32,
4543 SDValue(CurDAG->getMachineNode(PPC::LI, dl,
4544 MVT::i32,
4545 getI32Imm(0, dl)),
4546 0), Op.getValue(1));
4547 return true;
4548 case ISD::SETNE: {
4549 if (isPPC64) break;
4550 Op = SDValue(CurDAG->getMachineNode(PPC::NOR, dl, MVT::i32, Op, Op), 0);
4551 SDNode *AD = CurDAG->getMachineNode(PPC::ADDIC, dl, MVT::i32, MVT::Glue,
4552 Op, getI32Imm(~0U, dl));
4553 CurDAG->SelectNodeTo(N, PPC::SUBFE, MVT::i32, SDValue(AD, 0), Op,
4554 SDValue(AD, 1));
4555 return true;
4556 }
4557 case ISD::SETLT: {
4558 SDValue AD = SDValue(CurDAG->getMachineNode(PPC::ADDI, dl, MVT::i32, Op,
4559 getI32Imm(1, dl)), 0);
4560 SDValue AN = SDValue(CurDAG->getMachineNode(PPC::AND, dl, MVT::i32, AD,
4561 Op), 0);
4562 SDValue Ops[] = { AN, getI32Imm(1, dl), getI32Imm(31, dl),
4563 getI32Imm(31, dl) };
4564 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4565 return true;
4566 }
4567 case ISD::SETGT: {
4568 SDValue Ops[] = { Op, getI32Imm(1, dl), getI32Imm(31, dl),
4569 getI32Imm(31, dl) };
4570 Op = SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4571 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Op, getI32Imm(1, dl));
4572 return true;
4573 }
4574 }
4575 }
4576 }
4577
4578 // Altivec Vector compare instructions do not set any CR register by default and
4579 // vector compare operations return the same type as the operands.
4580 if (!IsStrict && LHS.getValueType().isVector()) {
4581 if (Subtarget->hasSPE())
4582 return false;
4583
4584 EVT VecVT = LHS.getValueType();
4585 bool Swap, Negate;
4586 unsigned int VCmpInst =
4587 getVCmpInst(VecVT.getSimpleVT(), CC, Subtarget->hasVSX(), Swap, Negate);
4588 if (Swap)
4589 std::swap(LHS, RHS);
4590
4591 EVT ResVT = VecVT.changeVectorElementTypeToInteger();
4592 if (Negate) {
4593 SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, ResVT, LHS, RHS), 0);
4594 CurDAG->SelectNodeTo(N, Subtarget->hasVSX() ? PPC::XXLNOR : PPC::VNOR,
4595 ResVT, VCmp, VCmp);
4596 return true;
4597 }
4598
4599 CurDAG->SelectNodeTo(N, VCmpInst, ResVT, LHS, RHS);
4600 return true;
4601 }
4602
4603 if (Subtarget->useCRBits())
4604 return false;
4605
4606 bool Inv;
4607 unsigned Idx = getCRIdxForSetCC(CC, Inv);
4608 SDValue CCReg = SelectCC(LHS, RHS, CC, dl, Chain);
4609 if (IsStrict)
4610 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), CCReg.getValue(1));
4611 SDValue IntCR;
4612
4613 // SPE e*cmp* instructions only set the 'gt' bit, so hard-code that
4614 // The correct compare instruction is already set by SelectCC()
4615 if (Subtarget->hasSPE() && LHS.getValueType().isFloatingPoint()) {
4616 Idx = 1;
4617 }
4618
4619 // Force the ccreg into CR7.
4620 SDValue CR7Reg = CurDAG->getRegister(PPC::CR7, MVT::i32);
4621
4622 SDValue InGlue; // Null incoming flag value.
4623 CCReg = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, CR7Reg, CCReg,
4624 InGlue).getValue(1);
4625
4626 IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg,
4627 CCReg), 0);
4628
4629 SDValue Ops[] = { IntCR, getI32Imm((32 - (3 - Idx)) & 31, dl),
4630 getI32Imm(31, dl), getI32Imm(31, dl) };
4631 if (!Inv) {
4632 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4633 return true;
4634 }
4635
4636 // Get the specified bit.
4637 SDValue Tmp =
4638 SDValue(CurDAG->getMachineNode(PPC::RLWINM, dl, MVT::i32, Ops), 0);
4639 CurDAG->SelectNodeTo(N, PPC::XORI, MVT::i32, Tmp, getI32Imm(1, dl));
4640 return true;
4641}
4642
4643/// Does this node represent a load/store node whose address can be represented
4644/// with a register plus an immediate that's a multiple of \p Val:
4645bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const {
4646 LoadSDNode *LDN = dyn_cast<LoadSDNode>(N);
4647 StoreSDNode *STN = dyn_cast<StoreSDNode>(N);
4648 MemIntrinsicSDNode *MIN = dyn_cast<MemIntrinsicSDNode>(N);
4649 SDValue AddrOp;
4650 if (LDN || (MIN && MIN->getOpcode() == PPCISD::LD_SPLAT))
4651 AddrOp = N->getOperand(1);
4652 else if (STN)
4653 AddrOp = STN->getOperand(2);
4654
4655 // If the address points a frame object or a frame object with an offset,
4656 // we need to check the object alignment.
4657 short Imm = 0;
4658 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(
4659 AddrOp.getOpcode() == ISD::ADD ? AddrOp.getOperand(0) :
4660 AddrOp)) {
4661 // If op0 is a frame index that is under aligned, we can't do it either,
4662 // because it is translated to r31 or r1 + slot + offset. We won't know the
4663 // slot number until the stack frame is finalized.
4664 const MachineFrameInfo &MFI = CurDAG->getMachineFunction().getFrameInfo();
4665 unsigned SlotAlign = MFI.getObjectAlign(FI->getIndex()).value();
4666 if ((SlotAlign % Val) != 0)
4667 return false;
4668
4669 // If we have an offset, we need further check on the offset.
4670 if (AddrOp.getOpcode() != ISD::ADD)
4671 return true;
4672 }
4673
4674 if (AddrOp.getOpcode() == ISD::ADD)
4675 return isIntS16Immediate(AddrOp.getOperand(1), Imm) && !(Imm % Val);
4676
4677 // If the address comes from the outside, the offset will be zero.
4678 return AddrOp.getOpcode() == ISD::CopyFromReg;
4679}
4680
4681void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
4682 // Transfer memoperands.
4683 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
4684 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
4685}
4686
4688 bool &NeedSwapOps, bool &IsUnCmp) {
4689
4690 assert(N->getOpcode() == ISD::SELECT_CC && "Expecting a SELECT_CC here.");
4691
4692 SDValue LHS = N->getOperand(0);
4693 SDValue RHS = N->getOperand(1);
4694 SDValue TrueRes = N->getOperand(2);
4695 SDValue FalseRes = N->getOperand(3);
4696 ConstantSDNode *TrueConst = dyn_cast<ConstantSDNode>(TrueRes);
4697 if (!TrueConst || (N->getSimpleValueType(0) != MVT::i64 &&
4698 N->getSimpleValueType(0) != MVT::i32))
4699 return false;
4700
4701 // We are looking for any of:
4702 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4703 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, cc2)), cc1)
4704 // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, 1, -1, cc2), seteq)
4705 // (select_cc lhs, rhs, 0, (select_cc [lr]hs, [lr]hs, -1, 1, cc2), seteq)
4706 int64_t TrueResVal = TrueConst->getSExtValue();
4707 if ((TrueResVal < -1 || TrueResVal > 1) ||
4708 (TrueResVal == -1 && FalseRes.getOpcode() != ISD::ZERO_EXTEND) ||
4709 (TrueResVal == 1 && FalseRes.getOpcode() != ISD::SIGN_EXTEND) ||
4710 (TrueResVal == 0 &&
4711 (FalseRes.getOpcode() != ISD::SELECT_CC || CC != ISD::SETEQ)))
4712 return false;
4713
4714 SDValue SetOrSelCC = FalseRes.getOpcode() == ISD::SELECT_CC
4715 ? FalseRes
4716 : FalseRes.getOperand(0);
4717 bool InnerIsSel = SetOrSelCC.getOpcode() == ISD::SELECT_CC;
4718 if (SetOrSelCC.getOpcode() != ISD::SETCC &&
4719 SetOrSelCC.getOpcode() != ISD::SELECT_CC)
4720 return false;
4721
4722 // Without this setb optimization, the outer SELECT_CC will be manually
4723 // selected to SELECT_CC_I4/SELECT_CC_I8 Pseudo, then expand-isel-pseudos pass
4724 // transforms pseudo instruction to isel instruction. When there are more than
4725 // one use for result like zext/sext, with current optimization we only see
4726 // isel is replaced by setb but can't see any significant gain. Since
4727 // setb has longer latency than original isel, we should avoid this. Another
4728 // point is that setb requires comparison always kept, it can break the
4729 // opportunity to get the comparison away if we have in future.
4730 if (!SetOrSelCC.hasOneUse() || (!InnerIsSel && !FalseRes.hasOneUse()))
4731 return false;
4732
4733 SDValue InnerLHS = SetOrSelCC.getOperand(0);
4734 SDValue InnerRHS = SetOrSelCC.getOperand(1);
4735 ISD::CondCode InnerCC =
4736 cast<CondCodeSDNode>(SetOrSelCC.getOperand(InnerIsSel ? 4 : 2))->get();
4737 // If the inner comparison is a select_cc, make sure the true/false values are
4738 // 1/-1 and canonicalize it if needed.
4739 if (InnerIsSel) {
4740 ConstantSDNode *SelCCTrueConst =
4741 dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(2));
4742 ConstantSDNode *SelCCFalseConst =
4743 dyn_cast<ConstantSDNode>(SetOrSelCC.getOperand(3));
4744 if (!SelCCTrueConst || !SelCCFalseConst)
4745 return false;
4746 int64_t SelCCTVal = SelCCTrueConst->getSExtValue();
4747 int64_t SelCCFVal = SelCCFalseConst->getSExtValue();
4748 // The values must be -1/1 (requiring a swap) or 1/-1.
4749 if (SelCCTVal == -1 && SelCCFVal == 1) {
4750 std::swap(InnerLHS, InnerRHS);
4751 } else if (SelCCTVal != 1 || SelCCFVal != -1)
4752 return false;
4753 }
4754
4755 // Canonicalize unsigned case
4756 if (InnerCC == ISD::SETULT || InnerCC == ISD::SETUGT) {
4757 IsUnCmp = true;
4758 InnerCC = (InnerCC == ISD::SETULT) ? ISD::SETLT : ISD::SETGT;
4759 }
4760
4761 bool InnerSwapped = false;
4762 if (LHS == InnerRHS && RHS == InnerLHS)
4763 InnerSwapped = true;
4764 else if (LHS != InnerLHS || RHS != InnerRHS)
4765 return false;
4766
4767 switch (CC) {
4768 // (select_cc lhs, rhs, 0, \
4769 // (select_cc [lr]hs, [lr]hs, 1, -1, setlt/setgt), seteq)
4770 case ISD::SETEQ:
4771 if (!InnerIsSel)
4772 return false;
4773 if (InnerCC != ISD::SETLT && InnerCC != ISD::SETGT)
4774 return false;
4775 NeedSwapOps = (InnerCC == ISD::SETGT) ? InnerSwapped : !InnerSwapped;
4776 break;
4777
4778 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4779 // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setgt)), setu?lt)
4780 // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setlt)), setu?lt)
4781 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?lt)
4782 // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setgt)), setu?lt)
4783 // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setlt)), setu?lt)
4784 case ISD::SETULT:
4785 if (!IsUnCmp && InnerCC != ISD::SETNE)
4786 return false;
4787 IsUnCmp = true;
4788 [[fallthrough]];
4789 case ISD::SETLT:
4790 if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETGT && !InnerSwapped) ||
4791 (InnerCC == ISD::SETLT && InnerSwapped))
4792 NeedSwapOps = (TrueResVal == 1);
4793 else
4794 return false;
4795 break;
4796
4797 // (select_cc lhs, rhs, 1, (sext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4798 // (select_cc lhs, rhs, 1, (sext (setcc lhs, rhs, setlt)), setu?gt)
4799 // (select_cc lhs, rhs, 1, (sext (setcc rhs, lhs, setgt)), setu?gt)
4800 // (select_cc lhs, rhs, -1, (zext (setcc [lr]hs, [lr]hs, setne)), setu?gt)
4801 // (select_cc lhs, rhs, -1, (zext (setcc lhs, rhs, setlt)), setu?gt)
4802 // (select_cc lhs, rhs, -1, (zext (setcc rhs, lhs, setgt)), setu?gt)
4803 case ISD::SETUGT:
4804 if (!IsUnCmp && InnerCC != ISD::SETNE)
4805 return false;
4806 IsUnCmp = true;
4807 [[fallthrough]];
4808 case ISD::SETGT:
4809 if (InnerCC == ISD::SETNE || (InnerCC == ISD::SETLT && !InnerSwapped) ||
4810 (InnerCC == ISD::SETGT && InnerSwapped))
4811 NeedSwapOps = (TrueResVal == -1);
4812 else
4813 return false;
4814 break;
4815
4816 default:
4817 return false;
4818 }
4819
4820 LLVM_DEBUG(dbgs() << "Found a node that can be lowered to a SETB: ");
4821 LLVM_DEBUG(N->dump());
4822
4823 return true;
4824}
4825
4826// Return true if it's a software square-root/divide operand.
4827static bool isSWTestOp(SDValue N) {
4828 if (N.getOpcode() == PPCISD::FTSQRT)
4829 return true;
4830 if (N.getNumOperands() < 1 || !isa<ConstantSDNode>(N.getOperand(0)) ||
4831 N.getOpcode() != ISD::INTRINSIC_WO_CHAIN)
4832 return false;
4833 switch (N.getConstantOperandVal(0)) {
4834 case Intrinsic::ppc_vsx_xvtdivdp:
4835 case Intrinsic::ppc_vsx_xvtdivsp:
4836 case Intrinsic::ppc_vsx_xvtsqrtdp:
4837 case Intrinsic::ppc_vsx_xvtsqrtsp:
4838 return true;
4839 }
4840 return false;
4841}
4842
4843bool PPCDAGToDAGISel::tryFoldSWTestBRCC(SDNode *N) {
4844 assert(N->getOpcode() == ISD::BR_CC && "ISD::BR_CC is expected.");
4845 // We are looking for following patterns, where `truncate to i1` actually has
4846 // the same semantic with `and 1`.
4847 // (br_cc seteq, (truncateToi1 SWTestOp), 0) -> (BCC PRED_NU, SWTestOp)
4848 // (br_cc seteq, (and SWTestOp, 2), 0) -> (BCC PRED_NE, SWTestOp)
4849 // (br_cc seteq, (and SWTestOp, 4), 0) -> (BCC PRED_LE, SWTestOp)
4850 // (br_cc seteq, (and SWTestOp, 8), 0) -> (BCC PRED_GE, SWTestOp)
4851 // (br_cc setne, (truncateToi1 SWTestOp), 0) -> (BCC PRED_UN, SWTestOp)
4852 // (br_cc setne, (and SWTestOp, 2), 0) -> (BCC PRED_EQ, SWTestOp)
4853 // (br_cc setne, (and SWTestOp, 4), 0) -> (BCC PRED_GT, SWTestOp)
4854 // (br_cc setne, (and SWTestOp, 8), 0) -> (BCC PRED_LT, SWTestOp)
4855 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
4856 if (CC != ISD::SETEQ && CC != ISD::SETNE)
4857 return false;
4858
4859 SDValue CmpRHS = N->getOperand(3);
4860 if (!isNullConstant(CmpRHS))
4861 return false;
4862
4863 SDValue CmpLHS = N->getOperand(2);
4864 if (CmpLHS.getNumOperands() < 1 || !isSWTestOp(CmpLHS.getOperand(0)))
4865 return false;
4866
4867 unsigned PCC = 0;
4868 bool IsCCNE = CC == ISD::SETNE;
4869 if (CmpLHS.getOpcode() == ISD::AND &&
4870 isa<ConstantSDNode>(CmpLHS.getOperand(1)))
4871 switch (CmpLHS.getConstantOperandVal(1)) {
4872 case 1:
4873 PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU;
4874 break;
4875 case 2:
4876 PCC = IsCCNE ? PPC::PRED_EQ : PPC::PRED_NE;
4877 break;
4878 case 4:
4879 PCC = IsCCNE ? PPC::PRED_GT : PPC::PRED_LE;
4880 break;
4881 case 8:
4882 PCC = IsCCNE ? PPC::PRED_LT : PPC::PRED_GE;
4883 break;
4884 default:
4885 return false;
4886 }
4887 else if (CmpLHS.getOpcode() == ISD::TRUNCATE &&
4888 CmpLHS.getValueType() == MVT::i1)
4889 PCC = IsCCNE ? PPC::PRED_UN : PPC::PRED_NU;
4890
4891 if (PCC) {
4892 SDLoc dl(N);
4893 SDValue Ops[] = {getI32Imm(PCC, dl), CmpLHS.getOperand(0), N->getOperand(4),
4894 N->getOperand(0)};
4895 CurDAG->SelectNodeTo(N, PPC::BCC, MVT::Other, Ops);
4896 return true;
4897 }
4898 return false;
4899}
4900
4901bool PPCDAGToDAGISel::trySelectLoopCountIntrinsic(SDNode *N) {
4902 // Sometimes the promoted value of the intrinsic is ANDed by some non-zero
4903 // value, for example when crbits is disabled. If so, select the
4904 // loop_decrement intrinsics now.
4905 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
4906 SDValue LHS = N->getOperand(2), RHS = N->getOperand(3);
4907
4908 if (LHS.getOpcode() != ISD::AND || !isa<ConstantSDNode>(LHS.getOperand(1)) ||
4909 isNullConstant(LHS.getOperand(1)))
4910 return false;
4911
4912 if (LHS.getOperand(0).getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4913 LHS.getOperand(0).getConstantOperandVal(1) != Intrinsic::loop_decrement)
4914 return false;
4915
4916 if (!isa<ConstantSDNode>(RHS))
4917 return false;
4918
4919 assert((CC == ISD::SETEQ || CC == ISD::SETNE) &&
4920 "Counter decrement comparison is not EQ or NE");
4921
4922 SDValue OldDecrement = LHS.getOperand(0);
4923 assert(OldDecrement.hasOneUse() && "loop decrement has more than one use!");
4924
4925 SDLoc DecrementLoc(OldDecrement);
4926 SDValue ChainInput = OldDecrement.getOperand(0);
4927 SDValue DecrementOps[] = {Subtarget->isPPC64() ? getI64Imm(1, DecrementLoc)
4928 : getI32Imm(1, DecrementLoc)};
4929 unsigned DecrementOpcode =
4930 Subtarget->isPPC64() ? PPC::DecreaseCTR8loop : PPC::DecreaseCTRloop;
4931 SDNode *NewDecrement = CurDAG->getMachineNode(DecrementOpcode, DecrementLoc,
4932 MVT::i1, DecrementOps);
4933
4934 unsigned Val = RHS->getAsZExtVal();
4935 bool IsBranchOnTrue = (CC == ISD::SETEQ && Val) || (CC == ISD::SETNE && !Val);
4936 unsigned Opcode = IsBranchOnTrue ? PPC::BC : PPC::BCn;
4937
4938 ReplaceUses(LHS.getValue(0), LHS.getOperand(1));
4939 CurDAG->RemoveDeadNode(LHS.getNode());
4940
4941 // Mark the old loop_decrement intrinsic as dead.
4942 ReplaceUses(OldDecrement.getValue(1), ChainInput);
4943 CurDAG->RemoveDeadNode(OldDecrement.getNode());
4944
4945 SDValue Chain = CurDAG->getNode(ISD::TokenFactor, SDLoc(N), MVT::Other,
4946 ChainInput, N->getOperand(0));
4947
4948 CurDAG->SelectNodeTo(N, Opcode, MVT::Other, SDValue(NewDecrement, 0),
4949 N->getOperand(4), Chain);
4950 return true;
4951}
4952
4953bool PPCDAGToDAGISel::tryAsSingleRLWINM(SDNode *N) {
4954 assert(N->getOpcode() == ISD::AND && "ISD::AND SDNode expected");
4955 unsigned Imm;
4956 if (!isInt32Immediate(N->getOperand(1), Imm))
4957 return false;
4958
4959 SDLoc dl(N);
4960 SDValue Val = N->getOperand(0);
4961 unsigned SH, MB, ME;
4962 // If this is an and of a value rotated between 0 and 31 bits and then and'd
4963 // with a mask, emit rlwinm
4964 if (isRotateAndMask(Val.getNode(), Imm, false, SH, MB, ME)) {
4965 Val = Val.getOperand(0);
4966 SDValue Ops[] = {Val, getI32Imm(SH, dl), getI32Imm(MB, dl),
4967 getI32Imm(ME, dl)};
4968 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4969 return true;
4970 }
4971
4972 // If this is just a masked value where the input is not handled, and
4973 // is not a rotate-left (handled by a pattern in the .td file), emit rlwinm
4974 if (isRunOfOnes(Imm, MB, ME) && Val.getOpcode() != ISD::ROTL) {
4975 SDValue Ops[] = {Val, getI32Imm(0, dl), getI32Imm(MB, dl),
4976 getI32Imm(ME, dl)};
4977 CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops);
4978 return true;
4979 }
4980
4981 // AND X, 0 -> 0, not "rlwinm 32".
4982