LLVM  14.0.0git
TargetLowering.cpp
Go to the documentation of this file.
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/GlobalVariable.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCExpr.h"
31 #include "llvm/Support/KnownBits.h"
35 #include <cctype>
36 using namespace llvm;
37 
38 /// NOTE: The TargetMachine owns TLOF.
40  : TargetLoweringBase(tm) {}
41 
42 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
43  return nullptr;
44 }
45 
48 }
49 
50 /// Check whether a given call node is in tail position within its function. If
51 /// so, it sets Chain to the input chain of the tail call.
53  SDValue &Chain) const {
54  const Function &F = DAG.getMachineFunction().getFunction();
55 
56  // First, check if tail calls have been disabled in this function.
57  if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
58  return false;
59 
60  // Conservatively require the attributes of the call to match those of
61  // the return. Ignore following attributes because they don't affect the
62  // call sequence.
63  AttrBuilder CallerAttrs(F.getContext(), F.getAttributes().getRetAttrs());
64  for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
65  Attribute::DereferenceableOrNull, Attribute::NoAlias,
66  Attribute::NonNull})
67  CallerAttrs.removeAttribute(Attr);
68 
69  if (CallerAttrs.hasAttributes())
70  return false;
71 
72  // It's not safe to eliminate the sign / zero extension of the return value.
73  if (CallerAttrs.contains(Attribute::ZExt) ||
74  CallerAttrs.contains(Attribute::SExt))
75  return false;
76 
77  // Check if the only use is a function return node.
78  return isUsedByReturnOnly(Node, Chain);
79 }
80 
82  const uint32_t *CallerPreservedMask,
83  const SmallVectorImpl<CCValAssign> &ArgLocs,
84  const SmallVectorImpl<SDValue> &OutVals) const {
85  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
86  const CCValAssign &ArgLoc = ArgLocs[I];
87  if (!ArgLoc.isRegLoc())
88  continue;
89  MCRegister Reg = ArgLoc.getLocReg();
90  // Only look at callee saved registers.
91  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
92  continue;
93  // Check that we pass the value used for the caller.
94  // (We look for a CopyFromReg reading a virtual register that is used
95  // for the function live-in value of register Reg)
96  SDValue Value = OutVals[I];
97  if (Value->getOpcode() != ISD::CopyFromReg)
98  return false;
99  Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
100  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
101  return false;
102  }
103  return true;
104 }
105 
106 /// Set CallLoweringInfo attribute flags based on a call instruction
107 /// and called function attributes.
109  unsigned ArgIdx) {
110  IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
111  IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
112  IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
113  IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
114  IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
115  IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
116  IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
117  IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
118  IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
119  IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
120  IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
121  IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
122  Alignment = Call->getParamStackAlign(ArgIdx);
123  IndirectType = nullptr;
125  "multiple ABI attributes?");
126  if (IsByVal) {
127  IndirectType = Call->getParamByValType(ArgIdx);
128  if (!Alignment)
129  Alignment = Call->getParamAlign(ArgIdx);
130  }
131  if (IsPreallocated)
132  IndirectType = Call->getParamPreallocatedType(ArgIdx);
133  if (IsInAlloca)
134  IndirectType = Call->getParamInAllocaType(ArgIdx);
135 }
136 
137 /// Generate a libcall taking the given operands as arguments and returning a
138 /// result of type RetVT.
139 std::pair<SDValue, SDValue>
141  ArrayRef<SDValue> Ops,
142  MakeLibCallOptions CallOptions,
143  const SDLoc &dl,
144  SDValue InChain) const {
145  if (!InChain)
146  InChain = DAG.getEntryNode();
147 
149  Args.reserve(Ops.size());
150 
152  for (unsigned i = 0; i < Ops.size(); ++i) {
153  SDValue NewOp = Ops[i];
154  Entry.Node = NewOp;
155  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
156  Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
157  CallOptions.IsSExt);
158  Entry.IsZExt = !Entry.IsSExt;
159 
160  if (CallOptions.IsSoften &&
162  Entry.IsSExt = Entry.IsZExt = false;
163  }
164  Args.push_back(Entry);
165  }
166 
167  if (LC == RTLIB::UNKNOWN_LIBCALL)
168  report_fatal_error("Unsupported library call operation!");
170  getPointerTy(DAG.getDataLayout()));
171 
172  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
174  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
175  bool zeroExtend = !signExtend;
176 
177  if (CallOptions.IsSoften &&
179  signExtend = zeroExtend = false;
180  }
181 
182  CLI.setDebugLoc(dl)
183  .setChain(InChain)
185  .setNoReturn(CallOptions.DoesNotReturn)
186  .setDiscardResult(!CallOptions.IsReturnValueUsed)
188  .setSExtResult(signExtend)
189  .setZExtResult(zeroExtend);
190  return LowerCallTo(CLI);
191 }
192 
194  std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
195  unsigned SrcAS, const AttributeList &FuncAttributes) const {
196  if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
197  return false;
198 
199  EVT VT = getOptimalMemOpType(Op, FuncAttributes);
200 
201  if (VT == MVT::Other) {
202  // Use the largest integer type whose alignment constraints are satisfied.
203  // We only need to check DstAlign here as SrcAlign is always greater or
204  // equal to DstAlign (or zero).
205  VT = MVT::i64;
206  if (Op.isFixedDstAlign())
207  while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
208  !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
209  VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
210  assert(VT.isInteger());
211 
212  // Find the largest legal integer type.
213  MVT LVT = MVT::i64;
214  while (!isTypeLegal(LVT))
215  LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
216  assert(LVT.isInteger());
217 
218  // If the type we've chosen is larger than the largest legal integer type
219  // then use that instead.
220  if (VT.bitsGT(LVT))
221  VT = LVT;
222  }
223 
224  unsigned NumMemOps = 0;
225  uint64_t Size = Op.size();
226  while (Size) {
227  unsigned VTSize = VT.getSizeInBits() / 8;
228  while (VTSize > Size) {
229  // For now, only use non-vector load / store's for the left-over pieces.
230  EVT NewVT = VT;
231  unsigned NewVTSize;
232 
233  bool Found = false;
234  if (VT.isVector() || VT.isFloatingPoint()) {
235  NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
236  if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
237  isSafeMemOpType(NewVT.getSimpleVT()))
238  Found = true;
239  else if (NewVT == MVT::i64 &&
242  // i64 is usually not legal on 32-bit targets, but f64 may be.
243  NewVT = MVT::f64;
244  Found = true;
245  }
246  }
247 
248  if (!Found) {
249  do {
250  NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
251  if (NewVT == MVT::i8)
252  break;
253  } while (!isSafeMemOpType(NewVT.getSimpleVT()));
254  }
255  NewVTSize = NewVT.getSizeInBits() / 8;
256 
257  // If the new VT cannot cover all of the remaining bits, then consider
258  // issuing a (or a pair of) unaligned and overlapping load / store.
259  bool Fast;
260  if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
262  VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
263  MachineMemOperand::MONone, &Fast) &&
264  Fast)
265  VTSize = Size;
266  else {
267  VT = NewVT;
268  VTSize = NewVTSize;
269  }
270  }
271 
272  if (++NumMemOps > Limit)
273  return false;
274 
275  MemOps.push_back(VT);
276  Size -= VTSize;
277  }
278 
279  return true;
280 }
281 
282 /// Soften the operands of a comparison. This code is shared among BR_CC,
283 /// SELECT_CC, and SETCC handlers.
285  SDValue &NewLHS, SDValue &NewRHS,
286  ISD::CondCode &CCCode,
287  const SDLoc &dl, const SDValue OldLHS,
288  const SDValue OldRHS) const {
289  SDValue Chain;
290  return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
291  OldRHS, Chain);
292 }
293 
295  SDValue &NewLHS, SDValue &NewRHS,
296  ISD::CondCode &CCCode,
297  const SDLoc &dl, const SDValue OldLHS,
298  const SDValue OldRHS,
299  SDValue &Chain,
300  bool IsSignaling) const {
301  // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
302  // not supporting it. We can update this code when libgcc provides such
303  // functions.
304 
305  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
306  && "Unsupported setcc type!");
307 
308  // Expand into one or more soft-fp libcall(s).
309  RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
310  bool ShouldInvertCC = false;
311  switch (CCCode) {
312  case ISD::SETEQ:
313  case ISD::SETOEQ:
314  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
315  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
316  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
317  break;
318  case ISD::SETNE:
319  case ISD::SETUNE:
320  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
321  (VT == MVT::f64) ? RTLIB::UNE_F64 :
322  (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
323  break;
324  case ISD::SETGE:
325  case ISD::SETOGE:
326  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
327  (VT == MVT::f64) ? RTLIB::OGE_F64 :
328  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
329  break;
330  case ISD::SETLT:
331  case ISD::SETOLT:
332  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
333  (VT == MVT::f64) ? RTLIB::OLT_F64 :
334  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
335  break;
336  case ISD::SETLE:
337  case ISD::SETOLE:
338  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
339  (VT == MVT::f64) ? RTLIB::OLE_F64 :
340  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
341  break;
342  case ISD::SETGT:
343  case ISD::SETOGT:
344  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
345  (VT == MVT::f64) ? RTLIB::OGT_F64 :
346  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
347  break;
348  case ISD::SETO:
349  ShouldInvertCC = true;
351  case ISD::SETUO:
352  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
353  (VT == MVT::f64) ? RTLIB::UO_F64 :
354  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
355  break;
356  case ISD::SETONE:
357  // SETONE = O && UNE
358  ShouldInvertCC = true;
360  case ISD::SETUEQ:
361  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
362  (VT == MVT::f64) ? RTLIB::UO_F64 :
363  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
364  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
365  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
366  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
367  break;
368  default:
369  // Invert CC for unordered comparisons
370  ShouldInvertCC = true;
371  switch (CCCode) {
372  case ISD::SETULT:
373  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
374  (VT == MVT::f64) ? RTLIB::OGE_F64 :
375  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
376  break;
377  case ISD::SETULE:
378  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
379  (VT == MVT::f64) ? RTLIB::OGT_F64 :
380  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
381  break;
382  case ISD::SETUGT:
383  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
384  (VT == MVT::f64) ? RTLIB::OLE_F64 :
385  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
386  break;
387  case ISD::SETUGE:
388  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
389  (VT == MVT::f64) ? RTLIB::OLT_F64 :
390  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
391  break;
392  default: llvm_unreachable("Do not know how to soften this setcc!");
393  }
394  }
395 
396  // Use the target specific return value for comparions lib calls.
397  EVT RetVT = getCmpLibcallReturnType();
398  SDValue Ops[2] = {NewLHS, NewRHS};
400  EVT OpsVT[2] = { OldLHS.getValueType(),
401  OldRHS.getValueType() };
402  CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
403  auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
404  NewLHS = Call.first;
405  NewRHS = DAG.getConstant(0, dl, RetVT);
406 
407  CCCode = getCmpLibcallCC(LC1);
408  if (ShouldInvertCC) {
409  assert(RetVT.isInteger());
410  CCCode = getSetCCInverse(CCCode, RetVT);
411  }
412 
413  if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
414  // Update Chain.
415  Chain = Call.second;
416  } else {
417  EVT SetCCVT =
418  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
419  SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
420  auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
421  CCCode = getCmpLibcallCC(LC2);
422  if (ShouldInvertCC)
423  CCCode = getSetCCInverse(CCCode, RetVT);
424  NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
425  if (Chain)
426  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
427  Call2.second);
428  NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
429  Tmp.getValueType(), Tmp, NewLHS);
430  NewRHS = SDValue();
431  }
432 }
433 
434 /// Return the entry encoding for a jump table in the current function. The
435 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
437  // In non-pic modes, just use the address of a block.
438  if (!isPositionIndependent())
440 
441  // In PIC mode, if the target supports a GPRel32 directive, use it.
442  if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
444 
445  // Otherwise, use a label difference.
447 }
448 
450  SelectionDAG &DAG) const {
451  // If our PIC model is GP relative, use the global offset table as the base.
452  unsigned JTEncoding = getJumpTableEncoding();
453 
454  if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
457 
458  return Table;
459 }
460 
461 /// This returns the relocation base for the given PIC jumptable, the same as
462 /// getPICJumpTableRelocBase, but as an MCExpr.
463 const MCExpr *
465  unsigned JTI,MCContext &Ctx) const{
466  // The normal PIC reloc base is the label at the start of the jump table.
467  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
468 }
469 
470 bool
472  const TargetMachine &TM = getTargetMachine();
473  const GlobalValue *GV = GA->getGlobal();
474 
475  // If the address is not even local to this DSO we will have to load it from
476  // a got and then add the offset.
477  if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
478  return false;
479 
480  // If the code is position independent we will have to add a base register.
481  if (isPositionIndependent())
482  return false;
483 
484  // Otherwise we can do it.
485  return true;
486 }
487 
488 //===----------------------------------------------------------------------===//
489 // Optimization Methods
490 //===----------------------------------------------------------------------===//
491 
492 /// If the specified instruction has a constant integer operand and there are
493 /// bits set in that constant that are not demanded, then clear those bits and
494 /// return true.
496  const APInt &DemandedBits,
497  const APInt &DemandedElts,
498  TargetLoweringOpt &TLO) const {
499  SDLoc DL(Op);
500  unsigned Opcode = Op.getOpcode();
501 
502  // Do target-specific constant optimization.
503  if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
504  return TLO.New.getNode();
505 
506  // FIXME: ISD::SELECT, ISD::SELECT_CC
507  switch (Opcode) {
508  default:
509  break;
510  case ISD::XOR:
511  case ISD::AND:
512  case ISD::OR: {
513  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
514  if (!Op1C || Op1C->isOpaque())
515  return false;
516 
517  // If this is a 'not' op, don't touch it because that's a canonical form.
518  const APInt &C = Op1C->getAPIntValue();
519  if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
520  return false;
521 
522  if (!C.isSubsetOf(DemandedBits)) {
523  EVT VT = Op.getValueType();
524  SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
525  SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
526  return TLO.CombineTo(Op, NewOp);
527  }
528 
529  break;
530  }
531  }
532 
533  return false;
534 }
535 
537  const APInt &DemandedBits,
538  TargetLoweringOpt &TLO) const {
539  EVT VT = Op.getValueType();
540  APInt DemandedElts = VT.isVector()
542  : APInt(1, 1);
543  return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
544 }
545 
546 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
547 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
548 /// generalized for targets with other types of implicit widening casts.
550  const APInt &Demanded,
551  TargetLoweringOpt &TLO) const {
552  assert(Op.getNumOperands() == 2 &&
553  "ShrinkDemandedOp only supports binary operators!");
554  assert(Op.getNode()->getNumValues() == 1 &&
555  "ShrinkDemandedOp only supports nodes with one result!");
556 
557  SelectionDAG &DAG = TLO.DAG;
558  SDLoc dl(Op);
559 
560  // Early return, as this function cannot handle vector types.
561  if (Op.getValueType().isVector())
562  return false;
563 
564  // Don't do this if the node has another user, which may require the
565  // full value.
566  if (!Op.getNode()->hasOneUse())
567  return false;
568 
569  // Search for the smallest integer type with free casts to and from
570  // Op's type. For expedience, just check power-of-2 integer types.
571  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
572  unsigned DemandedSize = Demanded.getActiveBits();
573  unsigned SmallVTBits = DemandedSize;
574  if (!isPowerOf2_32(SmallVTBits))
575  SmallVTBits = NextPowerOf2(SmallVTBits);
576  for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
577  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
578  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
579  TLI.isZExtFree(SmallVT, Op.getValueType())) {
580  // We found a type with free casts.
581  SDValue X = DAG.getNode(
582  Op.getOpcode(), dl, SmallVT,
583  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
584  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
585  assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
586  SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
587  return TLO.CombineTo(Op, Z);
588  }
589  }
590  return false;
591 }
592 
594  DAGCombinerInfo &DCI) const {
595  SelectionDAG &DAG = DCI.DAG;
596  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
597  !DCI.isBeforeLegalizeOps());
598  KnownBits Known;
599 
600  bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
601  if (Simplified) {
602  DCI.AddToWorklist(Op.getNode());
603  DCI.CommitTargetLoweringOpt(TLO);
604  }
605  return Simplified;
606 }
607 
609  KnownBits &Known,
610  TargetLoweringOpt &TLO,
611  unsigned Depth,
612  bool AssumeSingleUse) const {
613  EVT VT = Op.getValueType();
614 
615  // TODO: We can probably do more work on calculating the known bits and
616  // simplifying the operations for scalable vectors, but for now we just
617  // bail out.
618  if (VT.isScalableVector()) {
619  // Pretend we don't know anything for now.
620  Known = KnownBits(DemandedBits.getBitWidth());
621  return false;
622  }
623 
624  APInt DemandedElts = VT.isVector()
626  : APInt(1, 1);
627  return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
628  AssumeSingleUse);
629 }
630 
631 // TODO: Can we merge SelectionDAG::GetDemandedBits into this?
632 // TODO: Under what circumstances can we create nodes? Constant folding?
634  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
635  SelectionDAG &DAG, unsigned Depth) const {
636  // Limit search depth.
638  return SDValue();
639 
640  // Ignore UNDEFs.
641  if (Op.isUndef())
642  return SDValue();
643 
644  // Not demanding any bits/elts from Op.
645  if (DemandedBits == 0 || DemandedElts == 0)
646  return DAG.getUNDEF(Op.getValueType());
647 
648  bool IsLE = DAG.getDataLayout().isLittleEndian();
649  unsigned NumElts = DemandedElts.getBitWidth();
650  unsigned BitWidth = DemandedBits.getBitWidth();
651  KnownBits LHSKnown, RHSKnown;
652  switch (Op.getOpcode()) {
653  case ISD::BITCAST: {
654  SDValue Src = peekThroughBitcasts(Op.getOperand(0));
655  EVT SrcVT = Src.getValueType();
656  EVT DstVT = Op.getValueType();
657  if (SrcVT == DstVT)
658  return Src;
659 
660  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
661  unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
662  if (NumSrcEltBits == NumDstEltBits)
663  if (SDValue V = SimplifyMultipleUseDemandedBits(
664  Src, DemandedBits, DemandedElts, DAG, Depth + 1))
665  return DAG.getBitcast(DstVT, V);
666 
667  if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0) {
668  unsigned Scale = NumDstEltBits / NumSrcEltBits;
669  unsigned NumSrcElts = SrcVT.getVectorNumElements();
670  APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
671  APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
672  for (unsigned i = 0; i != Scale; ++i) {
673  unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
674  unsigned BitOffset = EltOffset * NumSrcEltBits;
675  APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset);
676  if (!Sub.isZero()) {
677  DemandedSrcBits |= Sub;
678  for (unsigned j = 0; j != NumElts; ++j)
679  if (DemandedElts[j])
680  DemandedSrcElts.setBit((j * Scale) + i);
681  }
682  }
683 
684  if (SDValue V = SimplifyMultipleUseDemandedBits(
685  Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
686  return DAG.getBitcast(DstVT, V);
687  }
688 
689  // TODO - bigendian once we have test coverage.
690  if (IsLE && (NumSrcEltBits % NumDstEltBits) == 0) {
691  unsigned Scale = NumSrcEltBits / NumDstEltBits;
692  unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
693  APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
694  APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
695  for (unsigned i = 0; i != NumElts; ++i)
696  if (DemandedElts[i]) {
697  unsigned Offset = (i % Scale) * NumDstEltBits;
698  DemandedSrcBits.insertBits(DemandedBits, Offset);
699  DemandedSrcElts.setBit(i / Scale);
700  }
701 
702  if (SDValue V = SimplifyMultipleUseDemandedBits(
703  Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
704  return DAG.getBitcast(DstVT, V);
705  }
706 
707  break;
708  }
709  case ISD::AND: {
710  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
711  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
712 
713  // If all of the demanded bits are known 1 on one side, return the other.
714  // These bits cannot contribute to the result of the 'and' in this
715  // context.
716  if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
717  return Op.getOperand(0);
718  if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
719  return Op.getOperand(1);
720  break;
721  }
722  case ISD::OR: {
723  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
724  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
725 
726  // If all of the demanded bits are known zero on one side, return the
727  // other. These bits cannot contribute to the result of the 'or' in this
728  // context.
729  if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
730  return Op.getOperand(0);
731  if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
732  return Op.getOperand(1);
733  break;
734  }
735  case ISD::XOR: {
736  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
737  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
738 
739  // If all of the demanded bits are known zero on one side, return the
740  // other.
741  if (DemandedBits.isSubsetOf(RHSKnown.Zero))
742  return Op.getOperand(0);
743  if (DemandedBits.isSubsetOf(LHSKnown.Zero))
744  return Op.getOperand(1);
745  break;
746  }
747  case ISD::SHL: {
748  // If we are only demanding sign bits then we can use the shift source
749  // directly.
750  if (const APInt *MaxSA =
751  DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
752  SDValue Op0 = Op.getOperand(0);
753  unsigned ShAmt = MaxSA->getZExtValue();
754  unsigned NumSignBits =
755  DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
756  unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
757  if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
758  return Op0;
759  }
760  break;
761  }
762  case ISD::SETCC: {
763  SDValue Op0 = Op.getOperand(0);
764  SDValue Op1 = Op.getOperand(1);
765  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
766  // If (1) we only need the sign-bit, (2) the setcc operands are the same
767  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
768  // -1, we may be able to bypass the setcc.
769  if (DemandedBits.isSignMask() &&
772  BooleanContent::ZeroOrNegativeOneBooleanContent) {
773  // If we're testing X < 0, then this compare isn't needed - just use X!
774  // FIXME: We're limiting to integer types here, but this should also work
775  // if we don't care about FP signed-zero. The use of SETLT with FP means
776  // that we don't care about NaNs.
777  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
779  return Op0;
780  }
781  break;
782  }
783  case ISD::SIGN_EXTEND_INREG: {
784  // If none of the extended bits are demanded, eliminate the sextinreg.
785  SDValue Op0 = Op.getOperand(0);
786  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
787  unsigned ExBits = ExVT.getScalarSizeInBits();
788  if (DemandedBits.getActiveBits() <= ExBits)
789  return Op0;
790  // If the input is already sign extended, just drop the extension.
791  unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
792  if (NumSignBits >= (BitWidth - ExBits + 1))
793  return Op0;
794  break;
795  }
799  // If we only want the lowest element and none of extended bits, then we can
800  // return the bitcasted source vector.
801  SDValue Src = Op.getOperand(0);
802  EVT SrcVT = Src.getValueType();
803  EVT DstVT = Op.getValueType();
804  if (IsLE && DemandedElts == 1 &&
805  DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
806  DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
807  return DAG.getBitcast(DstVT, Src);
808  }
809  break;
810  }
811  case ISD::INSERT_VECTOR_ELT: {
812  // If we don't demand the inserted element, return the base vector.
813  SDValue Vec = Op.getOperand(0);
814  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
815  EVT VecVT = Vec.getValueType();
816  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
817  !DemandedElts[CIdx->getZExtValue()])
818  return Vec;
819  break;
820  }
821  case ISD::INSERT_SUBVECTOR: {
822  SDValue Vec = Op.getOperand(0);
823  SDValue Sub = Op.getOperand(1);
824  uint64_t Idx = Op.getConstantOperandVal(2);
825  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
826  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
827  // If we don't demand the inserted subvector, return the base vector.
828  if (DemandedSubElts == 0)
829  return Vec;
830  // If this simply widens the lowest subvector, see if we can do it earlier.
831  if (Idx == 0 && Vec.isUndef()) {
832  if (SDValue NewSub = SimplifyMultipleUseDemandedBits(
833  Sub, DemandedBits, DemandedSubElts, DAG, Depth + 1))
834  return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
835  Op.getOperand(0), NewSub, Op.getOperand(2));
836  }
837  break;
838  }
839  case ISD::VECTOR_SHUFFLE: {
840  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
841 
842  // If all the demanded elts are from one operand and are inline,
843  // then we can use the operand directly.
844  bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
845  for (unsigned i = 0; i != NumElts; ++i) {
846  int M = ShuffleMask[i];
847  if (M < 0 || !DemandedElts[i])
848  continue;
849  AllUndef = false;
850  IdentityLHS &= (M == (int)i);
851  IdentityRHS &= ((M - NumElts) == i);
852  }
853 
854  if (AllUndef)
855  return DAG.getUNDEF(Op.getValueType());
856  if (IdentityLHS)
857  return Op.getOperand(0);
858  if (IdentityRHS)
859  return Op.getOperand(1);
860  break;
861  }
862  default:
863  if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
864  if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
865  Op, DemandedBits, DemandedElts, DAG, Depth))
866  return V;
867  break;
868  }
869  return SDValue();
870 }
871 
873  SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
874  unsigned Depth) const {
875  EVT VT = Op.getValueType();
876  APInt DemandedElts = VT.isVector()
878  : APInt(1, 1);
879  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
880  Depth);
881 }
882 
884  SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
885  unsigned Depth) const {
886  APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
887  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
888  Depth);
889 }
890 
891 /// Look at Op. At this point, we know that only the OriginalDemandedBits of the
892 /// result of Op are ever used downstream. If we can use this information to
893 /// simplify Op, create a new simplified DAG node and return true, returning the
894 /// original and new nodes in Old and New. Otherwise, analyze the expression and
895 /// return a mask of Known bits for the expression (used to simplify the
896 /// caller). The Known bits may only be accurate for those bits in the
897 /// OriginalDemandedBits and OriginalDemandedElts.
899  SDValue Op, const APInt &OriginalDemandedBits,
900  const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
901  unsigned Depth, bool AssumeSingleUse) const {
902  unsigned BitWidth = OriginalDemandedBits.getBitWidth();
903  assert(Op.getScalarValueSizeInBits() == BitWidth &&
904  "Mask size mismatches value type size!");
905 
906  // Don't know anything.
907  Known = KnownBits(BitWidth);
908 
909  // TODO: We can probably do more work on calculating the known bits and
910  // simplifying the operations for scalable vectors, but for now we just
911  // bail out.
912  if (Op.getValueType().isScalableVector())
913  return false;
914 
915  bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
916  unsigned NumElts = OriginalDemandedElts.getBitWidth();
917  assert((!Op.getValueType().isVector() ||
918  NumElts == Op.getValueType().getVectorNumElements()) &&
919  "Unexpected vector size");
920 
921  APInt DemandedBits = OriginalDemandedBits;
922  APInt DemandedElts = OriginalDemandedElts;
923  SDLoc dl(Op);
924  auto &DL = TLO.DAG.getDataLayout();
925 
926  // Undef operand.
927  if (Op.isUndef())
928  return false;
929 
930  if (Op.getOpcode() == ISD::Constant) {
931  // We know all of the bits for a constant!
932  Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
933  return false;
934  }
935 
936  if (Op.getOpcode() == ISD::ConstantFP) {
937  // We know all of the bits for a floating point constant!
938  Known = KnownBits::makeConstant(
939  cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
940  return false;
941  }
942 
943  // Other users may use these bits.
944  EVT VT = Op.getValueType();
945  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
946  if (Depth != 0) {
947  // If not at the root, Just compute the Known bits to
948  // simplify things downstream.
949  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
950  return false;
951  }
952  // If this is the root being simplified, allow it to have multiple uses,
953  // just set the DemandedBits/Elts to all bits.
955  DemandedElts = APInt::getAllOnes(NumElts);
956  } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
957  // Not demanding any bits/elts from Op.
958  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
959  } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
960  // Limit search depth.
961  return false;
962  }
963 
964  KnownBits Known2;
965  switch (Op.getOpcode()) {
966  case ISD::TargetConstant:
967  llvm_unreachable("Can't simplify this node");
968  case ISD::SCALAR_TO_VECTOR: {
969  if (!DemandedElts[0])
970  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
971 
972  KnownBits SrcKnown;
973  SDValue Src = Op.getOperand(0);
974  unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
975  APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
976  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
977  return true;
978 
979  // Upper elements are undef, so only get the knownbits if we just demand
980  // the bottom element.
981  if (DemandedElts == 1)
982  Known = SrcKnown.anyextOrTrunc(BitWidth);
983  break;
984  }
985  case ISD::BUILD_VECTOR:
986  // Collect the known bits that are shared by every demanded element.
987  // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
988  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
989  return false; // Don't fall through, will infinitely loop.
990  case ISD::LOAD: {
991  auto *LD = cast<LoadSDNode>(Op);
992  if (getTargetConstantFromLoad(LD)) {
993  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
994  return false; // Don't fall through, will infinitely loop.
995  }
996  if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
997  // If this is a ZEXTLoad and we are looking at the loaded value.
998  EVT MemVT = LD->getMemoryVT();
999  unsigned MemBits = MemVT.getScalarSizeInBits();
1000  Known.Zero.setBitsFrom(MemBits);
1001  return false; // Don't fall through, will infinitely loop.
1002  }
1003  break;
1004  }
1005  case ISD::INSERT_VECTOR_ELT: {
1006  SDValue Vec = Op.getOperand(0);
1007  SDValue Scl = Op.getOperand(1);
1008  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1009  EVT VecVT = Vec.getValueType();
1010 
1011  // If index isn't constant, assume we need all vector elements AND the
1012  // inserted element.
1013  APInt DemandedVecElts(DemandedElts);
1014  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1015  unsigned Idx = CIdx->getZExtValue();
1016  DemandedVecElts.clearBit(Idx);
1017 
1018  // Inserted element is not required.
1019  if (!DemandedElts[Idx])
1020  return TLO.CombineTo(Op, Vec);
1021  }
1022 
1023  KnownBits KnownScl;
1024  unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1025  APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1026  if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1027  return true;
1028 
1029  Known = KnownScl.anyextOrTrunc(BitWidth);
1030 
1031  KnownBits KnownVec;
1032  if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1033  Depth + 1))
1034  return true;
1035 
1036  if (!!DemandedVecElts)
1037  Known = KnownBits::commonBits(Known, KnownVec);
1038 
1039  return false;
1040  }
1041  case ISD::INSERT_SUBVECTOR: {
1042  // Demand any elements from the subvector and the remainder from the src its
1043  // inserted into.
1044  SDValue Src = Op.getOperand(0);
1045  SDValue Sub = Op.getOperand(1);
1046  uint64_t Idx = Op.getConstantOperandVal(2);
1047  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1048  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1049  APInt DemandedSrcElts = DemandedElts;
1050  DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
1051 
1052  KnownBits KnownSub, KnownSrc;
1053  if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1054  Depth + 1))
1055  return true;
1056  if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1057  Depth + 1))
1058  return true;
1059 
1060  Known.Zero.setAllBits();
1061  Known.One.setAllBits();
1062  if (!!DemandedSubElts)
1063  Known = KnownBits::commonBits(Known, KnownSub);
1064  if (!!DemandedSrcElts)
1065  Known = KnownBits::commonBits(Known, KnownSrc);
1066 
1067  // Attempt to avoid multi-use src if we don't need anything from it.
1068  if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1069  !DemandedSrcElts.isAllOnes()) {
1070  SDValue NewSub = SimplifyMultipleUseDemandedBits(
1071  Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1072  SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1073  Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1074  if (NewSub || NewSrc) {
1075  NewSub = NewSub ? NewSub : Sub;
1076  NewSrc = NewSrc ? NewSrc : Src;
1077  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1078  Op.getOperand(2));
1079  return TLO.CombineTo(Op, NewOp);
1080  }
1081  }
1082  break;
1083  }
1084  case ISD::EXTRACT_SUBVECTOR: {
1085  // Offset the demanded elts by the subvector index.
1086  SDValue Src = Op.getOperand(0);
1087  if (Src.getValueType().isScalableVector())
1088  break;
1089  uint64_t Idx = Op.getConstantOperandVal(1);
1090  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1091  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1092 
1093  if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1094  Depth + 1))
1095  return true;
1096 
1097  // Attempt to avoid multi-use src if we don't need anything from it.
1098  if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1099  SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1100  Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1101  if (DemandedSrc) {
1102  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1103  Op.getOperand(1));
1104  return TLO.CombineTo(Op, NewOp);
1105  }
1106  }
1107  break;
1108  }
1109  case ISD::CONCAT_VECTORS: {
1110  Known.Zero.setAllBits();
1111  Known.One.setAllBits();
1112  EVT SubVT = Op.getOperand(0).getValueType();
1113  unsigned NumSubVecs = Op.getNumOperands();
1114  unsigned NumSubElts = SubVT.getVectorNumElements();
1115  for (unsigned i = 0; i != NumSubVecs; ++i) {
1116  APInt DemandedSubElts =
1117  DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1118  if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1119  Known2, TLO, Depth + 1))
1120  return true;
1121  // Known bits are shared by every demanded subvector element.
1122  if (!!DemandedSubElts)
1123  Known = KnownBits::commonBits(Known, Known2);
1124  }
1125  break;
1126  }
1127  case ISD::VECTOR_SHUFFLE: {
1128  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1129 
1130  // Collect demanded elements from shuffle operands..
1131  APInt DemandedLHS(NumElts, 0);
1132  APInt DemandedRHS(NumElts, 0);
1133  for (unsigned i = 0; i != NumElts; ++i) {
1134  if (!DemandedElts[i])
1135  continue;
1136  int M = ShuffleMask[i];
1137  if (M < 0) {
1138  // For UNDEF elements, we don't know anything about the common state of
1139  // the shuffle result.
1140  DemandedLHS.clearAllBits();
1141  DemandedRHS.clearAllBits();
1142  break;
1143  }
1144  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1145  if (M < (int)NumElts)
1146  DemandedLHS.setBit(M);
1147  else
1148  DemandedRHS.setBit(M - NumElts);
1149  }
1150 
1151  if (!!DemandedLHS || !!DemandedRHS) {
1152  SDValue Op0 = Op.getOperand(0);
1153  SDValue Op1 = Op.getOperand(1);
1154 
1155  Known.Zero.setAllBits();
1156  Known.One.setAllBits();
1157  if (!!DemandedLHS) {
1158  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1159  Depth + 1))
1160  return true;
1161  Known = KnownBits::commonBits(Known, Known2);
1162  }
1163  if (!!DemandedRHS) {
1164  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1165  Depth + 1))
1166  return true;
1167  Known = KnownBits::commonBits(Known, Known2);
1168  }
1169 
1170  // Attempt to avoid multi-use ops if we don't need anything from them.
1171  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1172  Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1173  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1174  Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1175  if (DemandedOp0 || DemandedOp1) {
1176  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1177  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1178  SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1179  return TLO.CombineTo(Op, NewOp);
1180  }
1181  }
1182  break;
1183  }
1184  case ISD::AND: {
1185  SDValue Op0 = Op.getOperand(0);
1186  SDValue Op1 = Op.getOperand(1);
1187 
1188  // If the RHS is a constant, check to see if the LHS would be zero without
1189  // using the bits from the RHS. Below, we use knowledge about the RHS to
1190  // simplify the LHS, here we're using information from the LHS to simplify
1191  // the RHS.
1192  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1193  // Do not increment Depth here; that can cause an infinite loop.
1194  KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1195  // If the LHS already has zeros where RHSC does, this 'and' is dead.
1196  if ((LHSKnown.Zero & DemandedBits) ==
1197  (~RHSC->getAPIntValue() & DemandedBits))
1198  return TLO.CombineTo(Op, Op0);
1199 
1200  // If any of the set bits in the RHS are known zero on the LHS, shrink
1201  // the constant.
1202  if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1203  DemandedElts, TLO))
1204  return true;
1205 
1206  // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1207  // constant, but if this 'and' is only clearing bits that were just set by
1208  // the xor, then this 'and' can be eliminated by shrinking the mask of
1209  // the xor. For example, for a 32-bit X:
1210  // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1211  if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1212  LHSKnown.One == ~RHSC->getAPIntValue()) {
1213  SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1214  return TLO.CombineTo(Op, Xor);
1215  }
1216  }
1217 
1218  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1219  Depth + 1))
1220  return true;
1221  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1222  if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1223  Known2, TLO, Depth + 1))
1224  return true;
1225  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1226 
1227  // Attempt to avoid multi-use ops if we don't need anything from them.
1228  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1229  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1230  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1231  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1232  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1233  if (DemandedOp0 || DemandedOp1) {
1234  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1235  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1236  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1237  return TLO.CombineTo(Op, NewOp);
1238  }
1239  }
1240 
1241  // If all of the demanded bits are known one on one side, return the other.
1242  // These bits cannot contribute to the result of the 'and'.
1243  if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1244  return TLO.CombineTo(Op, Op0);
1245  if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1246  return TLO.CombineTo(Op, Op1);
1247  // If all of the demanded bits in the inputs are known zeros, return zero.
1248  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1249  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1250  // If the RHS is a constant, see if we can simplify it.
1251  if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1252  TLO))
1253  return true;
1254  // If the operation can be done in a smaller type, do so.
1255  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1256  return true;
1257 
1258  Known &= Known2;
1259  break;
1260  }
1261  case ISD::OR: {
1262  SDValue Op0 = Op.getOperand(0);
1263  SDValue Op1 = Op.getOperand(1);
1264 
1265  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1266  Depth + 1))
1267  return true;
1268  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1269  if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1270  Known2, TLO, Depth + 1))
1271  return true;
1272  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1273 
1274  // Attempt to avoid multi-use ops if we don't need anything from them.
1275  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1276  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1277  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1278  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1279  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1280  if (DemandedOp0 || DemandedOp1) {
1281  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1282  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1283  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1284  return TLO.CombineTo(Op, NewOp);
1285  }
1286  }
1287 
1288  // If all of the demanded bits are known zero on one side, return the other.
1289  // These bits cannot contribute to the result of the 'or'.
1290  if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1291  return TLO.CombineTo(Op, Op0);
1292  if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1293  return TLO.CombineTo(Op, Op1);
1294  // If the RHS is a constant, see if we can simplify it.
1295  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1296  return true;
1297  // If the operation can be done in a smaller type, do so.
1298  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1299  return true;
1300 
1301  Known |= Known2;
1302  break;
1303  }
1304  case ISD::XOR: {
1305  SDValue Op0 = Op.getOperand(0);
1306  SDValue Op1 = Op.getOperand(1);
1307 
1308  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1309  Depth + 1))
1310  return true;
1311  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1312  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1313  Depth + 1))
1314  return true;
1315  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1316 
1317  // Attempt to avoid multi-use ops if we don't need anything from them.
1318  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1319  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1320  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1321  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1322  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1323  if (DemandedOp0 || DemandedOp1) {
1324  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1325  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1326  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1327  return TLO.CombineTo(Op, NewOp);
1328  }
1329  }
1330 
1331  // If all of the demanded bits are known zero on one side, return the other.
1332  // These bits cannot contribute to the result of the 'xor'.
1333  if (DemandedBits.isSubsetOf(Known.Zero))
1334  return TLO.CombineTo(Op, Op0);
1335  if (DemandedBits.isSubsetOf(Known2.Zero))
1336  return TLO.CombineTo(Op, Op1);
1337  // If the operation can be done in a smaller type, do so.
1338  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1339  return true;
1340 
1341  // If all of the unknown bits are known to be zero on one side or the other
1342  // turn this into an *inclusive* or.
1343  // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1344  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1345  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1346 
1347  ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1348  if (C) {
1349  // If one side is a constant, and all of the set bits in the constant are
1350  // also known set on the other side, turn this into an AND, as we know
1351  // the bits will be cleared.
1352  // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1353  // NB: it is okay if more bits are known than are requested
1354  if (C->getAPIntValue() == Known2.One) {
1355  SDValue ANDC =
1356  TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1357  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1358  }
1359 
1360  // If the RHS is a constant, see if we can change it. Don't alter a -1
1361  // constant because that's a 'not' op, and that is better for combining
1362  // and codegen.
1363  if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
1364  // We're flipping all demanded bits. Flip the undemanded bits too.
1365  SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1366  return TLO.CombineTo(Op, New);
1367  }
1368  }
1369 
1370  // If we can't turn this into a 'not', try to shrink the constant.
1371  if (!C || !C->isAllOnes())
1372  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1373  return true;
1374 
1375  Known ^= Known2;
1376  break;
1377  }
1378  case ISD::SELECT:
1379  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1380  Depth + 1))
1381  return true;
1382  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1383  Depth + 1))
1384  return true;
1385  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1386  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1387 
1388  // If the operands are constants, see if we can simplify them.
1389  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1390  return true;
1391 
1392  // Only known if known in both the LHS and RHS.
1393  Known = KnownBits::commonBits(Known, Known2);
1394  break;
1395  case ISD::SELECT_CC:
1396  if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1397  Depth + 1))
1398  return true;
1399  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1400  Depth + 1))
1401  return true;
1402  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1403  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1404 
1405  // If the operands are constants, see if we can simplify them.
1406  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1407  return true;
1408 
1409  // Only known if known in both the LHS and RHS.
1410  Known = KnownBits::commonBits(Known, Known2);
1411  break;
1412  case ISD::SETCC: {
1413  SDValue Op0 = Op.getOperand(0);
1414  SDValue Op1 = Op.getOperand(1);
1415  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1416  // If (1) we only need the sign-bit, (2) the setcc operands are the same
1417  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1418  // -1, we may be able to bypass the setcc.
1419  if (DemandedBits.isSignMask() &&
1422  BooleanContent::ZeroOrNegativeOneBooleanContent) {
1423  // If we're testing X < 0, then this compare isn't needed - just use X!
1424  // FIXME: We're limiting to integer types here, but this should also work
1425  // if we don't care about FP signed-zero. The use of SETLT with FP means
1426  // that we don't care about NaNs.
1427  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1429  return TLO.CombineTo(Op, Op0);
1430 
1431  // TODO: Should we check for other forms of sign-bit comparisons?
1432  // Examples: X <= -1, X >= 0
1433  }
1434  if (getBooleanContents(Op0.getValueType()) ==
1436  BitWidth > 1)
1437  Known.Zero.setBitsFrom(1);
1438  break;
1439  }
1440  case ISD::SHL: {
1441  SDValue Op0 = Op.getOperand(0);
1442  SDValue Op1 = Op.getOperand(1);
1443  EVT ShiftVT = Op1.getValueType();
1444 
1445  if (const APInt *SA =
1446  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1447  unsigned ShAmt = SA->getZExtValue();
1448  if (ShAmt == 0)
1449  return TLO.CombineTo(Op, Op0);
1450 
1451  // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1452  // single shift. We can do this if the bottom bits (which are shifted
1453  // out) are never demanded.
1454  // TODO - support non-uniform vector amounts.
1455  if (Op0.getOpcode() == ISD::SRL) {
1456  if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1457  if (const APInt *SA2 =
1458  TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1459  unsigned C1 = SA2->getZExtValue();
1460  unsigned Opc = ISD::SHL;
1461  int Diff = ShAmt - C1;
1462  if (Diff < 0) {
1463  Diff = -Diff;
1464  Opc = ISD::SRL;
1465  }
1466  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1467  return TLO.CombineTo(
1468  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1469  }
1470  }
1471  }
1472 
1473  // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1474  // are not demanded. This will likely allow the anyext to be folded away.
1475  // TODO - support non-uniform vector amounts.
1476  if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1477  SDValue InnerOp = Op0.getOperand(0);
1478  EVT InnerVT = InnerOp.getValueType();
1479  unsigned InnerBits = InnerVT.getScalarSizeInBits();
1480  if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1481  isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1482  EVT ShTy = getShiftAmountTy(InnerVT, DL);
1483  if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1484  ShTy = InnerVT;
1485  SDValue NarrowShl =
1486  TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1487  TLO.DAG.getConstant(ShAmt, dl, ShTy));
1488  return TLO.CombineTo(
1489  Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1490  }
1491 
1492  // Repeat the SHL optimization above in cases where an extension
1493  // intervenes: (shl (anyext (shr x, c1)), c2) to
1494  // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1495  // aren't demanded (as above) and that the shifted upper c1 bits of
1496  // x aren't demanded.
1497  // TODO - support non-uniform vector amounts.
1498  if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1499  InnerOp.hasOneUse()) {
1500  if (const APInt *SA2 =
1501  TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1502  unsigned InnerShAmt = SA2->getZExtValue();
1503  if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1504  DemandedBits.getActiveBits() <=
1505  (InnerBits - InnerShAmt + ShAmt) &&
1506  DemandedBits.countTrailingZeros() >= ShAmt) {
1507  SDValue NewSA =
1508  TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1509  SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1510  InnerOp.getOperand(0));
1511  return TLO.CombineTo(
1512  Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1513  }
1514  }
1515  }
1516  }
1517 
1518  APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1519  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1520  Depth + 1))
1521  return true;
1522  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1523  Known.Zero <<= ShAmt;
1524  Known.One <<= ShAmt;
1525  // low bits known zero.
1526  Known.Zero.setLowBits(ShAmt);
1527 
1528  // Try shrinking the operation as long as the shift amount will still be
1529  // in range.
1530  if ((ShAmt < DemandedBits.getActiveBits()) &&
1531  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1532  return true;
1533  }
1534 
1535  // If we are only demanding sign bits then we can use the shift source
1536  // directly.
1537  if (const APInt *MaxSA =
1538  TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1539  unsigned ShAmt = MaxSA->getZExtValue();
1540  unsigned NumSignBits =
1541  TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1542  unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1543  if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1544  return TLO.CombineTo(Op, Op0);
1545  }
1546  break;
1547  }
1548  case ISD::SRL: {
1549  SDValue Op0 = Op.getOperand(0);
1550  SDValue Op1 = Op.getOperand(1);
1551  EVT ShiftVT = Op1.getValueType();
1552 
1553  if (const APInt *SA =
1554  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1555  unsigned ShAmt = SA->getZExtValue();
1556  if (ShAmt == 0)
1557  return TLO.CombineTo(Op, Op0);
1558 
1559  // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1560  // single shift. We can do this if the top bits (which are shifted out)
1561  // are never demanded.
1562  // TODO - support non-uniform vector amounts.
1563  if (Op0.getOpcode() == ISD::SHL) {
1564  if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1565  if (const APInt *SA2 =
1566  TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1567  unsigned C1 = SA2->getZExtValue();
1568  unsigned Opc = ISD::SRL;
1569  int Diff = ShAmt - C1;
1570  if (Diff < 0) {
1571  Diff = -Diff;
1572  Opc = ISD::SHL;
1573  }
1574  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1575  return TLO.CombineTo(
1576  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1577  }
1578  }
1579  }
1580 
1581  APInt InDemandedMask = (DemandedBits << ShAmt);
1582 
1583  // If the shift is exact, then it does demand the low bits (and knows that
1584  // they are zero).
1585  if (Op->getFlags().hasExact())
1586  InDemandedMask.setLowBits(ShAmt);
1587 
1588  // Compute the new bits that are at the top now.
1589  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1590  Depth + 1))
1591  return true;
1592  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1593  Known.Zero.lshrInPlace(ShAmt);
1594  Known.One.lshrInPlace(ShAmt);
1595  // High bits known zero.
1596  Known.Zero.setHighBits(ShAmt);
1597  }
1598  break;
1599  }
1600  case ISD::SRA: {
1601  SDValue Op0 = Op.getOperand(0);
1602  SDValue Op1 = Op.getOperand(1);
1603  EVT ShiftVT = Op1.getValueType();
1604 
1605  // If we only want bits that already match the signbit then we don't need
1606  // to shift.
1607  unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1608  if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1609  NumHiDemandedBits)
1610  return TLO.CombineTo(Op, Op0);
1611 
1612  // If this is an arithmetic shift right and only the low-bit is set, we can
1613  // always convert this into a logical shr, even if the shift amount is
1614  // variable. The low bit of the shift cannot be an input sign bit unless
1615  // the shift amount is >= the size of the datatype, which is undefined.
1616  if (DemandedBits.isOne())
1617  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1618 
1619  if (const APInt *SA =
1620  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1621  unsigned ShAmt = SA->getZExtValue();
1622  if (ShAmt == 0)
1623  return TLO.CombineTo(Op, Op0);
1624 
1625  APInt InDemandedMask = (DemandedBits << ShAmt);
1626 
1627  // If the shift is exact, then it does demand the low bits (and knows that
1628  // they are zero).
1629  if (Op->getFlags().hasExact())
1630  InDemandedMask.setLowBits(ShAmt);
1631 
1632  // If any of the demanded bits are produced by the sign extension, we also
1633  // demand the input sign bit.
1634  if (DemandedBits.countLeadingZeros() < ShAmt)
1635  InDemandedMask.setSignBit();
1636 
1637  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1638  Depth + 1))
1639  return true;
1640  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1641  Known.Zero.lshrInPlace(ShAmt);
1642  Known.One.lshrInPlace(ShAmt);
1643 
1644  // If the input sign bit is known to be zero, or if none of the top bits
1645  // are demanded, turn this into an unsigned shift right.
1646  if (Known.Zero[BitWidth - ShAmt - 1] ||
1647  DemandedBits.countLeadingZeros() >= ShAmt) {
1648  SDNodeFlags Flags;
1649  Flags.setExact(Op->getFlags().hasExact());
1650  return TLO.CombineTo(
1651  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1652  }
1653 
1654  int Log2 = DemandedBits.exactLogBase2();
1655  if (Log2 >= 0) {
1656  // The bit must come from the sign.
1657  SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1658  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1659  }
1660 
1661  if (Known.One[BitWidth - ShAmt - 1])
1662  // New bits are known one.
1663  Known.One.setHighBits(ShAmt);
1664 
1665  // Attempt to avoid multi-use ops if we don't need anything from them.
1666  if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1667  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1668  Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1669  if (DemandedOp0) {
1670  SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1671  return TLO.CombineTo(Op, NewOp);
1672  }
1673  }
1674  }
1675  break;
1676  }
1677  case ISD::FSHL:
1678  case ISD::FSHR: {
1679  SDValue Op0 = Op.getOperand(0);
1680  SDValue Op1 = Op.getOperand(1);
1681  SDValue Op2 = Op.getOperand(2);
1682  bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1683 
1684  if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1685  unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1686 
1687  // For fshl, 0-shift returns the 1st arg.
1688  // For fshr, 0-shift returns the 2nd arg.
1689  if (Amt == 0) {
1690  if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1691  Known, TLO, Depth + 1))
1692  return true;
1693  break;
1694  }
1695 
1696  // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1697  // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1698  APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1699  APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1700  if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1701  Depth + 1))
1702  return true;
1703  if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1704  Depth + 1))
1705  return true;
1706 
1707  Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1708  Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1709  Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1710  Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1711  Known.One |= Known2.One;
1712  Known.Zero |= Known2.Zero;
1713  }
1714 
1715  // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1716  if (isPowerOf2_32(BitWidth)) {
1717  APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1718  if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1719  Known2, TLO, Depth + 1))
1720  return true;
1721  }
1722  break;
1723  }
1724  case ISD::ROTL:
1725  case ISD::ROTR: {
1726  SDValue Op0 = Op.getOperand(0);
1727  SDValue Op1 = Op.getOperand(1);
1728  bool IsROTL = (Op.getOpcode() == ISD::ROTL);
1729 
1730  // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1731  if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1732  return TLO.CombineTo(Op, Op0);
1733 
1734  if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) {
1735  unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1736  unsigned RevAmt = BitWidth - Amt;
1737 
1738  // rotl: (Op0 << Amt) | (Op0 >> (BW - Amt))
1739  // rotr: (Op0 << (BW - Amt)) | (Op0 >> Amt)
1740  APInt Demanded0 = DemandedBits.rotr(IsROTL ? Amt : RevAmt);
1741  if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1742  Depth + 1))
1743  return true;
1744 
1745  // rot*(x, 0) --> x
1746  if (Amt == 0)
1747  return TLO.CombineTo(Op, Op0);
1748 
1749  // See if we don't demand either half of the rotated bits.
1750  if ((!TLO.LegalOperations() || isOperationLegal(ISD::SHL, VT)) &&
1751  DemandedBits.countTrailingZeros() >= (IsROTL ? Amt : RevAmt)) {
1752  Op1 = TLO.DAG.getConstant(IsROTL ? Amt : RevAmt, dl, Op1.getValueType());
1753  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, Op1));
1754  }
1755  if ((!TLO.LegalOperations() || isOperationLegal(ISD::SRL, VT)) &&
1756  DemandedBits.countLeadingZeros() >= (IsROTL ? RevAmt : Amt)) {
1757  Op1 = TLO.DAG.getConstant(IsROTL ? RevAmt : Amt, dl, Op1.getValueType());
1758  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1759  }
1760  }
1761 
1762  // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1763  if (isPowerOf2_32(BitWidth)) {
1764  APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1765  if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1766  Depth + 1))
1767  return true;
1768  }
1769  break;
1770  }
1771  case ISD::UMIN: {
1772  // Check if one arg is always less than (or equal) to the other arg.
1773  SDValue Op0 = Op.getOperand(0);
1774  SDValue Op1 = Op.getOperand(1);
1775  KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1776  KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1777  Known = KnownBits::umin(Known0, Known1);
1778  if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1779  return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1780  if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1781  return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1782  break;
1783  }
1784  case ISD::UMAX: {
1785  // Check if one arg is always greater than (or equal) to the other arg.
1786  SDValue Op0 = Op.getOperand(0);
1787  SDValue Op1 = Op.getOperand(1);
1788  KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1789  KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1790  Known = KnownBits::umax(Known0, Known1);
1791  if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1792  return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1793  if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1794  return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1795  break;
1796  }
1797  case ISD::BITREVERSE: {
1798  SDValue Src = Op.getOperand(0);
1799  APInt DemandedSrcBits = DemandedBits.reverseBits();
1800  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1801  Depth + 1))
1802  return true;
1803  Known.One = Known2.One.reverseBits();
1804  Known.Zero = Known2.Zero.reverseBits();
1805  break;
1806  }
1807  case ISD::BSWAP: {
1808  SDValue Src = Op.getOperand(0);
1809 
1810  // If the only bits demanded come from one byte of the bswap result,
1811  // just shift the input byte into position to eliminate the bswap.
1812  unsigned NLZ = DemandedBits.countLeadingZeros();
1813  unsigned NTZ = DemandedBits.countTrailingZeros();
1814 
1815  // Round NTZ down to the next byte. If we have 11 trailing zeros, then
1816  // we need all the bits down to bit 8. Likewise, round NLZ. If we
1817  // have 14 leading zeros, round to 8.
1818  NLZ = alignDown(NLZ, 8);
1819  NTZ = alignDown(NTZ, 8);
1820  // If we need exactly one byte, we can do this transformation.
1821  if (BitWidth - NLZ - NTZ == 8) {
1822  // Replace this with either a left or right shift to get the byte into
1823  // the right place.
1824  unsigned ShiftOpcode = NLZ > NTZ ? ISD::SRL : ISD::SHL;
1825  if (!TLO.LegalOperations() || isOperationLegal(ShiftOpcode, VT)) {
1826  EVT ShiftAmtTy = getShiftAmountTy(VT, DL);
1827  unsigned ShiftAmount = NLZ > NTZ ? NLZ - NTZ : NTZ - NLZ;
1828  SDValue ShAmt = TLO.DAG.getConstant(ShiftAmount, dl, ShiftAmtTy);
1829  SDValue NewOp = TLO.DAG.getNode(ShiftOpcode, dl, VT, Src, ShAmt);
1830  return TLO.CombineTo(Op, NewOp);
1831  }
1832  }
1833 
1834  APInt DemandedSrcBits = DemandedBits.byteSwap();
1835  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1836  Depth + 1))
1837  return true;
1838  Known.One = Known2.One.byteSwap();
1839  Known.Zero = Known2.Zero.byteSwap();
1840  break;
1841  }
1842  case ISD::CTPOP: {
1843  // If only 1 bit is demanded, replace with PARITY as long as we're before
1844  // op legalization.
1845  // FIXME: Limit to scalars for now.
1846  if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
1847  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1848  Op.getOperand(0)));
1849 
1850  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1851  break;
1852  }
1853  case ISD::SIGN_EXTEND_INREG: {
1854  SDValue Op0 = Op.getOperand(0);
1855  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1856  unsigned ExVTBits = ExVT.getScalarSizeInBits();
1857 
1858  // If we only care about the highest bit, don't bother shifting right.
1859  if (DemandedBits.isSignMask()) {
1860  unsigned MinSignedBits =
1861  TLO.DAG.ComputeMaxSignificantBits(Op0, DemandedElts, Depth + 1);
1862  bool AlreadySignExtended = ExVTBits >= MinSignedBits;
1863  // However if the input is already sign extended we expect the sign
1864  // extension to be dropped altogether later and do not simplify.
1865  if (!AlreadySignExtended) {
1866  // Compute the correct shift amount type, which must be getShiftAmountTy
1867  // for scalar types after legalization.
1868  SDValue ShiftAmt = TLO.DAG.getConstant(BitWidth - ExVTBits, dl,
1869  getShiftAmountTy(VT, DL));
1870  return TLO.CombineTo(Op,
1871  TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1872  }
1873  }
1874 
1875  // If none of the extended bits are demanded, eliminate the sextinreg.
1876  if (DemandedBits.getActiveBits() <= ExVTBits)
1877  return TLO.CombineTo(Op, Op0);
1878 
1879  APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1880 
1881  // Since the sign extended bits are demanded, we know that the sign
1882  // bit is demanded.
1883  InputDemandedBits.setBit(ExVTBits - 1);
1884 
1885  if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1886  return true;
1887  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1888 
1889  // If the sign bit of the input is known set or clear, then we know the
1890  // top bits of the result.
1891 
1892  // If the input sign bit is known zero, convert this into a zero extension.
1893  if (Known.Zero[ExVTBits - 1])
1894  return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1895 
1896  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1897  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1898  Known.One.setBitsFrom(ExVTBits);
1899  Known.Zero &= Mask;
1900  } else { // Input sign bit unknown
1901  Known.Zero &= Mask;
1902  Known.One &= Mask;
1903  }
1904  break;
1905  }
1906  case ISD::BUILD_PAIR: {
1907  EVT HalfVT = Op.getOperand(0).getValueType();
1908  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1909 
1910  APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1911  APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1912 
1913  KnownBits KnownLo, KnownHi;
1914 
1915  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1916  return true;
1917 
1918  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1919  return true;
1920 
1921  Known.Zero = KnownLo.Zero.zext(BitWidth) |
1922  KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1923 
1924  Known.One = KnownLo.One.zext(BitWidth) |
1925  KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1926  break;
1927  }
1928  case ISD::ZERO_EXTEND:
1930  SDValue Src = Op.getOperand(0);
1931  EVT SrcVT = Src.getValueType();
1932  unsigned InBits = SrcVT.getScalarSizeInBits();
1933  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1934  bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1935 
1936  // If none of the top bits are demanded, convert this into an any_extend.
1937  if (DemandedBits.getActiveBits() <= InBits) {
1938  // If we only need the non-extended bits of the bottom element
1939  // then we can just bitcast to the result.
1940  if (IsLE && IsVecInReg && DemandedElts == 1 &&
1941  VT.getSizeInBits() == SrcVT.getSizeInBits())
1942  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1943 
1944  unsigned Opc =
1946  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1947  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1948  }
1949 
1950  APInt InDemandedBits = DemandedBits.trunc(InBits);
1951  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1952  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1953  Depth + 1))
1954  return true;
1955  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1956  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1957  Known = Known.zext(BitWidth);
1958 
1959  // Attempt to avoid multi-use ops if we don't need anything from them.
1960  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1961  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1962  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1963  break;
1964  }
1965  case ISD::SIGN_EXTEND:
1967  SDValue Src = Op.getOperand(0);
1968  EVT SrcVT = Src.getValueType();
1969  unsigned InBits = SrcVT.getScalarSizeInBits();
1970  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1971  bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1972 
1973  // If none of the top bits are demanded, convert this into an any_extend.
1974  if (DemandedBits.getActiveBits() <= InBits) {
1975  // If we only need the non-extended bits of the bottom element
1976  // then we can just bitcast to the result.
1977  if (IsLE && IsVecInReg && DemandedElts == 1 &&
1978  VT.getSizeInBits() == SrcVT.getSizeInBits())
1979  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1980 
1981  unsigned Opc =
1983  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1984  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1985  }
1986 
1987  APInt InDemandedBits = DemandedBits.trunc(InBits);
1988  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1989 
1990  // Since some of the sign extended bits are demanded, we know that the sign
1991  // bit is demanded.
1992  InDemandedBits.setBit(InBits - 1);
1993 
1994  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1995  Depth + 1))
1996  return true;
1997  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1998  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1999 
2000  // If the sign bit is known one, the top bits match.
2001  Known = Known.sext(BitWidth);
2002 
2003  // If the sign bit is known zero, convert this to a zero extend.
2004  if (Known.isNonNegative()) {
2005  unsigned Opc =
2007  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
2008  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
2009  }
2010 
2011  // Attempt to avoid multi-use ops if we don't need anything from them.
2012  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2013  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2014  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2015  break;
2016  }
2017  case ISD::ANY_EXTEND:
2019  SDValue Src = Op.getOperand(0);
2020  EVT SrcVT = Src.getValueType();
2021  unsigned InBits = SrcVT.getScalarSizeInBits();
2022  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2023  bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
2024 
2025  // If we only need the bottom element then we can just bitcast.
2026  // TODO: Handle ANY_EXTEND?
2027  if (IsLE && IsVecInReg && DemandedElts == 1 &&
2028  VT.getSizeInBits() == SrcVT.getSizeInBits())
2029  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2030 
2031  APInt InDemandedBits = DemandedBits.trunc(InBits);
2032  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
2033  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
2034  Depth + 1))
2035  return true;
2036  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2037  assert(Known.getBitWidth() == InBits && "Src width has changed?");
2038  Known = Known.anyext(BitWidth);
2039 
2040  // Attempt to avoid multi-use ops if we don't need anything from them.
2041  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2042  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
2043  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
2044  break;
2045  }
2046  case ISD::TRUNCATE: {
2047  SDValue Src = Op.getOperand(0);
2048 
2049  // Simplify the input, using demanded bit information, and compute the known
2050  // zero/one bits live out.
2051  unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2052  APInt TruncMask = DemandedBits.zext(OperandBitWidth);
2053  if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2054  Depth + 1))
2055  return true;
2056  Known = Known.trunc(BitWidth);
2057 
2058  // Attempt to avoid multi-use ops if we don't need anything from them.
2059  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2060  Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2061  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2062 
2063  // If the input is only used by this truncate, see if we can shrink it based
2064  // on the known demanded bits.
2065  if (Src.getNode()->hasOneUse()) {
2066  switch (Src.getOpcode()) {
2067  default:
2068  break;
2069  case ISD::SRL:
2070  // Shrink SRL by a constant if none of the high bits shifted in are
2071  // demanded.
2072  if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2073  // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2074  // undesirable.
2075  break;
2076 
2077  const APInt *ShAmtC =
2078  TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2079  if (!ShAmtC || ShAmtC->uge(BitWidth))
2080  break;
2081  uint64_t ShVal = ShAmtC->getZExtValue();
2082 
2083  APInt HighBits =
2084  APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2085  HighBits.lshrInPlace(ShVal);
2086  HighBits = HighBits.trunc(BitWidth);
2087 
2088  if (!(HighBits & DemandedBits)) {
2089  // None of the shifted in bits are needed. Add a truncate of the
2090  // shift input, then shift it.
2091  SDValue NewShAmt = TLO.DAG.getConstant(
2092  ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2093  SDValue NewTrunc =
2094  TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2095  return TLO.CombineTo(
2096  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2097  }
2098  break;
2099  }
2100  }
2101 
2102  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2103  break;
2104  }
2105  case ISD::AssertZext: {
2106  // AssertZext demands all of the high bits, plus any of the low bits
2107  // demanded by its users.
2108  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2110  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2111  TLO, Depth + 1))
2112  return true;
2113  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2114 
2115  Known.Zero |= ~InMask;
2116  break;
2117  }
2118  case ISD::EXTRACT_VECTOR_ELT: {
2119  SDValue Src = Op.getOperand(0);
2120  SDValue Idx = Op.getOperand(1);
2121  ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2122  unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2123 
2124  if (SrcEltCnt.isScalable())
2125  return false;
2126 
2127  // Demand the bits from every vector element without a constant index.
2128  unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2129  APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
2130  if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2131  if (CIdx->getAPIntValue().ult(NumSrcElts))
2132  DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2133 
2134  // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2135  // anything about the extended bits.
2136  APInt DemandedSrcBits = DemandedBits;
2137  if (BitWidth > EltBitWidth)
2138  DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2139 
2140  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2141  Depth + 1))
2142  return true;
2143 
2144  // Attempt to avoid multi-use ops if we don't need anything from them.
2145  if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2146  if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2147  Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2148  SDValue NewOp =
2149  TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2150  return TLO.CombineTo(Op, NewOp);
2151  }
2152  }
2153 
2154  Known = Known2;
2155  if (BitWidth > EltBitWidth)
2156  Known = Known.anyext(BitWidth);
2157  break;
2158  }
2159  case ISD::BITCAST: {
2160  SDValue Src = Op.getOperand(0);
2161  EVT SrcVT = Src.getValueType();
2162  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2163 
2164  // If this is an FP->Int bitcast and if the sign bit is the only
2165  // thing demanded, turn this into a FGETSIGN.
2166  if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2167  DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2168  SrcVT.isFloatingPoint()) {
2169  bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2171  if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2172  SrcVT != MVT::f128) {
2173  // Cannot eliminate/lower SHL for f128 yet.
2174  EVT Ty = OpVTLegal ? VT : MVT::i32;
2175  // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2176  // place. We expect the SHL to be eliminated by other optimizations.
2177  SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2178  unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2179  if (!OpVTLegal && OpVTSizeInBits > 32)
2180  Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2181  unsigned ShVal = Op.getValueSizeInBits() - 1;
2182  SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2183  return TLO.CombineTo(Op,
2184  TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2185  }
2186  }
2187 
2188  // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2189  // Demand the elt/bit if any of the original elts/bits are demanded.
2190  if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0) {
2191  unsigned Scale = BitWidth / NumSrcEltBits;
2192  unsigned NumSrcElts = SrcVT.getVectorNumElements();
2193  APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2194  APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2195  for (unsigned i = 0; i != Scale; ++i) {
2196  unsigned EltOffset = IsLE ? i : (Scale - 1 - i);
2197  unsigned BitOffset = EltOffset * NumSrcEltBits;
2198  APInt Sub = DemandedBits.extractBits(NumSrcEltBits, BitOffset);
2199  if (!Sub.isZero()) {
2200  DemandedSrcBits |= Sub;
2201  for (unsigned j = 0; j != NumElts; ++j)
2202  if (DemandedElts[j])
2203  DemandedSrcElts.setBit((j * Scale) + i);
2204  }
2205  }
2206 
2207  APInt KnownSrcUndef, KnownSrcZero;
2208  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2209  KnownSrcZero, TLO, Depth + 1))
2210  return true;
2211 
2212  KnownBits KnownSrcBits;
2213  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2214  KnownSrcBits, TLO, Depth + 1))
2215  return true;
2216  } else if (IsLE && (NumSrcEltBits % BitWidth) == 0) {
2217  // TODO - bigendian once we have test coverage.
2218  unsigned Scale = NumSrcEltBits / BitWidth;
2219  unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2220  APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2221  APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2222  for (unsigned i = 0; i != NumElts; ++i)
2223  if (DemandedElts[i]) {
2224  unsigned Offset = (i % Scale) * BitWidth;
2225  DemandedSrcBits.insertBits(DemandedBits, Offset);
2226  DemandedSrcElts.setBit(i / Scale);
2227  }
2228 
2229  if (SrcVT.isVector()) {
2230  APInt KnownSrcUndef, KnownSrcZero;
2231  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2232  KnownSrcZero, TLO, Depth + 1))
2233  return true;
2234  }
2235 
2236  KnownBits KnownSrcBits;
2237  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2238  KnownSrcBits, TLO, Depth + 1))
2239  return true;
2240  }
2241 
2242  // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2243  // recursive call where Known may be useful to the caller.
2244  if (Depth > 0) {
2245  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2246  return false;
2247  }
2248  break;
2249  }
2250  case ISD::ADD:
2251  case ISD::MUL:
2252  case ISD::SUB: {
2253  // Add, Sub, and Mul don't demand any bits in positions beyond that
2254  // of the highest bit demanded of them.
2255  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2256  SDNodeFlags Flags = Op.getNode()->getFlags();
2257  unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2258  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2259  if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2260  Depth + 1) ||
2261  SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2262  Depth + 1) ||
2263  // See if the operation should be performed at a smaller bit width.
2264  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2265  if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2266  // Disable the nsw and nuw flags. We can no longer guarantee that we
2267  // won't wrap after simplification.
2268  Flags.setNoSignedWrap(false);
2269  Flags.setNoUnsignedWrap(false);
2270  SDValue NewOp =
2271  TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2272  return TLO.CombineTo(Op, NewOp);
2273  }
2274  return true;
2275  }
2276 
2277  // Attempt to avoid multi-use ops if we don't need anything from them.
2278  if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2279  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2280  Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2281  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2282  Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2283  if (DemandedOp0 || DemandedOp1) {
2284  Flags.setNoSignedWrap(false);
2285  Flags.setNoUnsignedWrap(false);
2286  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2287  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2288  SDValue NewOp =
2289  TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2290  return TLO.CombineTo(Op, NewOp);
2291  }
2292  }
2293 
2294  // If we have a constant operand, we may be able to turn it into -1 if we
2295  // do not demand the high bits. This can make the constant smaller to
2296  // encode, allow more general folding, or match specialized instruction
2297  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2298  // is probably not useful (and could be detrimental).
2300  APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2301  if (C && !C->isAllOnes() && !C->isOne() &&
2302  (C->getAPIntValue() | HighMask).isAllOnes()) {
2303  SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2304  // Disable the nsw and nuw flags. We can no longer guarantee that we
2305  // won't wrap after simplification.
2306  Flags.setNoSignedWrap(false);
2307  Flags.setNoUnsignedWrap(false);
2308  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2309  return TLO.CombineTo(Op, NewOp);
2310  }
2311 
2313  }
2314  default:
2315  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2316  if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2317  Known, TLO, Depth))
2318  return true;
2319  break;
2320  }
2321 
2322  // Just use computeKnownBits to compute output bits.
2323  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2324  break;
2325  }
2326 
2327  // If we know the value of all of the demanded bits, return this as a
2328  // constant.
2329  if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2330  // Avoid folding to a constant if any OpaqueConstant is involved.
2331  const SDNode *N = Op.getNode();
2332  for (SDNode *Op :
2334  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2335  if (C->isOpaque())
2336  return false;
2337  }
2338  if (VT.isInteger())
2339  return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2340  if (VT.isFloatingPoint())
2341  return TLO.CombineTo(
2342  Op,
2343  TLO.DAG.getConstantFP(
2344  APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2345  }
2346 
2347  return false;
2348 }
2349 
2351  const APInt &DemandedElts,
2352  APInt &KnownUndef,
2353  APInt &KnownZero,
2354  DAGCombinerInfo &DCI) const {
2355  SelectionDAG &DAG = DCI.DAG;
2356  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2357  !DCI.isBeforeLegalizeOps());
2358 
2359  bool Simplified =
2360  SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2361  if (Simplified) {
2362  DCI.AddToWorklist(Op.getNode());
2363  DCI.CommitTargetLoweringOpt(TLO);
2364  }
2365 
2366  return Simplified;
2367 }
2368 
2369 /// Given a vector binary operation and known undefined elements for each input
2370 /// operand, compute whether each element of the output is undefined.
2372  const APInt &UndefOp0,
2373  const APInt &UndefOp1) {
2374  EVT VT = BO.getValueType();
2375  assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
2376  "Vector binop only");
2377 
2378  EVT EltVT = VT.getVectorElementType();
2379  unsigned NumElts = VT.getVectorNumElements();
2380  assert(UndefOp0.getBitWidth() == NumElts &&
2381  UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
2382 
2383  auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2384  const APInt &UndefVals) {
2385  if (UndefVals[Index])
2386  return DAG.getUNDEF(EltVT);
2387 
2388  if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2389  // Try hard to make sure that the getNode() call is not creating temporary
2390  // nodes. Ignore opaque integers because they do not constant fold.
2391  SDValue Elt = BV->getOperand(Index);
2392  auto *C = dyn_cast<ConstantSDNode>(Elt);
2393  if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2394  return Elt;
2395  }
2396 
2397  return SDValue();
2398  };
2399 
2400  APInt KnownUndef = APInt::getZero(NumElts);
2401  for (unsigned i = 0; i != NumElts; ++i) {
2402  // If both inputs for this element are either constant or undef and match
2403  // the element type, compute the constant/undef result for this element of
2404  // the vector.
2405  // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2406  // not handle FP constants. The code within getNode() should be refactored
2407  // to avoid the danger of creating a bogus temporary node here.
2408  SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2409  SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2410  if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2411  if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2412  KnownUndef.setBit(i);
2413  }
2414  return KnownUndef;
2415 }
2416 
2418  SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2419  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2420  bool AssumeSingleUse) const {
2421  EVT VT = Op.getValueType();
2422  unsigned Opcode = Op.getOpcode();
2423  APInt DemandedElts = OriginalDemandedElts;
2424  unsigned NumElts = DemandedElts.getBitWidth();
2425  assert(VT.isVector() && "Expected vector op");
2426 
2427  KnownUndef = KnownZero = APInt::getZero(NumElts);
2428 
2429  // TODO: For now we assume we know nothing about scalable vectors.
2430  if (VT.isScalableVector())
2431  return false;
2432 
2433  assert(VT.getVectorNumElements() == NumElts &&
2434  "Mask size mismatches value type element count!");
2435 
2436  // Undef operand.
2437  if (Op.isUndef()) {
2438  KnownUndef.setAllBits();
2439  return false;
2440  }
2441 
2442  // If Op has other users, assume that all elements are needed.
2443  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2444  DemandedElts.setAllBits();
2445 
2446  // Not demanding any elements from Op.
2447  if (DemandedElts == 0) {
2448  KnownUndef.setAllBits();
2449  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2450  }
2451 
2452  // Limit search depth.
2454  return false;
2455 
2456  SDLoc DL(Op);
2457  unsigned EltSizeInBits = VT.getScalarSizeInBits();
2458  bool IsLE = TLO.DAG.getDataLayout().isLittleEndian();
2459 
2460  // Helper for demanding the specified elements and all the bits of both binary
2461  // operands.
2462  auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2463  SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2464  TLO.DAG, Depth + 1);
2465  SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2466  TLO.DAG, Depth + 1);
2467  if (NewOp0 || NewOp1) {
2468  SDValue NewOp = TLO.DAG.getNode(
2469  Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2470  return TLO.CombineTo(Op, NewOp);
2471  }
2472  return false;
2473  };
2474 
2475  switch (Opcode) {
2476  case ISD::SCALAR_TO_VECTOR: {
2477  if (!DemandedElts[0]) {
2478  KnownUndef.setAllBits();
2479  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2480  }
2481  SDValue ScalarSrc = Op.getOperand(0);
2482  if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2483  SDValue Src = ScalarSrc.getOperand(0);
2484  SDValue Idx = ScalarSrc.getOperand(1);
2485  EVT SrcVT = Src.getValueType();
2486 
2487  ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2488 
2489  if (SrcEltCnt.isScalable())
2490  return false;
2491 
2492  unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2493  if (isNullConstant(Idx)) {
2494  APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2495  APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2496  APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2497  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2498  TLO, Depth + 1))
2499  return true;
2500  }
2501  }
2502  KnownUndef.setHighBits(NumElts - 1);
2503  break;
2504  }
2505  case ISD::BITCAST: {
2506  SDValue Src = Op.getOperand(0);
2507  EVT SrcVT = Src.getValueType();
2508 
2509  // We only handle vectors here.
2510  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2511  if (!SrcVT.isVector())
2512  break;
2513 
2514  // Fast handling of 'identity' bitcasts.
2515  unsigned NumSrcElts = SrcVT.getVectorNumElements();
2516  if (NumSrcElts == NumElts)
2517  return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2518  KnownZero, TLO, Depth + 1);
2519 
2520  APInt SrcDemandedElts, SrcZero, SrcUndef;
2521 
2522  // Bitcast from 'large element' src vector to 'small element' vector, we
2523  // must demand a source element if any DemandedElt maps to it.
2524  if ((NumElts % NumSrcElts) == 0) {
2525  unsigned Scale = NumElts / NumSrcElts;
2526  SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2527  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2528  TLO, Depth + 1))
2529  return true;
2530 
2531  // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2532  // of the large element.
2533  // TODO - bigendian once we have test coverage.
2534  if (IsLE) {
2535  unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2536  APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
2537  for (unsigned i = 0; i != NumElts; ++i)
2538  if (DemandedElts[i]) {
2539  unsigned Ofs = (i % Scale) * EltSizeInBits;
2540  SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2541  }
2542 
2543  KnownBits Known;
2544  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2545  TLO, Depth + 1))
2546  return true;
2547  }
2548 
2549  // If the src element is zero/undef then all the output elements will be -
2550  // only demanded elements are guaranteed to be correct.
2551  for (unsigned i = 0; i != NumSrcElts; ++i) {
2552  if (SrcDemandedElts[i]) {
2553  if (SrcZero[i])
2554  KnownZero.setBits(i * Scale, (i + 1) * Scale);
2555  if (SrcUndef[i])
2556  KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2557  }
2558  }
2559  }
2560 
2561  // Bitcast from 'small element' src vector to 'large element' vector, we
2562  // demand all smaller source elements covered by the larger demanded element
2563  // of this vector.
2564  if ((NumSrcElts % NumElts) == 0) {
2565  unsigned Scale = NumSrcElts / NumElts;
2566  SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2567  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2568  TLO, Depth + 1))
2569  return true;
2570 
2571  // If all the src elements covering an output element are zero/undef, then
2572  // the output element will be as well, assuming it was demanded.
2573  for (unsigned i = 0; i != NumElts; ++i) {
2574  if (DemandedElts[i]) {
2575  if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
2576  KnownZero.setBit(i);
2577  if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
2578  KnownUndef.setBit(i);
2579  }
2580  }
2581  }
2582  break;
2583  }
2584  case ISD::BUILD_VECTOR: {
2585  // Check all elements and simplify any unused elements with UNDEF.
2586  if (!DemandedElts.isAllOnes()) {
2587  // Don't simplify BROADCASTS.
2588  if (llvm::any_of(Op->op_values(),
2589  [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2590  SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2591  bool Updated = false;
2592  for (unsigned i = 0; i != NumElts; ++i) {
2593  if (!DemandedElts[i] && !Ops[i].isUndef()) {
2594  Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2595  KnownUndef.setBit(i);
2596  Updated = true;
2597  }
2598  }
2599  if (Updated)
2600  return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2601  }
2602  }
2603  for (unsigned i = 0; i != NumElts; ++i) {
2604  SDValue SrcOp = Op.getOperand(i);
2605  if (SrcOp.isUndef()) {
2606  KnownUndef.setBit(i);
2607  } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2609  KnownZero.setBit(i);
2610  }
2611  }
2612  break;
2613  }
2614  case ISD::CONCAT_VECTORS: {
2615  EVT SubVT = Op.getOperand(0).getValueType();
2616  unsigned NumSubVecs = Op.getNumOperands();
2617  unsigned NumSubElts = SubVT.getVectorNumElements();
2618  for (unsigned i = 0; i != NumSubVecs; ++i) {
2619  SDValue SubOp = Op.getOperand(i);
2620  APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2621  APInt SubUndef, SubZero;
2622  if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2623  Depth + 1))
2624  return true;
2625  KnownUndef.insertBits(SubUndef, i * NumSubElts);
2626  KnownZero.insertBits(SubZero, i * NumSubElts);
2627  }
2628  break;
2629  }
2630  case ISD::INSERT_SUBVECTOR: {
2631  // Demand any elements from the subvector and the remainder from the src its
2632  // inserted into.
2633  SDValue Src = Op.getOperand(0);
2634  SDValue Sub = Op.getOperand(1);
2635  uint64_t Idx = Op.getConstantOperandVal(2);
2636  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2637  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2638  APInt DemandedSrcElts = DemandedElts;
2639  DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
2640 
2641  APInt SubUndef, SubZero;
2642  if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2643  Depth + 1))
2644  return true;
2645 
2646  // If none of the src operand elements are demanded, replace it with undef.
2647  if (!DemandedSrcElts && !Src.isUndef())
2648  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2649  TLO.DAG.getUNDEF(VT), Sub,
2650  Op.getOperand(2)));
2651 
2652  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2653  TLO, Depth + 1))
2654  return true;
2655  KnownUndef.insertBits(SubUndef, Idx);
2656  KnownZero.insertBits(SubZero, Idx);
2657 
2658  // Attempt to avoid multi-use ops if we don't need anything from them.
2659  if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
2660  SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2661  Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2662  SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2663  Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2664  if (NewSrc || NewSub) {
2665  NewSrc = NewSrc ? NewSrc : Src;
2666  NewSub = NewSub ? NewSub : Sub;
2667  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2668  NewSub, Op.getOperand(2));
2669  return TLO.CombineTo(Op, NewOp);
2670  }
2671  }
2672  break;
2673  }
2674  case ISD::EXTRACT_SUBVECTOR: {
2675  // Offset the demanded elts by the subvector index.
2676  SDValue Src = Op.getOperand(0);
2677  if (Src.getValueType().isScalableVector())
2678  break;
2679  uint64_t Idx = Op.getConstantOperandVal(1);
2680  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2681  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2682 
2683  APInt SrcUndef, SrcZero;
2684  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2685  Depth + 1))
2686  return true;
2687  KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2688  KnownZero = SrcZero.extractBits(NumElts, Idx);
2689 
2690  // Attempt to avoid multi-use ops if we don't need anything from them.
2691  if (!DemandedElts.isAllOnes()) {
2692  SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2693  Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2694  if (NewSrc) {
2695  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2696  Op.getOperand(1));
2697  return TLO.CombineTo(Op, NewOp);
2698  }
2699  }
2700  break;
2701  }
2702  case ISD::INSERT_VECTOR_ELT: {
2703  SDValue Vec = Op.getOperand(0);
2704  SDValue Scl = Op.getOperand(1);
2705  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2706 
2707  // For a legal, constant insertion index, if we don't need this insertion
2708  // then strip it, else remove it from the demanded elts.
2709  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2710  unsigned Idx = CIdx->getZExtValue();
2711  if (!DemandedElts[Idx])
2712  return TLO.CombineTo(Op, Vec);
2713 
2714  APInt DemandedVecElts(DemandedElts);
2715  DemandedVecElts.clearBit(Idx);
2716  if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2717  KnownZero, TLO, Depth + 1))
2718  return true;
2719 
2720  KnownUndef.setBitVal(Idx, Scl.isUndef());
2721 
2722  KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2723  break;
2724  }
2725 
2726  APInt VecUndef, VecZero;
2727  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2728  Depth + 1))
2729  return true;
2730  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2731  break;
2732  }
2733  case ISD::VSELECT: {
2734  // Try to transform the select condition based on the current demanded
2735  // elements.
2736  // TODO: If a condition element is undef, we can choose from one arm of the
2737  // select (and if one arm is undef, then we can propagate that to the
2738  // result).
2739  // TODO - add support for constant vselect masks (see IR version of this).
2740  APInt UnusedUndef, UnusedZero;
2741  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2742  UnusedZero, TLO, Depth + 1))
2743  return true;
2744 
2745  // See if we can simplify either vselect operand.
2746  APInt DemandedLHS(DemandedElts);
2747  APInt DemandedRHS(DemandedElts);
2748  APInt UndefLHS, ZeroLHS;
2749  APInt UndefRHS, ZeroRHS;
2750  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2751  ZeroLHS, TLO, Depth + 1))
2752  return true;
2753  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2754  ZeroRHS, TLO, Depth + 1))
2755  return true;
2756 
2757  KnownUndef = UndefLHS & UndefRHS;
2758  KnownZero = ZeroLHS & ZeroRHS;
2759  break;
2760  }
2761  case ISD::VECTOR_SHUFFLE: {
2762  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2763 
2764  // Collect demanded elements from shuffle operands..
2765  APInt DemandedLHS(NumElts, 0);
2766  APInt DemandedRHS(NumElts, 0);
2767  for (unsigned i = 0; i != NumElts; ++i) {
2768  int M = ShuffleMask[i];
2769  if (M < 0 || !DemandedElts[i])
2770  continue;
2771  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
2772  if (M < (int)NumElts)
2773  DemandedLHS.setBit(M);
2774  else
2775  DemandedRHS.setBit(M - NumElts);
2776  }
2777 
2778  // See if we can simplify either shuffle operand.
2779  APInt UndefLHS, ZeroLHS;
2780  APInt UndefRHS, ZeroRHS;
2781  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2782  ZeroLHS, TLO, Depth + 1))
2783  return true;
2784  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2785  ZeroRHS, TLO, Depth + 1))
2786  return true;
2787 
2788  // Simplify mask using undef elements from LHS/RHS.
2789  bool Updated = false;
2790  bool IdentityLHS = true, IdentityRHS = true;
2791  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2792  for (unsigned i = 0; i != NumElts; ++i) {
2793  int &M = NewMask[i];
2794  if (M < 0)
2795  continue;
2796  if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2797  (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2798  Updated = true;
2799  M = -1;
2800  }
2801  IdentityLHS &= (M < 0) || (M == (int)i);
2802  IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2803  }
2804 
2805  // Update legal shuffle masks based on demanded elements if it won't reduce
2806  // to Identity which can cause premature removal of the shuffle mask.
2807  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2808  SDValue LegalShuffle =
2809  buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2810  NewMask, TLO.DAG);
2811  if (LegalShuffle)
2812  return TLO.CombineTo(Op, LegalShuffle);
2813  }
2814 
2815  // Propagate undef/zero elements from LHS/RHS.
2816  for (unsigned i = 0; i != NumElts; ++i) {
2817  int M = ShuffleMask[i];
2818  if (M < 0) {
2819  KnownUndef.setBit(i);
2820  } else if (M < (int)NumElts) {
2821  if (UndefLHS[M])
2822  KnownUndef.setBit(i);
2823  if (ZeroLHS[M])
2824  KnownZero.setBit(i);
2825  } else {
2826  if (UndefRHS[M - NumElts])
2827  KnownUndef.setBit(i);
2828  if (ZeroRHS[M - NumElts])
2829  KnownZero.setBit(i);
2830  }
2831  }
2832  break;
2833  }
2837  APInt SrcUndef, SrcZero;
2838  SDValue Src = Op.getOperand(0);
2839  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2840  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2841  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2842  Depth + 1))
2843  return true;
2844  KnownZero = SrcZero.zextOrTrunc(NumElts);
2845  KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2846 
2847  if (IsLE && Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2848  Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2849  DemandedSrcElts == 1) {
2850  // aext - if we just need the bottom element then we can bitcast.
2851  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2852  }
2853 
2854  if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2855  // zext(undef) upper bits are guaranteed to be zero.
2856  if (DemandedElts.isSubsetOf(KnownUndef))
2857  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2858  KnownUndef.clearAllBits();
2859 
2860  // zext - if we just need the bottom element then we can mask:
2861  // zext(and(x,c)) -> and(x,c') iff the zext is the only user of the and.
2862  if (IsLE && DemandedSrcElts == 1 && Src.getOpcode() == ISD::AND &&
2863  Op->isOnlyUserOf(Src.getNode()) &&
2864  Op.getValueSizeInBits() == Src.getValueSizeInBits()) {
2865  SDLoc DL(Op);
2866  EVT SrcVT = Src.getValueType();
2867  EVT SrcSVT = SrcVT.getScalarType();
2868  SmallVector<SDValue> MaskElts;
2869  MaskElts.push_back(TLO.DAG.getAllOnesConstant(DL, SrcSVT));
2870  MaskElts.append(NumSrcElts - 1, TLO.DAG.getConstant(0, DL, SrcSVT));
2871  SDValue Mask = TLO.DAG.getBuildVector(SrcVT, DL, MaskElts);
2872  if (SDValue Fold = TLO.DAG.FoldConstantArithmetic(
2873  ISD::AND, DL, SrcVT, {Src.getOperand(1), Mask})) {
2874  Fold = TLO.DAG.getNode(ISD::AND, DL, SrcVT, Src.getOperand(0), Fold);
2875  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Fold));
2876  }
2877  }
2878  }
2879  break;
2880  }
2881 
2882  // TODO: There are more binop opcodes that could be handled here - MIN,
2883  // MAX, saturated math, etc.
2884  case ISD::ADD: {
2885  SDValue Op0 = Op.getOperand(0);
2886  SDValue Op1 = Op.getOperand(1);
2887  if (Op0 == Op1 && Op->isOnlyUserOf(Op0.getNode())) {
2888  APInt UndefLHS, ZeroLHS;
2889  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2890  Depth + 1, /*AssumeSingleUse*/ true))
2891  return true;
2892  }
2894  }
2895  case ISD::OR:
2896  case ISD::XOR:
2897  case ISD::SUB:
2898  case ISD::FADD:
2899  case ISD::FSUB:
2900  case ISD::FMUL:
2901  case ISD::FDIV:
2902  case ISD::FREM: {
2903  SDValue Op0 = Op.getOperand(0);
2904  SDValue Op1 = Op.getOperand(1);
2905 
2906  APInt UndefRHS, ZeroRHS;
2907  if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2908  Depth + 1))
2909  return true;
2910  APInt UndefLHS, ZeroLHS;
2911  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2912  Depth + 1))
2913  return true;
2914 
2915  KnownZero = ZeroLHS & ZeroRHS;
2916  KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2917 
2918  // Attempt to avoid multi-use ops if we don't need anything from them.
2919  // TODO - use KnownUndef to relax the demandedelts?
2920  if (!DemandedElts.isAllOnes())
2921  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2922  return true;
2923  break;
2924  }
2925  case ISD::SHL:
2926  case ISD::SRL:
2927  case ISD::SRA:
2928  case ISD::ROTL:
2929  case ISD::ROTR: {
2930  SDValue Op0 = Op.getOperand(0);
2931  SDValue Op1 = Op.getOperand(1);
2932 
2933  APInt UndefRHS, ZeroRHS;
2934  if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2935  Depth + 1))
2936  return true;
2937  APInt UndefLHS, ZeroLHS;
2938  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2939  Depth + 1))
2940  return true;
2941 
2942  KnownZero = ZeroLHS;
2943  KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2944 
2945  // Attempt to avoid multi-use ops if we don't need anything from them.
2946  // TODO - use KnownUndef to relax the demandedelts?
2947  if (!DemandedElts.isAllOnes())
2948  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2949  return true;
2950  break;
2951  }
2952  case ISD::MUL:
2953  case ISD::AND: {
2954  SDValue Op0 = Op.getOperand(0);
2955  SDValue Op1 = Op.getOperand(1);
2956 
2957  APInt SrcUndef, SrcZero;
2958  if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2959  Depth + 1))
2960  return true;
2961  if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2962  TLO, Depth + 1))
2963  return true;
2964 
2965  // If either side has a zero element, then the result element is zero, even
2966  // if the other is an UNDEF.
2967  // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2968  // and then handle 'and' nodes with the rest of the binop opcodes.
2969  KnownZero |= SrcZero;
2970  KnownUndef &= SrcUndef;
2971  KnownUndef &= ~KnownZero;
2972 
2973  // Attempt to avoid multi-use ops if we don't need anything from them.
2974  // TODO - use KnownUndef to relax the demandedelts?
2975  if (!DemandedElts.isAllOnes())
2976  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2977  return true;
2978  break;
2979  }
2980  case ISD::TRUNCATE:
2981  case ISD::SIGN_EXTEND:
2982  case ISD::ZERO_EXTEND:
2983  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2984  KnownZero, TLO, Depth + 1))
2985  return true;
2986 
2987  if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2988  // zext(undef) upper bits are guaranteed to be zero.
2989  if (DemandedElts.isSubsetOf(KnownUndef))
2990  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2991  KnownUndef.clearAllBits();
2992  }
2993  break;
2994  default: {
2995  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2996  if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2997  KnownZero, TLO, Depth))
2998  return true;
2999  } else {
3000  KnownBits Known;
3001  APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
3002  if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
3003  TLO, Depth, AssumeSingleUse))
3004  return true;
3005  }
3006  break;
3007  }
3008  }
3009  assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
3010 
3011  // Constant fold all undef cases.
3012  // TODO: Handle zero cases as well.
3013  if (DemandedElts.isSubsetOf(KnownUndef))
3014  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
3015 
3016  return false;
3017 }
3018 
3019 /// Determine which of the bits specified in Mask are known to be either zero or
3020 /// one and return them in the Known.
3022  KnownBits &Known,
3023  const APInt &DemandedElts,
3024  const SelectionDAG &DAG,
3025  unsigned Depth) const {
3026  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3027  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3028  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3029  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3030  "Should use MaskedValueIsZero if you don't know whether Op"
3031  " is a target node!");
3032  Known.resetAll();
3033 }
3034 
3036  GISelKnownBits &Analysis, Register R, KnownBits &Known,
3037  const APInt &DemandedElts, const MachineRegisterInfo &MRI,
3038  unsigned Depth) const {
3039  Known.resetAll();
3040 }
3041 
3043  const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
3044  // The low bits are known zero if the pointer is aligned.
3045  Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
3046 }
3047 
3049  GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
3050  unsigned Depth) const {
3051  return Align(1);
3052 }
3053 
3054 /// This method can be implemented by targets that want to expose additional
3055 /// information about sign bits to the DAG Combiner.
3057  const APInt &,
3058  const SelectionDAG &,
3059  unsigned Depth) const {
3060  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3061  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3062  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3063  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3064  "Should use ComputeNumSignBits if you don't know whether Op"
3065  " is a target node!");
3066  return 1;
3067 }
3068 
3070  GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
3071  const MachineRegisterInfo &MRI, unsigned Depth) const {
3072  return 1;
3073 }
3074 
3076  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3077  TargetLoweringOpt &TLO, unsigned Depth) const {
3078  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3079  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3080  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3081  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3082  "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3083  " is a target node!");
3084  return false;
3085 }
3086 
3088  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3089  KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3090  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3091  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3092  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3093  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3094  "Should use SimplifyDemandedBits if you don't know whether Op"
3095  " is a target node!");
3096  computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3097  return false;
3098 }
3099 
3101  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3102  SelectionDAG &DAG, unsigned Depth) const {
3103  assert(
3104  (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3105  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3106  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3107  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3108  "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3109  " is a target node!");
3110  return SDValue();
3111 }
3112 
3113 SDValue
3116  SelectionDAG &DAG) const {
3117  bool LegalMask = isShuffleMaskLegal(Mask, VT);
3118  if (!LegalMask) {
3119  std::swap(N0, N1);
3121  LegalMask = isShuffleMaskLegal(Mask, VT);
3122  }
3123 
3124  if (!LegalMask)
3125  return SDValue();
3126 
3127  return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3128 }
3129 
3131  return nullptr;
3132 }
3133 
3135  SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3136  bool PoisonOnly, unsigned Depth) const {
3137  assert(
3138  (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3139  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3140  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3141  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3142  "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3143  " is a target node!");
3144  return false;
3145 }
3146 
3148  const SelectionDAG &DAG,
3149  bool SNaN,
3150  unsigned Depth) const {
3151  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3152  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3153  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3154  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3155  "Should use isKnownNeverNaN if you don't know whether Op"
3156  " is a target node!");
3157  return false;
3158 }
3159 
3161  const APInt &DemandedElts,
3162  APInt &UndefElts,
3163  unsigned Depth) const {
3164  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3165  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3166  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3167  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3168  "Should use isSplatValue if you don't know whether Op"
3169  " is a target node!");
3170  return false;
3171 }
3172 
3173 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3174 // work with truncating build vectors and vectors with elements of less than
3175 // 8 bits.
3177  if (!N)
3178  return false;
3179 
3180  APInt CVal;
3181  if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3182  CVal = CN->getAPIntValue();
3183  } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3184  auto *CN = BV->getConstantSplatNode();
3185  if (!CN)
3186  return false;
3187 
3188  // If this is a truncating build vector, truncate the splat value.
3189  // Otherwise, we may fail to match the expected values below.
3190  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3191  CVal = CN->getAPIntValue();
3192  if (BVEltWidth < CVal.getBitWidth())
3193  CVal = CVal.trunc(BVEltWidth);
3194  } else {
3195  return false;
3196  }
3197 
3198  switch (getBooleanContents(N->getValueType(0))) {
3200  return CVal[0];
3202  return CVal.isOne();
3204  return CVal.isAllOnes();
3205  }
3206 
3207  llvm_unreachable("Invalid boolean contents");
3208 }
3209 
3211  if (!N)
3212  return false;
3213 
3214  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3215  if (!CN) {
3216  const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3217  if (!BV)
3218  return false;
3219 
3220  // Only interested in constant splats, we don't care about undef
3221  // elements in identifying boolean constants and getConstantSplatNode
3222  // returns NULL if all ops are undef;
3223  CN = BV->getConstantSplatNode();
3224  if (!CN)
3225  return false;
3226  }
3227 
3228  if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3229  return !CN->getAPIntValue()[0];
3230 
3231  return CN->isZero();
3232 }
3233 
3235  bool SExt) const {
3236  if (VT == MVT::i1)
3237  return N->isOne();
3238 
3240  switch (Cnt) {
3242  // An extended value of 1 is always true, unless its original type is i1,
3243  // in which case it will be sign extended to -1.
3244  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3247  return N->isAllOnes() && SExt;
3248  }
3249  llvm_unreachable("Unexpected enumeration.");
3250 }
3251 
3252 /// This helper function of SimplifySetCC tries to optimize the comparison when
3253 /// either operand of the SetCC node is a bitwise-and instruction.
3254 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3255  ISD::CondCode Cond, const SDLoc &DL,
3256  DAGCombinerInfo &DCI) const {
3257  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3258  std::swap(N0, N1);
3259 
3260  SelectionDAG &DAG = DCI.DAG;
3261  EVT OpVT = N0.getValueType();
3262  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3263  (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3264  return SDValue();
3265 
3266  // (X & Y) != 0 --> zextOrTrunc(X & Y)
3267  // iff everything but LSB is known zero:
3268  if (Cond == ISD::SETNE && isNullConstant(N1) &&
3271  unsigned NumEltBits = OpVT.getScalarSizeInBits();
3272  APInt UpperBits = APInt::getHighBitsSet(NumEltBits, NumEltBits - 1);
3273  if (DAG.MaskedValueIsZero(N0, UpperBits))
3274  return DAG.getBoolExtOrTrunc(N0, DL, VT, OpVT);
3275  }
3276 
3277  // Match these patterns in any of their permutations:
3278  // (X & Y) == Y
3279  // (X & Y) != Y
3280  SDValue X, Y;
3281  if (N0.getOperand(0) == N1) {
3282  X = N0.getOperand(1);
3283  Y = N0.getOperand(0);
3284  } else if (N0.getOperand(1) == N1) {
3285  X = N0.getOperand(0);
3286  Y = N0.getOperand(1);
3287  } else {
3288  return SDValue();
3289  }
3290 
3291  SDValue Zero = DAG.getConstant(0, DL, OpVT);
3292  if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3293  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3294  // Note that where Y is variable and is known to have at most one bit set
3295  // (for example, if it is Z & 1) we cannot do this; the expressions are not
3296  // equivalent when Y == 0.
3297  assert(OpVT.isInteger());
3298  Cond = ISD::getSetCCInverse(Cond, OpVT);
3299  if (DCI.isBeforeLegalizeOps() ||
3301  return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3302  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3303  // If the target supports an 'and-not' or 'and-complement' logic operation,
3304  // try to use that to make a comparison operation more efficient.
3305  // But don't do this transform if the mask is a single bit because there are
3306  // more efficient ways to deal with that case (for example, 'bt' on x86 or
3307  // 'rlwinm' on PPC).
3308 
3309  // Bail out if the compare operand that we want to turn into a zero is
3310  // already a zero (otherwise, infinite loop).
3311  auto *YConst = dyn_cast<ConstantSDNode>(Y);
3312  if (YConst && YConst->isZero())
3313  return SDValue();
3314 
3315  // Transform this into: ~X & Y == 0.
3316  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3317  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3318  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3319  }
3320 
3321  return SDValue();
3322 }
3323 
3324 /// There are multiple IR patterns that could be checking whether certain
3325 /// truncation of a signed number would be lossy or not. The pattern which is
3326 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
3327 /// We are looking for the following pattern: (KeptBits is a constant)
3328 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3329 /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3330 /// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3331 /// We will unfold it into the natural trunc+sext pattern:
3332 /// ((%x << C) a>> C) dstcond %x
3333 /// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
3334 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3335  EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3336  const SDLoc &DL) const {
3337  // We must be comparing with a constant.
3338  ConstantSDNode *C1;
3339  if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3340  return SDValue();
3341 
3342  // N0 should be: add %x, (1 << (KeptBits-1))
3343  if (N0->getOpcode() != ISD::ADD)
3344  return SDValue();
3345 
3346  // And we must be 'add'ing a constant.
3347  ConstantSDNode *C01;
3348  if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3349  return SDValue();
3350 
3351  SDValue X = N0->getOperand(0);
3352  EVT XVT = X.getValueType();
3353 
3354  // Validate constants ...
3355 
3356  APInt I1 = C1->getAPIntValue();
3357 
3358  ISD::CondCode NewCond;
3359  if (Cond == ISD::CondCode::SETULT) {
3360  NewCond = ISD::CondCode::SETEQ;
3361  } else if (Cond == ISD::CondCode::SETULE) {
3362  NewCond = ISD::CondCode::SETEQ;
3363  // But need to 'canonicalize' the constant.
3364  I1 += 1;
3365  } else if (Cond == ISD::CondCode::SETUGT) {
3366  NewCond = ISD::CondCode::SETNE;
3367  // But need to 'canonicalize' the constant.
3368  I1 += 1;
3369  } else if (Cond == ISD::CondCode::SETUGE) {
3370  NewCond = ISD::CondCode::SETNE;
3371  } else
3372  return SDValue();
3373 
3374  APInt I01 = C01->getAPIntValue();
3375 
3376  auto checkConstants = [&I1, &I01]() -> bool {
3377  // Both of them must be power-of-two, and the constant from setcc is bigger.
3378  return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3379  };
3380 
3381  if (checkConstants()) {
3382  // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3383  } else {
3384  // What if we invert constants? (and the target predicate)
3385  I1.negate();
3386  I01.negate();
3387  assert(XVT.isInteger());
3388  NewCond = getSetCCInverse(NewCond, XVT);
3389  if (!checkConstants())
3390  return SDValue();
3391  // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3392  }
3393 
3394  // They are power-of-two, so which bit is set?
3395  const unsigned KeptBits = I1.logBase2();
3396  const unsigned KeptBitsMinusOne = I01.logBase2();
3397 
3398  // Magic!
3399  if (KeptBits != (KeptBitsMinusOne + 1))
3400  return SDValue();
3401  assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
3402 
3403  // We don't want to do this in every single case.
3404  SelectionDAG &DAG = DCI.DAG;
3406  XVT, KeptBits))
3407  return SDValue();
3408 
3409  const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3410  assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable");
3411 
3412  // Unfold into: ((%x << C) a>> C) cond %x
3413  // Where 'cond' will be either 'eq' or 'ne'.
3414  SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3415  SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3416  SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3417  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3418 
3419  return T2;
3420 }
3421 
3422 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3423 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3424  EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3425  DAGCombinerInfo &DCI, const SDLoc &DL) const {
3426  assert(isConstOrConstSplat(N1C) &&
3427  isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
3428  "Should be a comparison with 0.");
3429  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3430  "Valid only for [in]equality comparisons.");
3431 
3432  unsigned NewShiftOpcode;
3433  SDValue X, C, Y;
3434 
3435  SelectionDAG &DAG = DCI.DAG;
3436  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3437 
3438  // Look for '(C l>>/<< Y)'.
3439  auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3440  // The shift should be one-use.
3441  if (!V.hasOneUse())
3442  return false;
3443  unsigned OldShiftOpcode = V.getOpcode();
3444  switch (OldShiftOpcode) {
3445  case ISD::SHL:
3446  NewShiftOpcode = ISD::SRL;
3447  break;
3448  case ISD::SRL:
3449  NewShiftOpcode = ISD::SHL;
3450  break;
3451  default:
3452  return false; // must be a logical shift.
3453  }
3454  // We should be shifting a constant.
3455  // FIXME: best to use isConstantOrConstantVector().
3456  C = V.getOperand(0);
3457  ConstantSDNode *CC =
3458  isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3459  if (!CC)
3460  return false;
3461  Y = V.getOperand(1);
3462 
3463  ConstantSDNode *XC =
3464  isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3465  return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3466  X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3467  };
3468 
3469  // LHS of comparison should be an one-use 'and'.
3470  if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3471  return SDValue();
3472 
3473  X = N0.getOperand(0);
3474  SDValue Mask = N0.getOperand(1);
3475 
3476  // 'and' is commutative!
3477  if (!Match(Mask)) {
3478  std::swap(X, Mask);
3479  if (!Match(Mask))
3480  return SDValue();
3481  }
3482 
3483  EVT VT = X.getValueType();
3484 
3485  // Produce:
3486  // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3487  SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3488  SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3489  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3490  return T2;
3491 }
3492 
3493 /// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3494 /// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3495 /// handle the commuted versions of these patterns.
3496 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3497  ISD::CondCode Cond, const SDLoc &DL,
3498  DAGCombinerInfo &DCI) const {
3499  unsigned BOpcode = N0.getOpcode();
3500  assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
3501  "Unexpected binop");
3502  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
3503 
3504  // (X + Y) == X --> Y == 0
3505  // (X - Y) == X --> Y == 0
3506  // (X ^ Y) == X --> Y == 0
3507  SelectionDAG &DAG = DCI.DAG;
3508  EVT OpVT = N0.getValueType();
3509  SDValue X = N0.getOperand(0);
3510  SDValue Y = N0.getOperand(1);
3511  if (X == N1)
3512  return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3513 
3514  if (Y != N1)
3515  return SDValue();
3516 
3517  // (X + Y) == Y --> X == 0
3518  // (X ^ Y) == Y --> X == 0
3519  if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3520  return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3521 
3522  // The shift would not be valid if the operands are boolean (i1).
3523  if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3524  return SDValue();
3525 
3526  // (X - Y) == Y --> X == Y << 1
3527  EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3528  !DCI.isBeforeLegalize());
3529  SDValue One = DAG.getConstant(1, DL, ShiftVT);
3530  SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3531  if (!DCI.isCalledByLegalizer())
3532  DCI.AddToWorklist(YShl1.getNode());
3533  return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3534 }
3535 
3537  SDValue N0, const APInt &C1,
3538  ISD::CondCode Cond, const SDLoc &dl,
3539  SelectionDAG &DAG) {
3540  // Look through truncs that don't change the value of a ctpop.
3541  // FIXME: Add vector support? Need to be careful with setcc result type below.
3542  SDValue CTPOP = N0;
3543  if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3545  CTPOP = N0.getOperand(0);
3546 
3547  if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3548  return SDValue();
3549 
3550  EVT CTVT = CTPOP.getValueType();
3551  SDValue CTOp = CTPOP.getOperand(0);
3552 
3553  // If this is a vector CTPOP, keep the CTPOP if it is legal.
3554  // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3555  if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3556  return SDValue();
3557 
3558  // (ctpop x) u< 2 -> (x & x-1) == 0
3559  // (ctpop x) u> 1 -> (x & x-1) != 0
3560  if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3561  unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3562  if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3563  return SDValue();
3564  if (C1 == 0 && (Cond == ISD::SETULT))
3565  return SDValue(); // This is handled elsewhere.
3566 
3567  unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3568 
3569  SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3570  SDValue Result = CTOp;
3571  for (unsigned i = 0; i < Passes; i++) {
3572  SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3573  Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3574  }
3576  return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3577  }
3578 
3579  // If ctpop is not supported, expand a power-of-2 comparison based on it.
3580  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3581  // For scalars, keep CTPOP if it is legal or custom.
3582  if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3583  return SDValue();
3584  // This is based on X86's custom lowering for CTPOP which produces more
3585  // instructions than the expansion here.
3586 
3587  // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3588  // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3589  SDValue Zero = DAG.getConstant(0, dl, CTVT);
3590  SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3591  assert(CTVT.isInteger());
3592  ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3593  SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3594  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3595  SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3596  SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3597  unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3598  return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3599  }
3600 
3601  return SDValue();
3602 }
3603 
3604 /// Try to simplify a setcc built with the specified operands and cc. If it is
3605 /// unable to simplify it, return a null SDValue.
3607  ISD::CondCode Cond, bool foldBooleans,
3608  DAGCombinerInfo &DCI,
3609  const SDLoc &dl) const {
3610  SelectionDAG &DAG = DCI.DAG;
3611  const DataLayout &Layout = DAG.getDataLayout();
3612  EVT OpVT = N0.getValueType();
3613 
3614  // Constant fold or commute setcc.
3615  if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3616  return Fold;
3617 
3618  // Ensure that the constant occurs on the RHS and fold constant comparisons.
3619  // TODO: Handle non-splat vector constants. All undef causes trouble.
3620  // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3621  // infinite loop here when we encounter one.
3623  if (isConstOrConstSplat(N0) &&
3624  (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3625  (DCI.isBeforeLegalizeOps() ||
3626  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3627  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3628 
3629  // If we have a subtract with the same 2 non-constant operands as this setcc
3630  // -- but in reverse order -- then try to commute the operands of this setcc
3631  // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3632  // instruction on some targets.
3633  if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3634  (DCI.isBeforeLegalizeOps() ||
3635  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3636  DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3637  !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3638  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3639 
3640  if (auto *N1C = isConstOrConstSplat(N1)) {
3641  const APInt &C1 = N1C->getAPIntValue();
3642 
3643  // Optimize some CTPOP cases.
3644  if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3645  return V;
3646 
3647  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3648  // equality comparison, then we're just comparing whether X itself is
3649  // zero.
3650  if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
3651  N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3653  if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3654  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3655  ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3656  if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3657  // (srl (ctlz x), 5) == 0 -> X != 0
3658  // (srl (ctlz x), 5) != 1 -> X != 0
3659  Cond = ISD::SETNE;
3660  } else {
3661  // (srl (ctlz x), 5) != 0 -> X == 0
3662  // (srl (ctlz x), 5) == 1 -> X == 0
3663  Cond = ISD::SETEQ;
3664  }
3665  SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3666  return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3667  Cond);
3668  }
3669  }
3670  }
3671  }
3672 
3673  // FIXME: Support vectors.
3674  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3675  const APInt &C1 = N1C->getAPIntValue();
3676 
3677  // (zext x) == C --> x == (trunc C)
3678  // (sext x) == C --> x == (trunc C)
3679  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3680  DCI.isBeforeLegalize() && N0->hasOneUse()) {
3681  unsigned MinBits = N0.getValueSizeInBits();
3682  SDValue PreExt;
3683  bool Signed = false;
3684  if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3685  // ZExt
3686  MinBits = N0->getOperand(0).getValueSizeInBits();
3687  PreExt = N0->getOperand(0);
3688  } else if (N0->getOpcode() == ISD::AND) {
3689  // DAGCombine turns costly ZExts into ANDs
3690  if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3691  if ((C->getAPIntValue()+1).isPowerOf2()) {
3692  MinBits = C->getAPIntValue().countTrailingOnes();
3693  PreExt = N0->getOperand(0);
3694  }
3695  } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3696  // SExt
3697  MinBits = N0->getOperand(0).getValueSizeInBits();
3698  PreExt = N0->getOperand(0);
3699  Signed = true;
3700  } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3701  // ZEXTLOAD / SEXTLOAD
3702  if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3703  MinBits = LN0->getMemoryVT().getSizeInBits();
3704  PreExt = N0;
3705  } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3706  Signed = true;
3707  MinBits = LN0->getMemoryVT().getSizeInBits();
3708  PreExt = N0;
3709  }
3710  }
3711 
3712  // Figure out how many bits we need to preserve this constant.
3713  unsigned ReqdBits = Signed ? C1.getMinSignedBits() : C1.getActiveBits();
3714 
3715  // Make sure we're not losing bits from the constant.
3716  if (MinBits > 0 &&
3717  MinBits < C1.getBitWidth() &&
3718  MinBits >= ReqdBits) {
3719  EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3720  if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3721  // Will get folded away.
3722  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3723  if (MinBits == 1 && C1 == 1)
3724  // Invert the condition.
3725  return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3727  SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3728  return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3729  }
3730 
3731  // If truncating the setcc operands is not desirable, we can still
3732  // simplify the expression in some cases:
3733  // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3734  // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3735  // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3736  // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3737  // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3738  // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3739  SDValue TopSetCC = N0->getOperand(0);
3740  unsigned N0Opc = N0->getOpcode();
3741  bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3742  if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3743  TopSetCC.getOpcode() == ISD::SETCC &&
3744  (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3745  (isConstFalseVal(N1C) ||
3746  isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3747 
3748  bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
3749  (!N1C->isZero() && Cond == ISD::SETNE);
3750 
3751  if (!Inverse)
3752  return TopSetCC;
3753 
3755  cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3756  TopSetCC.getOperand(0).getValueType());
3757  return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3758  TopSetCC.getOperand(1),
3759  InvCond);
3760  }
3761  }
3762  }
3763 
3764  // If the LHS is '(and load, const)', the RHS is 0, the test is for
3765  // equality or unsigned, and all 1 bits of the const are in the same
3766  // partial word, see if we can shorten the load.
3767  if (DCI.isBeforeLegalize() &&
3769  N0.getOpcode() == ISD::AND && C1 == 0 &&
3770  N0.getNode()->hasOneUse() &&
3771  isa<LoadSDNode>(N0.getOperand(0)) &&
3772  N0.getOperand(0).getNode()->hasOneUse() &&
3773  isa<ConstantSDNode>(N0.getOperand(1))) {
3774  LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3775  APInt bestMask;
3776  unsigned bestWidth = 0, bestOffset = 0;
3777  if (Lod->isSimple() && Lod->isUnindexed()) {
3778  unsigned origWidth = N0.getValueSizeInBits();
3779  unsigned maskWidth = origWidth;
3780  // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3781  // 8 bits, but have to be careful...
3782  if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3783  origWidth = Lod->getMemoryVT().getSizeInBits();
3784  const APInt &Mask = N0.getConstantOperandAPInt(1);
3785  for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3786  APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3787  for (unsigned offset=0; offset<origWidth/width; offset++) {
3788  if (Mask.isSubsetOf(newMask)) {
3789  if (Layout.isLittleEndian())
3790  bestOffset = (uint64_t)offset * (width/8);
3791  else
3792  bestOffset = (origWidth/width - offset - 1) * (width/8);
3793  bestMask = Mask.lshr(offset * (width/8) * 8);
3794  bestWidth = width;
3795  break;
3796  }
3797  newMask <<= width;
3798  }
3799  }
3800  }
3801  if (bestWidth) {
3802  EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3803  if (newVT.isRound() &&
3804  shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3805  SDValue Ptr = Lod->getBasePtr();
3806  if (bestOffset != 0)
3807  Ptr =
3808  DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3809  SDValue NewLoad =
3810  DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3811  Lod->getPointerInfo().getWithOffset(bestOffset),
3812  Lod->getOriginalAlign());
3813  return DAG.getSetCC(dl, VT,
3814  DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3815  DAG.getConstant(bestMask.trunc(bestWidth),
3816  dl, newVT)),
3817  DAG.getConstant(0LL, dl, newVT), Cond);
3818  }
3819  }
3820  }
3821 
3822  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3823  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3824  unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3825 
3826  // If the comparison constant has bits in the upper part, the
3827  // zero-extended value could never match.
3828  if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3829  C1.getBitWidth() - InSize))) {
3830  switch (Cond) {
3831  case ISD::SETUGT:
3832  case ISD::SETUGE:
3833  case ISD::SETEQ:
3834  return DAG.getConstant(0, dl, VT);
3835  case ISD::SETULT:
3836  case ISD::SETULE:
3837  case ISD::SETNE:
3838  return DAG.getConstant(1, dl, VT);
3839  case ISD::SETGT:
3840  case ISD::SETGE:
3841  // True if the sign bit of C1 is set.
3842  return DAG.getConstant(C1.isNegative(), dl, VT);
3843  case ISD::SETLT:
3844  case ISD::SETLE:
3845  // True if the sign bit of C1 isn't set.
3846  return DAG.getConstant(C1.isNonNegative(), dl, VT);
3847  default:
3848  break;
3849  }
3850  }
3851 
3852  // Otherwise, we can perform the comparison with the low bits.
3853  switch (Cond) {
3854  case ISD::SETEQ:
3855  case ISD::SETNE:
3856  case ISD::SETUGT:
3857  case ISD::SETUGE:
3858  case ISD::SETULT:
3859  case ISD::SETULE: {
3860  EVT newVT = N0.getOperand(0).getValueType();
3861  if (DCI.isBeforeLegalizeOps() ||
3862  (isOperationLegal(ISD::SETCC, newVT) &&
3863  isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3864  EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3865  SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3866 
3867  SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3868  NewConst, Cond);
3869  return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3870  }
3871  break;
3872  }
3873  default:
3874  break; // todo, be more careful with signed comparisons
3875  }
3876  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3877  (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3878  !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3879  OpVT)) {
3880  EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3881  unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3882  EVT ExtDstTy = N0.getValueType();
3883  unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3884 
3885  // If the constant doesn't fit into the number of bits for the source of
3886  // the sign extension, it is impossible for both sides to be equal.
3887  if (C1.getMinSignedBits() > ExtSrcTyBits)
3888  return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3889 
3890  assert(ExtDstTy == N0.getOperand(0).getValueType() &&
3891  ExtDstTy != ExtSrcTy && "Unexpected types!");
3892  APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3893  SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3894  DAG.getConstant(Imm, dl, ExtDstTy));
3895  if (!DCI.isCalledByLegalizer())
3896  DCI.AddToWorklist(ZextOp.getNode());
3897  // Otherwise, make this a use of a zext.
3898  return DAG.getSetCC(dl, VT, ZextOp,
3899  DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3900  } else if ((N1C->isZero() || N1C->isOne()) &&
3901  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3902  // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3903  if (N0.getOpcode() == ISD::SETCC &&
3904  isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3905  (N0.getValueType() == MVT::i1 ||
3908  bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3909  if (TrueWhenTrue)
3910  return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3911  // Invert the condition.
3912  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3913  CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3914  if (DCI.isBeforeLegalizeOps() ||
3916  return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3917  }
3918 
3919  if ((N0.getOpcode() == ISD::XOR ||
3920  (N0.getOpcode() == ISD::AND &&
3921  N0.getOperand(0).getOpcode() == ISD::XOR &&
3922  N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3923  isOneConstant(N0.getOperand(1))) {
3924  // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3925  // can only do this if the top bits are known zero.
3926  unsigned BitWidth = N0.getValueSizeInBits();
3927  if (DAG.MaskedValueIsZero(N0,
3929  BitWidth-1))) {
3930  // Okay, get the un-inverted input value.
3931  SDValue Val;
3932  if (N0.getOpcode() == ISD::XOR) {
3933  Val = N0.getOperand(0);
3934  } else {
3935  assert(N0.getOpcode() == ISD::AND &&
3936  N0.getOperand(0).getOpcode() == ISD::XOR);
3937  // ((X^1)&1)^1 -> X & 1
3938  Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3939  N0.getOperand(0).getOperand(0),
3940  N0.getOperand(1));
3941  }
3942 
3943  return DAG.getSetCC(dl, VT, Val, N1,
3945  }
3946  } else if (N1C->isOne()) {
3947  SDValue Op0 = N0;
3948  if (Op0.getOpcode() == ISD::TRUNCATE)
3949  Op0 = Op0.getOperand(0);
3950 
3951  if ((Op0.getOpcode() == ISD::XOR) &&
3952  Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3953  Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3954  SDValue XorLHS = Op0.getOperand(0);
3955  SDValue XorRHS = Op0.getOperand(1);
3956  // Ensure that the input setccs return an i1 type or 0/1 value.
3957  if (Op0.getValueType() == MVT::i1 ||
3958  (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3962  // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3964  return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3965  }
3966  }
3967  if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3968  // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3969  if (Op0.getValueType().bitsGT(VT))
3970  Op0 = DAG.getNode(ISD::AND, dl, VT,
3971  DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3972  DAG.getConstant(1, dl, VT));
3973  else if (Op0.getValueType().bitsLT(VT))
3974  Op0 = DAG.getNode(ISD::AND, dl, VT,
3975  DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3976  DAG.getConstant(1, dl, VT));
3977 
3978  return DAG.getSetCC(dl, VT, Op0,
3979  DAG.getConstant(0, dl, Op0.getValueType()),
3981  }
3982  if (Op0.getOpcode() == ISD::AssertZext &&
3983  cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3984  return DAG.getSetCC(dl, VT, Op0,
3985  DAG.getConstant(0, dl, Op0.getValueType()),
3987  }
3988  }
3989 
3990  // Given:
3991  // icmp eq/ne (urem %x, %y), 0
3992  // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3993  // icmp eq/ne %x, 0
3994  if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
3995  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3996  KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3997  KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3998  if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3999  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
4000  }
4001 
4002  // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
4003  // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
4004  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4005  N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
4006  N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
4007  N1C && N1C->isAllOnes()) {
4008  return DAG.getSetCC(dl, VT, N0.getOperand(0),
4009  DAG.getConstant(0, dl, OpVT),
4011  }
4012 
4013  if (SDValue V =
4014  optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
4015  return V;
4016  }
4017 
4018  // These simplifications apply to splat vectors as well.
4019  // TODO: Handle more splat vector cases.
4020  if (auto *N1C = isConstOrConstSplat(N1)) {
4021  const APInt &C1 = N1C->getAPIntValue();
4022 
4023  APInt MinVal, MaxVal;
4024  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
4025  if (ISD::isSignedIntSetCC(Cond)) {
4026  MinVal = APInt::getSignedMinValue(OperandBitSize);
4027  MaxVal = APInt::getSignedMaxValue(OperandBitSize);
4028  } else {
4029  MinVal = APInt::getMinValue(OperandBitSize);
4030  MaxVal = APInt::getMaxValue(OperandBitSize);
4031  }
4032 
4033  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
4034  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
4035  // X >= MIN --> true
4036  if (C1 == MinVal)
4037  return DAG.getBoolConstant(true, dl, VT, OpVT);
4038 
4039  if (!VT.isVector()) { // TODO: Support this for vectors.
4040  // X >= C0 --> X > (C0 - 1)
4041  APInt C = C1 - 1;
4043  if ((DCI.isBeforeLegalizeOps() ||
4044  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
4045  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
4046  isLegalICmpImmediate(C.getSExtValue())))) {
4047  return DAG.getSetCC(dl, VT, N0,
4048  DAG.getConstant(C, dl, N1.getValueType()),
4049  NewCC);
4050  }
4051  }
4052  }
4053 
4054  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
4055  // X <= MAX --> true
4056  if (C1 == MaxVal)
4057  return DAG.getBoolConstant(true, dl, VT, OpVT);
4058 
4059  // X <= C0 --> X < (C0 + 1)
4060  if (!VT.isVector()) { // TODO: Support this for vectors.
4061  APInt C = C1 + 1;
4063  if ((DCI.isBeforeLegalizeOps() ||
4064  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
4065  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
4066  isLegalICmpImmediate(C.getSExtValue())))) {
4067  return DAG.getSetCC(dl, VT, N0,
4068  DAG.getConstant(C, dl, N1.getValueType()),
4069  NewCC);
4070  }
4071  }
4072  }
4073 
4074  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
4075  if (C1 == MinVal)
4076  return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
4077 
4078  // TODO: Support this for vectors after legalize ops.
4079  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4080  // Canonicalize setlt X, Max --> setne X, Max
4081  if (C1 == MaxVal)
4082  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4083 
4084  // If we have setult X, 1, turn it into seteq X, 0
4085  if (C1 == MinVal+1)
4086  return DAG.getSetCC(dl, VT, N0,
4087  DAG.getConstant(MinVal, dl, N0.getValueType()),
4088  ISD::SETEQ);
4089  }
4090  }
4091 
4092  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
4093  if (C1 == MaxVal)
4094  return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
4095 
4096  // TODO: Support this for vectors after legalize ops.
4097  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4098  // Canonicalize setgt X, Min --> setne X, Min
4099  if (C1 == MinVal)
4100  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4101 
4102  // If we have setugt X, Max-1, turn it into seteq X, Max
4103  if (C1 == MaxVal-1)
4104  return DAG.getSetCC(dl, VT, N0,
4105  DAG.getConstant(MaxVal, dl, N0.getValueType()),
4106  ISD::SETEQ);
4107  }
4108  }
4109 
4110  if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4111  // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4112  if (C1.isZero())
4113  if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4114  VT, N0, N1, Cond, DCI, dl))
4115  return CC;
4116 
4117  // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4118  // For example, when high 32-bits of i64 X are known clear:
4119  // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
4120  // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
4121  bool CmpZero = N1C->getAPIntValue().isZero();
4122  bool CmpNegOne = N1C->getAPIntValue().isAllOnes();
4123  if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4124  // Match or(lo,shl(hi,bw/2)) pattern.
4125  auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4126  unsigned EltBits = V.getScalarValueSizeInBits();
4127  if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4128  return false;
4129  SDValue LHS = V.getOperand(0);
4130  SDValue RHS = V.getOperand(1);
4131  APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4132  // Unshifted element must have zero upperbits.
4133  if (RHS.getOpcode() == ISD::SHL &&
4134  isa<ConstantSDNode>(RHS.getOperand(1)) &&
4135  RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4136  DAG.MaskedValueIsZero(LHS, HiBits)) {
4137  Lo = LHS;
4138  Hi = RHS.getOperand(0);
4139  return true;
4140  }
4141  if (LHS.getOpcode() == ISD::SHL &&
4142  isa<ConstantSDNode>(LHS.getOperand(1)) &&
4143  LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4144  DAG.MaskedValueIsZero(RHS, HiBits)) {
4145  Lo = RHS;
4146  Hi = LHS.getOperand(0);
4147  return true;
4148  }
4149  return false;
4150  };
4151 
4152  auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4153  unsigned EltBits = N0.getScalarValueSizeInBits();
4154  unsigned HalfBits = EltBits / 2;
4155  APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4156  SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4157  SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4158  SDValue NewN0 =
4159  DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4160  SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4161  return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4162  };
4163 
4164  SDValue Lo, Hi;
4165  if (IsConcat(N0, Lo, Hi))
4166  return MergeConcat(Lo, Hi);
4167 
4168  if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4169  SDValue Lo0, Lo1, Hi0, Hi1;
4170  if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4171  IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4172  return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4173  DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4174  }
4175  }
4176  }
4177  }
4178 
4179  // If we have "setcc X, C0", check to see if we can shrink the immediate
4180  // by changing cc.
4181  // TODO: Support this for vectors after legalize ops.
4182  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4183  // SETUGT X, SINTMAX -> SETLT X, 0
4184  // SETUGE X, SINTMIN -> SETLT X, 0
4185  if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4186  (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4187  return DAG.getSetCC(dl, VT, N0,
4188  DAG.getConstant(0, dl, N1.getValueType()),
4189  ISD::SETLT);
4190 
4191  // SETULT X, SINTMIN -> SETGT X, -1
4192  // SETULE X, SINTMAX -> SETGT X, -1
4193  if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4194  (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4195  return DAG.getSetCC(dl, VT, N0,
4196  DAG.getAllOnesConstant(dl, N1.getValueType()),
4197  ISD::SETGT);
4198  }
4199  }
4200 
4201  // Back to non-vector simplifications.
4202  // TODO: Can we do these for vector splats?
4203  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4204  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4205  const APInt &C1 = N1C->getAPIntValue();
4206  EVT ShValTy = N0.getValueType();
4207 
4208  // Fold bit comparisons when we can. This will result in an
4209  // incorrect value when boolean false is negative one, unless
4210  // the bitsize is 1 in which case the false value is the same
4211  // in practice regardless of the representation.
4212  if ((VT.getSizeInBits() == 1 ||
4214  (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4215  (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4216  N0.getOpcode() == ISD::AND) {
4217  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4218  EVT ShiftTy =
4219  getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4220  if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4221  // Perform the xform if the AND RHS is a single bit.
4222  unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4223  if (AndRHS->getAPIntValue().isPowerOf2() &&
4224  !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4225  return DAG.getNode(ISD::TRUNCATE, dl, VT,
4226  DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4227  DAG.getConstant(ShCt, dl, ShiftTy)));
4228  }
4229  } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4230  // (X & 8) == 8 --> (X & 8) >> 3
4231  // Perform the xform if C1 is a single bit.
4232  unsigned ShCt = C1.logBase2();
4233  if (C1.isPowerOf2() &&
4234  !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4235  return DAG.getNode(ISD::TRUNCATE, dl, VT,
4236  DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4237  DAG.getConstant(ShCt, dl, ShiftTy)));
4238  }
4239  }
4240  }
4241  }
4242 
4243  if (C1.getMinSignedBits() <= 64 &&
4244  !isLegalICmpImmediate(C1.getSExtValue())) {
4245  EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4246  // (X & -256) == 256 -> (X >> 8) == 1
4247  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4248  N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4249  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4250  const APInt &AndRHSC = AndRHS->getAPIntValue();
4251  if (AndRHSC.isNegatedPowerOf2() && (AndRHSC & C1) == C1) {
4252  unsigned ShiftBits = AndRHSC.countTrailingZeros();
4253  if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4254  SDValue Shift =
4255  DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4256  DAG.getConstant(ShiftBits, dl, ShiftTy));
4257  SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4258  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4259  }
4260  }
4261  }
4262  } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4263  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4264  bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4265  // X < 0x100000000 -> (X >> 32) < 1
4266  // X >= 0x100000000 -> (X >> 32) >= 1
4267  // X <= 0x0ffffffff -> (X >> 32) < 1
4268  // X > 0x0ffffffff -> (X >> 32) >= 1
4269  unsigned ShiftBits;
4270  APInt NewC = C1;
4271  ISD::CondCode NewCond = Cond;
4272  if (AdjOne) {
4273  ShiftBits = C1.countTrailingOnes();
4274  NewC = NewC + 1;
4275  NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4276  } else {
4277  ShiftBits = C1.countTrailingZeros();
4278  }
4279  NewC.lshrInPlace(ShiftBits);
4280  if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4282  !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4283  SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4284  DAG.getConstant(ShiftBits, dl, ShiftTy));
4285  SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4286  return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4287  }
4288  }
4289  }
4290  }
4291 
4292  if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4293  auto *CFP = cast<ConstantFPSDNode>(N1);
4294  assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
4295 
4296  // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4297  // constant if knowing that the operand is non-nan is enough. We prefer to
4298  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4299  // materialize 0.0.
4300  if (Cond == ISD::SETO || Cond == ISD::SETUO)
4301  return DAG.getSetCC(dl, VT, N0, N0, Cond);
4302 
4303  // setcc (fneg x), C -> setcc swap(pred) x, -C
4304  if (N0.getOpcode() == ISD::FNEG) {
4306  if (DCI.isBeforeLegalizeOps() ||
4307  isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4308  SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4309  return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4310  }
4311  }
4312 
4313  // If the condition is not legal, see if we can find an equivalent one
4314  // which is legal.
4315  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4316  // If the comparison was an awkward floating-point == or != and one of
4317  // the comparison operands is infinity or negative infinity, convert the
4318  // condition to a less-awkward <= or >=.
4319  if (CFP->getValueAPF().isInfinity()) {
4320  bool IsNegInf = CFP->getValueAPF().isNegative();
4322  switch (Cond) {
4323  case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4324  case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4325  case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4326  case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4327  default: break;
4328  }
4329  if (NewCond != ISD::SETCC_INVALID &&
4330  isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4331  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4332  }
4333  }
4334  }
4335 
4336  if (N0 == N1) {
4337  // The sext(setcc()) => setcc() optimization relies on the appropriate
4338  // constant being emitted.
4339  assert(!N0.getValueType().isInteger() &&
4340  "Integer types should be handled by FoldSetCC");
4341 
4342  bool EqTrue = ISD::isTrueWhenEqual(Cond);
4343  unsigned UOF = ISD::getUnorderedFlavor(Cond);
4344  if (UOF == 2) // FP operators that are undefined on NaNs.
4345  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4346  if (UOF == unsigned(EqTrue))
4347  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4348  // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4349  // if it is not already.
4350  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4351  if (NewCond != Cond &&
4352  (DCI.isBeforeLegalizeOps() ||
4353  isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4354  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4355  }
4356 
4357  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4358  N0.getValueType().isInteger()) {
4359  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4360  N0.getOpcode() == ISD::XOR) {
4361  // Simplify (X+Y) == (X+Z) --> Y == Z
4362  if (N0.getOpcode() == N1.getOpcode()) {
4363  if (N0.getOperand(0) == N1.getOperand(0))
4364  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4365  if (N0.getOperand(1) == N1.getOperand(1))
4366  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4367  if (isCommutativeBinOp(N0.getOpcode())) {
4368  // If X op Y == Y op X, try other combinations.
4369  if (N0.getOperand(0) == N1.getOperand(1))
4370  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4371  Cond);
4372  if (N0.getOperand(1) == N1.getOperand(0))
4373  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4374  Cond);
4375  }
4376  }
4377 
4378  // If RHS is a legal immediate value for a compare instruction, we need
4379  // to be careful about increasing register pressure needlessly.
4380  bool LegalRHSImm = false;
4381 
4382  if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4383  if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4384  // Turn (X+C1) == C2 --> X == C2-C1
4385  if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4386  return DAG.getSetCC(dl, VT, N0.getOperand(0),
4387  DAG.getConstant(RHSC->getAPIntValue()-
4388  LHSR->getAPIntValue(),
4389  dl, N0.getValueType()), Cond);
4390  }
4391 
4392  // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4393  if (N0.getOpcode() == ISD::XOR)
4394  // If we know that all of the inverted bits are zero, don't bother
4395  // performing the inversion.
4396  if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4397  return
4398  DAG.getSetCC(dl, VT, N0.getOperand(0),
4399  DAG.getConstant(LHSR->getAPIntValue() ^
4400  RHSC->getAPIntValue(),
4401  dl, N0.getValueType()),
4402  Cond);
4403  }
4404 
4405  // Turn (C1-X) == C2 --> X == C1-C2
4406  if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4407  if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4408  return
4409  DAG.getSetCC(dl, VT, N0.getOperand(1),
4410  DAG.getConstant(SUBC->getAPIntValue() -
4411  RHSC->getAPIntValue(),
4412  dl, N0.getValueType()),
4413  Cond);
4414  }
4415  }
4416 
4417  // Could RHSC fold directly into a compare?
4418  if (RHSC->getValueType(0).getSizeInBits() <= 64)
4419  LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4420  }
4421 
4422  // (X+Y) == X --> Y == 0 and similar folds.
4423  // Don't do this if X is an immediate that can fold into a cmp
4424  // instruction and X+Y has other uses. It could be an induction variable
4425  // chain, and the transform would increase register pressure.
4426  if (!LegalRHSImm || N0.hasOneUse())
4427  if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4428  return V;
4429  }
4430 
4431  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4432  N1.getOpcode() == ISD::XOR)
4433  if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4434  return V;
4435 
4436  if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))