LLVM  14.0.0git
TargetLowering.cpp
Go to the documentation of this file.
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/GlobalVariable.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCExpr.h"
30 #include "llvm/Support/KnownBits.h"
34 #include <cctype>
35 using namespace llvm;
36 
37 /// NOTE: The TargetMachine owns TLOF.
39  : TargetLoweringBase(tm) {}
40 
41 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
42  return nullptr;
43 }
44 
47 }
48 
49 /// Check whether a given call node is in tail position within its function. If
50 /// so, it sets Chain to the input chain of the tail call.
52  SDValue &Chain) const {
53  const Function &F = DAG.getMachineFunction().getFunction();
54 
55  // First, check if tail calls have been disabled in this function.
56  if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
57  return false;
58 
59  // Conservatively require the attributes of the call to match those of
60  // the return. Ignore following attributes because they don't affect the
61  // call sequence.
62  AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);
63  for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
64  Attribute::DereferenceableOrNull, Attribute::NoAlias,
65  Attribute::NonNull})
66  CallerAttrs.removeAttribute(Attr);
67 
68  if (CallerAttrs.hasAttributes())
69  return false;
70 
71  // It's not safe to eliminate the sign / zero extension of the return value.
72  if (CallerAttrs.contains(Attribute::ZExt) ||
73  CallerAttrs.contains(Attribute::SExt))
74  return false;
75 
76  // Check if the only use is a function return node.
77  return isUsedByReturnOnly(Node, Chain);
78 }
79 
81  const uint32_t *CallerPreservedMask,
82  const SmallVectorImpl<CCValAssign> &ArgLocs,
83  const SmallVectorImpl<SDValue> &OutVals) const {
84  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
85  const CCValAssign &ArgLoc = ArgLocs[I];
86  if (!ArgLoc.isRegLoc())
87  continue;
88  MCRegister Reg = ArgLoc.getLocReg();
89  // Only look at callee saved registers.
90  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
91  continue;
92  // Check that we pass the value used for the caller.
93  // (We look for a CopyFromReg reading a virtual register that is used
94  // for the function live-in value of register Reg)
95  SDValue Value = OutVals[I];
96  if (Value->getOpcode() != ISD::CopyFromReg)
97  return false;
98  Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
99  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
100  return false;
101  }
102  return true;
103 }
104 
105 /// Set CallLoweringInfo attribute flags based on a call instruction
106 /// and called function attributes.
108  unsigned ArgIdx) {
109  IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
110  IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
111  IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
112  IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
113  IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
114  IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
115  IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
116  IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
117  IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
118  IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
119  IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
120  IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
121  Alignment = Call->getParamStackAlign(ArgIdx);
122  IndirectType = nullptr;
124  "multiple ABI attributes?");
125  if (IsByVal) {
126  IndirectType = Call->getParamByValType(ArgIdx);
127  if (!Alignment)
128  Alignment = Call->getParamAlign(ArgIdx);
129  }
130  if (IsPreallocated)
131  IndirectType = Call->getParamPreallocatedType(ArgIdx);
132  if (IsInAlloca)
133  IndirectType = Call->getParamInAllocaType(ArgIdx);
134 }
135 
136 /// Generate a libcall taking the given operands as arguments and returning a
137 /// result of type RetVT.
138 std::pair<SDValue, SDValue>
140  ArrayRef<SDValue> Ops,
141  MakeLibCallOptions CallOptions,
142  const SDLoc &dl,
143  SDValue InChain) const {
144  if (!InChain)
145  InChain = DAG.getEntryNode();
146 
148  Args.reserve(Ops.size());
149 
151  for (unsigned i = 0; i < Ops.size(); ++i) {
152  SDValue NewOp = Ops[i];
153  Entry.Node = NewOp;
154  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
155  Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
156  CallOptions.IsSExt);
157  Entry.IsZExt = !Entry.IsSExt;
158 
159  if (CallOptions.IsSoften &&
161  Entry.IsSExt = Entry.IsZExt = false;
162  }
163  Args.push_back(Entry);
164  }
165 
166  if (LC == RTLIB::UNKNOWN_LIBCALL)
167  report_fatal_error("Unsupported library call operation!");
169  getPointerTy(DAG.getDataLayout()));
170 
171  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
173  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
174  bool zeroExtend = !signExtend;
175 
176  if (CallOptions.IsSoften &&
178  signExtend = zeroExtend = false;
179  }
180 
181  CLI.setDebugLoc(dl)
182  .setChain(InChain)
184  .setNoReturn(CallOptions.DoesNotReturn)
185  .setDiscardResult(!CallOptions.IsReturnValueUsed)
187  .setSExtResult(signExtend)
188  .setZExtResult(zeroExtend);
189  return LowerCallTo(CLI);
190 }
191 
193  std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
194  unsigned SrcAS, const AttributeList &FuncAttributes) const {
195  if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
196  return false;
197 
198  EVT VT = getOptimalMemOpType(Op, FuncAttributes);
199 
200  if (VT == MVT::Other) {
201  // Use the largest integer type whose alignment constraints are satisfied.
202  // We only need to check DstAlign here as SrcAlign is always greater or
203  // equal to DstAlign (or zero).
204  VT = MVT::i64;
205  if (Op.isFixedDstAlign())
206  while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
207  !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
208  VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
209  assert(VT.isInteger());
210 
211  // Find the largest legal integer type.
212  MVT LVT = MVT::i64;
213  while (!isTypeLegal(LVT))
214  LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
215  assert(LVT.isInteger());
216 
217  // If the type we've chosen is larger than the largest legal integer type
218  // then use that instead.
219  if (VT.bitsGT(LVT))
220  VT = LVT;
221  }
222 
223  unsigned NumMemOps = 0;
224  uint64_t Size = Op.size();
225  while (Size) {
226  unsigned VTSize = VT.getSizeInBits() / 8;
227  while (VTSize > Size) {
228  // For now, only use non-vector load / store's for the left-over pieces.
229  EVT NewVT = VT;
230  unsigned NewVTSize;
231 
232  bool Found = false;
233  if (VT.isVector() || VT.isFloatingPoint()) {
234  NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
235  if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
236  isSafeMemOpType(NewVT.getSimpleVT()))
237  Found = true;
238  else if (NewVT == MVT::i64 &&
241  // i64 is usually not legal on 32-bit targets, but f64 may be.
242  NewVT = MVT::f64;
243  Found = true;
244  }
245  }
246 
247  if (!Found) {
248  do {
249  NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
250  if (NewVT == MVT::i8)
251  break;
252  } while (!isSafeMemOpType(NewVT.getSimpleVT()));
253  }
254  NewVTSize = NewVT.getSizeInBits() / 8;
255 
256  // If the new VT cannot cover all of the remaining bits, then consider
257  // issuing a (or a pair of) unaligned and overlapping load / store.
258  bool Fast;
259  if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
261  VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
262  MachineMemOperand::MONone, &Fast) &&
263  Fast)
264  VTSize = Size;
265  else {
266  VT = NewVT;
267  VTSize = NewVTSize;
268  }
269  }
270 
271  if (++NumMemOps > Limit)
272  return false;
273 
274  MemOps.push_back(VT);
275  Size -= VTSize;
276  }
277 
278  return true;
279 }
280 
281 /// Soften the operands of a comparison. This code is shared among BR_CC,
282 /// SELECT_CC, and SETCC handlers.
284  SDValue &NewLHS, SDValue &NewRHS,
285  ISD::CondCode &CCCode,
286  const SDLoc &dl, const SDValue OldLHS,
287  const SDValue OldRHS) const {
288  SDValue Chain;
289  return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
290  OldRHS, Chain);
291 }
292 
294  SDValue &NewLHS, SDValue &NewRHS,
295  ISD::CondCode &CCCode,
296  const SDLoc &dl, const SDValue OldLHS,
297  const SDValue OldRHS,
298  SDValue &Chain,
299  bool IsSignaling) const {
300  // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
301  // not supporting it. We can update this code when libgcc provides such
302  // functions.
303 
304  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
305  && "Unsupported setcc type!");
306 
307  // Expand into one or more soft-fp libcall(s).
308  RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
309  bool ShouldInvertCC = false;
310  switch (CCCode) {
311  case ISD::SETEQ:
312  case ISD::SETOEQ:
313  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
314  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
315  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
316  break;
317  case ISD::SETNE:
318  case ISD::SETUNE:
319  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
320  (VT == MVT::f64) ? RTLIB::UNE_F64 :
321  (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
322  break;
323  case ISD::SETGE:
324  case ISD::SETOGE:
325  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
326  (VT == MVT::f64) ? RTLIB::OGE_F64 :
327  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
328  break;
329  case ISD::SETLT:
330  case ISD::SETOLT:
331  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
332  (VT == MVT::f64) ? RTLIB::OLT_F64 :
333  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
334  break;
335  case ISD::SETLE:
336  case ISD::SETOLE:
337  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
338  (VT == MVT::f64) ? RTLIB::OLE_F64 :
339  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
340  break;
341  case ISD::SETGT:
342  case ISD::SETOGT:
343  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
344  (VT == MVT::f64) ? RTLIB::OGT_F64 :
345  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
346  break;
347  case ISD::SETO:
348  ShouldInvertCC = true;
350  case ISD::SETUO:
351  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
352  (VT == MVT::f64) ? RTLIB::UO_F64 :
353  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
354  break;
355  case ISD::SETONE:
356  // SETONE = O && UNE
357  ShouldInvertCC = true;
359  case ISD::SETUEQ:
360  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
361  (VT == MVT::f64) ? RTLIB::UO_F64 :
362  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
363  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
364  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
365  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
366  break;
367  default:
368  // Invert CC for unordered comparisons
369  ShouldInvertCC = true;
370  switch (CCCode) {
371  case ISD::SETULT:
372  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
373  (VT == MVT::f64) ? RTLIB::OGE_F64 :
374  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
375  break;
376  case ISD::SETULE:
377  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
378  (VT == MVT::f64) ? RTLIB::OGT_F64 :
379  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
380  break;
381  case ISD::SETUGT:
382  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
383  (VT == MVT::f64) ? RTLIB::OLE_F64 :
384  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
385  break;
386  case ISD::SETUGE:
387  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
388  (VT == MVT::f64) ? RTLIB::OLT_F64 :
389  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
390  break;
391  default: llvm_unreachable("Do not know how to soften this setcc!");
392  }
393  }
394 
395  // Use the target specific return value for comparions lib calls.
396  EVT RetVT = getCmpLibcallReturnType();
397  SDValue Ops[2] = {NewLHS, NewRHS};
399  EVT OpsVT[2] = { OldLHS.getValueType(),
400  OldRHS.getValueType() };
401  CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
402  auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
403  NewLHS = Call.first;
404  NewRHS = DAG.getConstant(0, dl, RetVT);
405 
406  CCCode = getCmpLibcallCC(LC1);
407  if (ShouldInvertCC) {
408  assert(RetVT.isInteger());
409  CCCode = getSetCCInverse(CCCode, RetVT);
410  }
411 
412  if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
413  // Update Chain.
414  Chain = Call.second;
415  } else {
416  EVT SetCCVT =
417  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
418  SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
419  auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
420  CCCode = getCmpLibcallCC(LC2);
421  if (ShouldInvertCC)
422  CCCode = getSetCCInverse(CCCode, RetVT);
423  NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
424  if (Chain)
425  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
426  Call2.second);
427  NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
428  Tmp.getValueType(), Tmp, NewLHS);
429  NewRHS = SDValue();
430  }
431 }
432 
433 /// Return the entry encoding for a jump table in the current function. The
434 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
436  // In non-pic modes, just use the address of a block.
437  if (!isPositionIndependent())
439 
440  // In PIC mode, if the target supports a GPRel32 directive, use it.
441  if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
443 
444  // Otherwise, use a label difference.
446 }
447 
449  SelectionDAG &DAG) const {
450  // If our PIC model is GP relative, use the global offset table as the base.
451  unsigned JTEncoding = getJumpTableEncoding();
452 
453  if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
456 
457  return Table;
458 }
459 
460 /// This returns the relocation base for the given PIC jumptable, the same as
461 /// getPICJumpTableRelocBase, but as an MCExpr.
462 const MCExpr *
464  unsigned JTI,MCContext &Ctx) const{
465  // The normal PIC reloc base is the label at the start of the jump table.
466  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
467 }
468 
469 bool
471  const TargetMachine &TM = getTargetMachine();
472  const GlobalValue *GV = GA->getGlobal();
473 
474  // If the address is not even local to this DSO we will have to load it from
475  // a got and then add the offset.
476  if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
477  return false;
478 
479  // If the code is position independent we will have to add a base register.
480  if (isPositionIndependent())
481  return false;
482 
483  // Otherwise we can do it.
484  return true;
485 }
486 
487 //===----------------------------------------------------------------------===//
488 // Optimization Methods
489 //===----------------------------------------------------------------------===//
490 
491 /// If the specified instruction has a constant integer operand and there are
492 /// bits set in that constant that are not demanded, then clear those bits and
493 /// return true.
495  const APInt &DemandedBits,
496  const APInt &DemandedElts,
497  TargetLoweringOpt &TLO) const {
498  SDLoc DL(Op);
499  unsigned Opcode = Op.getOpcode();
500 
501  // Do target-specific constant optimization.
502  if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
503  return TLO.New.getNode();
504 
505  // FIXME: ISD::SELECT, ISD::SELECT_CC
506  switch (Opcode) {
507  default:
508  break;
509  case ISD::XOR:
510  case ISD::AND:
511  case ISD::OR: {
512  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
513  if (!Op1C || Op1C->isOpaque())
514  return false;
515 
516  // If this is a 'not' op, don't touch it because that's a canonical form.
517  const APInt &C = Op1C->getAPIntValue();
518  if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
519  return false;
520 
521  if (!C.isSubsetOf(DemandedBits)) {
522  EVT VT = Op.getValueType();
523  SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
524  SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
525  return TLO.CombineTo(Op, NewOp);
526  }
527 
528  break;
529  }
530  }
531 
532  return false;
533 }
534 
536  const APInt &DemandedBits,
537  TargetLoweringOpt &TLO) const {
538  EVT VT = Op.getValueType();
539  APInt DemandedElts = VT.isVector()
541  : APInt(1, 1);
542  return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
543 }
544 
545 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
546 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
547 /// generalized for targets with other types of implicit widening casts.
549  const APInt &Demanded,
550  TargetLoweringOpt &TLO) const {
551  assert(Op.getNumOperands() == 2 &&
552  "ShrinkDemandedOp only supports binary operators!");
553  assert(Op.getNode()->getNumValues() == 1 &&
554  "ShrinkDemandedOp only supports nodes with one result!");
555 
556  SelectionDAG &DAG = TLO.DAG;
557  SDLoc dl(Op);
558 
559  // Early return, as this function cannot handle vector types.
560  if (Op.getValueType().isVector())
561  return false;
562 
563  // Don't do this if the node has another user, which may require the
564  // full value.
565  if (!Op.getNode()->hasOneUse())
566  return false;
567 
568  // Search for the smallest integer type with free casts to and from
569  // Op's type. For expedience, just check power-of-2 integer types.
570  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
571  unsigned DemandedSize = Demanded.getActiveBits();
572  unsigned SmallVTBits = DemandedSize;
573  if (!isPowerOf2_32(SmallVTBits))
574  SmallVTBits = NextPowerOf2(SmallVTBits);
575  for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
576  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
577  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
578  TLI.isZExtFree(SmallVT, Op.getValueType())) {
579  // We found a type with free casts.
580  SDValue X = DAG.getNode(
581  Op.getOpcode(), dl, SmallVT,
582  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
583  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
584  assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
585  SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
586  return TLO.CombineTo(Op, Z);
587  }
588  }
589  return false;
590 }
591 
593  DAGCombinerInfo &DCI) const {
594  SelectionDAG &DAG = DCI.DAG;
595  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
596  !DCI.isBeforeLegalizeOps());
597  KnownBits Known;
598 
599  bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
600  if (Simplified) {
601  DCI.AddToWorklist(Op.getNode());
602  DCI.CommitTargetLoweringOpt(TLO);
603  }
604  return Simplified;
605 }
606 
608  KnownBits &Known,
609  TargetLoweringOpt &TLO,
610  unsigned Depth,
611  bool AssumeSingleUse) const {
612  EVT VT = Op.getValueType();
613 
614  // TODO: We can probably do more work on calculating the known bits and
615  // simplifying the operations for scalable vectors, but for now we just
616  // bail out.
617  if (VT.isScalableVector()) {
618  // Pretend we don't know anything for now.
619  Known = KnownBits(DemandedBits.getBitWidth());
620  return false;
621  }
622 
623  APInt DemandedElts = VT.isVector()
625  : APInt(1, 1);
626  return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
627  AssumeSingleUse);
628 }
629 
630 // TODO: Can we merge SelectionDAG::GetDemandedBits into this?
631 // TODO: Under what circumstances can we create nodes? Constant folding?
633  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
634  SelectionDAG &DAG, unsigned Depth) const {
635  // Limit search depth.
637  return SDValue();
638 
639  // Ignore UNDEFs.
640  if (Op.isUndef())
641  return SDValue();
642 
643  // Not demanding any bits/elts from Op.
644  if (DemandedBits == 0 || DemandedElts == 0)
645  return DAG.getUNDEF(Op.getValueType());
646 
647  unsigned NumElts = DemandedElts.getBitWidth();
648  unsigned BitWidth = DemandedBits.getBitWidth();
649  KnownBits LHSKnown, RHSKnown;
650  switch (Op.getOpcode()) {
651  case ISD::BITCAST: {
652  SDValue Src = peekThroughBitcasts(Op.getOperand(0));
653  EVT SrcVT = Src.getValueType();
654  EVT DstVT = Op.getValueType();
655  if (SrcVT == DstVT)
656  return Src;
657 
658  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
659  unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
660  if (NumSrcEltBits == NumDstEltBits)
661  if (SDValue V = SimplifyMultipleUseDemandedBits(
662  Src, DemandedBits, DemandedElts, DAG, Depth + 1))
663  return DAG.getBitcast(DstVT, V);
664 
665  // TODO - bigendian once we have test coverage.
666  if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
667  DAG.getDataLayout().isLittleEndian()) {
668  unsigned Scale = NumDstEltBits / NumSrcEltBits;
669  unsigned NumSrcElts = SrcVT.getVectorNumElements();
670  APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
671  APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
672  for (unsigned i = 0; i != Scale; ++i) {
673  unsigned Offset = i * NumSrcEltBits;
674  APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
675  if (!Sub.isNullValue()) {
676  DemandedSrcBits |= Sub;
677  for (unsigned j = 0; j != NumElts; ++j)
678  if (DemandedElts[j])
679  DemandedSrcElts.setBit((j * Scale) + i);
680  }
681  }
682 
683  if (SDValue V = SimplifyMultipleUseDemandedBits(
684  Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
685  return DAG.getBitcast(DstVT, V);
686  }
687 
688  // TODO - bigendian once we have test coverage.
689  if ((NumSrcEltBits % NumDstEltBits) == 0 &&
690  DAG.getDataLayout().isLittleEndian()) {
691  unsigned Scale = NumSrcEltBits / NumDstEltBits;
692  unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
693  APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
694  APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
695  for (unsigned i = 0; i != NumElts; ++i)
696  if (DemandedElts[i]) {
697  unsigned Offset = (i % Scale) * NumDstEltBits;
698  DemandedSrcBits.insertBits(DemandedBits, Offset);
699  DemandedSrcElts.setBit(i / Scale);
700  }
701 
702  if (SDValue V = SimplifyMultipleUseDemandedBits(
703  Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
704  return DAG.getBitcast(DstVT, V);
705  }
706 
707  break;
708  }
709  case ISD::AND: {
710  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
711  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
712 
713  // If all of the demanded bits are known 1 on one side, return the other.
714  // These bits cannot contribute to the result of the 'and' in this
715  // context.
716  if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
717  return Op.getOperand(0);
718  if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
719  return Op.getOperand(1);
720  break;
721  }
722  case ISD::OR: {
723  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
724  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
725 
726  // If all of the demanded bits are known zero on one side, return the
727  // other. These bits cannot contribute to the result of the 'or' in this
728  // context.
729  if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
730  return Op.getOperand(0);
731  if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
732  return Op.getOperand(1);
733  break;
734  }
735  case ISD::XOR: {
736  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
737  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
738 
739  // If all of the demanded bits are known zero on one side, return the
740  // other.
741  if (DemandedBits.isSubsetOf(RHSKnown.Zero))
742  return Op.getOperand(0);
743  if (DemandedBits.isSubsetOf(LHSKnown.Zero))
744  return Op.getOperand(1);
745  break;
746  }
747  case ISD::SHL: {
748  // If we are only demanding sign bits then we can use the shift source
749  // directly.
750  if (const APInt *MaxSA =
751  DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
752  SDValue Op0 = Op.getOperand(0);
753  unsigned ShAmt = MaxSA->getZExtValue();
754  unsigned NumSignBits =
755  DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
756  unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
757  if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
758  return Op0;
759  }
760  break;
761  }
762  case ISD::SETCC: {
763  SDValue Op0 = Op.getOperand(0);
764  SDValue Op1 = Op.getOperand(1);
765  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
766  // If (1) we only need the sign-bit, (2) the setcc operands are the same
767  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
768  // -1, we may be able to bypass the setcc.
769  if (DemandedBits.isSignMask() &&
772  BooleanContent::ZeroOrNegativeOneBooleanContent) {
773  // If we're testing X < 0, then this compare isn't needed - just use X!
774  // FIXME: We're limiting to integer types here, but this should also work
775  // if we don't care about FP signed-zero. The use of SETLT with FP means
776  // that we don't care about NaNs.
777  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
779  return Op0;
780  }
781  break;
782  }
783  case ISD::SIGN_EXTEND_INREG: {
784  // If none of the extended bits are demanded, eliminate the sextinreg.
785  SDValue Op0 = Op.getOperand(0);
786  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
787  unsigned ExBits = ExVT.getScalarSizeInBits();
788  if (DemandedBits.getActiveBits() <= ExBits)
789  return Op0;
790  // If the input is already sign extended, just drop the extension.
791  unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
792  if (NumSignBits >= (BitWidth - ExBits + 1))
793  return Op0;
794  break;
795  }
799  // If we only want the lowest element and none of extended bits, then we can
800  // return the bitcasted source vector.
801  SDValue Src = Op.getOperand(0);
802  EVT SrcVT = Src.getValueType();
803  EVT DstVT = Op.getValueType();
804  if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
805  DAG.getDataLayout().isLittleEndian() &&
806  DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
807  return DAG.getBitcast(DstVT, Src);
808  }
809  break;
810  }
811  case ISD::INSERT_VECTOR_ELT: {
812  // If we don't demand the inserted element, return the base vector.
813  SDValue Vec = Op.getOperand(0);
814  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
815  EVT VecVT = Vec.getValueType();
816  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
817  !DemandedElts[CIdx->getZExtValue()])
818  return Vec;
819  break;
820  }
821  case ISD::INSERT_SUBVECTOR: {
822  // If we don't demand the inserted subvector, return the base vector.
823  SDValue Vec = Op.getOperand(0);
824  SDValue Sub = Op.getOperand(1);
825  uint64_t Idx = Op.getConstantOperandVal(2);
826  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
827  if (DemandedElts.extractBits(NumSubElts, Idx) == 0)
828  return Vec;
829  break;
830  }
831  case ISD::VECTOR_SHUFFLE: {
832  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
833 
834  // If all the demanded elts are from one operand and are inline,
835  // then we can use the operand directly.
836  bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
837  for (unsigned i = 0; i != NumElts; ++i) {
838  int M = ShuffleMask[i];
839  if (M < 0 || !DemandedElts[i])
840  continue;
841  AllUndef = false;
842  IdentityLHS &= (M == (int)i);
843  IdentityRHS &= ((M - NumElts) == i);
844  }
845 
846  if (AllUndef)
847  return DAG.getUNDEF(Op.getValueType());
848  if (IdentityLHS)
849  return Op.getOperand(0);
850  if (IdentityRHS)
851  return Op.getOperand(1);
852  break;
853  }
854  default:
855  if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
856  if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
857  Op, DemandedBits, DemandedElts, DAG, Depth))
858  return V;
859  break;
860  }
861  return SDValue();
862 }
863 
865  SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
866  unsigned Depth) const {
867  EVT VT = Op.getValueType();
868  APInt DemandedElts = VT.isVector()
870  : APInt(1, 1);
871  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
872  Depth);
873 }
874 
876  SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
877  unsigned Depth) const {
878  APInt DemandedBits = APInt::getAllOnesValue(Op.getScalarValueSizeInBits());
879  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
880  Depth);
881 }
882 
883 /// Look at Op. At this point, we know that only the OriginalDemandedBits of the
884 /// result of Op are ever used downstream. If we can use this information to
885 /// simplify Op, create a new simplified DAG node and return true, returning the
886 /// original and new nodes in Old and New. Otherwise, analyze the expression and
887 /// return a mask of Known bits for the expression (used to simplify the
888 /// caller). The Known bits may only be accurate for those bits in the
889 /// OriginalDemandedBits and OriginalDemandedElts.
891  SDValue Op, const APInt &OriginalDemandedBits,
892  const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
893  unsigned Depth, bool AssumeSingleUse) const {
894  unsigned BitWidth = OriginalDemandedBits.getBitWidth();
895  assert(Op.getScalarValueSizeInBits() == BitWidth &&
896  "Mask size mismatches value type size!");
897 
898  // Don't know anything.
899  Known = KnownBits(BitWidth);
900 
901  // TODO: We can probably do more work on calculating the known bits and
902  // simplifying the operations for scalable vectors, but for now we just
903  // bail out.
904  if (Op.getValueType().isScalableVector())
905  return false;
906 
907  unsigned NumElts = OriginalDemandedElts.getBitWidth();
908  assert((!Op.getValueType().isVector() ||
909  NumElts == Op.getValueType().getVectorNumElements()) &&
910  "Unexpected vector size");
911 
912  APInt DemandedBits = OriginalDemandedBits;
913  APInt DemandedElts = OriginalDemandedElts;
914  SDLoc dl(Op);
915  auto &DL = TLO.DAG.getDataLayout();
916 
917  // Undef operand.
918  if (Op.isUndef())
919  return false;
920 
921  if (Op.getOpcode() == ISD::Constant) {
922  // We know all of the bits for a constant!
923  Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
924  return false;
925  }
926 
927  if (Op.getOpcode() == ISD::ConstantFP) {
928  // We know all of the bits for a floating point constant!
929  Known = KnownBits::makeConstant(
930  cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
931  return false;
932  }
933 
934  // Other users may use these bits.
935  EVT VT = Op.getValueType();
936  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
937  if (Depth != 0) {
938  // If not at the root, Just compute the Known bits to
939  // simplify things downstream.
940  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
941  return false;
942  }
943  // If this is the root being simplified, allow it to have multiple uses,
944  // just set the DemandedBits/Elts to all bits.
946  DemandedElts = APInt::getAllOnesValue(NumElts);
947  } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
948  // Not demanding any bits/elts from Op.
949  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
950  } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
951  // Limit search depth.
952  return false;
953  }
954 
955  KnownBits Known2;
956  switch (Op.getOpcode()) {
957  case ISD::TargetConstant:
958  llvm_unreachable("Can't simplify this node");
959  case ISD::SCALAR_TO_VECTOR: {
960  if (!DemandedElts[0])
961  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
962 
963  KnownBits SrcKnown;
964  SDValue Src = Op.getOperand(0);
965  unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
966  APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
967  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
968  return true;
969 
970  // Upper elements are undef, so only get the knownbits if we just demand
971  // the bottom element.
972  if (DemandedElts == 1)
973  Known = SrcKnown.anyextOrTrunc(BitWidth);
974  break;
975  }
976  case ISD::BUILD_VECTOR:
977  // Collect the known bits that are shared by every demanded element.
978  // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
979  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
980  return false; // Don't fall through, will infinitely loop.
981  case ISD::LOAD: {
982  auto *LD = cast<LoadSDNode>(Op);
983  if (getTargetConstantFromLoad(LD)) {
984  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
985  return false; // Don't fall through, will infinitely loop.
986  }
987  if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
988  // If this is a ZEXTLoad and we are looking at the loaded value.
989  EVT MemVT = LD->getMemoryVT();
990  unsigned MemBits = MemVT.getScalarSizeInBits();
991  Known.Zero.setBitsFrom(MemBits);
992  return false; // Don't fall through, will infinitely loop.
993  }
994  break;
995  }
996  case ISD::INSERT_VECTOR_ELT: {
997  SDValue Vec = Op.getOperand(0);
998  SDValue Scl = Op.getOperand(1);
999  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1000  EVT VecVT = Vec.getValueType();
1001 
1002  // If index isn't constant, assume we need all vector elements AND the
1003  // inserted element.
1004  APInt DemandedVecElts(DemandedElts);
1005  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1006  unsigned Idx = CIdx->getZExtValue();
1007  DemandedVecElts.clearBit(Idx);
1008 
1009  // Inserted element is not required.
1010  if (!DemandedElts[Idx])
1011  return TLO.CombineTo(Op, Vec);
1012  }
1013 
1014  KnownBits KnownScl;
1015  unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1016  APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1017  if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1018  return true;
1019 
1020  Known = KnownScl.anyextOrTrunc(BitWidth);
1021 
1022  KnownBits KnownVec;
1023  if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1024  Depth + 1))
1025  return true;
1026 
1027  if (!!DemandedVecElts)
1028  Known = KnownBits::commonBits(Known, KnownVec);
1029 
1030  return false;
1031  }
1032  case ISD::INSERT_SUBVECTOR: {
1033  // Demand any elements from the subvector and the remainder from the src its
1034  // inserted into.
1035  SDValue Src = Op.getOperand(0);
1036  SDValue Sub = Op.getOperand(1);
1037  uint64_t Idx = Op.getConstantOperandVal(2);
1038  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1039  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1040  APInt DemandedSrcElts = DemandedElts;
1041  DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
1042 
1043  KnownBits KnownSub, KnownSrc;
1044  if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1045  Depth + 1))
1046  return true;
1047  if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1048  Depth + 1))
1049  return true;
1050 
1051  Known.Zero.setAllBits();
1052  Known.One.setAllBits();
1053  if (!!DemandedSubElts)
1054  Known = KnownBits::commonBits(Known, KnownSub);
1055  if (!!DemandedSrcElts)
1056  Known = KnownBits::commonBits(Known, KnownSrc);
1057 
1058  // Attempt to avoid multi-use src if we don't need anything from it.
1059  if (!DemandedBits.isAllOnesValue() || !DemandedSubElts.isAllOnesValue() ||
1060  !DemandedSrcElts.isAllOnesValue()) {
1061  SDValue NewSub = SimplifyMultipleUseDemandedBits(
1062  Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1063  SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1064  Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1065  if (NewSub || NewSrc) {
1066  NewSub = NewSub ? NewSub : Sub;
1067  NewSrc = NewSrc ? NewSrc : Src;
1068  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1069  Op.getOperand(2));
1070  return TLO.CombineTo(Op, NewOp);
1071  }
1072  }
1073  break;
1074  }
1075  case ISD::EXTRACT_SUBVECTOR: {
1076  // Offset the demanded elts by the subvector index.
1077  SDValue Src = Op.getOperand(0);
1078  if (Src.getValueType().isScalableVector())
1079  break;
1080  uint64_t Idx = Op.getConstantOperandVal(1);
1081  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1082  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1083 
1084  if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1085  Depth + 1))
1086  return true;
1087 
1088  // Attempt to avoid multi-use src if we don't need anything from it.
1089  if (!DemandedBits.isAllOnesValue() || !DemandedSrcElts.isAllOnesValue()) {
1090  SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1091  Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1092  if (DemandedSrc) {
1093  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1094  Op.getOperand(1));
1095  return TLO.CombineTo(Op, NewOp);
1096  }
1097  }
1098  break;
1099  }
1100  case ISD::CONCAT_VECTORS: {
1101  Known.Zero.setAllBits();
1102  Known.One.setAllBits();
1103  EVT SubVT = Op.getOperand(0).getValueType();
1104  unsigned NumSubVecs = Op.getNumOperands();
1105  unsigned NumSubElts = SubVT.getVectorNumElements();
1106  for (unsigned i = 0; i != NumSubVecs; ++i) {
1107  APInt DemandedSubElts =
1108  DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1109  if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1110  Known2, TLO, Depth + 1))
1111  return true;
1112  // Known bits are shared by every demanded subvector element.
1113  if (!!DemandedSubElts)
1114  Known = KnownBits::commonBits(Known, Known2);
1115  }
1116  break;
1117  }
1118  case ISD::VECTOR_SHUFFLE: {
1119  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1120 
1121  // Collect demanded elements from shuffle operands..
1122  APInt DemandedLHS(NumElts, 0);
1123  APInt DemandedRHS(NumElts, 0);
1124  for (unsigned i = 0; i != NumElts; ++i) {
1125  if (!DemandedElts[i])
1126  continue;
1127  int M = ShuffleMask[i];
1128  if (M < 0) {
1129  // For UNDEF elements, we don't know anything about the common state of
1130  // the shuffle result.
1131  DemandedLHS.clearAllBits();
1132  DemandedRHS.clearAllBits();
1133  break;
1134  }
1135  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1136  if (M < (int)NumElts)
1137  DemandedLHS.setBit(M);
1138  else
1139  DemandedRHS.setBit(M - NumElts);
1140  }
1141 
1142  if (!!DemandedLHS || !!DemandedRHS) {
1143  SDValue Op0 = Op.getOperand(0);
1144  SDValue Op1 = Op.getOperand(1);
1145 
1146  Known.Zero.setAllBits();
1147  Known.One.setAllBits();
1148  if (!!DemandedLHS) {
1149  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1150  Depth + 1))
1151  return true;
1152  Known = KnownBits::commonBits(Known, Known2);
1153  }
1154  if (!!DemandedRHS) {
1155  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1156  Depth + 1))
1157  return true;
1158  Known = KnownBits::commonBits(Known, Known2);
1159  }
1160 
1161  // Attempt to avoid multi-use ops if we don't need anything from them.
1162  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1163  Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1164  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1165  Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1166  if (DemandedOp0 || DemandedOp1) {
1167  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1168  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1169  SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1170  return TLO.CombineTo(Op, NewOp);
1171  }
1172  }
1173  break;
1174  }
1175  case ISD::AND: {
1176  SDValue Op0 = Op.getOperand(0);
1177  SDValue Op1 = Op.getOperand(1);
1178 
1179  // If the RHS is a constant, check to see if the LHS would be zero without
1180  // using the bits from the RHS. Below, we use knowledge about the RHS to
1181  // simplify the LHS, here we're using information from the LHS to simplify
1182  // the RHS.
1183  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1184  // Do not increment Depth here; that can cause an infinite loop.
1185  KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1186  // If the LHS already has zeros where RHSC does, this 'and' is dead.
1187  if ((LHSKnown.Zero & DemandedBits) ==
1188  (~RHSC->getAPIntValue() & DemandedBits))
1189  return TLO.CombineTo(Op, Op0);
1190 
1191  // If any of the set bits in the RHS are known zero on the LHS, shrink
1192  // the constant.
1193  if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1194  DemandedElts, TLO))
1195  return true;
1196 
1197  // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1198  // constant, but if this 'and' is only clearing bits that were just set by
1199  // the xor, then this 'and' can be eliminated by shrinking the mask of
1200  // the xor. For example, for a 32-bit X:
1201  // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1202  if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1203  LHSKnown.One == ~RHSC->getAPIntValue()) {
1204  SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1205  return TLO.CombineTo(Op, Xor);
1206  }
1207  }
1208 
1209  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1210  Depth + 1))
1211  return true;
1212  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1213  if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1214  Known2, TLO, Depth + 1))
1215  return true;
1216  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1217 
1218  // Attempt to avoid multi-use ops if we don't need anything from them.
1219  if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1220  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1221  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1222  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1223  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1224  if (DemandedOp0 || DemandedOp1) {
1225  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1226  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1227  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1228  return TLO.CombineTo(Op, NewOp);
1229  }
1230  }
1231 
1232  // If all of the demanded bits are known one on one side, return the other.
1233  // These bits cannot contribute to the result of the 'and'.
1234  if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1235  return TLO.CombineTo(Op, Op0);
1236  if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1237  return TLO.CombineTo(Op, Op1);
1238  // If all of the demanded bits in the inputs are known zeros, return zero.
1239  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1240  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1241  // If the RHS is a constant, see if we can simplify it.
1242  if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1243  TLO))
1244  return true;
1245  // If the operation can be done in a smaller type, do so.
1246  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1247  return true;
1248 
1249  Known &= Known2;
1250  break;
1251  }
1252  case ISD::OR: {
1253  SDValue Op0 = Op.getOperand(0);
1254  SDValue Op1 = Op.getOperand(1);
1255 
1256  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1257  Depth + 1))
1258  return true;
1259  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1260  if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1261  Known2, TLO, Depth + 1))
1262  return true;
1263  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1264 
1265  // Attempt to avoid multi-use ops if we don't need anything from them.
1266  if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1267  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1268  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1269  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1270  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1271  if (DemandedOp0 || DemandedOp1) {
1272  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1273  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1274  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1275  return TLO.CombineTo(Op, NewOp);
1276  }
1277  }
1278 
1279  // If all of the demanded bits are known zero on one side, return the other.
1280  // These bits cannot contribute to the result of the 'or'.
1281  if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1282  return TLO.CombineTo(Op, Op0);
1283  if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1284  return TLO.CombineTo(Op, Op1);
1285  // If the RHS is a constant, see if we can simplify it.
1286  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1287  return true;
1288  // If the operation can be done in a smaller type, do so.
1289  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1290  return true;
1291 
1292  Known |= Known2;
1293  break;
1294  }
1295  case ISD::XOR: {
1296  SDValue Op0 = Op.getOperand(0);
1297  SDValue Op1 = Op.getOperand(1);
1298 
1299  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1300  Depth + 1))
1301  return true;
1302  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1303  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1304  Depth + 1))
1305  return true;
1306  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1307 
1308  // Attempt to avoid multi-use ops if we don't need anything from them.
1309  if (!DemandedBits.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1310  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1311  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1312  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1313  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1314  if (DemandedOp0 || DemandedOp1) {
1315  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1316  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1317  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1318  return TLO.CombineTo(Op, NewOp);
1319  }
1320  }
1321 
1322  // If all of the demanded bits are known zero on one side, return the other.
1323  // These bits cannot contribute to the result of the 'xor'.
1324  if (DemandedBits.isSubsetOf(Known.Zero))
1325  return TLO.CombineTo(Op, Op0);
1326  if (DemandedBits.isSubsetOf(Known2.Zero))
1327  return TLO.CombineTo(Op, Op1);
1328  // If the operation can be done in a smaller type, do so.
1329  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1330  return true;
1331 
1332  // If all of the unknown bits are known to be zero on one side or the other
1333  // turn this into an *inclusive* or.
1334  // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1335  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1336  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1337 
1338  ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1339  if (C) {
1340  // If one side is a constant, and all of the set bits in the constant are
1341  // also known set on the other side, turn this into an AND, as we know
1342  // the bits will be cleared.
1343  // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1344  // NB: it is okay if more bits are known than are requested
1345  if (C->getAPIntValue() == Known2.One) {
1346  SDValue ANDC =
1347  TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1348  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1349  }
1350 
1351  // If the RHS is a constant, see if we can change it. Don't alter a -1
1352  // constant because that's a 'not' op, and that is better for combining
1353  // and codegen.
1354  if (!C->isAllOnesValue() &&
1355  DemandedBits.isSubsetOf(C->getAPIntValue())) {
1356  // We're flipping all demanded bits. Flip the undemanded bits too.
1357  SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1358  return TLO.CombineTo(Op, New);
1359  }
1360  }
1361 
1362  // If we can't turn this into a 'not', try to shrink the constant.
1363  if (!C || !C->isAllOnesValue())
1364  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1365  return true;
1366 
1367  Known ^= Known2;
1368  break;
1369  }
1370  case ISD::SELECT:
1371  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1372  Depth + 1))
1373  return true;
1374  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1375  Depth + 1))
1376  return true;
1377  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1378  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1379 
1380  // If the operands are constants, see if we can simplify them.
1381  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1382  return true;
1383 
1384  // Only known if known in both the LHS and RHS.
1385  Known = KnownBits::commonBits(Known, Known2);
1386  break;
1387  case ISD::SELECT_CC:
1388  if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1389  Depth + 1))
1390  return true;
1391  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1392  Depth + 1))
1393  return true;
1394  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1395  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1396 
1397  // If the operands are constants, see if we can simplify them.
1398  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1399  return true;
1400 
1401  // Only known if known in both the LHS and RHS.
1402  Known = KnownBits::commonBits(Known, Known2);
1403  break;
1404  case ISD::SETCC: {
1405  SDValue Op0 = Op.getOperand(0);
1406  SDValue Op1 = Op.getOperand(1);
1407  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1408  // If (1) we only need the sign-bit, (2) the setcc operands are the same
1409  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1410  // -1, we may be able to bypass the setcc.
1411  if (DemandedBits.isSignMask() &&
1414  BooleanContent::ZeroOrNegativeOneBooleanContent) {
1415  // If we're testing X < 0, then this compare isn't needed - just use X!
1416  // FIXME: We're limiting to integer types here, but this should also work
1417  // if we don't care about FP signed-zero. The use of SETLT with FP means
1418  // that we don't care about NaNs.
1419  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1421  return TLO.CombineTo(Op, Op0);
1422 
1423  // TODO: Should we check for other forms of sign-bit comparisons?
1424  // Examples: X <= -1, X >= 0
1425  }
1426  if (getBooleanContents(Op0.getValueType()) ==
1428  BitWidth > 1)
1429  Known.Zero.setBitsFrom(1);
1430  break;
1431  }
1432  case ISD::SHL: {
1433  SDValue Op0 = Op.getOperand(0);
1434  SDValue Op1 = Op.getOperand(1);
1435  EVT ShiftVT = Op1.getValueType();
1436 
1437  if (const APInt *SA =
1438  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1439  unsigned ShAmt = SA->getZExtValue();
1440  if (ShAmt == 0)
1441  return TLO.CombineTo(Op, Op0);
1442 
1443  // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1444  // single shift. We can do this if the bottom bits (which are shifted
1445  // out) are never demanded.
1446  // TODO - support non-uniform vector amounts.
1447  if (Op0.getOpcode() == ISD::SRL) {
1448  if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1449  if (const APInt *SA2 =
1450  TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1451  unsigned C1 = SA2->getZExtValue();
1452  unsigned Opc = ISD::SHL;
1453  int Diff = ShAmt - C1;
1454  if (Diff < 0) {
1455  Diff = -Diff;
1456  Opc = ISD::SRL;
1457  }
1458  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1459  return TLO.CombineTo(
1460  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1461  }
1462  }
1463  }
1464 
1465  // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1466  // are not demanded. This will likely allow the anyext to be folded away.
1467  // TODO - support non-uniform vector amounts.
1468  if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1469  SDValue InnerOp = Op0.getOperand(0);
1470  EVT InnerVT = InnerOp.getValueType();
1471  unsigned InnerBits = InnerVT.getScalarSizeInBits();
1472  if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1473  isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1474  EVT ShTy = getShiftAmountTy(InnerVT, DL);
1475  if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1476  ShTy = InnerVT;
1477  SDValue NarrowShl =
1478  TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1479  TLO.DAG.getConstant(ShAmt, dl, ShTy));
1480  return TLO.CombineTo(
1481  Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1482  }
1483 
1484  // Repeat the SHL optimization above in cases where an extension
1485  // intervenes: (shl (anyext (shr x, c1)), c2) to
1486  // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1487  // aren't demanded (as above) and that the shifted upper c1 bits of
1488  // x aren't demanded.
1489  // TODO - support non-uniform vector amounts.
1490  if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1491  InnerOp.hasOneUse()) {
1492  if (const APInt *SA2 =
1493  TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1494  unsigned InnerShAmt = SA2->getZExtValue();
1495  if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1496  DemandedBits.getActiveBits() <=
1497  (InnerBits - InnerShAmt + ShAmt) &&
1498  DemandedBits.countTrailingZeros() >= ShAmt) {
1499  SDValue NewSA =
1500  TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1501  SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1502  InnerOp.getOperand(0));
1503  return TLO.CombineTo(
1504  Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1505  }
1506  }
1507  }
1508  }
1509 
1510  APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1511  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1512  Depth + 1))
1513  return true;
1514  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1515  Known.Zero <<= ShAmt;
1516  Known.One <<= ShAmt;
1517  // low bits known zero.
1518  Known.Zero.setLowBits(ShAmt);
1519 
1520  // Try shrinking the operation as long as the shift amount will still be
1521  // in range.
1522  if ((ShAmt < DemandedBits.getActiveBits()) &&
1523  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1524  return true;
1525  }
1526 
1527  // If we are only demanding sign bits then we can use the shift source
1528  // directly.
1529  if (const APInt *MaxSA =
1530  TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1531  unsigned ShAmt = MaxSA->getZExtValue();
1532  unsigned NumSignBits =
1533  TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1534  unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1535  if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1536  return TLO.CombineTo(Op, Op0);
1537  }
1538  break;
1539  }
1540  case ISD::SRL: {
1541  SDValue Op0 = Op.getOperand(0);
1542  SDValue Op1 = Op.getOperand(1);
1543  EVT ShiftVT = Op1.getValueType();
1544 
1545  if (const APInt *SA =
1546  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1547  unsigned ShAmt = SA->getZExtValue();
1548  if (ShAmt == 0)
1549  return TLO.CombineTo(Op, Op0);
1550 
1551  // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1552  // single shift. We can do this if the top bits (which are shifted out)
1553  // are never demanded.
1554  // TODO - support non-uniform vector amounts.
1555  if (Op0.getOpcode() == ISD::SHL) {
1556  if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1557  if (const APInt *SA2 =
1558  TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1559  unsigned C1 = SA2->getZExtValue();
1560  unsigned Opc = ISD::SRL;
1561  int Diff = ShAmt - C1;
1562  if (Diff < 0) {
1563  Diff = -Diff;
1564  Opc = ISD::SHL;
1565  }
1566  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1567  return TLO.CombineTo(
1568  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1569  }
1570  }
1571  }
1572 
1573  APInt InDemandedMask = (DemandedBits << ShAmt);
1574 
1575  // If the shift is exact, then it does demand the low bits (and knows that
1576  // they are zero).
1577  if (Op->getFlags().hasExact())
1578  InDemandedMask.setLowBits(ShAmt);
1579 
1580  // Compute the new bits that are at the top now.
1581  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1582  Depth + 1))
1583  return true;
1584  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1585  Known.Zero.lshrInPlace(ShAmt);
1586  Known.One.lshrInPlace(ShAmt);
1587  // High bits known zero.
1588  Known.Zero.setHighBits(ShAmt);
1589  }
1590  break;
1591  }
1592  case ISD::SRA: {
1593  SDValue Op0 = Op.getOperand(0);
1594  SDValue Op1 = Op.getOperand(1);
1595  EVT ShiftVT = Op1.getValueType();
1596 
1597  // If we only want bits that already match the signbit then we don't need
1598  // to shift.
1599  unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1600  if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1601  NumHiDemandedBits)
1602  return TLO.CombineTo(Op, Op0);
1603 
1604  // If this is an arithmetic shift right and only the low-bit is set, we can
1605  // always convert this into a logical shr, even if the shift amount is
1606  // variable. The low bit of the shift cannot be an input sign bit unless
1607  // the shift amount is >= the size of the datatype, which is undefined.
1608  if (DemandedBits.isOneValue())
1609  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1610 
1611  if (const APInt *SA =
1612  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1613  unsigned ShAmt = SA->getZExtValue();
1614  if (ShAmt == 0)
1615  return TLO.CombineTo(Op, Op0);
1616 
1617  APInt InDemandedMask = (DemandedBits << ShAmt);
1618 
1619  // If the shift is exact, then it does demand the low bits (and knows that
1620  // they are zero).
1621  if (Op->getFlags().hasExact())
1622  InDemandedMask.setLowBits(ShAmt);
1623 
1624  // If any of the demanded bits are produced by the sign extension, we also
1625  // demand the input sign bit.
1626  if (DemandedBits.countLeadingZeros() < ShAmt)
1627  InDemandedMask.setSignBit();
1628 
1629  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1630  Depth + 1))
1631  return true;
1632  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1633  Known.Zero.lshrInPlace(ShAmt);
1634  Known.One.lshrInPlace(ShAmt);
1635 
1636  // If the input sign bit is known to be zero, or if none of the top bits
1637  // are demanded, turn this into an unsigned shift right.
1638  if (Known.Zero[BitWidth - ShAmt - 1] ||
1639  DemandedBits.countLeadingZeros() >= ShAmt) {
1640  SDNodeFlags Flags;
1641  Flags.setExact(Op->getFlags().hasExact());
1642  return TLO.CombineTo(
1643  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1644  }
1645 
1646  int Log2 = DemandedBits.exactLogBase2();
1647  if (Log2 >= 0) {
1648  // The bit must come from the sign.
1649  SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1650  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1651  }
1652 
1653  if (Known.One[BitWidth - ShAmt - 1])
1654  // New bits are known one.
1655  Known.One.setHighBits(ShAmt);
1656 
1657  // Attempt to avoid multi-use ops if we don't need anything from them.
1658  if (!InDemandedMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
1659  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1660  Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1661  if (DemandedOp0) {
1662  SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1663  return TLO.CombineTo(Op, NewOp);
1664  }
1665  }
1666  }
1667  break;
1668  }
1669  case ISD::FSHL:
1670  case ISD::FSHR: {
1671  SDValue Op0 = Op.getOperand(0);
1672  SDValue Op1 = Op.getOperand(1);
1673  SDValue Op2 = Op.getOperand(2);
1674  bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1675 
1676  if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1677  unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1678 
1679  // For fshl, 0-shift returns the 1st arg.
1680  // For fshr, 0-shift returns the 2nd arg.
1681  if (Amt == 0) {
1682  if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1683  Known, TLO, Depth + 1))
1684  return true;
1685  break;
1686  }
1687 
1688  // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1689  // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1690  APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1691  APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1692  if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1693  Depth + 1))
1694  return true;
1695  if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1696  Depth + 1))
1697  return true;
1698 
1699  Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1700  Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1701  Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1702  Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1703  Known.One |= Known2.One;
1704  Known.Zero |= Known2.Zero;
1705  }
1706 
1707  // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1708  if (isPowerOf2_32(BitWidth)) {
1709  APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1710  if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1711  Known2, TLO, Depth + 1))
1712  return true;
1713  }
1714  break;
1715  }
1716  case ISD::ROTL:
1717  case ISD::ROTR: {
1718  SDValue Op0 = Op.getOperand(0);
1719  SDValue Op1 = Op.getOperand(1);
1720 
1721  // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1722  if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1723  return TLO.CombineTo(Op, Op0);
1724 
1725  // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1726  if (isPowerOf2_32(BitWidth)) {
1727  APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1728  if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1729  Depth + 1))
1730  return true;
1731  }
1732  break;
1733  }
1734  case ISD::UMIN: {
1735  // Check if one arg is always less than (or equal) to the other arg.
1736  SDValue Op0 = Op.getOperand(0);
1737  SDValue Op1 = Op.getOperand(1);
1738  KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1739  KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1740  Known = KnownBits::umin(Known0, Known1);
1741  if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1742  return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1743  if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1744  return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1745  break;
1746  }
1747  case ISD::UMAX: {
1748  // Check if one arg is always greater than (or equal) to the other arg.
1749  SDValue Op0 = Op.getOperand(0);
1750  SDValue Op1 = Op.getOperand(1);
1751  KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1752  KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1753  Known = KnownBits::umax(Known0, Known1);
1754  if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1755  return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1756  if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1757  return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1758  break;
1759  }
1760  case ISD::BITREVERSE: {
1761  SDValue Src = Op.getOperand(0);
1762  APInt DemandedSrcBits = DemandedBits.reverseBits();
1763  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1764  Depth + 1))
1765  return true;
1766  Known.One = Known2.One.reverseBits();
1767  Known.Zero = Known2.Zero.reverseBits();
1768  break;
1769  }
1770  case ISD::BSWAP: {
1771  SDValue Src = Op.getOperand(0);
1772  APInt DemandedSrcBits = DemandedBits.byteSwap();
1773  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1774  Depth + 1))
1775  return true;
1776  Known.One = Known2.One.byteSwap();
1777  Known.Zero = Known2.Zero.byteSwap();
1778  break;
1779  }
1780  case ISD::CTPOP: {
1781  // If only 1 bit is demanded, replace with PARITY as long as we're before
1782  // op legalization.
1783  // FIXME: Limit to scalars for now.
1784  if (DemandedBits.isOneValue() && !TLO.LegalOps && !VT.isVector())
1785  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1786  Op.getOperand(0)));
1787 
1788  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1789  break;
1790  }
1791  case ISD::SIGN_EXTEND_INREG: {
1792  SDValue Op0 = Op.getOperand(0);
1793  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1794  unsigned ExVTBits = ExVT.getScalarSizeInBits();
1795 
1796  // If we only care about the highest bit, don't bother shifting right.
1797  if (DemandedBits.isSignMask()) {
1798  unsigned NumSignBits =
1799  TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1800  bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1801  // However if the input is already sign extended we expect the sign
1802  // extension to be dropped altogether later and do not simplify.
1803  if (!AlreadySignExtended) {
1804  // Compute the correct shift amount type, which must be getShiftAmountTy
1805  // for scalar types after legalization.
1806  EVT ShiftAmtTy = VT;
1807  if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1808  ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1809 
1810  SDValue ShiftAmt =
1811  TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1812  return TLO.CombineTo(Op,
1813  TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1814  }
1815  }
1816 
1817  // If none of the extended bits are demanded, eliminate the sextinreg.
1818  if (DemandedBits.getActiveBits() <= ExVTBits)
1819  return TLO.CombineTo(Op, Op0);
1820 
1821  APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1822 
1823  // Since the sign extended bits are demanded, we know that the sign
1824  // bit is demanded.
1825  InputDemandedBits.setBit(ExVTBits - 1);
1826 
1827  if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1828  return true;
1829  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1830 
1831  // If the sign bit of the input is known set or clear, then we know the
1832  // top bits of the result.
1833 
1834  // If the input sign bit is known zero, convert this into a zero extension.
1835  if (Known.Zero[ExVTBits - 1])
1836  return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1837 
1838  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1839  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1840  Known.One.setBitsFrom(ExVTBits);
1841  Known.Zero &= Mask;
1842  } else { // Input sign bit unknown
1843  Known.Zero &= Mask;
1844  Known.One &= Mask;
1845  }
1846  break;
1847  }
1848  case ISD::BUILD_PAIR: {
1849  EVT HalfVT = Op.getOperand(0).getValueType();
1850  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1851 
1852  APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1853  APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1854 
1855  KnownBits KnownLo, KnownHi;
1856 
1857  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1858  return true;
1859 
1860  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1861  return true;
1862 
1863  Known.Zero = KnownLo.Zero.zext(BitWidth) |
1864  KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1865 
1866  Known.One = KnownLo.One.zext(BitWidth) |
1867  KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1868  break;
1869  }
1870  case ISD::ZERO_EXTEND:
1872  SDValue Src = Op.getOperand(0);
1873  EVT SrcVT = Src.getValueType();
1874  unsigned InBits = SrcVT.getScalarSizeInBits();
1875  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1876  bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1877 
1878  // If none of the top bits are demanded, convert this into an any_extend.
1879  if (DemandedBits.getActiveBits() <= InBits) {
1880  // If we only need the non-extended bits of the bottom element
1881  // then we can just bitcast to the result.
1882  if (IsVecInReg && DemandedElts == 1 &&
1883  VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1885  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1886 
1887  unsigned Opc =
1889  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1890  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1891  }
1892 
1893  APInt InDemandedBits = DemandedBits.trunc(InBits);
1894  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1895  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1896  Depth + 1))
1897  return true;
1898  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1899  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1900  Known = Known.zext(BitWidth);
1901 
1902  // Attempt to avoid multi-use ops if we don't need anything from them.
1903  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1904  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1905  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1906  break;
1907  }
1908  case ISD::SIGN_EXTEND:
1910  SDValue Src = Op.getOperand(0);
1911  EVT SrcVT = Src.getValueType();
1912  unsigned InBits = SrcVT.getScalarSizeInBits();
1913  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1914  bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1915 
1916  // If none of the top bits are demanded, convert this into an any_extend.
1917  if (DemandedBits.getActiveBits() <= InBits) {
1918  // If we only need the non-extended bits of the bottom element
1919  // then we can just bitcast to the result.
1920  if (IsVecInReg && DemandedElts == 1 &&
1921  VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1923  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1924 
1925  unsigned Opc =
1927  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1928  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1929  }
1930 
1931  APInt InDemandedBits = DemandedBits.trunc(InBits);
1932  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1933 
1934  // Since some of the sign extended bits are demanded, we know that the sign
1935  // bit is demanded.
1936  InDemandedBits.setBit(InBits - 1);
1937 
1938  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1939  Depth + 1))
1940  return true;
1941  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1942  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1943 
1944  // If the sign bit is known one, the top bits match.
1945  Known = Known.sext(BitWidth);
1946 
1947  // If the sign bit is known zero, convert this to a zero extend.
1948  if (Known.isNonNegative()) {
1949  unsigned Opc =
1951  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1952  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1953  }
1954 
1955  // Attempt to avoid multi-use ops if we don't need anything from them.
1956  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1957  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1958  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1959  break;
1960  }
1961  case ISD::ANY_EXTEND:
1963  SDValue Src = Op.getOperand(0);
1964  EVT SrcVT = Src.getValueType();
1965  unsigned InBits = SrcVT.getScalarSizeInBits();
1966  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1967  bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1968 
1969  // If we only need the bottom element then we can just bitcast.
1970  // TODO: Handle ANY_EXTEND?
1971  if (IsVecInReg && DemandedElts == 1 &&
1972  VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1974  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1975 
1976  APInt InDemandedBits = DemandedBits.trunc(InBits);
1977  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1978  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1979  Depth + 1))
1980  return true;
1981  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1982  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1983  Known = Known.anyext(BitWidth);
1984 
1985  // Attempt to avoid multi-use ops if we don't need anything from them.
1986  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1987  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1988  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1989  break;
1990  }
1991  case ISD::TRUNCATE: {
1992  SDValue Src = Op.getOperand(0);
1993 
1994  // Simplify the input, using demanded bit information, and compute the known
1995  // zero/one bits live out.
1996  unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
1997  APInt TruncMask = DemandedBits.zext(OperandBitWidth);
1998  if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
1999  Depth + 1))
2000  return true;
2001  Known = Known.trunc(BitWidth);
2002 
2003  // Attempt to avoid multi-use ops if we don't need anything from them.
2004  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2005  Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2006  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2007 
2008  // If the input is only used by this truncate, see if we can shrink it based
2009  // on the known demanded bits.
2010  if (Src.getNode()->hasOneUse()) {
2011  switch (Src.getOpcode()) {
2012  default:
2013  break;
2014  case ISD::SRL:
2015  // Shrink SRL by a constant if none of the high bits shifted in are
2016  // demanded.
2017  if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2018  // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2019  // undesirable.
2020  break;
2021 
2022  const APInt *ShAmtC =
2023  TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2024  if (!ShAmtC || ShAmtC->uge(BitWidth))
2025  break;
2026  uint64_t ShVal = ShAmtC->getZExtValue();
2027 
2028  APInt HighBits =
2029  APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2030  HighBits.lshrInPlace(ShVal);
2031  HighBits = HighBits.trunc(BitWidth);
2032 
2033  if (!(HighBits & DemandedBits)) {
2034  // None of the shifted in bits are needed. Add a truncate of the
2035  // shift input, then shift it.
2036  SDValue NewShAmt = TLO.DAG.getConstant(
2037  ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2038  SDValue NewTrunc =
2039  TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2040  return TLO.CombineTo(
2041  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2042  }
2043  break;
2044  }
2045  }
2046 
2047  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2048  break;
2049  }
2050  case ISD::AssertZext: {
2051  // AssertZext demands all of the high bits, plus any of the low bits
2052  // demanded by its users.
2053  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2055  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2056  TLO, Depth + 1))
2057  return true;
2058  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2059 
2060  Known.Zero |= ~InMask;
2061  break;
2062  }
2063  case ISD::EXTRACT_VECTOR_ELT: {
2064  SDValue Src = Op.getOperand(0);
2065  SDValue Idx = Op.getOperand(1);
2066  ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2067  unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2068 
2069  if (SrcEltCnt.isScalable())
2070  return false;
2071 
2072  // Demand the bits from every vector element without a constant index.
2073  unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2074  APInt DemandedSrcElts = APInt::getAllOnesValue(NumSrcElts);
2075  if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2076  if (CIdx->getAPIntValue().ult(NumSrcElts))
2077  DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2078 
2079  // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2080  // anything about the extended bits.
2081  APInt DemandedSrcBits = DemandedBits;
2082  if (BitWidth > EltBitWidth)
2083  DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2084 
2085  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2086  Depth + 1))
2087  return true;
2088 
2089  // Attempt to avoid multi-use ops if we don't need anything from them.
2090  if (!DemandedSrcBits.isAllOnesValue() ||
2091  !DemandedSrcElts.isAllOnesValue()) {
2092  if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2093  Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2094  SDValue NewOp =
2095  TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2096  return TLO.CombineTo(Op, NewOp);
2097  }
2098  }
2099 
2100  Known = Known2;
2101  if (BitWidth > EltBitWidth)
2102  Known = Known.anyext(BitWidth);
2103  break;
2104  }
2105  case ISD::BITCAST: {
2106  SDValue Src = Op.getOperand(0);
2107  EVT SrcVT = Src.getValueType();
2108  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2109 
2110  // If this is an FP->Int bitcast and if the sign bit is the only
2111  // thing demanded, turn this into a FGETSIGN.
2112  if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2113  DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2114  SrcVT.isFloatingPoint()) {
2115  bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2117  if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2118  SrcVT != MVT::f128) {
2119  // Cannot eliminate/lower SHL for f128 yet.
2120  EVT Ty = OpVTLegal ? VT : MVT::i32;
2121  // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2122  // place. We expect the SHL to be eliminated by other optimizations.
2123  SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2124  unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2125  if (!OpVTLegal && OpVTSizeInBits > 32)
2126  Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2127  unsigned ShVal = Op.getValueSizeInBits() - 1;
2128  SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2129  return TLO.CombineTo(Op,
2130  TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2131  }
2132  }
2133 
2134  // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2135  // Demand the elt/bit if any of the original elts/bits are demanded.
2136  // TODO - bigendian once we have test coverage.
2137  if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
2138  TLO.DAG.getDataLayout().isLittleEndian()) {
2139  unsigned Scale = BitWidth / NumSrcEltBits;
2140  unsigned NumSrcElts = SrcVT.getVectorNumElements();
2141  APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2142  APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2143  for (unsigned i = 0; i != Scale; ++i) {
2144  unsigned Offset = i * NumSrcEltBits;
2145  APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
2146  if (!Sub.isNullValue()) {
2147  DemandedSrcBits |= Sub;
2148  for (unsigned j = 0; j != NumElts; ++j)
2149  if (DemandedElts[j])
2150  DemandedSrcElts.setBit((j * Scale) + i);
2151  }
2152  }
2153 
2154  APInt KnownSrcUndef, KnownSrcZero;
2155  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2156  KnownSrcZero, TLO, Depth + 1))
2157  return true;
2158 
2159  KnownBits KnownSrcBits;
2160  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2161  KnownSrcBits, TLO, Depth + 1))
2162  return true;
2163  } else if ((NumSrcEltBits % BitWidth) == 0 &&
2164  TLO.DAG.getDataLayout().isLittleEndian()) {
2165  unsigned Scale = NumSrcEltBits / BitWidth;
2166  unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2167  APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits);
2168  APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts);
2169  for (unsigned i = 0; i != NumElts; ++i)
2170  if (DemandedElts[i]) {
2171  unsigned Offset = (i % Scale) * BitWidth;
2172  DemandedSrcBits.insertBits(DemandedBits, Offset);
2173  DemandedSrcElts.setBit(i / Scale);
2174  }
2175 
2176  if (SrcVT.isVector()) {
2177  APInt KnownSrcUndef, KnownSrcZero;
2178  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2179  KnownSrcZero, TLO, Depth + 1))
2180  return true;
2181  }
2182 
2183  KnownBits KnownSrcBits;
2184  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2185  KnownSrcBits, TLO, Depth + 1))
2186  return true;
2187  }
2188 
2189  // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2190  // recursive call where Known may be useful to the caller.
2191  if (Depth > 0) {
2192  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2193  return false;
2194  }
2195  break;
2196  }
2197  case ISD::ADD:
2198  case ISD::MUL:
2199  case ISD::SUB: {
2200  // Add, Sub, and Mul don't demand any bits in positions beyond that
2201  // of the highest bit demanded of them.
2202  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2203  SDNodeFlags Flags = Op.getNode()->getFlags();
2204  unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2205  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2206  if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2207  Depth + 1) ||
2208  SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2209  Depth + 1) ||
2210  // See if the operation should be performed at a smaller bit width.
2211  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2212  if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2213  // Disable the nsw and nuw flags. We can no longer guarantee that we
2214  // won't wrap after simplification.
2215  Flags.setNoSignedWrap(false);
2216  Flags.setNoUnsignedWrap(false);
2217  SDValue NewOp =
2218  TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2219  return TLO.CombineTo(Op, NewOp);
2220  }
2221  return true;
2222  }
2223 
2224  // Attempt to avoid multi-use ops if we don't need anything from them.
2225  if (!LoMask.isAllOnesValue() || !DemandedElts.isAllOnesValue()) {
2226  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2227  Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2228  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2229  Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2230  if (DemandedOp0 || DemandedOp1) {
2231  Flags.setNoSignedWrap(false);
2232  Flags.setNoUnsignedWrap(false);
2233  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2234  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2235  SDValue NewOp =
2236  TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2237  return TLO.CombineTo(Op, NewOp);
2238  }
2239  }
2240 
2241  // If we have a constant operand, we may be able to turn it into -1 if we
2242  // do not demand the high bits. This can make the constant smaller to
2243  // encode, allow more general folding, or match specialized instruction
2244  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2245  // is probably not useful (and could be detrimental).
2247  APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2248  if (C && !C->isAllOnesValue() && !C->isOne() &&
2249  (C->getAPIntValue() | HighMask).isAllOnesValue()) {
2250  SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2251  // Disable the nsw and nuw flags. We can no longer guarantee that we
2252  // won't wrap after simplification.
2253  Flags.setNoSignedWrap(false);
2254  Flags.setNoUnsignedWrap(false);
2255  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2256  return TLO.CombineTo(Op, NewOp);
2257  }
2258 
2260  }
2261  default:
2262  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2263  if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2264  Known, TLO, Depth))
2265  return true;
2266  break;
2267  }
2268 
2269  // Just use computeKnownBits to compute output bits.
2270  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2271  break;
2272  }
2273 
2274  // If we know the value of all of the demanded bits, return this as a
2275  // constant.
2276  if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2277  // Avoid folding to a constant if any OpaqueConstant is involved.
2278  const SDNode *N = Op.getNode();
2279  for (SDNode *Op :
2281  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2282  if (C->isOpaque())
2283  return false;
2284  }
2285  if (VT.isInteger())
2286  return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2287  if (VT.isFloatingPoint())
2288  return TLO.CombineTo(
2289  Op,
2290  TLO.DAG.getConstantFP(
2291  APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2292  }
2293 
2294  return false;
2295 }
2296 
2298  const APInt &DemandedElts,
2299  APInt &KnownUndef,
2300  APInt &KnownZero,
2301  DAGCombinerInfo &DCI) const {
2302  SelectionDAG &DAG = DCI.DAG;
2303  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2304  !DCI.isBeforeLegalizeOps());
2305 
2306  bool Simplified =
2307  SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2308  if (Simplified) {
2309  DCI.AddToWorklist(Op.getNode());
2310  DCI.CommitTargetLoweringOpt(TLO);
2311  }
2312 
2313  return Simplified;
2314 }
2315 
2316 /// Given a vector binary operation and known undefined elements for each input
2317 /// operand, compute whether each element of the output is undefined.
2319  const APInt &UndefOp0,
2320  const APInt &UndefOp1) {
2321  EVT VT = BO.getValueType();
2322  assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
2323  "Vector binop only");
2324 
2325  EVT EltVT = VT.getVectorElementType();
2326  unsigned NumElts = VT.getVectorNumElements();
2327  assert(UndefOp0.getBitWidth() == NumElts &&
2328  UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
2329 
2330  auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2331  const APInt &UndefVals) {
2332  if (UndefVals[Index])
2333  return DAG.getUNDEF(EltVT);
2334 
2335  if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2336  // Try hard to make sure that the getNode() call is not creating temporary
2337  // nodes. Ignore opaque integers because they do not constant fold.
2338  SDValue Elt = BV->getOperand(Index);
2339  auto *C = dyn_cast<ConstantSDNode>(Elt);
2340  if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2341  return Elt;
2342  }
2343 
2344  return SDValue();
2345  };
2346 
2347  APInt KnownUndef = APInt::getNullValue(NumElts);
2348  for (unsigned i = 0; i != NumElts; ++i) {
2349  // If both inputs for this element are either constant or undef and match
2350  // the element type, compute the constant/undef result for this element of
2351  // the vector.
2352  // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2353  // not handle FP constants. The code within getNode() should be refactored
2354  // to avoid the danger of creating a bogus temporary node here.
2355  SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2356  SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2357  if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2358  if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2359  KnownUndef.setBit(i);
2360  }
2361  return KnownUndef;
2362 }
2363 
2365  SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2366  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2367  bool AssumeSingleUse) const {
2368  EVT VT = Op.getValueType();
2369  unsigned Opcode = Op.getOpcode();
2370  APInt DemandedElts = OriginalDemandedElts;
2371  unsigned NumElts = DemandedElts.getBitWidth();
2372  assert(VT.isVector() && "Expected vector op");
2373 
2374  KnownUndef = KnownZero = APInt::getNullValue(NumElts);
2375 
2376  // TODO: For now we assume we know nothing about scalable vectors.
2377  if (VT.isScalableVector())
2378  return false;
2379 
2380  assert(VT.getVectorNumElements() == NumElts &&
2381  "Mask size mismatches value type element count!");
2382 
2383  // Undef operand.
2384  if (Op.isUndef()) {
2385  KnownUndef.setAllBits();
2386  return false;
2387  }
2388 
2389  // If Op has other users, assume that all elements are needed.
2390  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2391  DemandedElts.setAllBits();
2392 
2393  // Not demanding any elements from Op.
2394  if (DemandedElts == 0) {
2395  KnownUndef.setAllBits();
2396  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2397  }
2398 
2399  // Limit search depth.
2401  return false;
2402 
2403  SDLoc DL(Op);
2404  unsigned EltSizeInBits = VT.getScalarSizeInBits();
2405 
2406  // Helper for demanding the specified elements and all the bits of both binary
2407  // operands.
2408  auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2409  SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2410  TLO.DAG, Depth + 1);
2411  SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2412  TLO.DAG, Depth + 1);
2413  if (NewOp0 || NewOp1) {
2414  SDValue NewOp = TLO.DAG.getNode(
2415  Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2416  return TLO.CombineTo(Op, NewOp);
2417  }
2418  return false;
2419  };
2420 
2421  switch (Opcode) {
2422  case ISD::SCALAR_TO_VECTOR: {
2423  if (!DemandedElts[0]) {
2424  KnownUndef.setAllBits();
2425  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2426  }
2427  SDValue ScalarSrc = Op.getOperand(0);
2428  if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2429  SDValue Src = ScalarSrc.getOperand(0);
2430  SDValue Idx = ScalarSrc.getOperand(1);
2431  EVT SrcVT = Src.getValueType();
2432 
2433  ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2434 
2435  if (SrcEltCnt.isScalable())
2436  return false;
2437 
2438  unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2439  if (isNullConstant(Idx)) {
2440  APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2441  APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2442  APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2443  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2444  TLO, Depth + 1))
2445  return true;
2446  }
2447  }
2448  KnownUndef.setHighBits(NumElts - 1);
2449  break;
2450  }
2451  case ISD::BITCAST: {
2452  SDValue Src = Op.getOperand(0);
2453  EVT SrcVT = Src.getValueType();
2454 
2455  // We only handle vectors here.
2456  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2457  if (!SrcVT.isVector())
2458  break;
2459 
2460  // Fast handling of 'identity' bitcasts.
2461  unsigned NumSrcElts = SrcVT.getVectorNumElements();
2462  if (NumSrcElts == NumElts)
2463  return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2464  KnownZero, TLO, Depth + 1);
2465 
2466  APInt SrcZero, SrcUndef;
2467  APInt SrcDemandedElts = APInt::getNullValue(NumSrcElts);
2468 
2469  // Bitcast from 'large element' src vector to 'small element' vector, we
2470  // must demand a source element if any DemandedElt maps to it.
2471  if ((NumElts % NumSrcElts) == 0) {
2472  unsigned Scale = NumElts / NumSrcElts;
2473  for (unsigned i = 0; i != NumElts; ++i)
2474  if (DemandedElts[i])
2475  SrcDemandedElts.setBit(i / Scale);
2476 
2477  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2478  TLO, Depth + 1))
2479  return true;
2480 
2481  // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2482  // of the large element.
2483  // TODO - bigendian once we have test coverage.
2484  if (TLO.DAG.getDataLayout().isLittleEndian()) {
2485  unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2486  APInt SrcDemandedBits = APInt::getNullValue(SrcEltSizeInBits);
2487  for (unsigned i = 0; i != NumElts; ++i)
2488  if (DemandedElts[i]) {
2489  unsigned Ofs = (i % Scale) * EltSizeInBits;
2490  SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2491  }
2492 
2493  KnownBits Known;
2494  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2495  TLO, Depth + 1))
2496  return true;
2497  }
2498 
2499  // If the src element is zero/undef then all the output elements will be -
2500  // only demanded elements are guaranteed to be correct.
2501  for (unsigned i = 0; i != NumSrcElts; ++i) {
2502  if (SrcDemandedElts[i]) {
2503  if (SrcZero[i])
2504  KnownZero.setBits(i * Scale, (i + 1) * Scale);
2505  if (SrcUndef[i])
2506  KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2507  }
2508  }
2509  }
2510 
2511  // Bitcast from 'small element' src vector to 'large element' vector, we
2512  // demand all smaller source elements covered by the larger demanded element
2513  // of this vector.
2514  if ((NumSrcElts % NumElts) == 0) {
2515  unsigned Scale = NumSrcElts / NumElts;
2516  for (unsigned i = 0; i != NumElts; ++i)
2517  if (DemandedElts[i])
2518  SrcDemandedElts.setBits(i * Scale, (i + 1) * Scale);
2519 
2520  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2521  TLO, Depth + 1))
2522  return true;
2523 
2524  // If all the src elements covering an output element are zero/undef, then
2525  // the output element will be as well, assuming it was demanded.
2526  for (unsigned i = 0; i != NumElts; ++i) {
2527  if (DemandedElts[i]) {
2528  if (SrcZero.extractBits(Scale, i * Scale).isAllOnesValue())
2529  KnownZero.setBit(i);
2530  if (SrcUndef.extractBits(Scale, i * Scale).isAllOnesValue())
2531  KnownUndef.setBit(i);
2532  }
2533  }
2534  }
2535  break;
2536  }
2537  case ISD::BUILD_VECTOR: {
2538  // Check all elements and simplify any unused elements with UNDEF.
2539  if (!DemandedElts.isAllOnesValue()) {
2540  // Don't simplify BROADCASTS.
2541  if (llvm::any_of(Op->op_values(),
2542  [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2543  SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2544  bool Updated = false;
2545  for (unsigned i = 0; i != NumElts; ++i) {
2546  if (!DemandedElts[i] && !Ops[i].isUndef()) {
2547  Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2548  KnownUndef.setBit(i);
2549  Updated = true;
2550  }
2551  }
2552  if (Updated)
2553  return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2554  }
2555  }
2556  for (unsigned i = 0; i != NumElts; ++i) {
2557  SDValue SrcOp = Op.getOperand(i);
2558  if (SrcOp.isUndef()) {
2559  KnownUndef.setBit(i);
2560  } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2562  KnownZero.setBit(i);
2563  }
2564  }
2565  break;
2566  }
2567  case ISD::CONCAT_VECTORS: {
2568  EVT SubVT = Op.getOperand(0).getValueType();
2569  unsigned NumSubVecs = Op.getNumOperands();
2570  unsigned NumSubElts = SubVT.getVectorNumElements();
2571  for (unsigned i = 0; i != NumSubVecs; ++i) {
2572  SDValue SubOp = Op.getOperand(i);
2573  APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2574  APInt SubUndef, SubZero;
2575  if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2576  Depth + 1))
2577  return true;
2578  KnownUndef.insertBits(SubUndef, i * NumSubElts);
2579  KnownZero.insertBits(SubZero, i * NumSubElts);
2580  }
2581  break;
2582  }
2583  case ISD::INSERT_SUBVECTOR: {
2584  // Demand any elements from the subvector and the remainder from the src its
2585  // inserted into.
2586  SDValue Src = Op.getOperand(0);
2587  SDValue Sub = Op.getOperand(1);
2588  uint64_t Idx = Op.getConstantOperandVal(2);
2589  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2590  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2591  APInt DemandedSrcElts = DemandedElts;
2592  DemandedSrcElts.insertBits(APInt::getNullValue(NumSubElts), Idx);
2593 
2594  APInt SubUndef, SubZero;
2595  if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2596  Depth + 1))
2597  return true;
2598 
2599  // If none of the src operand elements are demanded, replace it with undef.
2600  if (!DemandedSrcElts && !Src.isUndef())
2601  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2602  TLO.DAG.getUNDEF(VT), Sub,
2603  Op.getOperand(2)));
2604 
2605  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2606  TLO, Depth + 1))
2607  return true;
2608  KnownUndef.insertBits(SubUndef, Idx);
2609  KnownZero.insertBits(SubZero, Idx);
2610 
2611  // Attempt to avoid multi-use ops if we don't need anything from them.
2612  if (!DemandedSrcElts.isAllOnesValue() ||
2613  !DemandedSubElts.isAllOnesValue()) {
2614  SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2615  Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2616  SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2617  Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2618  if (NewSrc || NewSub) {
2619  NewSrc = NewSrc ? NewSrc : Src;
2620  NewSub = NewSub ? NewSub : Sub;
2621  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2622  NewSub, Op.getOperand(2));
2623  return TLO.CombineTo(Op, NewOp);
2624  }
2625  }
2626  break;
2627  }
2628  case ISD::EXTRACT_SUBVECTOR: {
2629  // Offset the demanded elts by the subvector index.
2630  SDValue Src = Op.getOperand(0);
2631  if (Src.getValueType().isScalableVector())
2632  break;
2633  uint64_t Idx = Op.getConstantOperandVal(1);
2634  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2635  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2636 
2637  APInt SrcUndef, SrcZero;
2638  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2639  Depth + 1))
2640  return true;
2641  KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2642  KnownZero = SrcZero.extractBits(NumElts, Idx);
2643 
2644  // Attempt to avoid multi-use ops if we don't need anything from them.
2645  if (!DemandedElts.isAllOnesValue()) {
2646  SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2647  Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2648  if (NewSrc) {
2649  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2650  Op.getOperand(1));
2651  return TLO.CombineTo(Op, NewOp);
2652  }
2653  }
2654  break;
2655  }
2656  case ISD::INSERT_VECTOR_ELT: {
2657  SDValue Vec = Op.getOperand(0);
2658  SDValue Scl = Op.getOperand(1);
2659  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2660 
2661  // For a legal, constant insertion index, if we don't need this insertion
2662  // then strip it, else remove it from the demanded elts.
2663  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2664  unsigned Idx = CIdx->getZExtValue();
2665  if (!DemandedElts[Idx])
2666  return TLO.CombineTo(Op, Vec);
2667 
2668  APInt DemandedVecElts(DemandedElts);
2669  DemandedVecElts.clearBit(Idx);
2670  if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2671  KnownZero, TLO, Depth + 1))
2672  return true;
2673 
2674  KnownUndef.setBitVal(Idx, Scl.isUndef());
2675 
2676  KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2677  break;
2678  }
2679 
2680  APInt VecUndef, VecZero;
2681  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2682  Depth + 1))
2683  return true;
2684  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2685  break;
2686  }
2687  case ISD::VSELECT: {
2688  // Try to transform the select condition based on the current demanded
2689  // elements.
2690  // TODO: If a condition element is undef, we can choose from one arm of the
2691  // select (and if one arm is undef, then we can propagate that to the
2692  // result).
2693  // TODO - add support for constant vselect masks (see IR version of this).
2694  APInt UnusedUndef, UnusedZero;
2695  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2696  UnusedZero, TLO, Depth + 1))
2697  return true;
2698 
2699  // See if we can simplify either vselect operand.
2700  APInt DemandedLHS(DemandedElts);
2701  APInt DemandedRHS(DemandedElts);
2702  APInt UndefLHS, ZeroLHS;
2703  APInt UndefRHS, ZeroRHS;
2704  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2705  ZeroLHS, TLO, Depth + 1))
2706  return true;
2707  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2708  ZeroRHS, TLO, Depth + 1))
2709  return true;
2710 
2711  KnownUndef = UndefLHS & UndefRHS;
2712  KnownZero = ZeroLHS & ZeroRHS;
2713  break;
2714  }
2715  case ISD::VECTOR_SHUFFLE: {
2716  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2717 
2718  // Collect demanded elements from shuffle operands..
2719  APInt DemandedLHS(NumElts, 0);
2720  APInt DemandedRHS(NumElts, 0);
2721  for (unsigned i = 0; i != NumElts; ++i) {
2722  int M = ShuffleMask[i];
2723  if (M < 0 || !DemandedElts[i])
2724  continue;
2725  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
2726  if (M < (int)NumElts)
2727  DemandedLHS.setBit(M);
2728  else
2729  DemandedRHS.setBit(M - NumElts);
2730  }
2731 
2732  // See if we can simplify either shuffle operand.
2733  APInt UndefLHS, ZeroLHS;
2734  APInt UndefRHS, ZeroRHS;
2735  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2736  ZeroLHS, TLO, Depth + 1))
2737  return true;
2738  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2739  ZeroRHS, TLO, Depth + 1))
2740  return true;
2741 
2742  // Simplify mask using undef elements from LHS/RHS.
2743  bool Updated = false;
2744  bool IdentityLHS = true, IdentityRHS = true;
2745  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2746  for (unsigned i = 0; i != NumElts; ++i) {
2747  int &M = NewMask[i];
2748  if (M < 0)
2749  continue;
2750  if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2751  (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2752  Updated = true;
2753  M = -1;
2754  }
2755  IdentityLHS &= (M < 0) || (M == (int)i);
2756  IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2757  }
2758 
2759  // Update legal shuffle masks based on demanded elements if it won't reduce
2760  // to Identity which can cause premature removal of the shuffle mask.
2761  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2762  SDValue LegalShuffle =
2763  buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2764  NewMask, TLO.DAG);
2765  if (LegalShuffle)
2766  return TLO.CombineTo(Op, LegalShuffle);
2767  }
2768 
2769  // Propagate undef/zero elements from LHS/RHS.
2770  for (unsigned i = 0; i != NumElts; ++i) {
2771  int M = ShuffleMask[i];
2772  if (M < 0) {
2773  KnownUndef.setBit(i);
2774  } else if (M < (int)NumElts) {
2775  if (UndefLHS[M])
2776  KnownUndef.setBit(i);
2777  if (ZeroLHS[M])
2778  KnownZero.setBit(i);
2779  } else {
2780  if (UndefRHS[M - NumElts])
2781  KnownUndef.setBit(i);
2782  if (ZeroRHS[M - NumElts])
2783  KnownZero.setBit(i);
2784  }
2785  }
2786  break;
2787  }
2791  APInt SrcUndef, SrcZero;
2792  SDValue Src = Op.getOperand(0);
2793  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2794  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2795  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2796  Depth + 1))
2797  return true;
2798  KnownZero = SrcZero.zextOrTrunc(NumElts);
2799  KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2800 
2801  if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2802  Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2803  DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2804  // aext - if we just need the bottom element then we can bitcast.
2805  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2806  }
2807 
2808  if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2809  // zext(undef) upper bits are guaranteed to be zero.
2810  if (DemandedElts.isSubsetOf(KnownUndef))
2811  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2812  KnownUndef.clearAllBits();
2813  }
2814  break;
2815  }
2816 
2817  // TODO: There are more binop opcodes that could be handled here - MIN,
2818  // MAX, saturated math, etc.
2819  case ISD::OR:
2820  case ISD::XOR:
2821  case ISD::ADD:
2822  case ISD::SUB:
2823  case ISD::FADD:
2824  case ISD::FSUB:
2825  case ISD::FMUL:
2826  case ISD::FDIV:
2827  case ISD::FREM: {
2828  SDValue Op0 = Op.getOperand(0);
2829  SDValue Op1 = Op.getOperand(1);
2830 
2831  APInt UndefRHS, ZeroRHS;
2832  if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2833  Depth + 1))
2834  return true;
2835  APInt UndefLHS, ZeroLHS;
2836  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2837  Depth + 1))
2838  return true;
2839 
2840  KnownZero = ZeroLHS & ZeroRHS;
2841  KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2842 
2843  // Attempt to avoid multi-use ops if we don't need anything from them.
2844  // TODO - use KnownUndef to relax the demandedelts?
2845  if (!DemandedElts.isAllOnesValue())
2846  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2847  return true;
2848  break;
2849  }
2850  case ISD::SHL:
2851  case ISD::SRL:
2852  case ISD::SRA:
2853  case ISD::ROTL:
2854  case ISD::ROTR: {
2855  SDValue Op0 = Op.getOperand(0);
2856  SDValue Op1 = Op.getOperand(1);
2857 
2858  APInt UndefRHS, ZeroRHS;
2859  if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2860  Depth + 1))
2861  return true;
2862  APInt UndefLHS, ZeroLHS;
2863  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2864  Depth + 1))
2865  return true;
2866 
2867  KnownZero = ZeroLHS;
2868  KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2869 
2870  // Attempt to avoid multi-use ops if we don't need anything from them.
2871  // TODO - use KnownUndef to relax the demandedelts?
2872  if (!DemandedElts.isAllOnesValue())
2873  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2874  return true;
2875  break;
2876  }
2877  case ISD::MUL:
2878  case ISD::AND: {
2879  SDValue Op0 = Op.getOperand(0);
2880  SDValue Op1 = Op.getOperand(1);
2881 
2882  APInt SrcUndef, SrcZero;
2883  if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2884  Depth + 1))
2885  return true;
2886  if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2887  TLO, Depth + 1))
2888  return true;
2889 
2890  // If either side has a zero element, then the result element is zero, even
2891  // if the other is an UNDEF.
2892  // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2893  // and then handle 'and' nodes with the rest of the binop opcodes.
2894  KnownZero |= SrcZero;
2895  KnownUndef &= SrcUndef;
2896  KnownUndef &= ~KnownZero;
2897 
2898  // Attempt to avoid multi-use ops if we don't need anything from them.
2899  // TODO - use KnownUndef to relax the demandedelts?
2900  if (!DemandedElts.isAllOnesValue())
2901  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2902  return true;
2903  break;
2904  }
2905  case ISD::TRUNCATE:
2906  case ISD::SIGN_EXTEND:
2907  case ISD::ZERO_EXTEND:
2908  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2909  KnownZero, TLO, Depth + 1))
2910  return true;
2911 
2912  if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2913  // zext(undef) upper bits are guaranteed to be zero.
2914  if (DemandedElts.isSubsetOf(KnownUndef))
2915  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2916  KnownUndef.clearAllBits();
2917  }
2918  break;
2919  default: {
2920  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2921  if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2922  KnownZero, TLO, Depth))
2923  return true;
2924  } else {
2925  KnownBits Known;
2926  APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits);
2927  if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2928  TLO, Depth, AssumeSingleUse))
2929  return true;
2930  }
2931  break;
2932  }
2933  }
2934  assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
2935 
2936  // Constant fold all undef cases.
2937  // TODO: Handle zero cases as well.
2938  if (DemandedElts.isSubsetOf(KnownUndef))
2939  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2940 
2941  return false;
2942 }
2943 
2944 /// Determine which of the bits specified in Mask are known to be either zero or
2945 /// one and return them in the Known.
2947  KnownBits &Known,
2948  const APInt &DemandedElts,
2949  const SelectionDAG &DAG,
2950  unsigned Depth) const {
2951  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2952  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2953  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2954  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2955  "Should use MaskedValueIsZero if you don't know whether Op"
2956  " is a target node!");
2957  Known.resetAll();
2958 }
2959 
2961  GISelKnownBits &Analysis, Register R, KnownBits &Known,
2962  const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2963  unsigned Depth) const {
2964  Known.resetAll();
2965 }
2966 
2968  const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
2969  // The low bits are known zero if the pointer is aligned.
2970  Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
2971 }
2972 
2974  GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
2975  unsigned Depth) const {
2976  return Align(1);
2977 }
2978 
2979 /// This method can be implemented by targets that want to expose additional
2980 /// information about sign bits to the DAG Combiner.
2982  const APInt &,
2983  const SelectionDAG &,
2984  unsigned Depth) const {
2985  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2986  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2987  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2988  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2989  "Should use ComputeNumSignBits if you don't know whether Op"
2990  " is a target node!");
2991  return 1;
2992 }
2993 
2995  GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
2996  const MachineRegisterInfo &MRI, unsigned Depth) const {
2997  return 1;
2998 }
2999 
3001  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3002  TargetLoweringOpt &TLO, unsigned Depth) const {
3003  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3004  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3005  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3006  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3007  "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3008  " is a target node!");
3009  return false;
3010 }
3011 
3013  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3014  KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3015  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3016  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3017  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3018  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3019  "Should use SimplifyDemandedBits if you don't know whether Op"
3020  " is a target node!");
3021  computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3022  return false;
3023 }
3024 
3026  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3027  SelectionDAG &DAG, unsigned Depth) const {
3028  assert(
3029  (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3030  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3031  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3032  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3033  "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3034  " is a target node!");
3035  return SDValue();
3036 }
3037 
3038 SDValue
3041  SelectionDAG &DAG) const {
3042  bool LegalMask = isShuffleMaskLegal(Mask, VT);
3043  if (!LegalMask) {
3044  std::swap(N0, N1);
3046  LegalMask = isShuffleMaskLegal(Mask, VT);
3047  }
3048 
3049  if (!LegalMask)
3050  return SDValue();
3051 
3052  return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3053 }
3054 
3056  return nullptr;
3057 }
3058 
3060  SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3061  bool PoisonOnly, unsigned Depth) const {
3062  assert(
3063  (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3064  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3065  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3066  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3067  "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3068  " is a target node!");
3069  return false;
3070 }
3071 
3073  const SelectionDAG &DAG,
3074  bool SNaN,
3075  unsigned Depth) const {
3076  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3077  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3078  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3079  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3080  "Should use isKnownNeverNaN if you don't know whether Op"
3081  " is a target node!");
3082  return false;
3083 }
3084 
3085 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3086 // work with truncating build vectors and vectors with elements of less than
3087 // 8 bits.
3089  if (!N)
3090  return false;
3091 
3092  APInt CVal;
3093  if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3094  CVal = CN->getAPIntValue();
3095  } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3096  auto *CN = BV->getConstantSplatNode();
3097  if (!CN)
3098  return false;
3099 
3100  // If this is a truncating build vector, truncate the splat value.
3101  // Otherwise, we may fail to match the expected values below.
3102  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3103  CVal = CN->getAPIntValue();
3104  if (BVEltWidth < CVal.getBitWidth())
3105  CVal = CVal.trunc(BVEltWidth);
3106  } else {
3107  return false;
3108  }
3109 
3110  switch (getBooleanContents(N->getValueType(0))) {
3112  return CVal[0];
3114  return CVal.isOneValue();
3116  return CVal.isAllOnesValue();
3117  }
3118 
3119  llvm_unreachable("Invalid boolean contents");
3120 }
3121 
3123  if (!N)
3124  return false;
3125 
3126  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3127  if (!CN) {
3128  const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3129  if (!BV)
3130  return false;
3131 
3132  // Only interested in constant splats, we don't care about undef
3133  // elements in identifying boolean constants and getConstantSplatNode
3134  // returns NULL if all ops are undef;
3135  CN = BV->getConstantSplatNode();
3136  if (!CN)
3137  return false;
3138  }
3139 
3140  if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3141  return !CN->getAPIntValue()[0];
3142 
3143  return CN->isNullValue();
3144 }
3145 
3147  bool SExt) const {
3148  if (VT == MVT::i1)
3149  return N->isOne();
3150 
3152  switch (Cnt) {
3154  // An extended value of 1 is always true, unless its original type is i1,
3155  // in which case it will be sign extended to -1.
3156  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3159  return N->isAllOnesValue() && SExt;
3160  }
3161  llvm_unreachable("Unexpected enumeration.");
3162 }
3163 
3164 /// This helper function of SimplifySetCC tries to optimize the comparison when
3165 /// either operand of the SetCC node is a bitwise-and instruction.
3166 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3167  ISD::CondCode Cond, const SDLoc &DL,
3168  DAGCombinerInfo &DCI) const {
3169  // Match these patterns in any of their permutations:
3170  // (X & Y) == Y
3171  // (X & Y) != Y
3172  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3173  std::swap(N0, N1);
3174 
3175  EVT OpVT = N0.getValueType();
3176  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3177  (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3178  return SDValue();
3179 
3180  SDValue X, Y;
3181  if (N0.getOperand(0) == N1) {
3182  X = N0.getOperand(1);
3183  Y = N0.getOperand(0);
3184  } else if (N0.getOperand(1) == N1) {
3185  X = N0.getOperand(0);
3186  Y = N0.getOperand(1);
3187  } else {
3188  return SDValue();
3189  }
3190 
3191  SelectionDAG &DAG = DCI.DAG;
3192  SDValue Zero = DAG.getConstant(0, DL, OpVT);
3193  if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3194  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3195  // Note that where Y is variable and is known to have at most one bit set
3196  // (for example, if it is Z & 1) we cannot do this; the expressions are not
3197  // equivalent when Y == 0.
3198  assert(OpVT.isInteger());
3199  Cond = ISD::getSetCCInverse(Cond, OpVT);
3200  if (DCI.isBeforeLegalizeOps() ||
3202  return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3203  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3204  // If the target supports an 'and-not' or 'and-complement' logic operation,
3205  // try to use that to make a comparison operation more efficient.
3206  // But don't do this transform if the mask is a single bit because there are
3207  // more efficient ways to deal with that case (for example, 'bt' on x86 or
3208  // 'rlwinm' on PPC).
3209 
3210  // Bail out if the compare operand that we want to turn into a zero is
3211  // already a zero (otherwise, infinite loop).
3212  auto *YConst = dyn_cast<ConstantSDNode>(Y);
3213  if (YConst && YConst->isNullValue())
3214  return SDValue();
3215 
3216  // Transform this into: ~X & Y == 0.
3217  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3218  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3219  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3220  }
3221 
3222  return SDValue();
3223 }
3224 
3225 /// There are multiple IR patterns that could be checking whether certain
3226 /// truncation of a signed number would be lossy or not. The pattern which is
3227 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
3228 /// We are looking for the following pattern: (KeptBits is a constant)
3229 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3230 /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3231 /// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3232 /// We will unfold it into the natural trunc+sext pattern:
3233 /// ((%x << C) a>> C) dstcond %x
3234 /// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
3235 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3236  EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3237  const SDLoc &DL) const {
3238  // We must be comparing with a constant.
3239  ConstantSDNode *C1;
3240  if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3241  return SDValue();
3242 
3243  // N0 should be: add %x, (1 << (KeptBits-1))
3244  if (N0->getOpcode() != ISD::ADD)
3245  return SDValue();
3246 
3247  // And we must be 'add'ing a constant.
3248  ConstantSDNode *C01;
3249  if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3250  return SDValue();
3251 
3252  SDValue X = N0->getOperand(0);
3253  EVT XVT = X.getValueType();
3254 
3255  // Validate constants ...
3256 
3257  APInt I1 = C1->getAPIntValue();
3258 
3259  ISD::CondCode NewCond;
3260  if (Cond == ISD::CondCode::SETULT) {
3261  NewCond = ISD::CondCode::SETEQ;
3262  } else if (Cond == ISD::CondCode::SETULE) {
3263  NewCond = ISD::CondCode::SETEQ;
3264  // But need to 'canonicalize' the constant.
3265  I1 += 1;
3266  } else if (Cond == ISD::CondCode::SETUGT) {
3267  NewCond = ISD::CondCode::SETNE;
3268  // But need to 'canonicalize' the constant.
3269  I1 += 1;
3270  } else if (Cond == ISD::CondCode::SETUGE) {
3271  NewCond = ISD::CondCode::SETNE;
3272  } else
3273  return SDValue();
3274 
3275  APInt I01 = C01->getAPIntValue();
3276 
3277  auto checkConstants = [&I1, &I01]() -> bool {
3278  // Both of them must be power-of-two, and the constant from setcc is bigger.
3279  return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3280  };
3281 
3282  if (checkConstants()) {
3283  // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3284  } else {
3285  // What if we invert constants? (and the target predicate)
3286  I1.negate();
3287  I01.negate();
3288  assert(XVT.isInteger());
3289  NewCond = getSetCCInverse(NewCond, XVT);
3290  if (!checkConstants())
3291  return SDValue();
3292  // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3293  }
3294 
3295  // They are power-of-two, so which bit is set?
3296  const unsigned KeptBits = I1.logBase2();
3297  const unsigned KeptBitsMinusOne = I01.logBase2();
3298 
3299  // Magic!
3300  if (KeptBits != (KeptBitsMinusOne + 1))
3301  return SDValue();
3302  assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
3303 
3304  // We don't want to do this in every single case.
3305  SelectionDAG &DAG = DCI.DAG;
3307  XVT, KeptBits))
3308  return SDValue();
3309 
3310  const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3311  assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable");
3312 
3313  // Unfold into: ((%x << C) a>> C) cond %x
3314  // Where 'cond' will be either 'eq' or 'ne'.
3315  SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3316  SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3317  SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3318  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3319 
3320  return T2;
3321 }
3322 
3323 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3324 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3325  EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3326  DAGCombinerInfo &DCI, const SDLoc &DL) const {
3327  assert(isConstOrConstSplat(N1C) &&
3328  isConstOrConstSplat(N1C)->getAPIntValue().isNullValue() &&
3329  "Should be a comparison with 0.");
3330  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3331  "Valid only for [in]equality comparisons.");
3332 
3333  unsigned NewShiftOpcode;
3334  SDValue X, C, Y;
3335 
3336  SelectionDAG &DAG = DCI.DAG;
3337  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3338 
3339  // Look for '(C l>>/<< Y)'.
3340  auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3341  // The shift should be one-use.
3342  if (!V.hasOneUse())
3343  return false;
3344  unsigned OldShiftOpcode = V.getOpcode();
3345  switch (OldShiftOpcode) {
3346  case ISD::SHL:
3347  NewShiftOpcode = ISD::SRL;
3348  break;
3349  case ISD::SRL:
3350  NewShiftOpcode = ISD::SHL;
3351  break;
3352  default:
3353  return false; // must be a logical shift.
3354  }
3355  // We should be shifting a constant.
3356  // FIXME: best to use isConstantOrConstantVector().
3357  C = V.getOperand(0);
3358  ConstantSDNode *CC =
3359  isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3360  if (!CC)
3361  return false;
3362  Y = V.getOperand(1);
3363 
3364  ConstantSDNode *XC =
3365  isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3366  return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3367  X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3368  };
3369 
3370  // LHS of comparison should be an one-use 'and'.
3371  if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3372  return SDValue();
3373 
3374  X = N0.getOperand(0);
3375  SDValue Mask = N0.getOperand(1);
3376 
3377  // 'and' is commutative!
3378  if (!Match(Mask)) {
3379  std::swap(X, Mask);
3380  if (!Match(Mask))
3381  return SDValue();
3382  }
3383 
3384  EVT VT = X.getValueType();
3385 
3386  // Produce:
3387  // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3388  SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3389  SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3390  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3391  return T2;
3392 }
3393 
3394 /// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3395 /// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3396 /// handle the commuted versions of these patterns.
3397 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3398  ISD::CondCode Cond, const SDLoc &DL,
3399  DAGCombinerInfo &DCI) const {
3400  unsigned BOpcode = N0.getOpcode();
3401  assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
3402  "Unexpected binop");
3403  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
3404 
3405  // (X + Y) == X --> Y == 0
3406  // (X - Y) == X --> Y == 0
3407  // (X ^ Y) == X --> Y == 0
3408  SelectionDAG &DAG = DCI.DAG;
3409  EVT OpVT = N0.getValueType();
3410  SDValue X = N0.getOperand(0);
3411  SDValue Y = N0.getOperand(1);
3412  if (X == N1)
3413  return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3414 
3415  if (Y != N1)
3416  return SDValue();
3417 
3418  // (X + Y) == Y --> X == 0
3419  // (X ^ Y) == Y --> X == 0
3420  if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3421  return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3422 
3423  // The shift would not be valid if the operands are boolean (i1).
3424  if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3425  return SDValue();
3426 
3427  // (X - Y) == Y --> X == Y << 1
3428  EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3429  !DCI.isBeforeLegalize());
3430  SDValue One = DAG.getConstant(1, DL, ShiftVT);
3431  SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3432  if (!DCI.isCalledByLegalizer())
3433  DCI.AddToWorklist(YShl1.getNode());
3434  return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3435 }
3436 
3438  SDValue N0, const APInt &C1,
3439  ISD::CondCode Cond, const SDLoc &dl,
3440  SelectionDAG &DAG) {
3441  // Look through truncs that don't change the value of a ctpop.
3442  // FIXME: Add vector support? Need to be careful with setcc result type below.
3443  SDValue CTPOP = N0;
3444  if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3446  CTPOP = N0.getOperand(0);
3447 
3448  if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3449  return SDValue();
3450 
3451  EVT CTVT = CTPOP.getValueType();
3452  SDValue CTOp = CTPOP.getOperand(0);
3453 
3454  // If this is a vector CTPOP, keep the CTPOP if it is legal.
3455  // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3456  if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3457  return SDValue();
3458 
3459  // (ctpop x) u< 2 -> (x & x-1) == 0
3460  // (ctpop x) u> 1 -> (x & x-1) != 0
3461  if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3462  unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3463  if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3464  return SDValue();
3465  if (C1 == 0 && (Cond == ISD::SETULT))
3466  return SDValue(); // This is handled elsewhere.
3467 
3468  unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3469 
3470  SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3471  SDValue Result = CTOp;
3472  for (unsigned i = 0; i < Passes; i++) {
3473  SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3474  Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3475  }
3477  return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3478  }
3479 
3480  // If ctpop is not supported, expand a power-of-2 comparison based on it.
3481  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3482  // For scalars, keep CTPOP if it is legal or custom.
3483  if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3484  return SDValue();
3485  // This is based on X86's custom lowering for CTPOP which produces more
3486  // instructions than the expansion here.
3487 
3488  // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3489  // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3490  SDValue Zero = DAG.getConstant(0, dl, CTVT);
3491  SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3492  assert(CTVT.isInteger());
3493  ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3494  SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3495  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3496  SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3497  SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3498  unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3499  return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3500  }
3501 
3502  return SDValue();
3503 }
3504 
3505 /// Try to simplify a setcc built with the specified operands and cc. If it is
3506 /// unable to simplify it, return a null SDValue.
3508  ISD::CondCode Cond, bool foldBooleans,
3509  DAGCombinerInfo &DCI,
3510  const SDLoc &dl) const {
3511  SelectionDAG &DAG = DCI.DAG;
3512  const DataLayout &Layout = DAG.getDataLayout();
3513  EVT OpVT = N0.getValueType();
3514 
3515  // Constant fold or commute setcc.
3516  if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3517  return Fold;
3518 
3519  // Ensure that the constant occurs on the RHS and fold constant comparisons.
3520  // TODO: Handle non-splat vector constants. All undef causes trouble.
3521  // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3522  // infinite loop here when we encounter one.
3524  if (isConstOrConstSplat(N0) &&
3525  (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3526  (DCI.isBeforeLegalizeOps() ||
3527  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3528  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3529 
3530  // If we have a subtract with the same 2 non-constant operands as this setcc
3531  // -- but in reverse order -- then try to commute the operands of this setcc
3532  // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3533  // instruction on some targets.
3534  if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3535  (DCI.isBeforeLegalizeOps() ||
3536  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3537  DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3538  !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3539  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3540 
3541  if (auto *N1C = isConstOrConstSplat(N1)) {
3542  const APInt &C1 = N1C->getAPIntValue();
3543 
3544  // Optimize some CTPOP cases.
3545  if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3546  return V;
3547 
3548  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3549  // equality comparison, then we're just comparing whether X itself is
3550  // zero.
3551  if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) &&
3552  N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3554  if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3555  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3556  ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3557  if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3558  // (srl (ctlz x), 5) == 0 -> X != 0
3559  // (srl (ctlz x), 5) != 1 -> X != 0
3560  Cond = ISD::SETNE;
3561  } else {
3562  // (srl (ctlz x), 5) != 0 -> X == 0
3563  // (srl (ctlz x), 5) == 1 -> X == 0
3564  Cond = ISD::SETEQ;
3565  }
3566  SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3567  return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3568  Cond);
3569  }
3570  }
3571  }
3572  }
3573 
3574  // FIXME: Support vectors.
3575  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3576  const APInt &C1 = N1C->getAPIntValue();
3577 
3578  // (zext x) == C --> x == (trunc C)
3579  // (sext x) == C --> x == (trunc C)
3580  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3581  DCI.isBeforeLegalize() && N0->hasOneUse()) {
3582  unsigned MinBits = N0.getValueSizeInBits();
3583  SDValue PreExt;
3584  bool Signed = false;
3585  if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3586  // ZExt
3587  MinBits = N0->getOperand(0).getValueSizeInBits();
3588  PreExt = N0->getOperand(0);
3589  } else if (N0->getOpcode() == ISD::AND) {
3590  // DAGCombine turns costly ZExts into ANDs
3591  if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3592  if ((C->getAPIntValue()+1).isPowerOf2()) {
3593  MinBits = C->getAPIntValue().countTrailingOnes();
3594  PreExt = N0->getOperand(0);
3595  }
3596  } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3597  // SExt
3598  MinBits = N0->getOperand(0).getValueSizeInBits();
3599  PreExt = N0->getOperand(0);
3600  Signed = true;
3601  } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3602  // ZEXTLOAD / SEXTLOAD
3603  if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3604  MinBits = LN0->getMemoryVT().getSizeInBits();
3605  PreExt = N0;
3606  } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3607  Signed = true;
3608  MinBits = LN0->getMemoryVT().getSizeInBits();
3609  PreExt = N0;
3610  }
3611  }
3612 
3613  // Figure out how many bits we need to preserve this constant.
3614  unsigned ReqdBits = Signed ?
3615  C1.getBitWidth() - C1.getNumSignBits() + 1 :
3616  C1.getActiveBits();
3617 
3618  // Make sure we're not losing bits from the constant.
3619  if (MinBits > 0 &&
3620  MinBits < C1.getBitWidth() &&
3621  MinBits >= ReqdBits) {
3622  EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3623  if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3624  // Will get folded away.
3625  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3626  if (MinBits == 1 && C1 == 1)
3627  // Invert the condition.
3628  return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3630  SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3631  return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3632  }
3633 
3634  // If truncating the setcc operands is not desirable, we can still
3635  // simplify the expression in some cases:
3636  // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3637  // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3638  // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3639  // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3640  // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3641  // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3642  SDValue TopSetCC = N0->getOperand(0);
3643  unsigned N0Opc = N0->getOpcode();
3644  bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3645  if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3646  TopSetCC.getOpcode() == ISD::SETCC &&
3647  (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3648  (isConstFalseVal(N1C) ||
3649  isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3650 
3651  bool Inverse = (N1C->isNullValue() && Cond == ISD::SETEQ) ||
3652  (!N1C->isNullValue() && Cond == ISD::SETNE);
3653 
3654  if (!Inverse)
3655  return TopSetCC;
3656 
3658  cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3659  TopSetCC.getOperand(0).getValueType());
3660  return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3661  TopSetCC.getOperand(1),
3662  InvCond);
3663  }
3664  }
3665  }
3666 
3667  // If the LHS is '(and load, const)', the RHS is 0, the test is for
3668  // equality or unsigned, and all 1 bits of the const are in the same
3669  // partial word, see if we can shorten the load.
3670  if (DCI.isBeforeLegalize() &&
3672  N0.getOpcode() == ISD::AND && C1 == 0 &&
3673  N0.getNode()->hasOneUse() &&
3674  isa<LoadSDNode>(N0.getOperand(0)) &&
3675  N0.getOperand(0).getNode()->hasOneUse() &&
3676  isa<ConstantSDNode>(N0.getOperand(1))) {
3677  LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3678  APInt bestMask;
3679  unsigned bestWidth = 0, bestOffset = 0;
3680  if (Lod->isSimple() && Lod->isUnindexed()) {
3681  unsigned origWidth = N0.getValueSizeInBits();
3682  unsigned maskWidth = origWidth;
3683  // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3684  // 8 bits, but have to be careful...
3685  if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3686  origWidth = Lod->getMemoryVT().getSizeInBits();
3687  const APInt &Mask = N0.getConstantOperandAPInt(1);
3688  for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3689  APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3690  for (unsigned offset=0; offset<origWidth/width; offset++) {
3691  if (Mask.isSubsetOf(newMask)) {
3692  if (Layout.isLittleEndian())
3693  bestOffset = (uint64_t)offset * (width/8);
3694  else
3695  bestOffset = (origWidth/width - offset - 1) * (width/8);
3696  bestMask = Mask.lshr(offset * (width/8) * 8);
3697  bestWidth = width;
3698  break;
3699  }
3700  newMask <<= width;
3701  }
3702  }
3703  }
3704  if (bestWidth) {
3705  EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3706  if (newVT.isRound() &&
3707  shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3708  SDValue Ptr = Lod->getBasePtr();
3709  if (bestOffset != 0)
3710  Ptr =
3711  DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3712  SDValue NewLoad =
3713  DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3714  Lod->getPointerInfo().getWithOffset(bestOffset),
3715  Lod->getOriginalAlign());
3716  return DAG.getSetCC(dl, VT,
3717  DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3718  DAG.getConstant(bestMask.trunc(bestWidth),
3719  dl, newVT)),
3720  DAG.getConstant(0LL, dl, newVT), Cond);
3721  }
3722  }
3723  }
3724 
3725  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3726  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3727  unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3728 
3729  // If the comparison constant has bits in the upper part, the
3730  // zero-extended value could never match.
3731  if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3732  C1.getBitWidth() - InSize))) {
3733  switch (Cond) {
3734  case ISD::SETUGT:
3735  case ISD::SETUGE:
3736  case ISD::SETEQ:
3737  return DAG.getConstant(0, dl, VT);
3738  case ISD::SETULT:
3739  case ISD::SETULE:
3740  case ISD::SETNE:
3741  return DAG.getConstant(1, dl, VT);
3742  case ISD::SETGT:
3743  case ISD::SETGE:
3744  // True if the sign bit of C1 is set.
3745  return DAG.getConstant(C1.isNegative(), dl, VT);
3746  case ISD::SETLT:
3747  case ISD::SETLE:
3748  // True if the sign bit of C1 isn't set.
3749  return DAG.getConstant(C1.isNonNegative(), dl, VT);
3750  default:
3751  break;
3752  }
3753  }
3754 
3755  // Otherwise, we can perform the comparison with the low bits.
3756  switch (Cond) {
3757  case ISD::SETEQ:
3758  case ISD::SETNE:
3759  case ISD::SETUGT:
3760  case ISD::SETUGE:
3761  case ISD::SETULT:
3762  case ISD::SETULE: {
3763  EVT newVT = N0.getOperand(0).getValueType();
3764  if (DCI.isBeforeLegalizeOps() ||
3765  (isOperationLegal(ISD::SETCC, newVT) &&
3766  isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3767  EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3768  SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3769 
3770  SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3771  NewConst, Cond);
3772  return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3773  }
3774  break;
3775  }
3776  default:
3777  break; // todo, be more careful with signed comparisons
3778  }
3779  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3780  (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3781  !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3782  OpVT)) {
3783  EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3784  unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3785  EVT ExtDstTy = N0.getValueType();
3786  unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3787 
3788  // If the constant doesn't fit into the number of bits for the source of
3789  // the sign extension, it is impossible for both sides to be equal.
3790  if (C1.getMinSignedBits() > ExtSrcTyBits)
3791  return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3792 
3793  assert(ExtDstTy == N0.getOperand(0).getValueType() &&
3794  ExtDstTy != ExtSrcTy && "Unexpected types!");
3795  APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3796  SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3797  DAG.getConstant(Imm, dl, ExtDstTy));
3798  if (!DCI.isCalledByLegalizer())
3799  DCI.AddToWorklist(ZextOp.getNode());
3800  // Otherwise, make this a use of a zext.
3801  return DAG.getSetCC(dl, VT, ZextOp,
3802  DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3803  } else if ((N1C->isNullValue() || N1C->isOne()) &&
3804  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3805  // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3806  if (N0.getOpcode() == ISD::SETCC &&
3807  isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3808  (N0.getValueType() == MVT::i1 ||
3811  bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3812  if (TrueWhenTrue)
3813  return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3814  // Invert the condition.
3815  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3816  CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3817  if (DCI.isBeforeLegalizeOps() ||
3819  return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3820  }
3821 
3822  if ((N0.getOpcode() == ISD::XOR ||
3823  (N0.getOpcode() == ISD::AND &&
3824  N0.getOperand(0).getOpcode() == ISD::XOR &&
3825  N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3826  isOneConstant(N0.getOperand(1))) {
3827  // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3828  // can only do this if the top bits are known zero.
3829  unsigned BitWidth = N0.getValueSizeInBits();
3830  if (DAG.MaskedValueIsZero(N0,
3832  BitWidth-1))) {
3833  // Okay, get the un-inverted input value.
3834  SDValue Val;
3835  if (N0.getOpcode() == ISD::XOR) {
3836  Val = N0.getOperand(0);
3837  } else {
3838  assert(N0.getOpcode() == ISD::AND &&
3839  N0.getOperand(0).getOpcode() == ISD::XOR);
3840  // ((X^1)&1)^1 -> X & 1
3841  Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3842  N0.getOperand(0).getOperand(0),
3843  N0.getOperand(1));
3844  }
3845 
3846  return DAG.getSetCC(dl, VT, Val, N1,
3848  }
3849  } else if (N1C->isOne()) {
3850  SDValue Op0 = N0;
3851  if (Op0.getOpcode() == ISD::TRUNCATE)
3852  Op0 = Op0.getOperand(0);
3853 
3854  if ((Op0.getOpcode() == ISD::XOR) &&
3855  Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3856  Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3857  SDValue XorLHS = Op0.getOperand(0);
3858  SDValue XorRHS = Op0.getOperand(1);
3859  // Ensure that the input setccs return an i1 type or 0/1 value.
3860  if (Op0.getValueType() == MVT::i1 ||
3861  (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3865  // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3867  return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3868  }
3869  }
3870  if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3871  // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3872  if (Op0.getValueType().bitsGT(VT))
3873  Op0 = DAG.getNode(ISD::AND, dl, VT,
3874  DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3875  DAG.getConstant(1, dl, VT));
3876  else if (Op0.getValueType().bitsLT(VT))
3877  Op0 = DAG.getNode(ISD::AND, dl, VT,
3878  DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3879  DAG.getConstant(1, dl, VT));
3880 
3881  return DAG.getSetCC(dl, VT, Op0,
3882  DAG.getConstant(0, dl, Op0.getValueType()),
3884  }
3885  if (Op0.getOpcode() == ISD::AssertZext &&
3886  cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3887  return DAG.getSetCC(dl, VT, Op0,
3888  DAG.getConstant(0, dl, Op0.getValueType()),
3890  }
3891  }
3892 
3893  // Given:
3894  // icmp eq/ne (urem %x, %y), 0
3895  // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3896  // icmp eq/ne %x, 0
3897  if (N0.getOpcode() == ISD::UREM && N1C->isNullValue() &&
3898  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3899  KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3900  KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3901  if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3902  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3903  }
3904 
3905  if (SDValue V =
3906  optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3907  return V;
3908  }
3909 
3910  // These simplifications apply to splat vectors as well.
3911  // TODO: Handle more splat vector cases.
3912  if (auto *N1C = isConstOrConstSplat(N1)) {
3913  const APInt &C1 = N1C->getAPIntValue();
3914 
3915  APInt MinVal, MaxVal;
3916  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3917  if (ISD::isSignedIntSetCC(Cond)) {
3918  MinVal = APInt::getSignedMinValue(OperandBitSize);
3919  MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3920  } else {
3921  MinVal = APInt::getMinValue(OperandBitSize);
3922  MaxVal = APInt::getMaxValue(OperandBitSize);
3923  }
3924 
3925  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3926  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3927  // X >= MIN --> true
3928  if (C1 == MinVal)
3929  return DAG.getBoolConstant(true, dl, VT, OpVT);
3930 
3931  if (!VT.isVector()) { // TODO: Support this for vectors.
3932  // X >= C0 --> X > (C0 - 1)
3933  APInt C = C1 - 1;
3935  if ((DCI.isBeforeLegalizeOps() ||
3936  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3937  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3938  isLegalICmpImmediate(C.getSExtValue())))) {
3939  return DAG.getSetCC(dl, VT, N0,
3940  DAG.getConstant(C, dl, N1.getValueType()),
3941  NewCC);
3942  }
3943  }
3944  }
3945 
3946  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3947  // X <= MAX --> true
3948  if (C1 == MaxVal)
3949  return DAG.getBoolConstant(true, dl, VT, OpVT);
3950 
3951  // X <= C0 --> X < (C0 + 1)
3952  if (!VT.isVector()) { // TODO: Support this for vectors.
3953  APInt C = C1 + 1;
3955  if ((DCI.isBeforeLegalizeOps() ||
3956  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3957  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3958  isLegalICmpImmediate(C.getSExtValue())))) {
3959  return DAG.getSetCC(dl, VT, N0,
3960  DAG.getConstant(C, dl, N1.getValueType()),
3961  NewCC);
3962  }
3963  }
3964  }
3965 
3966  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3967  if (C1 == MinVal)
3968  return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3969 
3970  // TODO: Support this for vectors after legalize ops.
3971  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3972  // Canonicalize setlt X, Max --> setne X, Max
3973  if (C1 == MaxVal)
3974  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3975 
3976  // If we have setult X, 1, turn it into seteq X, 0
3977  if (C1 == MinVal+1)
3978  return DAG.getSetCC(dl, VT, N0,
3979  DAG.getConstant(MinVal, dl, N0.getValueType()),
3980  ISD::SETEQ);
3981  }
3982  }
3983 
3984  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3985  if (C1 == MaxVal)
3986  return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3987 
3988  // TODO: Support this for vectors after legalize ops.
3989  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3990  // Canonicalize setgt X, Min --> setne X, Min
3991  if (C1 == MinVal)
3992  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3993 
3994  // If we have setugt X, Max-1, turn it into seteq X, Max
3995  if (C1 == MaxVal-1)
3996  return DAG.getSetCC(dl, VT, N0,
3997  DAG.getConstant(MaxVal, dl, N0.getValueType()),
3998  ISD::SETEQ);
3999  }
4000  }
4001 
4002  if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4003  // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4004  if (C1.isNullValue())
4005  if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4006  VT, N0, N1, Cond, DCI, dl))
4007  return CC;
4008 
4009  // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4010  // For example, when high 32-bits of i64 X are known clear:
4011  // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
4012  // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
4013  bool CmpZero = N1C->getAPIntValue().isNullValue();
4014  bool CmpNegOne = N1C->getAPIntValue().isAllOnesValue();
4015  if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4016  // Match or(lo,shl(hi,bw/2)) pattern.
4017  auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4018  unsigned EltBits = V.getScalarValueSizeInBits();
4019  if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4020  return false;
4021  SDValue LHS = V.getOperand(0);
4022  SDValue RHS = V.getOperand(1);
4023  APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4024  // Unshifted element must have zero upperbits.
4025  if (RHS.getOpcode() == ISD::SHL &&
4026  isa<ConstantSDNode>(RHS.getOperand(1)) &&
4027  RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4028  DAG.MaskedValueIsZero(LHS, HiBits)) {
4029  Lo = LHS;
4030  Hi = RHS.getOperand(0);
4031  return true;
4032  }
4033  if (LHS.getOpcode() == ISD::SHL &&
4034  isa<ConstantSDNode>(LHS.getOperand(1)) &&
4035  LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4036  DAG.MaskedValueIsZero(RHS, HiBits)) {
4037  Lo = RHS;
4038  Hi = LHS.getOperand(0);
4039  return true;
4040  }
4041  return false;
4042  };
4043 
4044  auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4045  unsigned EltBits = N0.getScalarValueSizeInBits();
4046  unsigned HalfBits = EltBits / 2;
4047  APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4048  SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4049  SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4050  SDValue NewN0 =
4051  DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4052  SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4053  return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4054  };
4055 
4056  SDValue Lo, Hi;
4057  if (IsConcat(N0, Lo, Hi))
4058  return MergeConcat(Lo, Hi);
4059 
4060  if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4061  SDValue Lo0, Lo1, Hi0, Hi1;
4062  if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4063  IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4064  return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4065  DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4066  }
4067  }
4068  }
4069  }
4070 
4071  // If we have "setcc X, C0", check to see if we can shrink the immediate
4072  // by changing cc.
4073  // TODO: Support this for vectors after legalize ops.
4074  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4075  // SETUGT X, SINTMAX -> SETLT X, 0
4076  // SETUGE X, SINTMIN -> SETLT X, 0
4077  if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4078  (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4079  return DAG.getSetCC(dl, VT, N0,
4080  DAG.getConstant(0, dl, N1.getValueType()),
4081  ISD::SETLT);
4082 
4083  // SETULT X, SINTMIN -> SETGT X, -1
4084  // SETULE X, SINTMAX -> SETGT X, -1
4085  if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4086  (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4087  return DAG.getSetCC(dl, VT, N0,
4088  DAG.getAllOnesConstant(dl, N1.getValueType()),
4089  ISD::SETGT);
4090  }
4091  }
4092 
4093  // Back to non-vector simplifications.
4094  // TODO: Can we do these for vector splats?
4095  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4096  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4097  const APInt &C1 = N1C->getAPIntValue();
4098  EVT ShValTy = N0.getValueType();
4099 
4100  // Fold bit comparisons when we can. This will result in an
4101  // incorrect value when boolean false is negative one, unless
4102  // the bitsize is 1 in which case the false value is the same
4103  // in practice regardless of the representation.
4104  if ((VT.getSizeInBits() == 1 ||
4106  (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4107  (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4108  N0.getOpcode() == ISD::AND) {
4109  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4110  EVT ShiftTy =
4111  getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4112  if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4113  // Perform the xform if the AND RHS is a single bit.
4114  unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4115  if (AndRHS->getAPIntValue().isPowerOf2() &&
4116  !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4117  return DAG.getNode(ISD::TRUNCATE, dl, VT,
4118  DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4119  DAG.getConstant(ShCt, dl, ShiftTy)));
4120  }
4121  } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4122  // (X & 8) == 8 --> (X & 8) >> 3
4123  // Perform the xform if C1 is a single bit.
4124  unsigned ShCt = C1.logBase2();
4125  if (C1.isPowerOf2() &&
4126  !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4127  return DAG.getNode(ISD::TRUNCATE, dl, VT,
4128  DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4129  DAG.getConstant(ShCt, dl, ShiftTy)));
4130  }
4131  }
4132  }
4133  }
4134 
4135  if (C1.getMinSignedBits() <= 64 &&
4136  !isLegalICmpImmediate(C1.getSExtValue())) {
4137  EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4138  // (X & -256) == 256 -> (X >> 8) == 1
4139  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4140  N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4141  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4142  const APInt &AndRHSC = AndRHS->getAPIntValue();
4143  if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
4144  unsigned ShiftBits = AndRHSC.countTrailingZeros();
4145  if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4146  SDValue Shift =
4147  DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4148  DAG.getConstant(ShiftBits, dl, ShiftTy));
4149  SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4150  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4151  }
4152  }
4153  }
4154  } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4155  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4156  bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4157  // X < 0x100000000 -> (X >> 32) < 1
4158  // X >= 0x100000000 -> (X >> 32) >= 1
4159  // X <= 0x0ffffffff -> (X >> 32) < 1
4160  // X > 0x0ffffffff -> (X >> 32) >= 1
4161  unsigned ShiftBits;
4162  APInt NewC = C1;
4163  ISD::CondCode NewCond = Cond;
4164  if (AdjOne) {
4165  ShiftBits = C1.countTrailingOnes();
4166  NewC = NewC + 1;
4167  NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4168  } else {
4169  ShiftBits = C1.countTrailingZeros();
4170  }
4171  NewC.lshrInPlace(ShiftBits);
4172  if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4174  !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4175  SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4176  DAG.getConstant(ShiftBits, dl, ShiftTy));
4177  SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4178  return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4179  }
4180  }
4181  }
4182  }
4183 
4184  if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4185  auto *CFP = cast<ConstantFPSDNode>(N1);
4186  assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
4187 
4188  // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4189  // constant if knowing that the operand is non-nan is enough. We prefer to
4190  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4191  // materialize 0.0.
4192  if (Cond == ISD::SETO || Cond == ISD::SETUO)
4193  return DAG.getSetCC(dl, VT, N0, N0, Cond);
4194 
4195  // setcc (fneg x), C -> setcc swap(pred) x, -C
4196  if (N0.getOpcode() == ISD::FNEG) {
4198  if (DCI.isBeforeLegalizeOps() ||
4199  isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4200  SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4201  return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4202  }
4203  }
4204 
4205  // If the condition is not legal, see if we can find an equivalent one
4206  // which is legal.
4207  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4208  // If the comparison was an awkward floating-point == or != and one of
4209  // the comparison operands is infinity or negative infinity, convert the
4210  // condition to a less-awkward <= or >=.
4211  if (CFP->getValueAPF().isInfinity()) {
4212  bool IsNegInf = CFP->getValueAPF().isNegative();
4214  switch (Cond) {
4215  case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4216  case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4217  case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4218  case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4219  default: break;
4220  }
4221  if (NewCond != ISD::SETCC_INVALID &&
4222  isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4223  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4224  }
4225  }
4226  }
4227 
4228  if (N0 == N1) {
4229  // The sext(setcc()) => setcc() optimization relies on the appropriate
4230  // constant being emitted.
4231  assert(!N0.getValueType().isInteger() &&
4232  "Integer types should be handled by FoldSetCC");
4233 
4234  bool EqTrue = ISD::isTrueWhenEqual(Cond);
4235  unsigned UOF = ISD::getUnorderedFlavor(Cond);
4236  if (UOF == 2) // FP operators that are undefined on NaNs.
4237  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4238  if (UOF == unsigned(EqTrue))
4239  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4240  // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4241  // if it is not already.
4242  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4243  if (NewCond != Cond &&
4244  (DCI.isBeforeLegalizeOps() ||
4245  isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4246  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4247  }
4248 
4249  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4250  N0.getValueType().isInteger()) {
4251  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4252  N0.getOpcode() == ISD::XOR) {
4253  // Simplify (X+Y) == (X+Z) --> Y == Z
4254  if (N0.getOpcode() == N1.getOpcode()) {
4255  if (N0.getOperand(0) == N1.getOperand(0))
4256  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4257  if (N0.getOperand(1) == N1.getOperand(1))
4258  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4259  if (isCommutativeBinOp(N0.getOpcode())) {
4260  // If X op Y == Y op X, try other combinations.
4261  if (N0.getOperand(0) == N1.getOperand(1))
4262  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4263  Cond);
4264  if (N0.getOperand(1) == N1.getOperand(0))
4265  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4266  Cond);
4267  }
4268  }
4269 
4270  // If RHS is a legal immediate value for a compare instruction, we need
4271  // to be careful about increasing register pressure needlessly.
4272  bool LegalRHSImm = false;
4273 
4274  if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4275  if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4276  // Turn (X+C1) == C2 --> X == C2-C1
4277  if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4278  return DAG.getSetCC(dl, VT, N0.getOperand(0),
4279  DAG.getConstant(RHSC->getAPIntValue()-
4280  LHSR->getAPIntValue(),
4281  dl, N0.getValueType()), Cond);
4282  }
4283 
4284  // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4285  if (N0.getOpcode() == ISD::XOR)
4286  // If we know that all of the inverted bits are zero, don't bother
4287  // performing the inversion.
4288  if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4289  return
4290  DAG.getSetCC(dl, VT, N0.getOperand(0),
4291  DAG.getConstant(LHSR->getAPIntValue() ^
4292  RHSC->getAPIntValue(),
4293  dl, N0.getValueType()),
4294  Cond);
4295  }
4296 
4297  // Turn (C1-X) == C2 --> X == C1-C2
4298  if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4299  if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4300  return
4301  DAG.getSetCC(dl, VT, N0.getOperand(1),
4302  DAG.getConstant(SUBC->getAPIntValue() -
4303  RHSC->getAPIntValue(),
4304  dl, N0.getValueType()),
4305  Cond);
4306  }
4307  }
4308 
4309  // Could RHSC fold directly into a compare?
4310  if (RHSC->getValueType(0).getSizeInBits() <= 64)
4311  LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4312  }
4313 
4314  // (X+Y) == X --> Y == 0 and similar folds.
4315  // Don't do this if X is an immediate that can fold into a cmp
4316  // instruction and X+Y has other uses. It could be an induction variable
4317  // chain, and the transform would increase register pressure.
4318  if (!LegalRHSImm || N0.hasOneUse())
4319  if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4320  return V;
4321  }
4322 
4323  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4324  N1.getOpcode() == ISD::XOR)
4325  if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4326  return V;
4327 
4328  if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4329  return V;
4330  }
4331 
4332  // Fold remainder of division by a constant.
4333  if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4334  N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4336 
4337  // When division is cheap or optimizing for minimum size,
4338  // fall through to DIVREM creation by skipping this fold.
4339  if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) {
4340  if (N0.getOpcode() == ISD::UREM) {
4341  if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4342  return Folded;
4343  } else if (N0.getOpcode() == ISD::SREM) {
4344  if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4345  return Folded;
4346  }
4347  }
4348  }
4349 
4350  // Fold away ALL boolean setcc's.
4351  if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4352  SDValue Temp;
4353  switch (Cond) {
4354  default: llvm_unreachable("Unknown integer setcc!");
4355  case ISD::SETEQ: // X == Y -> ~(X^Y)
4356  Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4357  N0 = DAG.getNOT(dl, Temp, OpVT);
4358  if (!DCI.isCalledByLegalizer())
4359  DCI.AddToWorklist(Temp.getNode());
4360  break;
4361  case ISD::SETNE: // X != Y --> (X^Y)
4362  N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4363  break;
4364  case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4365  case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4366  Temp = DAG.getNOT(dl, N0, OpVT);
4367  N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4368  if (!DCI.isCalledByLegalizer())
4369  DCI.AddToWorklist(Temp.getNode());
4370  break;
4371  case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4372  case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4373  Temp = DAG.getNOT(dl, N1, OpVT);
4374  N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4375  if (!DCI.isCalledByLegalizer())
4376  DCI.AddToWorklist(Temp.getNode());
4377  break;
4378  case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4379  case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4380  Temp = DAG.getNOT(dl, N0, OpVT);
4381  N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4382  if (!DCI.isCalledByLegalizer())
4383  DCI.AddToWorklist(Temp.getNode());
4384  break;
4385  case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4386  case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4387  Temp = DAG.getNOT(dl, N1, OpVT);
4388  N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4389  break;
4390  }
4391  if (VT.getScalarType() != MVT::i1) {
4392  if (!DCI.isCalledByLegalizer())
4393  DCI.AddToWorklist(N0.getNode());
4394  // FIXME: If running after legalize, we probably can't do this.
4396  N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4397  }
4398  return N0;
4399  }
4400 
4401  // Could not fold it.
4402  return SDValue();
4403 }
4404 
4405 /// Returns true (and the GlobalValue and the offset) if the node is a
4406 /// GlobalAddress + offset.
4408  int64_t &Offset) const {
4409 
4410  SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4411 
4412  if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4413  GA = GASD->getGlobal();
4414  Offset += GASD->getOffset();
4415  return true;
4416  }
4417 
4418  if (N->getOpcode() == ISD::ADD) {
4419  SDValue N1 = N->getOperand(0);
4420  SDValue N2 = N->getOperand(1);
4421  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4422  if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4423  Offset += V->getSExtValue();
4424  return true;
4425  }
4426  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4427  if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4428  Offset += V->getSExtValue();
4429  return true;
4430  }
4431  }
4432  }
4433 
4434  return false;
4435 }
4436 
4438  DAGCombinerInfo &DCI) const {
4439  // Default implementation: no optimization.
4440  return SDValue();
4441 }
4442 
4443 //===-----------------------------------