LLVM  14.0.0git
TargetLowering.cpp
Go to the documentation of this file.
1 //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This implements the TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/IR/DataLayout.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/GlobalVariable.h"
26 #include "llvm/IR/LLVMContext.h"
27 #include "llvm/MC/MCAsmInfo.h"
28 #include "llvm/MC/MCExpr.h"
31 #include "llvm/Support/KnownBits.h"
35 #include <cctype>
36 using namespace llvm;
37 
38 /// NOTE: The TargetMachine owns TLOF.
40  : TargetLoweringBase(tm) {}
41 
42 const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
43  return nullptr;
44 }
45 
48 }
49 
50 /// Check whether a given call node is in tail position within its function. If
51 /// so, it sets Chain to the input chain of the tail call.
53  SDValue &Chain) const {
54  const Function &F = DAG.getMachineFunction().getFunction();
55 
56  // First, check if tail calls have been disabled in this function.
57  if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
58  return false;
59 
60  // Conservatively require the attributes of the call to match those of
61  // the return. Ignore following attributes because they don't affect the
62  // call sequence.
63  AttrBuilder CallerAttrs(F.getAttributes(), AttributeList::ReturnIndex);
64  for (const auto &Attr : {Attribute::Alignment, Attribute::Dereferenceable,
65  Attribute::DereferenceableOrNull, Attribute::NoAlias,
66  Attribute::NonNull})
67  CallerAttrs.removeAttribute(Attr);
68 
69  if (CallerAttrs.hasAttributes())
70  return false;
71 
72  // It's not safe to eliminate the sign / zero extension of the return value.
73  if (CallerAttrs.contains(Attribute::ZExt) ||
74  CallerAttrs.contains(Attribute::SExt))
75  return false;
76 
77  // Check if the only use is a function return node.
78  return isUsedByReturnOnly(Node, Chain);
79 }
80 
82  const uint32_t *CallerPreservedMask,
83  const SmallVectorImpl<CCValAssign> &ArgLocs,
84  const SmallVectorImpl<SDValue> &OutVals) const {
85  for (unsigned I = 0, E = ArgLocs.size(); I != E; ++I) {
86  const CCValAssign &ArgLoc = ArgLocs[I];
87  if (!ArgLoc.isRegLoc())
88  continue;
89  MCRegister Reg = ArgLoc.getLocReg();
90  // Only look at callee saved registers.
91  if (MachineOperand::clobbersPhysReg(CallerPreservedMask, Reg))
92  continue;
93  // Check that we pass the value used for the caller.
94  // (We look for a CopyFromReg reading a virtual register that is used
95  // for the function live-in value of register Reg)
96  SDValue Value = OutVals[I];
97  if (Value->getOpcode() != ISD::CopyFromReg)
98  return false;
99  Register ArgReg = cast<RegisterSDNode>(Value->getOperand(1))->getReg();
100  if (MRI.getLiveInPhysReg(ArgReg) != Reg)
101  return false;
102  }
103  return true;
104 }
105 
106 /// Set CallLoweringInfo attribute flags based on a call instruction
107 /// and called function attributes.
109  unsigned ArgIdx) {
110  IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt);
111  IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt);
112  IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg);
113  IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet);
114  IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest);
115  IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal);
116  IsPreallocated = Call->paramHasAttr(ArgIdx, Attribute::Preallocated);
117  IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca);
118  IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned);
119  IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf);
120  IsSwiftAsync = Call->paramHasAttr(ArgIdx, Attribute::SwiftAsync);
121  IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError);
122  Alignment = Call->getParamStackAlign(ArgIdx);
123  IndirectType = nullptr;
125  "multiple ABI attributes?");
126  if (IsByVal) {
127  IndirectType = Call->getParamByValType(ArgIdx);
128  if (!Alignment)
129  Alignment = Call->getParamAlign(ArgIdx);
130  }
131  if (IsPreallocated)
132  IndirectType = Call->getParamPreallocatedType(ArgIdx);
133  if (IsInAlloca)
134  IndirectType = Call->getParamInAllocaType(ArgIdx);
135 }
136 
137 /// Generate a libcall taking the given operands as arguments and returning a
138 /// result of type RetVT.
139 std::pair<SDValue, SDValue>
141  ArrayRef<SDValue> Ops,
142  MakeLibCallOptions CallOptions,
143  const SDLoc &dl,
144  SDValue InChain) const {
145  if (!InChain)
146  InChain = DAG.getEntryNode();
147 
149  Args.reserve(Ops.size());
150 
152  for (unsigned i = 0; i < Ops.size(); ++i) {
153  SDValue NewOp = Ops[i];
154  Entry.Node = NewOp;
155  Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext());
156  Entry.IsSExt = shouldSignExtendTypeInLibCall(NewOp.getValueType(),
157  CallOptions.IsSExt);
158  Entry.IsZExt = !Entry.IsSExt;
159 
160  if (CallOptions.IsSoften &&
162  Entry.IsSExt = Entry.IsZExt = false;
163  }
164  Args.push_back(Entry);
165  }
166 
167  if (LC == RTLIB::UNKNOWN_LIBCALL)
168  report_fatal_error("Unsupported library call operation!");
170  getPointerTy(DAG.getDataLayout()));
171 
172  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
174  bool signExtend = shouldSignExtendTypeInLibCall(RetVT, CallOptions.IsSExt);
175  bool zeroExtend = !signExtend;
176 
177  if (CallOptions.IsSoften &&
179  signExtend = zeroExtend = false;
180  }
181 
182  CLI.setDebugLoc(dl)
183  .setChain(InChain)
185  .setNoReturn(CallOptions.DoesNotReturn)
186  .setDiscardResult(!CallOptions.IsReturnValueUsed)
188  .setSExtResult(signExtend)
189  .setZExtResult(zeroExtend);
190  return LowerCallTo(CLI);
191 }
192 
194  std::vector<EVT> &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS,
195  unsigned SrcAS, const AttributeList &FuncAttributes) const {
196  if (Op.isMemcpyWithFixedDstAlign() && Op.getSrcAlign() < Op.getDstAlign())
197  return false;
198 
199  EVT VT = getOptimalMemOpType(Op, FuncAttributes);
200 
201  if (VT == MVT::Other) {
202  // Use the largest integer type whose alignment constraints are satisfied.
203  // We only need to check DstAlign here as SrcAlign is always greater or
204  // equal to DstAlign (or zero).
205  VT = MVT::i64;
206  if (Op.isFixedDstAlign())
207  while (Op.getDstAlign() < (VT.getSizeInBits() / 8) &&
208  !allowsMisalignedMemoryAccesses(VT, DstAS, Op.getDstAlign()))
209  VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1);
210  assert(VT.isInteger());
211 
212  // Find the largest legal integer type.
213  MVT LVT = MVT::i64;
214  while (!isTypeLegal(LVT))
215  LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1);
216  assert(LVT.isInteger());
217 
218  // If the type we've chosen is larger than the largest legal integer type
219  // then use that instead.
220  if (VT.bitsGT(LVT))
221  VT = LVT;
222  }
223 
224  unsigned NumMemOps = 0;
225  uint64_t Size = Op.size();
226  while (Size) {
227  unsigned VTSize = VT.getSizeInBits() / 8;
228  while (VTSize > Size) {
229  // For now, only use non-vector load / store's for the left-over pieces.
230  EVT NewVT = VT;
231  unsigned NewVTSize;
232 
233  bool Found = false;
234  if (VT.isVector() || VT.isFloatingPoint()) {
235  NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32;
236  if (isOperationLegalOrCustom(ISD::STORE, NewVT) &&
237  isSafeMemOpType(NewVT.getSimpleVT()))
238  Found = true;
239  else if (NewVT == MVT::i64 &&
242  // i64 is usually not legal on 32-bit targets, but f64 may be.
243  NewVT = MVT::f64;
244  Found = true;
245  }
246  }
247 
248  if (!Found) {
249  do {
250  NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1);
251  if (NewVT == MVT::i8)
252  break;
253  } while (!isSafeMemOpType(NewVT.getSimpleVT()));
254  }
255  NewVTSize = NewVT.getSizeInBits() / 8;
256 
257  // If the new VT cannot cover all of the remaining bits, then consider
258  // issuing a (or a pair of) unaligned and overlapping load / store.
259  bool Fast;
260  if (NumMemOps && Op.allowOverlap() && NewVTSize < Size &&
262  VT, DstAS, Op.isFixedDstAlign() ? Op.getDstAlign() : Align(1),
263  MachineMemOperand::MONone, &Fast) &&
264  Fast)
265  VTSize = Size;
266  else {
267  VT = NewVT;
268  VTSize = NewVTSize;
269  }
270  }
271 
272  if (++NumMemOps > Limit)
273  return false;
274 
275  MemOps.push_back(VT);
276  Size -= VTSize;
277  }
278 
279  return true;
280 }
281 
282 /// Soften the operands of a comparison. This code is shared among BR_CC,
283 /// SELECT_CC, and SETCC handlers.
285  SDValue &NewLHS, SDValue &NewRHS,
286  ISD::CondCode &CCCode,
287  const SDLoc &dl, const SDValue OldLHS,
288  const SDValue OldRHS) const {
289  SDValue Chain;
290  return softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, dl, OldLHS,
291  OldRHS, Chain);
292 }
293 
295  SDValue &NewLHS, SDValue &NewRHS,
296  ISD::CondCode &CCCode,
297  const SDLoc &dl, const SDValue OldLHS,
298  const SDValue OldRHS,
299  SDValue &Chain,
300  bool IsSignaling) const {
301  // FIXME: Currently we cannot really respect all IEEE predicates due to libgcc
302  // not supporting it. We can update this code when libgcc provides such
303  // functions.
304 
305  assert((VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128 || VT == MVT::ppcf128)
306  && "Unsupported setcc type!");
307 
308  // Expand into one or more soft-fp libcall(s).
309  RTLIB::Libcall LC1 = RTLIB::UNKNOWN_LIBCALL, LC2 = RTLIB::UNKNOWN_LIBCALL;
310  bool ShouldInvertCC = false;
311  switch (CCCode) {
312  case ISD::SETEQ:
313  case ISD::SETOEQ:
314  LC1 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
315  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
316  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
317  break;
318  case ISD::SETNE:
319  case ISD::SETUNE:
320  LC1 = (VT == MVT::f32) ? RTLIB::UNE_F32 :
321  (VT == MVT::f64) ? RTLIB::UNE_F64 :
322  (VT == MVT::f128) ? RTLIB::UNE_F128 : RTLIB::UNE_PPCF128;
323  break;
324  case ISD::SETGE:
325  case ISD::SETOGE:
326  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
327  (VT == MVT::f64) ? RTLIB::OGE_F64 :
328  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
329  break;
330  case ISD::SETLT:
331  case ISD::SETOLT:
332  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
333  (VT == MVT::f64) ? RTLIB::OLT_F64 :
334  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
335  break;
336  case ISD::SETLE:
337  case ISD::SETOLE:
338  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
339  (VT == MVT::f64) ? RTLIB::OLE_F64 :
340  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
341  break;
342  case ISD::SETGT:
343  case ISD::SETOGT:
344  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
345  (VT == MVT::f64) ? RTLIB::OGT_F64 :
346  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
347  break;
348  case ISD::SETO:
349  ShouldInvertCC = true;
351  case ISD::SETUO:
352  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
353  (VT == MVT::f64) ? RTLIB::UO_F64 :
354  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
355  break;
356  case ISD::SETONE:
357  // SETONE = O && UNE
358  ShouldInvertCC = true;
360  case ISD::SETUEQ:
361  LC1 = (VT == MVT::f32) ? RTLIB::UO_F32 :
362  (VT == MVT::f64) ? RTLIB::UO_F64 :
363  (VT == MVT::f128) ? RTLIB::UO_F128 : RTLIB::UO_PPCF128;
364  LC2 = (VT == MVT::f32) ? RTLIB::OEQ_F32 :
365  (VT == MVT::f64) ? RTLIB::OEQ_F64 :
366  (VT == MVT::f128) ? RTLIB::OEQ_F128 : RTLIB::OEQ_PPCF128;
367  break;
368  default:
369  // Invert CC for unordered comparisons
370  ShouldInvertCC = true;
371  switch (CCCode) {
372  case ISD::SETULT:
373  LC1 = (VT == MVT::f32) ? RTLIB::OGE_F32 :
374  (VT == MVT::f64) ? RTLIB::OGE_F64 :
375  (VT == MVT::f128) ? RTLIB::OGE_F128 : RTLIB::OGE_PPCF128;
376  break;
377  case ISD::SETULE:
378  LC1 = (VT == MVT::f32) ? RTLIB::OGT_F32 :
379  (VT == MVT::f64) ? RTLIB::OGT_F64 :
380  (VT == MVT::f128) ? RTLIB::OGT_F128 : RTLIB::OGT_PPCF128;
381  break;
382  case ISD::SETUGT:
383  LC1 = (VT == MVT::f32) ? RTLIB::OLE_F32 :
384  (VT == MVT::f64) ? RTLIB::OLE_F64 :
385  (VT == MVT::f128) ? RTLIB::OLE_F128 : RTLIB::OLE_PPCF128;
386  break;
387  case ISD::SETUGE:
388  LC1 = (VT == MVT::f32) ? RTLIB::OLT_F32 :
389  (VT == MVT::f64) ? RTLIB::OLT_F64 :
390  (VT == MVT::f128) ? RTLIB::OLT_F128 : RTLIB::OLT_PPCF128;
391  break;
392  default: llvm_unreachable("Do not know how to soften this setcc!");
393  }
394  }
395 
396  // Use the target specific return value for comparions lib calls.
397  EVT RetVT = getCmpLibcallReturnType();
398  SDValue Ops[2] = {NewLHS, NewRHS};
400  EVT OpsVT[2] = { OldLHS.getValueType(),
401  OldRHS.getValueType() };
402  CallOptions.setTypeListBeforeSoften(OpsVT, RetVT, true);
403  auto Call = makeLibCall(DAG, LC1, RetVT, Ops, CallOptions, dl, Chain);
404  NewLHS = Call.first;
405  NewRHS = DAG.getConstant(0, dl, RetVT);
406 
407  CCCode = getCmpLibcallCC(LC1);
408  if (ShouldInvertCC) {
409  assert(RetVT.isInteger());
410  CCCode = getSetCCInverse(CCCode, RetVT);
411  }
412 
413  if (LC2 == RTLIB::UNKNOWN_LIBCALL) {
414  // Update Chain.
415  Chain = Call.second;
416  } else {
417  EVT SetCCVT =
418  getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), RetVT);
419  SDValue Tmp = DAG.getSetCC(dl, SetCCVT, NewLHS, NewRHS, CCCode);
420  auto Call2 = makeLibCall(DAG, LC2, RetVT, Ops, CallOptions, dl, Chain);
421  CCCode = getCmpLibcallCC(LC2);
422  if (ShouldInvertCC)
423  CCCode = getSetCCInverse(CCCode, RetVT);
424  NewLHS = DAG.getSetCC(dl, SetCCVT, Call2.first, NewRHS, CCCode);
425  if (Chain)
426  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Call.second,
427  Call2.second);
428  NewLHS = DAG.getNode(ShouldInvertCC ? ISD::AND : ISD::OR, dl,
429  Tmp.getValueType(), Tmp, NewLHS);
430  NewRHS = SDValue();
431  }
432 }
433 
434 /// Return the entry encoding for a jump table in the current function. The
435 /// returned value is a member of the MachineJumpTableInfo::JTEntryKind enum.
437  // In non-pic modes, just use the address of a block.
438  if (!isPositionIndependent())
440 
441  // In PIC mode, if the target supports a GPRel32 directive, use it.
442  if (getTargetMachine().getMCAsmInfo()->getGPRel32Directive() != nullptr)
444 
445  // Otherwise, use a label difference.
447 }
448 
450  SelectionDAG &DAG) const {
451  // If our PIC model is GP relative, use the global offset table as the base.
452  unsigned JTEncoding = getJumpTableEncoding();
453 
454  if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
457 
458  return Table;
459 }
460 
461 /// This returns the relocation base for the given PIC jumptable, the same as
462 /// getPICJumpTableRelocBase, but as an MCExpr.
463 const MCExpr *
465  unsigned JTI,MCContext &Ctx) const{
466  // The normal PIC reloc base is the label at the start of the jump table.
467  return MCSymbolRefExpr::create(MF->getJTISymbol(JTI, Ctx), Ctx);
468 }
469 
470 bool
472  const TargetMachine &TM = getTargetMachine();
473  const GlobalValue *GV = GA->getGlobal();
474 
475  // If the address is not even local to this DSO we will have to load it from
476  // a got and then add the offset.
477  if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
478  return false;
479 
480  // If the code is position independent we will have to add a base register.
481  if (isPositionIndependent())
482  return false;
483 
484  // Otherwise we can do it.
485  return true;
486 }
487 
488 //===----------------------------------------------------------------------===//
489 // Optimization Methods
490 //===----------------------------------------------------------------------===//
491 
492 /// If the specified instruction has a constant integer operand and there are
493 /// bits set in that constant that are not demanded, then clear those bits and
494 /// return true.
496  const APInt &DemandedBits,
497  const APInt &DemandedElts,
498  TargetLoweringOpt &TLO) const {
499  SDLoc DL(Op);
500  unsigned Opcode = Op.getOpcode();
501 
502  // Do target-specific constant optimization.
503  if (targetShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
504  return TLO.New.getNode();
505 
506  // FIXME: ISD::SELECT, ISD::SELECT_CC
507  switch (Opcode) {
508  default:
509  break;
510  case ISD::XOR:
511  case ISD::AND:
512  case ISD::OR: {
513  auto *Op1C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
514  if (!Op1C || Op1C->isOpaque())
515  return false;
516 
517  // If this is a 'not' op, don't touch it because that's a canonical form.
518  const APInt &C = Op1C->getAPIntValue();
519  if (Opcode == ISD::XOR && DemandedBits.isSubsetOf(C))
520  return false;
521 
522  if (!C.isSubsetOf(DemandedBits)) {
523  EVT VT = Op.getValueType();
524  SDValue NewC = TLO.DAG.getConstant(DemandedBits & C, DL, VT);
525  SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC);
526  return TLO.CombineTo(Op, NewOp);
527  }
528 
529  break;
530  }
531  }
532 
533  return false;
534 }
535 
537  const APInt &DemandedBits,
538  TargetLoweringOpt &TLO) const {
539  EVT VT = Op.getValueType();
540  APInt DemandedElts = VT.isVector()
542  : APInt(1, 1);
543  return ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO);
544 }
545 
546 /// Convert x+y to (VT)((SmallVT)x+(SmallVT)y) if the casts are free.
547 /// This uses isZExtFree and ZERO_EXTEND for the widening cast, but it could be
548 /// generalized for targets with other types of implicit widening casts.
550  const APInt &Demanded,
551  TargetLoweringOpt &TLO) const {
552  assert(Op.getNumOperands() == 2 &&
553  "ShrinkDemandedOp only supports binary operators!");
554  assert(Op.getNode()->getNumValues() == 1 &&
555  "ShrinkDemandedOp only supports nodes with one result!");
556 
557  SelectionDAG &DAG = TLO.DAG;
558  SDLoc dl(Op);
559 
560  // Early return, as this function cannot handle vector types.
561  if (Op.getValueType().isVector())
562  return false;
563 
564  // Don't do this if the node has another user, which may require the
565  // full value.
566  if (!Op.getNode()->hasOneUse())
567  return false;
568 
569  // Search for the smallest integer type with free casts to and from
570  // Op's type. For expedience, just check power-of-2 integer types.
571  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
572  unsigned DemandedSize = Demanded.getActiveBits();
573  unsigned SmallVTBits = DemandedSize;
574  if (!isPowerOf2_32(SmallVTBits))
575  SmallVTBits = NextPowerOf2(SmallVTBits);
576  for (; SmallVTBits < BitWidth; SmallVTBits = NextPowerOf2(SmallVTBits)) {
577  EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), SmallVTBits);
578  if (TLI.isTruncateFree(Op.getValueType(), SmallVT) &&
579  TLI.isZExtFree(SmallVT, Op.getValueType())) {
580  // We found a type with free casts.
581  SDValue X = DAG.getNode(
582  Op.getOpcode(), dl, SmallVT,
583  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(0)),
584  DAG.getNode(ISD::TRUNCATE, dl, SmallVT, Op.getOperand(1)));
585  assert(DemandedSize <= SmallVTBits && "Narrowed below demanded bits?");
586  SDValue Z = DAG.getNode(ISD::ANY_EXTEND, dl, Op.getValueType(), X);
587  return TLO.CombineTo(Op, Z);
588  }
589  }
590  return false;
591 }
592 
594  DAGCombinerInfo &DCI) const {
595  SelectionDAG &DAG = DCI.DAG;
596  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
597  !DCI.isBeforeLegalizeOps());
598  KnownBits Known;
599 
600  bool Simplified = SimplifyDemandedBits(Op, DemandedBits, Known, TLO);
601  if (Simplified) {
602  DCI.AddToWorklist(Op.getNode());
603  DCI.CommitTargetLoweringOpt(TLO);
604  }
605  return Simplified;
606 }
607 
609  KnownBits &Known,
610  TargetLoweringOpt &TLO,
611  unsigned Depth,
612  bool AssumeSingleUse) const {
613  EVT VT = Op.getValueType();
614 
615  // TODO: We can probably do more work on calculating the known bits and
616  // simplifying the operations for scalable vectors, but for now we just
617  // bail out.
618  if (VT.isScalableVector()) {
619  // Pretend we don't know anything for now.
620  Known = KnownBits(DemandedBits.getBitWidth());
621  return false;
622  }
623 
624  APInt DemandedElts = VT.isVector()
626  : APInt(1, 1);
627  return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, Depth,
628  AssumeSingleUse);
629 }
630 
631 // TODO: Can we merge SelectionDAG::GetDemandedBits into this?
632 // TODO: Under what circumstances can we create nodes? Constant folding?
634  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
635  SelectionDAG &DAG, unsigned Depth) const {
636  // Limit search depth.
638  return SDValue();
639 
640  // Ignore UNDEFs.
641  if (Op.isUndef())
642  return SDValue();
643 
644  // Not demanding any bits/elts from Op.
645  if (DemandedBits == 0 || DemandedElts == 0)
646  return DAG.getUNDEF(Op.getValueType());
647 
648  unsigned NumElts = DemandedElts.getBitWidth();
649  unsigned BitWidth = DemandedBits.getBitWidth();
650  KnownBits LHSKnown, RHSKnown;
651  switch (Op.getOpcode()) {
652  case ISD::BITCAST: {
653  SDValue Src = peekThroughBitcasts(Op.getOperand(0));
654  EVT SrcVT = Src.getValueType();
655  EVT DstVT = Op.getValueType();
656  if (SrcVT == DstVT)
657  return Src;
658 
659  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
660  unsigned NumDstEltBits = DstVT.getScalarSizeInBits();
661  if (NumSrcEltBits == NumDstEltBits)
662  if (SDValue V = SimplifyMultipleUseDemandedBits(
663  Src, DemandedBits, DemandedElts, DAG, Depth + 1))
664  return DAG.getBitcast(DstVT, V);
665 
666  // TODO - bigendian once we have test coverage.
667  if (SrcVT.isVector() && (NumDstEltBits % NumSrcEltBits) == 0 &&
668  DAG.getDataLayout().isLittleEndian()) {
669  unsigned Scale = NumDstEltBits / NumSrcEltBits;
670  unsigned NumSrcElts = SrcVT.getVectorNumElements();
671  APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
672  APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
673  for (unsigned i = 0; i != Scale; ++i) {
674  unsigned Offset = i * NumSrcEltBits;
675  APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
676  if (!Sub.isZero()) {
677  DemandedSrcBits |= Sub;
678  for (unsigned j = 0; j != NumElts; ++j)
679  if (DemandedElts[j])
680  DemandedSrcElts.setBit((j * Scale) + i);
681  }
682  }
683 
684  if (SDValue V = SimplifyMultipleUseDemandedBits(
685  Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
686  return DAG.getBitcast(DstVT, V);
687  }
688 
689  // TODO - bigendian once we have test coverage.
690  if ((NumSrcEltBits % NumDstEltBits) == 0 &&
691  DAG.getDataLayout().isLittleEndian()) {
692  unsigned Scale = NumSrcEltBits / NumDstEltBits;
693  unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
694  APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
695  APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
696  for (unsigned i = 0; i != NumElts; ++i)
697  if (DemandedElts[i]) {
698  unsigned Offset = (i % Scale) * NumDstEltBits;
699  DemandedSrcBits.insertBits(DemandedBits, Offset);
700  DemandedSrcElts.setBit(i / Scale);
701  }
702 
703  if (SDValue V = SimplifyMultipleUseDemandedBits(
704  Src, DemandedSrcBits, DemandedSrcElts, DAG, Depth + 1))
705  return DAG.getBitcast(DstVT, V);
706  }
707 
708  break;
709  }
710  case ISD::AND: {
711  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
712  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
713 
714  // If all of the demanded bits are known 1 on one side, return the other.
715  // These bits cannot contribute to the result of the 'and' in this
716  // context.
717  if (DemandedBits.isSubsetOf(LHSKnown.Zero | RHSKnown.One))
718  return Op.getOperand(0);
719  if (DemandedBits.isSubsetOf(RHSKnown.Zero | LHSKnown.One))
720  return Op.getOperand(1);
721  break;
722  }
723  case ISD::OR: {
724  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
725  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
726 
727  // If all of the demanded bits are known zero on one side, return the
728  // other. These bits cannot contribute to the result of the 'or' in this
729  // context.
730  if (DemandedBits.isSubsetOf(LHSKnown.One | RHSKnown.Zero))
731  return Op.getOperand(0);
732  if (DemandedBits.isSubsetOf(RHSKnown.One | LHSKnown.Zero))
733  return Op.getOperand(1);
734  break;
735  }
736  case ISD::XOR: {
737  LHSKnown = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
738  RHSKnown = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
739 
740  // If all of the demanded bits are known zero on one side, return the
741  // other.
742  if (DemandedBits.isSubsetOf(RHSKnown.Zero))
743  return Op.getOperand(0);
744  if (DemandedBits.isSubsetOf(LHSKnown.Zero))
745  return Op.getOperand(1);
746  break;
747  }
748  case ISD::SHL: {
749  // If we are only demanding sign bits then we can use the shift source
750  // directly.
751  if (const APInt *MaxSA =
752  DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
753  SDValue Op0 = Op.getOperand(0);
754  unsigned ShAmt = MaxSA->getZExtValue();
755  unsigned NumSignBits =
756  DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
757  unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
758  if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
759  return Op0;
760  }
761  break;
762  }
763  case ISD::SETCC: {
764  SDValue Op0 = Op.getOperand(0);
765  SDValue Op1 = Op.getOperand(1);
766  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
767  // If (1) we only need the sign-bit, (2) the setcc operands are the same
768  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
769  // -1, we may be able to bypass the setcc.
770  if (DemandedBits.isSignMask() &&
773  BooleanContent::ZeroOrNegativeOneBooleanContent) {
774  // If we're testing X < 0, then this compare isn't needed - just use X!
775  // FIXME: We're limiting to integer types here, but this should also work
776  // if we don't care about FP signed-zero. The use of SETLT with FP means
777  // that we don't care about NaNs.
778  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
780  return Op0;
781  }
782  break;
783  }
784  case ISD::SIGN_EXTEND_INREG: {
785  // If none of the extended bits are demanded, eliminate the sextinreg.
786  SDValue Op0 = Op.getOperand(0);
787  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
788  unsigned ExBits = ExVT.getScalarSizeInBits();
789  if (DemandedBits.getActiveBits() <= ExBits)
790  return Op0;
791  // If the input is already sign extended, just drop the extension.
792  unsigned NumSignBits = DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
793  if (NumSignBits >= (BitWidth - ExBits + 1))
794  return Op0;
795  break;
796  }
800  // If we only want the lowest element and none of extended bits, then we can
801  // return the bitcasted source vector.
802  SDValue Src = Op.getOperand(0);
803  EVT SrcVT = Src.getValueType();
804  EVT DstVT = Op.getValueType();
805  if (DemandedElts == 1 && DstVT.getSizeInBits() == SrcVT.getSizeInBits() &&
806  DAG.getDataLayout().isLittleEndian() &&
807  DemandedBits.getActiveBits() <= SrcVT.getScalarSizeInBits()) {
808  return DAG.getBitcast(DstVT, Src);
809  }
810  break;
811  }
812  case ISD::INSERT_VECTOR_ELT: {
813  // If we don't demand the inserted element, return the base vector.
814  SDValue Vec = Op.getOperand(0);
815  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
816  EVT VecVT = Vec.getValueType();
817  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
818  !DemandedElts[CIdx->getZExtValue()])
819  return Vec;
820  break;
821  }
822  case ISD::INSERT_SUBVECTOR: {
823  SDValue Vec = Op.getOperand(0);
824  SDValue Sub = Op.getOperand(1);
825  uint64_t Idx = Op.getConstantOperandVal(2);
826  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
827  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
828  // If we don't demand the inserted subvector, return the base vector.
829  if (DemandedSubElts == 0)
830  return Vec;
831  // If this simply widens the lowest subvector, see if we can do it earlier.
832  if (Idx == 0 && Vec.isUndef()) {
833  if (SDValue NewSub = SimplifyMultipleUseDemandedBits(
834  Sub, DemandedBits, DemandedSubElts, DAG, Depth + 1))
835  return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
836  Op.getOperand(0), NewSub, Op.getOperand(2));
837  }
838  break;
839  }
840  case ISD::VECTOR_SHUFFLE: {
841  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
842 
843  // If all the demanded elts are from one operand and are inline,
844  // then we can use the operand directly.
845  bool AllUndef = true, IdentityLHS = true, IdentityRHS = true;
846  for (unsigned i = 0; i != NumElts; ++i) {
847  int M = ShuffleMask[i];
848  if (M < 0 || !DemandedElts[i])
849  continue;
850  AllUndef = false;
851  IdentityLHS &= (M == (int)i);
852  IdentityRHS &= ((M - NumElts) == i);
853  }
854 
855  if (AllUndef)
856  return DAG.getUNDEF(Op.getValueType());
857  if (IdentityLHS)
858  return Op.getOperand(0);
859  if (IdentityRHS)
860  return Op.getOperand(1);
861  break;
862  }
863  default:
864  if (Op.getOpcode() >= ISD::BUILTIN_OP_END)
865  if (SDValue V = SimplifyMultipleUseDemandedBitsForTargetNode(
866  Op, DemandedBits, DemandedElts, DAG, Depth))
867  return V;
868  break;
869  }
870  return SDValue();
871 }
872 
874  SDValue Op, const APInt &DemandedBits, SelectionDAG &DAG,
875  unsigned Depth) const {
876  EVT VT = Op.getValueType();
877  APInt DemandedElts = VT.isVector()
879  : APInt(1, 1);
880  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
881  Depth);
882 }
883 
885  SDValue Op, const APInt &DemandedElts, SelectionDAG &DAG,
886  unsigned Depth) const {
887  APInt DemandedBits = APInt::getAllOnes(Op.getScalarValueSizeInBits());
888  return SimplifyMultipleUseDemandedBits(Op, DemandedBits, DemandedElts, DAG,
889  Depth);
890 }
891 
892 /// Look at Op. At this point, we know that only the OriginalDemandedBits of the
893 /// result of Op are ever used downstream. If we can use this information to
894 /// simplify Op, create a new simplified DAG node and return true, returning the
895 /// original and new nodes in Old and New. Otherwise, analyze the expression and
896 /// return a mask of Known bits for the expression (used to simplify the
897 /// caller). The Known bits may only be accurate for those bits in the
898 /// OriginalDemandedBits and OriginalDemandedElts.
900  SDValue Op, const APInt &OriginalDemandedBits,
901  const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
902  unsigned Depth, bool AssumeSingleUse) const {
903  unsigned BitWidth = OriginalDemandedBits.getBitWidth();
904  assert(Op.getScalarValueSizeInBits() == BitWidth &&
905  "Mask size mismatches value type size!");
906 
907  // Don't know anything.
908  Known = KnownBits(BitWidth);
909 
910  // TODO: We can probably do more work on calculating the known bits and
911  // simplifying the operations for scalable vectors, but for now we just
912  // bail out.
913  if (Op.getValueType().isScalableVector())
914  return false;
915 
916  unsigned NumElts = OriginalDemandedElts.getBitWidth();
917  assert((!Op.getValueType().isVector() ||
918  NumElts == Op.getValueType().getVectorNumElements()) &&
919  "Unexpected vector size");
920 
921  APInt DemandedBits = OriginalDemandedBits;
922  APInt DemandedElts = OriginalDemandedElts;
923  SDLoc dl(Op);
924  auto &DL = TLO.DAG.getDataLayout();
925 
926  // Undef operand.
927  if (Op.isUndef())
928  return false;
929 
930  if (Op.getOpcode() == ISD::Constant) {
931  // We know all of the bits for a constant!
932  Known = KnownBits::makeConstant(cast<ConstantSDNode>(Op)->getAPIntValue());
933  return false;
934  }
935 
936  if (Op.getOpcode() == ISD::ConstantFP) {
937  // We know all of the bits for a floating point constant!
938  Known = KnownBits::makeConstant(
939  cast<ConstantFPSDNode>(Op)->getValueAPF().bitcastToAPInt());
940  return false;
941  }
942 
943  // Other users may use these bits.
944  EVT VT = Op.getValueType();
945  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse) {
946  if (Depth != 0) {
947  // If not at the root, Just compute the Known bits to
948  // simplify things downstream.
949  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
950  return false;
951  }
952  // If this is the root being simplified, allow it to have multiple uses,
953  // just set the DemandedBits/Elts to all bits.
955  DemandedElts = APInt::getAllOnes(NumElts);
956  } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
957  // Not demanding any bits/elts from Op.
958  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
959  } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
960  // Limit search depth.
961  return false;
962  }
963 
964  KnownBits Known2;
965  switch (Op.getOpcode()) {
966  case ISD::TargetConstant:
967  llvm_unreachable("Can't simplify this node");
968  case ISD::SCALAR_TO_VECTOR: {
969  if (!DemandedElts[0])
970  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
971 
972  KnownBits SrcKnown;
973  SDValue Src = Op.getOperand(0);
974  unsigned SrcBitWidth = Src.getScalarValueSizeInBits();
975  APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth);
976  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1))
977  return true;
978 
979  // Upper elements are undef, so only get the knownbits if we just demand
980  // the bottom element.
981  if (DemandedElts == 1)
982  Known = SrcKnown.anyextOrTrunc(BitWidth);
983  break;
984  }
985  case ISD::BUILD_VECTOR:
986  // Collect the known bits that are shared by every demanded element.
987  // TODO: Call SimplifyDemandedBits for non-constant demanded elements.
988  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
989  return false; // Don't fall through, will infinitely loop.
990  case ISD::LOAD: {
991  auto *LD = cast<LoadSDNode>(Op);
992  if (getTargetConstantFromLoad(LD)) {
993  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
994  return false; // Don't fall through, will infinitely loop.
995  }
996  if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) {
997  // If this is a ZEXTLoad and we are looking at the loaded value.
998  EVT MemVT = LD->getMemoryVT();
999  unsigned MemBits = MemVT.getScalarSizeInBits();
1000  Known.Zero.setBitsFrom(MemBits);
1001  return false; // Don't fall through, will infinitely loop.
1002  }
1003  break;
1004  }
1005  case ISD::INSERT_VECTOR_ELT: {
1006  SDValue Vec = Op.getOperand(0);
1007  SDValue Scl = Op.getOperand(1);
1008  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
1009  EVT VecVT = Vec.getValueType();
1010 
1011  // If index isn't constant, assume we need all vector elements AND the
1012  // inserted element.
1013  APInt DemandedVecElts(DemandedElts);
1014  if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
1015  unsigned Idx = CIdx->getZExtValue();
1016  DemandedVecElts.clearBit(Idx);
1017 
1018  // Inserted element is not required.
1019  if (!DemandedElts[Idx])
1020  return TLO.CombineTo(Op, Vec);
1021  }
1022 
1023  KnownBits KnownScl;
1024  unsigned NumSclBits = Scl.getScalarValueSizeInBits();
1025  APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits);
1026  if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
1027  return true;
1028 
1029  Known = KnownScl.anyextOrTrunc(BitWidth);
1030 
1031  KnownBits KnownVec;
1032  if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO,
1033  Depth + 1))
1034  return true;
1035 
1036  if (!!DemandedVecElts)
1037  Known = KnownBits::commonBits(Known, KnownVec);
1038 
1039  return false;
1040  }
1041  case ISD::INSERT_SUBVECTOR: {
1042  // Demand any elements from the subvector and the remainder from the src its
1043  // inserted into.
1044  SDValue Src = Op.getOperand(0);
1045  SDValue Sub = Op.getOperand(1);
1046  uint64_t Idx = Op.getConstantOperandVal(2);
1047  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
1048  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
1049  APInt DemandedSrcElts = DemandedElts;
1050  DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
1051 
1052  KnownBits KnownSub, KnownSrc;
1053  if (SimplifyDemandedBits(Sub, DemandedBits, DemandedSubElts, KnownSub, TLO,
1054  Depth + 1))
1055  return true;
1056  if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, KnownSrc, TLO,
1057  Depth + 1))
1058  return true;
1059 
1060  Known.Zero.setAllBits();
1061  Known.One.setAllBits();
1062  if (!!DemandedSubElts)
1063  Known = KnownBits::commonBits(Known, KnownSub);
1064  if (!!DemandedSrcElts)
1065  Known = KnownBits::commonBits(Known, KnownSrc);
1066 
1067  // Attempt to avoid multi-use src if we don't need anything from it.
1068  if (!DemandedBits.isAllOnes() || !DemandedSubElts.isAllOnes() ||
1069  !DemandedSrcElts.isAllOnes()) {
1070  SDValue NewSub = SimplifyMultipleUseDemandedBits(
1071  Sub, DemandedBits, DemandedSubElts, TLO.DAG, Depth + 1);
1072  SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1073  Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1074  if (NewSub || NewSrc) {
1075  NewSub = NewSub ? NewSub : Sub;
1076  NewSrc = NewSrc ? NewSrc : Src;
1077  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc, NewSub,
1078  Op.getOperand(2));
1079  return TLO.CombineTo(Op, NewOp);
1080  }
1081  }
1082  break;
1083  }
1084  case ISD::EXTRACT_SUBVECTOR: {
1085  // Offset the demanded elts by the subvector index.
1086  SDValue Src = Op.getOperand(0);
1087  if (Src.getValueType().isScalableVector())
1088  break;
1089  uint64_t Idx = Op.getConstantOperandVal(1);
1090  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
1091  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
1092 
1093  if (SimplifyDemandedBits(Src, DemandedBits, DemandedSrcElts, Known, TLO,
1094  Depth + 1))
1095  return true;
1096 
1097  // Attempt to avoid multi-use src if we don't need anything from it.
1098  if (!DemandedBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
1099  SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
1100  Src, DemandedBits, DemandedSrcElts, TLO.DAG, Depth + 1);
1101  if (DemandedSrc) {
1102  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc,
1103  Op.getOperand(1));
1104  return TLO.CombineTo(Op, NewOp);
1105  }
1106  }
1107  break;
1108  }
1109  case ISD::CONCAT_VECTORS: {
1110  Known.Zero.setAllBits();
1111  Known.One.setAllBits();
1112  EVT SubVT = Op.getOperand(0).getValueType();
1113  unsigned NumSubVecs = Op.getNumOperands();
1114  unsigned NumSubElts = SubVT.getVectorNumElements();
1115  for (unsigned i = 0; i != NumSubVecs; ++i) {
1116  APInt DemandedSubElts =
1117  DemandedElts.extractBits(NumSubElts, i * NumSubElts);
1118  if (SimplifyDemandedBits(Op.getOperand(i), DemandedBits, DemandedSubElts,
1119  Known2, TLO, Depth + 1))
1120  return true;
1121  // Known bits are shared by every demanded subvector element.
1122  if (!!DemandedSubElts)
1123  Known = KnownBits::commonBits(Known, Known2);
1124  }
1125  break;
1126  }
1127  case ISD::VECTOR_SHUFFLE: {
1128  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
1129 
1130  // Collect demanded elements from shuffle operands..
1131  APInt DemandedLHS(NumElts, 0);
1132  APInt DemandedRHS(NumElts, 0);
1133  for (unsigned i = 0; i != NumElts; ++i) {
1134  if (!DemandedElts[i])
1135  continue;
1136  int M = ShuffleMask[i];
1137  if (M < 0) {
1138  // For UNDEF elements, we don't know anything about the common state of
1139  // the shuffle result.
1140  DemandedLHS.clearAllBits();
1141  DemandedRHS.clearAllBits();
1142  break;
1143  }
1144  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
1145  if (M < (int)NumElts)
1146  DemandedLHS.setBit(M);
1147  else
1148  DemandedRHS.setBit(M - NumElts);
1149  }
1150 
1151  if (!!DemandedLHS || !!DemandedRHS) {
1152  SDValue Op0 = Op.getOperand(0);
1153  SDValue Op1 = Op.getOperand(1);
1154 
1155  Known.Zero.setAllBits();
1156  Known.One.setAllBits();
1157  if (!!DemandedLHS) {
1158  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedLHS, Known2, TLO,
1159  Depth + 1))
1160  return true;
1161  Known = KnownBits::commonBits(Known, Known2);
1162  }
1163  if (!!DemandedRHS) {
1164  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedRHS, Known2, TLO,
1165  Depth + 1))
1166  return true;
1167  Known = KnownBits::commonBits(Known, Known2);
1168  }
1169 
1170  // Attempt to avoid multi-use ops if we don't need anything from them.
1171  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1172  Op0, DemandedBits, DemandedLHS, TLO.DAG, Depth + 1);
1173  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1174  Op1, DemandedBits, DemandedRHS, TLO.DAG, Depth + 1);
1175  if (DemandedOp0 || DemandedOp1) {
1176  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1177  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1178  SDValue NewOp = TLO.DAG.getVectorShuffle(VT, dl, Op0, Op1, ShuffleMask);
1179  return TLO.CombineTo(Op, NewOp);
1180  }
1181  }
1182  break;
1183  }
1184  case ISD::AND: {
1185  SDValue Op0 = Op.getOperand(0);
1186  SDValue Op1 = Op.getOperand(1);
1187 
1188  // If the RHS is a constant, check to see if the LHS would be zero without
1189  // using the bits from the RHS. Below, we use knowledge about the RHS to
1190  // simplify the LHS, here we're using information from the LHS to simplify
1191  // the RHS.
1192  if (ConstantSDNode *RHSC = isConstOrConstSplat(Op1)) {
1193  // Do not increment Depth here; that can cause an infinite loop.
1194  KnownBits LHSKnown = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth);
1195  // If the LHS already has zeros where RHSC does, this 'and' is dead.
1196  if ((LHSKnown.Zero & DemandedBits) ==
1197  (~RHSC->getAPIntValue() & DemandedBits))
1198  return TLO.CombineTo(Op, Op0);
1199 
1200  // If any of the set bits in the RHS are known zero on the LHS, shrink
1201  // the constant.
1202  if (ShrinkDemandedConstant(Op, ~LHSKnown.Zero & DemandedBits,
1203  DemandedElts, TLO))
1204  return true;
1205 
1206  // Bitwise-not (xor X, -1) is a special case: we don't usually shrink its
1207  // constant, but if this 'and' is only clearing bits that were just set by
1208  // the xor, then this 'and' can be eliminated by shrinking the mask of
1209  // the xor. For example, for a 32-bit X:
1210  // and (xor (srl X, 31), -1), 1 --> xor (srl X, 31), 1
1211  if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
1212  LHSKnown.One == ~RHSC->getAPIntValue()) {
1213  SDValue Xor = TLO.DAG.getNode(ISD::XOR, dl, VT, Op0.getOperand(0), Op1);
1214  return TLO.CombineTo(Op, Xor);
1215  }
1216  }
1217 
1218  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1219  Depth + 1))
1220  return true;
1221  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1222  if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts,
1223  Known2, TLO, Depth + 1))
1224  return true;
1225  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1226 
1227  // Attempt to avoid multi-use ops if we don't need anything from them.
1228  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1229  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1230  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1231  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1232  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1233  if (DemandedOp0 || DemandedOp1) {
1234  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1235  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1236  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1237  return TLO.CombineTo(Op, NewOp);
1238  }
1239  }
1240 
1241  // If all of the demanded bits are known one on one side, return the other.
1242  // These bits cannot contribute to the result of the 'and'.
1243  if (DemandedBits.isSubsetOf(Known2.Zero | Known.One))
1244  return TLO.CombineTo(Op, Op0);
1245  if (DemandedBits.isSubsetOf(Known.Zero | Known2.One))
1246  return TLO.CombineTo(Op, Op1);
1247  // If all of the demanded bits in the inputs are known zeros, return zero.
1248  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1249  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, dl, VT));
1250  // If the RHS is a constant, see if we can simplify it.
1251  if (ShrinkDemandedConstant(Op, ~Known2.Zero & DemandedBits, DemandedElts,
1252  TLO))
1253  return true;
1254  // If the operation can be done in a smaller type, do so.
1255  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1256  return true;
1257 
1258  Known &= Known2;
1259  break;
1260  }
1261  case ISD::OR: {
1262  SDValue Op0 = Op.getOperand(0);
1263  SDValue Op1 = Op.getOperand(1);
1264 
1265  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1266  Depth + 1))
1267  return true;
1268  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1269  if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts,
1270  Known2, TLO, Depth + 1))
1271  return true;
1272  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1273 
1274  // Attempt to avoid multi-use ops if we don't need anything from them.
1275  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1276  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1277  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1278  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1279  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1280  if (DemandedOp0 || DemandedOp1) {
1281  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1282  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1283  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1284  return TLO.CombineTo(Op, NewOp);
1285  }
1286  }
1287 
1288  // If all of the demanded bits are known zero on one side, return the other.
1289  // These bits cannot contribute to the result of the 'or'.
1290  if (DemandedBits.isSubsetOf(Known2.One | Known.Zero))
1291  return TLO.CombineTo(Op, Op0);
1292  if (DemandedBits.isSubsetOf(Known.One | Known2.Zero))
1293  return TLO.CombineTo(Op, Op1);
1294  // If the RHS is a constant, see if we can simplify it.
1295  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1296  return true;
1297  // If the operation can be done in a smaller type, do so.
1298  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1299  return true;
1300 
1301  Known |= Known2;
1302  break;
1303  }
1304  case ISD::XOR: {
1305  SDValue Op0 = Op.getOperand(0);
1306  SDValue Op1 = Op.getOperand(1);
1307 
1308  if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO,
1309  Depth + 1))
1310  return true;
1311  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1312  if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO,
1313  Depth + 1))
1314  return true;
1315  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1316 
1317  // Attempt to avoid multi-use ops if we don't need anything from them.
1318  if (!DemandedBits.isAllOnes() || !DemandedElts.isAllOnes()) {
1319  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1320  Op0, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1321  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
1322  Op1, DemandedBits, DemandedElts, TLO.DAG, Depth + 1);
1323  if (DemandedOp0 || DemandedOp1) {
1324  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
1325  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
1326  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1);
1327  return TLO.CombineTo(Op, NewOp);
1328  }
1329  }
1330 
1331  // If all of the demanded bits are known zero on one side, return the other.
1332  // These bits cannot contribute to the result of the 'xor'.
1333  if (DemandedBits.isSubsetOf(Known.Zero))
1334  return TLO.CombineTo(Op, Op0);
1335  if (DemandedBits.isSubsetOf(Known2.Zero))
1336  return TLO.CombineTo(Op, Op1);
1337  // If the operation can be done in a smaller type, do so.
1338  if (ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1339  return true;
1340 
1341  // If all of the unknown bits are known to be zero on one side or the other
1342  // turn this into an *inclusive* or.
1343  // e.g. (A & C1)^(B & C2) -> (A & C1)|(B & C2) iff C1&C2 == 0
1344  if (DemandedBits.isSubsetOf(Known.Zero | Known2.Zero))
1345  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::OR, dl, VT, Op0, Op1));
1346 
1347  ConstantSDNode* C = isConstOrConstSplat(Op1, DemandedElts);
1348  if (C) {
1349  // If one side is a constant, and all of the set bits in the constant are
1350  // also known set on the other side, turn this into an AND, as we know
1351  // the bits will be cleared.
1352  // e.g. (X | C1) ^ C2 --> (X | C1) & ~C2 iff (C1&C2) == C2
1353  // NB: it is okay if more bits are known than are requested
1354  if (C->getAPIntValue() == Known2.One) {
1355  SDValue ANDC =
1356  TLO.DAG.getConstant(~C->getAPIntValue() & DemandedBits, dl, VT);
1357  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::AND, dl, VT, Op0, ANDC));
1358  }
1359 
1360  // If the RHS is a constant, see if we can change it. Don't alter a -1
1361  // constant because that's a 'not' op, and that is better for combining
1362  // and codegen.
1363  if (!C->isAllOnes() && DemandedBits.isSubsetOf(C->getAPIntValue())) {
1364  // We're flipping all demanded bits. Flip the undemanded bits too.
1365  SDValue New = TLO.DAG.getNOT(dl, Op0, VT);
1366  return TLO.CombineTo(Op, New);
1367  }
1368  }
1369 
1370  // If we can't turn this into a 'not', try to shrink the constant.
1371  if (!C || !C->isAllOnes())
1372  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1373  return true;
1374 
1375  Known ^= Known2;
1376  break;
1377  }
1378  case ISD::SELECT:
1379  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known, TLO,
1380  Depth + 1))
1381  return true;
1382  if (SimplifyDemandedBits(Op.getOperand(1), DemandedBits, Known2, TLO,
1383  Depth + 1))
1384  return true;
1385  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1386  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1387 
1388  // If the operands are constants, see if we can simplify them.
1389  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1390  return true;
1391 
1392  // Only known if known in both the LHS and RHS.
1393  Known = KnownBits::commonBits(Known, Known2);
1394  break;
1395  case ISD::SELECT_CC:
1396  if (SimplifyDemandedBits(Op.getOperand(3), DemandedBits, Known, TLO,
1397  Depth + 1))
1398  return true;
1399  if (SimplifyDemandedBits(Op.getOperand(2), DemandedBits, Known2, TLO,
1400  Depth + 1))
1401  return true;
1402  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1403  assert(!Known2.hasConflict() && "Bits known to be one AND zero?");
1404 
1405  // If the operands are constants, see if we can simplify them.
1406  if (ShrinkDemandedConstant(Op, DemandedBits, DemandedElts, TLO))
1407  return true;
1408 
1409  // Only known if known in both the LHS and RHS.
1410  Known = KnownBits::commonBits(Known, Known2);
1411  break;
1412  case ISD::SETCC: {
1413  SDValue Op0 = Op.getOperand(0);
1414  SDValue Op1 = Op.getOperand(1);
1415  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
1416  // If (1) we only need the sign-bit, (2) the setcc operands are the same
1417  // width as the setcc result, and (3) the result of a setcc conforms to 0 or
1418  // -1, we may be able to bypass the setcc.
1419  if (DemandedBits.isSignMask() &&
1422  BooleanContent::ZeroOrNegativeOneBooleanContent) {
1423  // If we're testing X < 0, then this compare isn't needed - just use X!
1424  // FIXME: We're limiting to integer types here, but this should also work
1425  // if we don't care about FP signed-zero. The use of SETLT with FP means
1426  // that we don't care about NaNs.
1427  if (CC == ISD::SETLT && Op1.getValueType().isInteger() &&
1429  return TLO.CombineTo(Op, Op0);
1430 
1431  // TODO: Should we check for other forms of sign-bit comparisons?
1432  // Examples: X <= -1, X >= 0
1433  }
1434  if (getBooleanContents(Op0.getValueType()) ==
1436  BitWidth > 1)
1437  Known.Zero.setBitsFrom(1);
1438  break;
1439  }
1440  case ISD::SHL: {
1441  SDValue Op0 = Op.getOperand(0);
1442  SDValue Op1 = Op.getOperand(1);
1443  EVT ShiftVT = Op1.getValueType();
1444 
1445  if (const APInt *SA =
1446  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1447  unsigned ShAmt = SA->getZExtValue();
1448  if (ShAmt == 0)
1449  return TLO.CombineTo(Op, Op0);
1450 
1451  // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
1452  // single shift. We can do this if the bottom bits (which are shifted
1453  // out) are never demanded.
1454  // TODO - support non-uniform vector amounts.
1455  if (Op0.getOpcode() == ISD::SRL) {
1456  if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
1457  if (const APInt *SA2 =
1458  TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1459  unsigned C1 = SA2->getZExtValue();
1460  unsigned Opc = ISD::SHL;
1461  int Diff = ShAmt - C1;
1462  if (Diff < 0) {
1463  Diff = -Diff;
1464  Opc = ISD::SRL;
1465  }
1466  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1467  return TLO.CombineTo(
1468  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1469  }
1470  }
1471  }
1472 
1473  // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits
1474  // are not demanded. This will likely allow the anyext to be folded away.
1475  // TODO - support non-uniform vector amounts.
1476  if (Op0.getOpcode() == ISD::ANY_EXTEND) {
1477  SDValue InnerOp = Op0.getOperand(0);
1478  EVT InnerVT = InnerOp.getValueType();
1479  unsigned InnerBits = InnerVT.getScalarSizeInBits();
1480  if (ShAmt < InnerBits && DemandedBits.getActiveBits() <= InnerBits &&
1481  isTypeDesirableForOp(ISD::SHL, InnerVT)) {
1482  EVT ShTy = getShiftAmountTy(InnerVT, DL);
1483  if (!APInt(BitWidth, ShAmt).isIntN(ShTy.getSizeInBits()))
1484  ShTy = InnerVT;
1485  SDValue NarrowShl =
1486  TLO.DAG.getNode(ISD::SHL, dl, InnerVT, InnerOp,
1487  TLO.DAG.getConstant(ShAmt, dl, ShTy));
1488  return TLO.CombineTo(
1489  Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, NarrowShl));
1490  }
1491 
1492  // Repeat the SHL optimization above in cases where an extension
1493  // intervenes: (shl (anyext (shr x, c1)), c2) to
1494  // (shl (anyext x), c2-c1). This requires that the bottom c1 bits
1495  // aren't demanded (as above) and that the shifted upper c1 bits of
1496  // x aren't demanded.
1497  // TODO - support non-uniform vector amounts.
1498  if (Op0.hasOneUse() && InnerOp.getOpcode() == ISD::SRL &&
1499  InnerOp.hasOneUse()) {
1500  if (const APInt *SA2 =
1501  TLO.DAG.getValidShiftAmountConstant(InnerOp, DemandedElts)) {
1502  unsigned InnerShAmt = SA2->getZExtValue();
1503  if (InnerShAmt < ShAmt && InnerShAmt < InnerBits &&
1504  DemandedBits.getActiveBits() <=
1505  (InnerBits - InnerShAmt + ShAmt) &&
1506  DemandedBits.countTrailingZeros() >= ShAmt) {
1507  SDValue NewSA =
1508  TLO.DAG.getConstant(ShAmt - InnerShAmt, dl, ShiftVT);
1509  SDValue NewExt = TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT,
1510  InnerOp.getOperand(0));
1511  return TLO.CombineTo(
1512  Op, TLO.DAG.getNode(ISD::SHL, dl, VT, NewExt, NewSA));
1513  }
1514  }
1515  }
1516  }
1517 
1518  APInt InDemandedMask = DemandedBits.lshr(ShAmt);
1519  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1520  Depth + 1))
1521  return true;
1522  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1523  Known.Zero <<= ShAmt;
1524  Known.One <<= ShAmt;
1525  // low bits known zero.
1526  Known.Zero.setLowBits(ShAmt);
1527 
1528  // Try shrinking the operation as long as the shift amount will still be
1529  // in range.
1530  if ((ShAmt < DemandedBits.getActiveBits()) &&
1531  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO))
1532  return true;
1533  }
1534 
1535  // If we are only demanding sign bits then we can use the shift source
1536  // directly.
1537  if (const APInt *MaxSA =
1538  TLO.DAG.getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
1539  unsigned ShAmt = MaxSA->getZExtValue();
1540  unsigned NumSignBits =
1541  TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1542  unsigned UpperDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1543  if (NumSignBits > ShAmt && (NumSignBits - ShAmt) >= (UpperDemandedBits))
1544  return TLO.CombineTo(Op, Op0);
1545  }
1546  break;
1547  }
1548  case ISD::SRL: {
1549  SDValue Op0 = Op.getOperand(0);
1550  SDValue Op1 = Op.getOperand(1);
1551  EVT ShiftVT = Op1.getValueType();
1552 
1553  if (const APInt *SA =
1554  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1555  unsigned ShAmt = SA->getZExtValue();
1556  if (ShAmt == 0)
1557  return TLO.CombineTo(Op, Op0);
1558 
1559  // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
1560  // single shift. We can do this if the top bits (which are shifted out)
1561  // are never demanded.
1562  // TODO - support non-uniform vector amounts.
1563  if (Op0.getOpcode() == ISD::SHL) {
1564  if (!DemandedBits.intersects(APInt::getHighBitsSet(BitWidth, ShAmt))) {
1565  if (const APInt *SA2 =
1566  TLO.DAG.getValidShiftAmountConstant(Op0, DemandedElts)) {
1567  unsigned C1 = SA2->getZExtValue();
1568  unsigned Opc = ISD::SRL;
1569  int Diff = ShAmt - C1;
1570  if (Diff < 0) {
1571  Diff = -Diff;
1572  Opc = ISD::SHL;
1573  }
1574  SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT);
1575  return TLO.CombineTo(
1576  Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA));
1577  }
1578  }
1579  }
1580 
1581  APInt InDemandedMask = (DemandedBits << ShAmt);
1582 
1583  // If the shift is exact, then it does demand the low bits (and knows that
1584  // they are zero).
1585  if (Op->getFlags().hasExact())
1586  InDemandedMask.setLowBits(ShAmt);
1587 
1588  // Compute the new bits that are at the top now.
1589  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1590  Depth + 1))
1591  return true;
1592  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1593  Known.Zero.lshrInPlace(ShAmt);
1594  Known.One.lshrInPlace(ShAmt);
1595  // High bits known zero.
1596  Known.Zero.setHighBits(ShAmt);
1597  }
1598  break;
1599  }
1600  case ISD::SRA: {
1601  SDValue Op0 = Op.getOperand(0);
1602  SDValue Op1 = Op.getOperand(1);
1603  EVT ShiftVT = Op1.getValueType();
1604 
1605  // If we only want bits that already match the signbit then we don't need
1606  // to shift.
1607  unsigned NumHiDemandedBits = BitWidth - DemandedBits.countTrailingZeros();
1608  if (TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1) >=
1609  NumHiDemandedBits)
1610  return TLO.CombineTo(Op, Op0);
1611 
1612  // If this is an arithmetic shift right and only the low-bit is set, we can
1613  // always convert this into a logical shr, even if the shift amount is
1614  // variable. The low bit of the shift cannot be an input sign bit unless
1615  // the shift amount is >= the size of the datatype, which is undefined.
1616  if (DemandedBits.isOne())
1617  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1));
1618 
1619  if (const APInt *SA =
1620  TLO.DAG.getValidShiftAmountConstant(Op, DemandedElts)) {
1621  unsigned ShAmt = SA->getZExtValue();
1622  if (ShAmt == 0)
1623  return TLO.CombineTo(Op, Op0);
1624 
1625  APInt InDemandedMask = (DemandedBits << ShAmt);
1626 
1627  // If the shift is exact, then it does demand the low bits (and knows that
1628  // they are zero).
1629  if (Op->getFlags().hasExact())
1630  InDemandedMask.setLowBits(ShAmt);
1631 
1632  // If any of the demanded bits are produced by the sign extension, we also
1633  // demand the input sign bit.
1634  if (DemandedBits.countLeadingZeros() < ShAmt)
1635  InDemandedMask.setSignBit();
1636 
1637  if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO,
1638  Depth + 1))
1639  return true;
1640  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1641  Known.Zero.lshrInPlace(ShAmt);
1642  Known.One.lshrInPlace(ShAmt);
1643 
1644  // If the input sign bit is known to be zero, or if none of the top bits
1645  // are demanded, turn this into an unsigned shift right.
1646  if (Known.Zero[BitWidth - ShAmt - 1] ||
1647  DemandedBits.countLeadingZeros() >= ShAmt) {
1648  SDNodeFlags Flags;
1649  Flags.setExact(Op->getFlags().hasExact());
1650  return TLO.CombineTo(
1651  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1, Flags));
1652  }
1653 
1654  int Log2 = DemandedBits.exactLogBase2();
1655  if (Log2 >= 0) {
1656  // The bit must come from the sign.
1657  SDValue NewSA = TLO.DAG.getConstant(BitWidth - 1 - Log2, dl, ShiftVT);
1658  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, NewSA));
1659  }
1660 
1661  if (Known.One[BitWidth - ShAmt - 1])
1662  // New bits are known one.
1663  Known.One.setHighBits(ShAmt);
1664 
1665  // Attempt to avoid multi-use ops if we don't need anything from them.
1666  if (!InDemandedMask.isAllOnes() || !DemandedElts.isAllOnes()) {
1667  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
1668  Op0, InDemandedMask, DemandedElts, TLO.DAG, Depth + 1);
1669  if (DemandedOp0) {
1670  SDValue NewOp = TLO.DAG.getNode(ISD::SRA, dl, VT, DemandedOp0, Op1);
1671  return TLO.CombineTo(Op, NewOp);
1672  }
1673  }
1674  }
1675  break;
1676  }
1677  case ISD::FSHL:
1678  case ISD::FSHR: {
1679  SDValue Op0 = Op.getOperand(0);
1680  SDValue Op1 = Op.getOperand(1);
1681  SDValue Op2 = Op.getOperand(2);
1682  bool IsFSHL = (Op.getOpcode() == ISD::FSHL);
1683 
1684  if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) {
1685  unsigned Amt = SA->getAPIntValue().urem(BitWidth);
1686 
1687  // For fshl, 0-shift returns the 1st arg.
1688  // For fshr, 0-shift returns the 2nd arg.
1689  if (Amt == 0) {
1690  if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts,
1691  Known, TLO, Depth + 1))
1692  return true;
1693  break;
1694  }
1695 
1696  // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt))
1697  // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt)
1698  APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt));
1699  APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt);
1700  if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO,
1701  Depth + 1))
1702  return true;
1703  if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO,
1704  Depth + 1))
1705  return true;
1706 
1707  Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt));
1708  Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt));
1709  Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1710  Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt);
1711  Known.One |= Known2.One;
1712  Known.Zero |= Known2.Zero;
1713  }
1714 
1715  // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1716  if (isPowerOf2_32(BitWidth)) {
1717  APInt DemandedAmtBits(Op2.getScalarValueSizeInBits(), BitWidth - 1);
1718  if (SimplifyDemandedBits(Op2, DemandedAmtBits, DemandedElts,
1719  Known2, TLO, Depth + 1))
1720  return true;
1721  }
1722  break;
1723  }
1724  case ISD::ROTL:
1725  case ISD::ROTR: {
1726  SDValue Op0 = Op.getOperand(0);
1727  SDValue Op1 = Op.getOperand(1);
1728 
1729  // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
1730  if (BitWidth == TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1))
1731  return TLO.CombineTo(Op, Op0);
1732 
1733  // For pow-2 bitwidths we only demand the bottom modulo amt bits.
1734  if (isPowerOf2_32(BitWidth)) {
1735  APInt DemandedAmtBits(Op1.getScalarValueSizeInBits(), BitWidth - 1);
1736  if (SimplifyDemandedBits(Op1, DemandedAmtBits, DemandedElts, Known2, TLO,
1737  Depth + 1))
1738  return true;
1739  }
1740  break;
1741  }
1742  case ISD::UMIN: {
1743  // Check if one arg is always less than (or equal) to the other arg.
1744  SDValue Op0 = Op.getOperand(0);
1745  SDValue Op1 = Op.getOperand(1);
1746  KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1747  KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1748  Known = KnownBits::umin(Known0, Known1);
1749  if (Optional<bool> IsULE = KnownBits::ule(Known0, Known1))
1750  return TLO.CombineTo(Op, IsULE.getValue() ? Op0 : Op1);
1751  if (Optional<bool> IsULT = KnownBits::ult(Known0, Known1))
1752  return TLO.CombineTo(Op, IsULT.getValue() ? Op0 : Op1);
1753  break;
1754  }
1755  case ISD::UMAX: {
1756  // Check if one arg is always greater than (or equal) to the other arg.
1757  SDValue Op0 = Op.getOperand(0);
1758  SDValue Op1 = Op.getOperand(1);
1759  KnownBits Known0 = TLO.DAG.computeKnownBits(Op0, DemandedElts, Depth + 1);
1760  KnownBits Known1 = TLO.DAG.computeKnownBits(Op1, DemandedElts, Depth + 1);
1761  Known = KnownBits::umax(Known0, Known1);
1762  if (Optional<bool> IsUGE = KnownBits::uge(Known0, Known1))
1763  return TLO.CombineTo(Op, IsUGE.getValue() ? Op0 : Op1);
1764  if (Optional<bool> IsUGT = KnownBits::ugt(Known0, Known1))
1765  return TLO.CombineTo(Op, IsUGT.getValue() ? Op0 : Op1);
1766  break;
1767  }
1768  case ISD::BITREVERSE: {
1769  SDValue Src = Op.getOperand(0);
1770  APInt DemandedSrcBits = DemandedBits.reverseBits();
1771  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1772  Depth + 1))
1773  return true;
1774  Known.One = Known2.One.reverseBits();
1775  Known.Zero = Known2.Zero.reverseBits();
1776  break;
1777  }
1778  case ISD::BSWAP: {
1779  SDValue Src = Op.getOperand(0);
1780  APInt DemandedSrcBits = DemandedBits.byteSwap();
1781  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO,
1782  Depth + 1))
1783  return true;
1784  Known.One = Known2.One.byteSwap();
1785  Known.Zero = Known2.Zero.byteSwap();
1786  break;
1787  }
1788  case ISD::CTPOP: {
1789  // If only 1 bit is demanded, replace with PARITY as long as we're before
1790  // op legalization.
1791  // FIXME: Limit to scalars for now.
1792  if (DemandedBits.isOne() && !TLO.LegalOps && !VT.isVector())
1793  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::PARITY, dl, VT,
1794  Op.getOperand(0)));
1795 
1796  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
1797  break;
1798  }
1799  case ISD::SIGN_EXTEND_INREG: {
1800  SDValue Op0 = Op.getOperand(0);
1801  EVT ExVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1802  unsigned ExVTBits = ExVT.getScalarSizeInBits();
1803 
1804  // If we only care about the highest bit, don't bother shifting right.
1805  if (DemandedBits.isSignMask()) {
1806  unsigned NumSignBits =
1807  TLO.DAG.ComputeNumSignBits(Op0, DemandedElts, Depth + 1);
1808  bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1;
1809  // However if the input is already sign extended we expect the sign
1810  // extension to be dropped altogether later and do not simplify.
1811  if (!AlreadySignExtended) {
1812  // Compute the correct shift amount type, which must be getShiftAmountTy
1813  // for scalar types after legalization.
1814  EVT ShiftAmtTy = VT;
1815  if (TLO.LegalTypes() && !ShiftAmtTy.isVector())
1816  ShiftAmtTy = getShiftAmountTy(ShiftAmtTy, DL);
1817 
1818  SDValue ShiftAmt =
1819  TLO.DAG.getConstant(BitWidth - ExVTBits, dl, ShiftAmtTy);
1820  return TLO.CombineTo(Op,
1821  TLO.DAG.getNode(ISD::SHL, dl, VT, Op0, ShiftAmt));
1822  }
1823  }
1824 
1825  // If none of the extended bits are demanded, eliminate the sextinreg.
1826  if (DemandedBits.getActiveBits() <= ExVTBits)
1827  return TLO.CombineTo(Op, Op0);
1828 
1829  APInt InputDemandedBits = DemandedBits.getLoBits(ExVTBits);
1830 
1831  // Since the sign extended bits are demanded, we know that the sign
1832  // bit is demanded.
1833  InputDemandedBits.setBit(ExVTBits - 1);
1834 
1835  if (SimplifyDemandedBits(Op0, InputDemandedBits, Known, TLO, Depth + 1))
1836  return true;
1837  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1838 
1839  // If the sign bit of the input is known set or clear, then we know the
1840  // top bits of the result.
1841 
1842  // If the input sign bit is known zero, convert this into a zero extension.
1843  if (Known.Zero[ExVTBits - 1])
1844  return TLO.CombineTo(Op, TLO.DAG.getZeroExtendInReg(Op0, dl, ExVT));
1845 
1846  APInt Mask = APInt::getLowBitsSet(BitWidth, ExVTBits);
1847  if (Known.One[ExVTBits - 1]) { // Input sign bit known set
1848  Known.One.setBitsFrom(ExVTBits);
1849  Known.Zero &= Mask;
1850  } else { // Input sign bit unknown
1851  Known.Zero &= Mask;
1852  Known.One &= Mask;
1853  }
1854  break;
1855  }
1856  case ISD::BUILD_PAIR: {
1857  EVT HalfVT = Op.getOperand(0).getValueType();
1858  unsigned HalfBitWidth = HalfVT.getScalarSizeInBits();
1859 
1860  APInt MaskLo = DemandedBits.getLoBits(HalfBitWidth).trunc(HalfBitWidth);
1861  APInt MaskHi = DemandedBits.getHiBits(HalfBitWidth).trunc(HalfBitWidth);
1862 
1863  KnownBits KnownLo, KnownHi;
1864 
1865  if (SimplifyDemandedBits(Op.getOperand(0), MaskLo, KnownLo, TLO, Depth + 1))
1866  return true;
1867 
1868  if (SimplifyDemandedBits(Op.getOperand(1), MaskHi, KnownHi, TLO, Depth + 1))
1869  return true;
1870 
1871  Known.Zero = KnownLo.Zero.zext(BitWidth) |
1872  KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth);
1873 
1874  Known.One = KnownLo.One.zext(BitWidth) |
1875  KnownHi.One.zext(BitWidth).shl(HalfBitWidth);
1876  break;
1877  }
1878  case ISD::ZERO_EXTEND:
1880  SDValue Src = Op.getOperand(0);
1881  EVT SrcVT = Src.getValueType();
1882  unsigned InBits = SrcVT.getScalarSizeInBits();
1883  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1884  bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
1885 
1886  // If none of the top bits are demanded, convert this into an any_extend.
1887  if (DemandedBits.getActiveBits() <= InBits) {
1888  // If we only need the non-extended bits of the bottom element
1889  // then we can just bitcast to the result.
1890  if (IsVecInReg && DemandedElts == 1 &&
1891  VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1893  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1894 
1895  unsigned Opc =
1897  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1898  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1899  }
1900 
1901  APInt InDemandedBits = DemandedBits.trunc(InBits);
1902  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1903  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1904  Depth + 1))
1905  return true;
1906  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1907  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1908  Known = Known.zext(BitWidth);
1909 
1910  // Attempt to avoid multi-use ops if we don't need anything from them.
1911  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1912  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1913  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1914  break;
1915  }
1916  case ISD::SIGN_EXTEND:
1918  SDValue Src = Op.getOperand(0);
1919  EVT SrcVT = Src.getValueType();
1920  unsigned InBits = SrcVT.getScalarSizeInBits();
1921  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1922  bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG;
1923 
1924  // If none of the top bits are demanded, convert this into an any_extend.
1925  if (DemandedBits.getActiveBits() <= InBits) {
1926  // If we only need the non-extended bits of the bottom element
1927  // then we can just bitcast to the result.
1928  if (IsVecInReg && DemandedElts == 1 &&
1929  VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1931  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1932 
1933  unsigned Opc =
1935  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1936  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1937  }
1938 
1939  APInt InDemandedBits = DemandedBits.trunc(InBits);
1940  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1941 
1942  // Since some of the sign extended bits are demanded, we know that the sign
1943  // bit is demanded.
1944  InDemandedBits.setBit(InBits - 1);
1945 
1946  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1947  Depth + 1))
1948  return true;
1949  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1950  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1951 
1952  // If the sign bit is known one, the top bits match.
1953  Known = Known.sext(BitWidth);
1954 
1955  // If the sign bit is known zero, convert this to a zero extend.
1956  if (Known.isNonNegative()) {
1957  unsigned Opc =
1959  if (!TLO.LegalOperations() || isOperationLegal(Opc, VT))
1960  return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src));
1961  }
1962 
1963  // Attempt to avoid multi-use ops if we don't need anything from them.
1964  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1965  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1966  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1967  break;
1968  }
1969  case ISD::ANY_EXTEND:
1971  SDValue Src = Op.getOperand(0);
1972  EVT SrcVT = Src.getValueType();
1973  unsigned InBits = SrcVT.getScalarSizeInBits();
1974  unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
1975  bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG;
1976 
1977  // If we only need the bottom element then we can just bitcast.
1978  // TODO: Handle ANY_EXTEND?
1979  if (IsVecInReg && DemandedElts == 1 &&
1980  VT.getSizeInBits() == SrcVT.getSizeInBits() &&
1982  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
1983 
1984  APInt InDemandedBits = DemandedBits.trunc(InBits);
1985  APInt InDemandedElts = DemandedElts.zextOrSelf(InElts);
1986  if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO,
1987  Depth + 1))
1988  return true;
1989  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
1990  assert(Known.getBitWidth() == InBits && "Src width has changed?");
1991  Known = Known.anyext(BitWidth);
1992 
1993  // Attempt to avoid multi-use ops if we don't need anything from them.
1994  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
1995  Src, InDemandedBits, InDemandedElts, TLO.DAG, Depth + 1))
1996  return TLO.CombineTo(Op, TLO.DAG.getNode(Op.getOpcode(), dl, VT, NewSrc));
1997  break;
1998  }
1999  case ISD::TRUNCATE: {
2000  SDValue Src = Op.getOperand(0);
2001 
2002  // Simplify the input, using demanded bit information, and compute the known
2003  // zero/one bits live out.
2004  unsigned OperandBitWidth = Src.getScalarValueSizeInBits();
2005  APInt TruncMask = DemandedBits.zext(OperandBitWidth);
2006  if (SimplifyDemandedBits(Src, TruncMask, DemandedElts, Known, TLO,
2007  Depth + 1))
2008  return true;
2009  Known = Known.trunc(BitWidth);
2010 
2011  // Attempt to avoid multi-use ops if we don't need anything from them.
2012  if (SDValue NewSrc = SimplifyMultipleUseDemandedBits(
2013  Src, TruncMask, DemandedElts, TLO.DAG, Depth + 1))
2014  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, NewSrc));
2015 
2016  // If the input is only used by this truncate, see if we can shrink it based
2017  // on the known demanded bits.
2018  if (Src.getNode()->hasOneUse()) {
2019  switch (Src.getOpcode()) {
2020  default:
2021  break;
2022  case ISD::SRL:
2023  // Shrink SRL by a constant if none of the high bits shifted in are
2024  // demanded.
2025  if (TLO.LegalTypes() && !isTypeDesirableForOp(ISD::SRL, VT))
2026  // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is
2027  // undesirable.
2028  break;
2029 
2030  const APInt *ShAmtC =
2031  TLO.DAG.getValidShiftAmountConstant(Src, DemandedElts);
2032  if (!ShAmtC || ShAmtC->uge(BitWidth))
2033  break;
2034  uint64_t ShVal = ShAmtC->getZExtValue();
2035 
2036  APInt HighBits =
2037  APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth);
2038  HighBits.lshrInPlace(ShVal);
2039  HighBits = HighBits.trunc(BitWidth);
2040 
2041  if (!(HighBits & DemandedBits)) {
2042  // None of the shifted in bits are needed. Add a truncate of the
2043  // shift input, then shift it.
2044  SDValue NewShAmt = TLO.DAG.getConstant(
2045  ShVal, dl, getShiftAmountTy(VT, DL, TLO.LegalTypes()));
2046  SDValue NewTrunc =
2047  TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0));
2048  return TLO.CombineTo(
2049  Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, NewShAmt));
2050  }
2051  break;
2052  }
2053  }
2054 
2055  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2056  break;
2057  }
2058  case ISD::AssertZext: {
2059  // AssertZext demands all of the high bits, plus any of the low bits
2060  // demanded by its users.
2061  EVT ZVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2063  if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known,
2064  TLO, Depth + 1))
2065  return true;
2066  assert(!Known.hasConflict() && "Bits known to be one AND zero?");
2067 
2068  Known.Zero |= ~InMask;
2069  break;
2070  }
2071  case ISD::EXTRACT_VECTOR_ELT: {
2072  SDValue Src = Op.getOperand(0);
2073  SDValue Idx = Op.getOperand(1);
2074  ElementCount SrcEltCnt = Src.getValueType().getVectorElementCount();
2075  unsigned EltBitWidth = Src.getScalarValueSizeInBits();
2076 
2077  if (SrcEltCnt.isScalable())
2078  return false;
2079 
2080  // Demand the bits from every vector element without a constant index.
2081  unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2082  APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
2083  if (auto *CIdx = dyn_cast<ConstantSDNode>(Idx))
2084  if (CIdx->getAPIntValue().ult(NumSrcElts))
2085  DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, CIdx->getZExtValue());
2086 
2087  // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
2088  // anything about the extended bits.
2089  APInt DemandedSrcBits = DemandedBits;
2090  if (BitWidth > EltBitWidth)
2091  DemandedSrcBits = DemandedSrcBits.trunc(EltBitWidth);
2092 
2093  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, Known2, TLO,
2094  Depth + 1))
2095  return true;
2096 
2097  // Attempt to avoid multi-use ops if we don't need anything from them.
2098  if (!DemandedSrcBits.isAllOnes() || !DemandedSrcElts.isAllOnes()) {
2099  if (SDValue DemandedSrc = SimplifyMultipleUseDemandedBits(
2100  Src, DemandedSrcBits, DemandedSrcElts, TLO.DAG, Depth + 1)) {
2101  SDValue NewOp =
2102  TLO.DAG.getNode(Op.getOpcode(), dl, VT, DemandedSrc, Idx);
2103  return TLO.CombineTo(Op, NewOp);
2104  }
2105  }
2106 
2107  Known = Known2;
2108  if (BitWidth > EltBitWidth)
2109  Known = Known.anyext(BitWidth);
2110  break;
2111  }
2112  case ISD::BITCAST: {
2113  SDValue Src = Op.getOperand(0);
2114  EVT SrcVT = Src.getValueType();
2115  unsigned NumSrcEltBits = SrcVT.getScalarSizeInBits();
2116 
2117  // If this is an FP->Int bitcast and if the sign bit is the only
2118  // thing demanded, turn this into a FGETSIGN.
2119  if (!TLO.LegalOperations() && !VT.isVector() && !SrcVT.isVector() &&
2120  DemandedBits == APInt::getSignMask(Op.getValueSizeInBits()) &&
2121  SrcVT.isFloatingPoint()) {
2122  bool OpVTLegal = isOperationLegalOrCustom(ISD::FGETSIGN, VT);
2124  if ((OpVTLegal || i32Legal) && VT.isSimple() && SrcVT != MVT::f16 &&
2125  SrcVT != MVT::f128) {
2126  // Cannot eliminate/lower SHL for f128 yet.
2127  EVT Ty = OpVTLegal ? VT : MVT::i32;
2128  // Make a FGETSIGN + SHL to move the sign bit into the appropriate
2129  // place. We expect the SHL to be eliminated by other optimizations.
2130  SDValue Sign = TLO.DAG.getNode(ISD::FGETSIGN, dl, Ty, Src);
2131  unsigned OpVTSizeInBits = Op.getValueSizeInBits();
2132  if (!OpVTLegal && OpVTSizeInBits > 32)
2133  Sign = TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Sign);
2134  unsigned ShVal = Op.getValueSizeInBits() - 1;
2135  SDValue ShAmt = TLO.DAG.getConstant(ShVal, dl, VT);
2136  return TLO.CombineTo(Op,
2137  TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt));
2138  }
2139  }
2140 
2141  // Bitcast from a vector using SimplifyDemanded Bits/VectorElts.
2142  // Demand the elt/bit if any of the original elts/bits are demanded.
2143  // TODO - bigendian once we have test coverage.
2144  if (SrcVT.isVector() && (BitWidth % NumSrcEltBits) == 0 &&
2145  TLO.DAG.getDataLayout().isLittleEndian()) {
2146  unsigned Scale = BitWidth / NumSrcEltBits;
2147  unsigned NumSrcElts = SrcVT.getVectorNumElements();
2148  APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2149  APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2150  for (unsigned i = 0; i != Scale; ++i) {
2151  unsigned Offset = i * NumSrcEltBits;
2152  APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset);
2153  if (!Sub.isZero()) {
2154  DemandedSrcBits |= Sub;
2155  for (unsigned j = 0; j != NumElts; ++j)
2156  if (DemandedElts[j])
2157  DemandedSrcElts.setBit((j * Scale) + i);
2158  }
2159  }
2160 
2161  APInt KnownSrcUndef, KnownSrcZero;
2162  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2163  KnownSrcZero, TLO, Depth + 1))
2164  return true;
2165 
2166  KnownBits KnownSrcBits;
2167  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2168  KnownSrcBits, TLO, Depth + 1))
2169  return true;
2170  } else if ((NumSrcEltBits % BitWidth) == 0 &&
2171  TLO.DAG.getDataLayout().isLittleEndian()) {
2172  unsigned Scale = NumSrcEltBits / BitWidth;
2173  unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1;
2174  APInt DemandedSrcBits = APInt::getZero(NumSrcEltBits);
2175  APInt DemandedSrcElts = APInt::getZero(NumSrcElts);
2176  for (unsigned i = 0; i != NumElts; ++i)
2177  if (DemandedElts[i]) {
2178  unsigned Offset = (i % Scale) * BitWidth;
2179  DemandedSrcBits.insertBits(DemandedBits, Offset);
2180  DemandedSrcElts.setBit(i / Scale);
2181  }
2182 
2183  if (SrcVT.isVector()) {
2184  APInt KnownSrcUndef, KnownSrcZero;
2185  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef,
2186  KnownSrcZero, TLO, Depth + 1))
2187  return true;
2188  }
2189 
2190  KnownBits KnownSrcBits;
2191  if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts,
2192  KnownSrcBits, TLO, Depth + 1))
2193  return true;
2194  }
2195 
2196  // If this is a bitcast, let computeKnownBits handle it. Only do this on a
2197  // recursive call where Known may be useful to the caller.
2198  if (Depth > 0) {
2199  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2200  return false;
2201  }
2202  break;
2203  }
2204  case ISD::ADD:
2205  case ISD::MUL:
2206  case ISD::SUB: {
2207  // Add, Sub, and Mul don't demand any bits in positions beyond that
2208  // of the highest bit demanded of them.
2209  SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1);
2210  SDNodeFlags Flags = Op.getNode()->getFlags();
2211  unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros();
2212  APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ);
2213  if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO,
2214  Depth + 1) ||
2215  SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO,
2216  Depth + 1) ||
2217  // See if the operation should be performed at a smaller bit width.
2218  ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) {
2219  if (Flags.hasNoSignedWrap() || Flags.hasNoUnsignedWrap()) {
2220  // Disable the nsw and nuw flags. We can no longer guarantee that we
2221  // won't wrap after simplification.
2222  Flags.setNoSignedWrap(false);
2223  Flags.setNoUnsignedWrap(false);
2224  SDValue NewOp =
2225  TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2226  return TLO.CombineTo(Op, NewOp);
2227  }
2228  return true;
2229  }
2230 
2231  // Attempt to avoid multi-use ops if we don't need anything from them.
2232  if (!LoMask.isAllOnes() || !DemandedElts.isAllOnes()) {
2233  SDValue DemandedOp0 = SimplifyMultipleUseDemandedBits(
2234  Op0, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2235  SDValue DemandedOp1 = SimplifyMultipleUseDemandedBits(
2236  Op1, LoMask, DemandedElts, TLO.DAG, Depth + 1);
2237  if (DemandedOp0 || DemandedOp1) {
2238  Flags.setNoSignedWrap(false);
2239  Flags.setNoUnsignedWrap(false);
2240  Op0 = DemandedOp0 ? DemandedOp0 : Op0;
2241  Op1 = DemandedOp1 ? DemandedOp1 : Op1;
2242  SDValue NewOp =
2243  TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags);
2244  return TLO.CombineTo(Op, NewOp);
2245  }
2246  }
2247 
2248  // If we have a constant operand, we may be able to turn it into -1 if we
2249  // do not demand the high bits. This can make the constant smaller to
2250  // encode, allow more general folding, or match specialized instruction
2251  // patterns (eg, 'blsr' on x86). Don't bother changing 1 to -1 because that
2252  // is probably not useful (and could be detrimental).
2254  APInt HighMask = APInt::getHighBitsSet(BitWidth, DemandedBitsLZ);
2255  if (C && !C->isAllOnes() && !C->isOne() &&
2256  (C->getAPIntValue() | HighMask).isAllOnes()) {
2257  SDValue Neg1 = TLO.DAG.getAllOnesConstant(dl, VT);
2258  // Disable the nsw and nuw flags. We can no longer guarantee that we
2259  // won't wrap after simplification.
2260  Flags.setNoSignedWrap(false);
2261  Flags.setNoUnsignedWrap(false);
2262  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Neg1, Flags);
2263  return TLO.CombineTo(Op, NewOp);
2264  }
2265 
2267  }
2268  default:
2269  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2270  if (SimplifyDemandedBitsForTargetNode(Op, DemandedBits, DemandedElts,
2271  Known, TLO, Depth))
2272  return true;
2273  break;
2274  }
2275 
2276  // Just use computeKnownBits to compute output bits.
2277  Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth);
2278  break;
2279  }
2280 
2281  // If we know the value of all of the demanded bits, return this as a
2282  // constant.
2283  if (DemandedBits.isSubsetOf(Known.Zero | Known.One)) {
2284  // Avoid folding to a constant if any OpaqueConstant is involved.
2285  const SDNode *N = Op.getNode();
2286  for (SDNode *Op :
2288  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op))
2289  if (C->isOpaque())
2290  return false;
2291  }
2292  if (VT.isInteger())
2293  return TLO.CombineTo(Op, TLO.DAG.getConstant(Known.One, dl, VT));
2294  if (VT.isFloatingPoint())
2295  return TLO.CombineTo(
2296  Op,
2297  TLO.DAG.getConstantFP(
2298  APFloat(TLO.DAG.EVTToAPFloatSemantics(VT), Known.One), dl, VT));
2299  }
2300 
2301  return false;
2302 }
2303 
2305  const APInt &DemandedElts,
2306  APInt &KnownUndef,
2307  APInt &KnownZero,
2308  DAGCombinerInfo &DCI) const {
2309  SelectionDAG &DAG = DCI.DAG;
2310  TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
2311  !DCI.isBeforeLegalizeOps());
2312 
2313  bool Simplified =
2314  SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, TLO);
2315  if (Simplified) {
2316  DCI.AddToWorklist(Op.getNode());
2317  DCI.CommitTargetLoweringOpt(TLO);
2318  }
2319 
2320  return Simplified;
2321 }
2322 
2323 /// Given a vector binary operation and known undefined elements for each input
2324 /// operand, compute whether each element of the output is undefined.
2326  const APInt &UndefOp0,
2327  const APInt &UndefOp1) {
2328  EVT VT = BO.getValueType();
2329  assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() &&
2330  "Vector binop only");
2331 
2332  EVT EltVT = VT.getVectorElementType();
2333  unsigned NumElts = VT.getVectorNumElements();
2334  assert(UndefOp0.getBitWidth() == NumElts &&
2335  UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis");
2336 
2337  auto getUndefOrConstantElt = [&](SDValue V, unsigned Index,
2338  const APInt &UndefVals) {
2339  if (UndefVals[Index])
2340  return DAG.getUNDEF(EltVT);
2341 
2342  if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2343  // Try hard to make sure that the getNode() call is not creating temporary
2344  // nodes. Ignore opaque integers because they do not constant fold.
2345  SDValue Elt = BV->getOperand(Index);
2346  auto *C = dyn_cast<ConstantSDNode>(Elt);
2347  if (isa<ConstantFPSDNode>(Elt) || Elt.isUndef() || (C && !C->isOpaque()))
2348  return Elt;
2349  }
2350 
2351  return SDValue();
2352  };
2353 
2354  APInt KnownUndef = APInt::getZero(NumElts);
2355  for (unsigned i = 0; i != NumElts; ++i) {
2356  // If both inputs for this element are either constant or undef and match
2357  // the element type, compute the constant/undef result for this element of
2358  // the vector.
2359  // TODO: Ideally we would use FoldConstantArithmetic() here, but that does
2360  // not handle FP constants. The code within getNode() should be refactored
2361  // to avoid the danger of creating a bogus temporary node here.
2362  SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0);
2363  SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1);
2364  if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT)
2365  if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef())
2366  KnownUndef.setBit(i);
2367  }
2368  return KnownUndef;
2369 }
2370 
2372  SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef,
2373  APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth,
2374  bool AssumeSingleUse) const {
2375  EVT VT = Op.getValueType();
2376  unsigned Opcode = Op.getOpcode();
2377  APInt DemandedElts = OriginalDemandedElts;
2378  unsigned NumElts = DemandedElts.getBitWidth();
2379  assert(VT.isVector() && "Expected vector op");
2380 
2381  KnownUndef = KnownZero = APInt::getZero(NumElts);
2382 
2383  // TODO: For now we assume we know nothing about scalable vectors.
2384  if (VT.isScalableVector())
2385  return false;
2386 
2387  assert(VT.getVectorNumElements() == NumElts &&
2388  "Mask size mismatches value type element count!");
2389 
2390  // Undef operand.
2391  if (Op.isUndef()) {
2392  KnownUndef.setAllBits();
2393  return false;
2394  }
2395 
2396  // If Op has other users, assume that all elements are needed.
2397  if (!Op.getNode()->hasOneUse() && !AssumeSingleUse)
2398  DemandedElts.setAllBits();
2399 
2400  // Not demanding any elements from Op.
2401  if (DemandedElts == 0) {
2402  KnownUndef.setAllBits();
2403  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2404  }
2405 
2406  // Limit search depth.
2408  return false;
2409 
2410  SDLoc DL(Op);
2411  unsigned EltSizeInBits = VT.getScalarSizeInBits();
2412 
2413  // Helper for demanding the specified elements and all the bits of both binary
2414  // operands.
2415  auto SimplifyDemandedVectorEltsBinOp = [&](SDValue Op0, SDValue Op1) {
2416  SDValue NewOp0 = SimplifyMultipleUseDemandedVectorElts(Op0, DemandedElts,
2417  TLO.DAG, Depth + 1);
2418  SDValue NewOp1 = SimplifyMultipleUseDemandedVectorElts(Op1, DemandedElts,
2419  TLO.DAG, Depth + 1);
2420  if (NewOp0 || NewOp1) {
2421  SDValue NewOp = TLO.DAG.getNode(
2422  Opcode, SDLoc(Op), VT, NewOp0 ? NewOp0 : Op0, NewOp1 ? NewOp1 : Op1);
2423  return TLO.CombineTo(Op, NewOp);
2424  }
2425  return false;
2426  };
2427 
2428  switch (Opcode) {
2429  case ISD::SCALAR_TO_VECTOR: {
2430  if (!DemandedElts[0]) {
2431  KnownUndef.setAllBits();
2432  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2433  }
2434  SDValue ScalarSrc = Op.getOperand(0);
2435  if (ScalarSrc.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
2436  SDValue Src = ScalarSrc.getOperand(0);
2437  SDValue Idx = ScalarSrc.getOperand(1);
2438  EVT SrcVT = Src.getValueType();
2439 
2440  ElementCount SrcEltCnt = SrcVT.getVectorElementCount();
2441 
2442  if (SrcEltCnt.isScalable())
2443  return false;
2444 
2445  unsigned NumSrcElts = SrcEltCnt.getFixedValue();
2446  if (isNullConstant(Idx)) {
2447  APInt SrcDemandedElts = APInt::getOneBitSet(NumSrcElts, 0);
2448  APInt SrcUndef = KnownUndef.zextOrTrunc(NumSrcElts);
2449  APInt SrcZero = KnownZero.zextOrTrunc(NumSrcElts);
2450  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2451  TLO, Depth + 1))
2452  return true;
2453  }
2454  }
2455  KnownUndef.setHighBits(NumElts - 1);
2456  break;
2457  }
2458  case ISD::BITCAST: {
2459  SDValue Src = Op.getOperand(0);
2460  EVT SrcVT = Src.getValueType();
2461 
2462  // We only handle vectors here.
2463  // TODO - investigate calling SimplifyDemandedBits/ComputeKnownBits?
2464  if (!SrcVT.isVector())
2465  break;
2466 
2467  // Fast handling of 'identity' bitcasts.
2468  unsigned NumSrcElts = SrcVT.getVectorNumElements();
2469  if (NumSrcElts == NumElts)
2470  return SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef,
2471  KnownZero, TLO, Depth + 1);
2472 
2473  APInt SrcDemandedElts, SrcZero, SrcUndef;
2474 
2475  // Bitcast from 'large element' src vector to 'small element' vector, we
2476  // must demand a source element if any DemandedElt maps to it.
2477  if ((NumElts % NumSrcElts) == 0) {
2478  unsigned Scale = NumElts / NumSrcElts;
2479  SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2480  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2481  TLO, Depth + 1))
2482  return true;
2483 
2484  // Try calling SimplifyDemandedBits, converting demanded elts to the bits
2485  // of the large element.
2486  // TODO - bigendian once we have test coverage.
2487  if (TLO.DAG.getDataLayout().isLittleEndian()) {
2488  unsigned SrcEltSizeInBits = SrcVT.getScalarSizeInBits();
2489  APInt SrcDemandedBits = APInt::getZero(SrcEltSizeInBits);
2490  for (unsigned i = 0; i != NumElts; ++i)
2491  if (DemandedElts[i]) {
2492  unsigned Ofs = (i % Scale) * EltSizeInBits;
2493  SrcDemandedBits.setBits(Ofs, Ofs + EltSizeInBits);
2494  }
2495 
2496  KnownBits Known;
2497  if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcDemandedElts, Known,
2498  TLO, Depth + 1))
2499  return true;
2500  }
2501 
2502  // If the src element is zero/undef then all the output elements will be -
2503  // only demanded elements are guaranteed to be correct.
2504  for (unsigned i = 0; i != NumSrcElts; ++i) {
2505  if (SrcDemandedElts[i]) {
2506  if (SrcZero[i])
2507  KnownZero.setBits(i * Scale, (i + 1) * Scale);
2508  if (SrcUndef[i])
2509  KnownUndef.setBits(i * Scale, (i + 1) * Scale);
2510  }
2511  }
2512  }
2513 
2514  // Bitcast from 'small element' src vector to 'large element' vector, we
2515  // demand all smaller source elements covered by the larger demanded element
2516  // of this vector.
2517  if ((NumSrcElts % NumElts) == 0) {
2518  unsigned Scale = NumSrcElts / NumElts;
2519  SrcDemandedElts = APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2520  if (SimplifyDemandedVectorElts(Src, SrcDemandedElts, SrcUndef, SrcZero,
2521  TLO, Depth + 1))
2522  return true;
2523 
2524  // If all the src elements covering an output element are zero/undef, then
2525  // the output element will be as well, assuming it was demanded.
2526  for (unsigned i = 0; i != NumElts; ++i) {
2527  if (DemandedElts[i]) {
2528  if (SrcZero.extractBits(Scale, i * Scale).isAllOnes())
2529  KnownZero.setBit(i);
2530  if (SrcUndef.extractBits(Scale, i * Scale).isAllOnes())
2531  KnownUndef.setBit(i);
2532  }
2533  }
2534  }
2535  break;
2536  }
2537  case ISD::BUILD_VECTOR: {
2538  // Check all elements and simplify any unused elements with UNDEF.
2539  if (!DemandedElts.isAllOnes()) {
2540  // Don't simplify BROADCASTS.
2541  if (llvm::any_of(Op->op_values(),
2542  [&](SDValue Elt) { return Op.getOperand(0) != Elt; })) {
2543  SmallVector<SDValue, 32> Ops(Op->op_begin(), Op->op_end());
2544  bool Updated = false;
2545  for (unsigned i = 0; i != NumElts; ++i) {
2546  if (!DemandedElts[i] && !Ops[i].isUndef()) {
2547  Ops[i] = TLO.DAG.getUNDEF(Ops[0].getValueType());
2548  KnownUndef.setBit(i);
2549  Updated = true;
2550  }
2551  }
2552  if (Updated)
2553  return TLO.CombineTo(Op, TLO.DAG.getBuildVector(VT, DL, Ops));
2554  }
2555  }
2556  for (unsigned i = 0; i != NumElts; ++i) {
2557  SDValue SrcOp = Op.getOperand(i);
2558  if (SrcOp.isUndef()) {
2559  KnownUndef.setBit(i);
2560  } else if (EltSizeInBits == SrcOp.getScalarValueSizeInBits() &&
2562  KnownZero.setBit(i);
2563  }
2564  }
2565  break;
2566  }
2567  case ISD::CONCAT_VECTORS: {
2568  EVT SubVT = Op.getOperand(0).getValueType();
2569  unsigned NumSubVecs = Op.getNumOperands();
2570  unsigned NumSubElts = SubVT.getVectorNumElements();
2571  for (unsigned i = 0; i != NumSubVecs; ++i) {
2572  SDValue SubOp = Op.getOperand(i);
2573  APInt SubElts = DemandedElts.extractBits(NumSubElts, i * NumSubElts);
2574  APInt SubUndef, SubZero;
2575  if (SimplifyDemandedVectorElts(SubOp, SubElts, SubUndef, SubZero, TLO,
2576  Depth + 1))
2577  return true;
2578  KnownUndef.insertBits(SubUndef, i * NumSubElts);
2579  KnownZero.insertBits(SubZero, i * NumSubElts);
2580  }
2581  break;
2582  }
2583  case ISD::INSERT_SUBVECTOR: {
2584  // Demand any elements from the subvector and the remainder from the src its
2585  // inserted into.
2586  SDValue Src = Op.getOperand(0);
2587  SDValue Sub = Op.getOperand(1);
2588  uint64_t Idx = Op.getConstantOperandVal(2);
2589  unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
2590  APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
2591  APInt DemandedSrcElts = DemandedElts;
2592  DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
2593 
2594  APInt SubUndef, SubZero;
2595  if (SimplifyDemandedVectorElts(Sub, DemandedSubElts, SubUndef, SubZero, TLO,
2596  Depth + 1))
2597  return true;
2598 
2599  // If none of the src operand elements are demanded, replace it with undef.
2600  if (!DemandedSrcElts && !Src.isUndef())
2601  return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2602  TLO.DAG.getUNDEF(VT), Sub,
2603  Op.getOperand(2)));
2604 
2605  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownUndef, KnownZero,
2606  TLO, Depth + 1))
2607  return true;
2608  KnownUndef.insertBits(SubUndef, Idx);
2609  KnownZero.insertBits(SubZero, Idx);
2610 
2611  // Attempt to avoid multi-use ops if we don't need anything from them.
2612  if (!DemandedSrcElts.isAllOnes() || !DemandedSubElts.isAllOnes()) {
2613  SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2614  Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2615  SDValue NewSub = SimplifyMultipleUseDemandedVectorElts(
2616  Sub, DemandedSubElts, TLO.DAG, Depth + 1);
2617  if (NewSrc || NewSub) {
2618  NewSrc = NewSrc ? NewSrc : Src;
2619  NewSub = NewSub ? NewSub : Sub;
2620  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2621  NewSub, Op.getOperand(2));
2622  return TLO.CombineTo(Op, NewOp);
2623  }
2624  }
2625  break;
2626  }
2627  case ISD::EXTRACT_SUBVECTOR: {
2628  // Offset the demanded elts by the subvector index.
2629  SDValue Src = Op.getOperand(0);
2630  if (Src.getValueType().isScalableVector())
2631  break;
2632  uint64_t Idx = Op.getConstantOperandVal(1);
2633  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2634  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
2635 
2636  APInt SrcUndef, SrcZero;
2637  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2638  Depth + 1))
2639  return true;
2640  KnownUndef = SrcUndef.extractBits(NumElts, Idx);
2641  KnownZero = SrcZero.extractBits(NumElts, Idx);
2642 
2643  // Attempt to avoid multi-use ops if we don't need anything from them.
2644  if (!DemandedElts.isAllOnes()) {
2645  SDValue NewSrc = SimplifyMultipleUseDemandedVectorElts(
2646  Src, DemandedSrcElts, TLO.DAG, Depth + 1);
2647  if (NewSrc) {
2648  SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, NewSrc,
2649  Op.getOperand(1));
2650  return TLO.CombineTo(Op, NewOp);
2651  }
2652  }
2653  break;
2654  }
2655  case ISD::INSERT_VECTOR_ELT: {
2656  SDValue Vec = Op.getOperand(0);
2657  SDValue Scl = Op.getOperand(1);
2658  auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
2659 
2660  // For a legal, constant insertion index, if we don't need this insertion
2661  // then strip it, else remove it from the demanded elts.
2662  if (CIdx && CIdx->getAPIntValue().ult(NumElts)) {
2663  unsigned Idx = CIdx->getZExtValue();
2664  if (!DemandedElts[Idx])
2665  return TLO.CombineTo(Op, Vec);
2666 
2667  APInt DemandedVecElts(DemandedElts);
2668  DemandedVecElts.clearBit(Idx);
2669  if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
2670  KnownZero, TLO, Depth + 1))
2671  return true;
2672 
2673  KnownUndef.setBitVal(Idx, Scl.isUndef());
2674 
2675  KnownZero.setBitVal(Idx, isNullConstant(Scl) || isNullFPConstant(Scl));
2676  break;
2677  }
2678 
2679  APInt VecUndef, VecZero;
2680  if (SimplifyDemandedVectorElts(Vec, DemandedElts, VecUndef, VecZero, TLO,
2681  Depth + 1))
2682  return true;
2683  // Without knowing the insertion index we can't set KnownUndef/KnownZero.
2684  break;
2685  }
2686  case ISD::VSELECT: {
2687  // Try to transform the select condition based on the current demanded
2688  // elements.
2689  // TODO: If a condition element is undef, we can choose from one arm of the
2690  // select (and if one arm is undef, then we can propagate that to the
2691  // result).
2692  // TODO - add support for constant vselect masks (see IR version of this).
2693  APInt UnusedUndef, UnusedZero;
2694  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UnusedUndef,
2695  UnusedZero, TLO, Depth + 1))
2696  return true;
2697 
2698  // See if we can simplify either vselect operand.
2699  APInt DemandedLHS(DemandedElts);
2700  APInt DemandedRHS(DemandedElts);
2701  APInt UndefLHS, ZeroLHS;
2702  APInt UndefRHS, ZeroRHS;
2703  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedLHS, UndefLHS,
2704  ZeroLHS, TLO, Depth + 1))
2705  return true;
2706  if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedRHS, UndefRHS,
2707  ZeroRHS, TLO, Depth + 1))
2708  return true;
2709 
2710  KnownUndef = UndefLHS & UndefRHS;
2711  KnownZero = ZeroLHS & ZeroRHS;
2712  break;
2713  }
2714  case ISD::VECTOR_SHUFFLE: {
2715  ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(Op)->getMask();
2716 
2717  // Collect demanded elements from shuffle operands..
2718  APInt DemandedLHS(NumElts, 0);
2719  APInt DemandedRHS(NumElts, 0);
2720  for (unsigned i = 0; i != NumElts; ++i) {
2721  int M = ShuffleMask[i];
2722  if (M < 0 || !DemandedElts[i])
2723  continue;
2724  assert(0 <= M && M < (int)(2 * NumElts) && "Shuffle index out of range");
2725  if (M < (int)NumElts)
2726  DemandedLHS.setBit(M);
2727  else
2728  DemandedRHS.setBit(M - NumElts);
2729  }
2730 
2731  // See if we can simplify either shuffle operand.
2732  APInt UndefLHS, ZeroLHS;
2733  APInt UndefRHS, ZeroRHS;
2734  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, UndefLHS,
2735  ZeroLHS, TLO, Depth + 1))
2736  return true;
2737  if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, UndefRHS,
2738  ZeroRHS, TLO, Depth + 1))
2739  return true;
2740 
2741  // Simplify mask using undef elements from LHS/RHS.
2742  bool Updated = false;
2743  bool IdentityLHS = true, IdentityRHS = true;
2744  SmallVector<int, 32> NewMask(ShuffleMask.begin(), ShuffleMask.end());
2745  for (unsigned i = 0; i != NumElts; ++i) {
2746  int &M = NewMask[i];
2747  if (M < 0)
2748  continue;
2749  if (!DemandedElts[i] || (M < (int)NumElts && UndefLHS[M]) ||
2750  (M >= (int)NumElts && UndefRHS[M - NumElts])) {
2751  Updated = true;
2752  M = -1;
2753  }
2754  IdentityLHS &= (M < 0) || (M == (int)i);
2755  IdentityRHS &= (M < 0) || ((M - NumElts) == i);
2756  }
2757 
2758  // Update legal shuffle masks based on demanded elements if it won't reduce
2759  // to Identity which can cause premature removal of the shuffle mask.
2760  if (Updated && !IdentityLHS && !IdentityRHS && !TLO.LegalOps) {
2761  SDValue LegalShuffle =
2762  buildLegalVectorShuffle(VT, DL, Op.getOperand(0), Op.getOperand(1),
2763  NewMask, TLO.DAG);
2764  if (LegalShuffle)
2765  return TLO.CombineTo(Op, LegalShuffle);
2766  }
2767 
2768  // Propagate undef/zero elements from LHS/RHS.
2769  for (unsigned i = 0; i != NumElts; ++i) {
2770  int M = ShuffleMask[i];
2771  if (M < 0) {
2772  KnownUndef.setBit(i);
2773  } else if (M < (int)NumElts) {
2774  if (UndefLHS[M])
2775  KnownUndef.setBit(i);
2776  if (ZeroLHS[M])
2777  KnownZero.setBit(i);
2778  } else {
2779  if (UndefRHS[M - NumElts])
2780  KnownUndef.setBit(i);
2781  if (ZeroRHS[M - NumElts])
2782  KnownZero.setBit(i);
2783  }
2784  }
2785  break;
2786  }
2790  APInt SrcUndef, SrcZero;
2791  SDValue Src = Op.getOperand(0);
2792  unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2793  APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts);
2794  if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO,
2795  Depth + 1))
2796  return true;
2797  KnownZero = SrcZero.zextOrTrunc(NumElts);
2798  KnownUndef = SrcUndef.zextOrTrunc(NumElts);
2799 
2800  if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
2801  Op.getValueSizeInBits() == Src.getValueSizeInBits() &&
2802  DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) {
2803  // aext - if we just need the bottom element then we can bitcast.
2804  return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src));
2805  }
2806 
2807  if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) {
2808  // zext(undef) upper bits are guaranteed to be zero.
2809  if (DemandedElts.isSubsetOf(KnownUndef))
2810  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2811  KnownUndef.clearAllBits();
2812  }
2813  break;
2814  }
2815 
2816  // TODO: There are more binop opcodes that could be handled here - MIN,
2817  // MAX, saturated math, etc.
2818  case ISD::OR:
2819  case ISD::XOR:
2820  case ISD::ADD:
2821  case ISD::SUB:
2822  case ISD::FADD:
2823  case ISD::FSUB:
2824  case ISD::FMUL:
2825  case ISD::FDIV:
2826  case ISD::FREM: {
2827  SDValue Op0 = Op.getOperand(0);
2828  SDValue Op1 = Op.getOperand(1);
2829 
2830  APInt UndefRHS, ZeroRHS;
2831  if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2832  Depth + 1))
2833  return true;
2834  APInt UndefLHS, ZeroLHS;
2835  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2836  Depth + 1))
2837  return true;
2838 
2839  KnownZero = ZeroLHS & ZeroRHS;
2840  KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS);
2841 
2842  // Attempt to avoid multi-use ops if we don't need anything from them.
2843  // TODO - use KnownUndef to relax the demandedelts?
2844  if (!DemandedElts.isAllOnes())
2845  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2846  return true;
2847  break;
2848  }
2849  case ISD::SHL:
2850  case ISD::SRL:
2851  case ISD::SRA:
2852  case ISD::ROTL:
2853  case ISD::ROTR: {
2854  SDValue Op0 = Op.getOperand(0);
2855  SDValue Op1 = Op.getOperand(1);
2856 
2857  APInt UndefRHS, ZeroRHS;
2858  if (SimplifyDemandedVectorElts(Op1, DemandedElts, UndefRHS, ZeroRHS, TLO,
2859  Depth + 1))
2860  return true;
2861  APInt UndefLHS, ZeroLHS;
2862  if (SimplifyDemandedVectorElts(Op0, DemandedElts, UndefLHS, ZeroLHS, TLO,
2863  Depth + 1))
2864  return true;
2865 
2866  KnownZero = ZeroLHS;
2867  KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop?
2868 
2869  // Attempt to avoid multi-use ops if we don't need anything from them.
2870  // TODO - use KnownUndef to relax the demandedelts?
2871  if (!DemandedElts.isAllOnes())
2872  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2873  return true;
2874  break;
2875  }
2876  case ISD::MUL:
2877  case ISD::AND: {
2878  SDValue Op0 = Op.getOperand(0);
2879  SDValue Op1 = Op.getOperand(1);
2880 
2881  APInt SrcUndef, SrcZero;
2882  if (SimplifyDemandedVectorElts(Op1, DemandedElts, SrcUndef, SrcZero, TLO,
2883  Depth + 1))
2884  return true;
2885  if (SimplifyDemandedVectorElts(Op0, DemandedElts, KnownUndef, KnownZero,
2886  TLO, Depth + 1))
2887  return true;
2888 
2889  // If either side has a zero element, then the result element is zero, even
2890  // if the other is an UNDEF.
2891  // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros
2892  // and then handle 'and' nodes with the rest of the binop opcodes.
2893  KnownZero |= SrcZero;
2894  KnownUndef &= SrcUndef;
2895  KnownUndef &= ~KnownZero;
2896 
2897  // Attempt to avoid multi-use ops if we don't need anything from them.
2898  // TODO - use KnownUndef to relax the demandedelts?
2899  if (!DemandedElts.isAllOnes())
2900  if (SimplifyDemandedVectorEltsBinOp(Op0, Op1))
2901  return true;
2902  break;
2903  }
2904  case ISD::TRUNCATE:
2905  case ISD::SIGN_EXTEND:
2906  case ISD::ZERO_EXTEND:
2907  if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef,
2908  KnownZero, TLO, Depth + 1))
2909  return true;
2910 
2911  if (Op.getOpcode() == ISD::ZERO_EXTEND) {
2912  // zext(undef) upper bits are guaranteed to be zero.
2913  if (DemandedElts.isSubsetOf(KnownUndef))
2914  return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
2915  KnownUndef.clearAllBits();
2916  }
2917  break;
2918  default: {
2919  if (Op.getOpcode() >= ISD::BUILTIN_OP_END) {
2920  if (SimplifyDemandedVectorEltsForTargetNode(Op, DemandedElts, KnownUndef,
2921  KnownZero, TLO, Depth))
2922  return true;
2923  } else {
2924  KnownBits Known;
2925  APInt DemandedBits = APInt::getAllOnes(EltSizeInBits);
2926  if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known,
2927  TLO, Depth, AssumeSingleUse))
2928  return true;
2929  }
2930  break;
2931  }
2932  }
2933  assert((KnownUndef & KnownZero) == 0 && "Elements flagged as undef AND zero");
2934 
2935  // Constant fold all undef cases.
2936  // TODO: Handle zero cases as well.
2937  if (DemandedElts.isSubsetOf(KnownUndef))
2938  return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
2939 
2940  return false;
2941 }
2942 
2943 /// Determine which of the bits specified in Mask are known to be either zero or
2944 /// one and return them in the Known.
2946  KnownBits &Known,
2947  const APInt &DemandedElts,
2948  const SelectionDAG &DAG,
2949  unsigned Depth) const {
2950  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2951  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2952  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2953  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2954  "Should use MaskedValueIsZero if you don't know whether Op"
2955  " is a target node!");
2956  Known.resetAll();
2957 }
2958 
2960  GISelKnownBits &Analysis, Register R, KnownBits &Known,
2961  const APInt &DemandedElts, const MachineRegisterInfo &MRI,
2962  unsigned Depth) const {
2963  Known.resetAll();
2964 }
2965 
2967  const int FrameIdx, KnownBits &Known, const MachineFunction &MF) const {
2968  // The low bits are known zero if the pointer is aligned.
2969  Known.Zero.setLowBits(Log2(MF.getFrameInfo().getObjectAlign(FrameIdx)));
2970 }
2971 
2973  GISelKnownBits &Analysis, Register R, const MachineRegisterInfo &MRI,
2974  unsigned Depth) const {
2975  return Align(1);
2976 }
2977 
2978 /// This method can be implemented by targets that want to expose additional
2979 /// information about sign bits to the DAG Combiner.
2981  const APInt &,
2982  const SelectionDAG &,
2983  unsigned Depth) const {
2984  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
2985  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2986  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
2987  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
2988  "Should use ComputeNumSignBits if you don't know whether Op"
2989  " is a target node!");
2990  return 1;
2991 }
2992 
2994  GISelKnownBits &Analysis, Register R, const APInt &DemandedElts,
2995  const MachineRegisterInfo &MRI, unsigned Depth) const {
2996  return 1;
2997 }
2998 
3000  SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
3001  TargetLoweringOpt &TLO, unsigned Depth) const {
3002  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3003  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3004  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3005  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3006  "Should use SimplifyDemandedVectorElts if you don't know whether Op"
3007  " is a target node!");
3008  return false;
3009 }
3010 
3012  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3013  KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const {
3014  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3015  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3016  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3017  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3018  "Should use SimplifyDemandedBits if you don't know whether Op"
3019  " is a target node!");
3020  computeKnownBitsForTargetNode(Op, Known, DemandedElts, TLO.DAG, Depth);
3021  return false;
3022 }
3023 
3025  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
3026  SelectionDAG &DAG, unsigned Depth) const {
3027  assert(
3028  (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3029  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3030  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3031  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3032  "Should use SimplifyMultipleUseDemandedBits if you don't know whether Op"
3033  " is a target node!");
3034  return SDValue();
3035 }
3036 
3037 SDValue
3040  SelectionDAG &DAG) const {
3041  bool LegalMask = isShuffleMaskLegal(Mask, VT);
3042  if (!LegalMask) {
3043  std::swap(N0, N1);
3045  LegalMask = isShuffleMaskLegal(Mask, VT);
3046  }
3047 
3048  if (!LegalMask)
3049  return SDValue();
3050 
3051  return DAG.getVectorShuffle(VT, DL, N0, N1, Mask);
3052 }
3053 
3055  return nullptr;
3056 }
3057 
3059  SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
3060  bool PoisonOnly, unsigned Depth) const {
3061  assert(
3062  (Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3063  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3064  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3065  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3066  "Should use isGuaranteedNotToBeUndefOrPoison if you don't know whether Op"
3067  " is a target node!");
3068  return false;
3069 }
3070 
3072  const SelectionDAG &DAG,
3073  bool SNaN,
3074  unsigned Depth) const {
3075  assert((Op.getOpcode() >= ISD::BUILTIN_OP_END ||
3076  Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3077  Op.getOpcode() == ISD::INTRINSIC_W_CHAIN ||
3078  Op.getOpcode() == ISD::INTRINSIC_VOID) &&
3079  "Should use isKnownNeverNaN if you don't know whether Op"
3080  " is a target node!");
3081  return false;
3082 }
3083 
3084 // FIXME: Ideally, this would use ISD::isConstantSplatVector(), but that must
3085 // work with truncating build vectors and vectors with elements of less than
3086 // 8 bits.
3088  if (!N)
3089  return false;
3090 
3091  APInt CVal;
3092  if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
3093  CVal = CN->getAPIntValue();
3094  } else if (auto *BV = dyn_cast<BuildVectorSDNode>(N)) {
3095  auto *CN = BV->getConstantSplatNode();
3096  if (!CN)
3097  return false;
3098 
3099  // If this is a truncating build vector, truncate the splat value.
3100  // Otherwise, we may fail to match the expected values below.
3101  unsigned BVEltWidth = BV->getValueType(0).getScalarSizeInBits();
3102  CVal = CN->getAPIntValue();
3103  if (BVEltWidth < CVal.getBitWidth())
3104  CVal = CVal.trunc(BVEltWidth);
3105  } else {
3106  return false;
3107  }
3108 
3109  switch (getBooleanContents(N->getValueType(0))) {
3111  return CVal[0];
3113  return CVal.isOne();
3115  return CVal.isAllOnes();
3116  }
3117 
3118  llvm_unreachable("Invalid boolean contents");
3119 }
3120 
3122  if (!N)
3123  return false;
3124 
3125  const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N);
3126  if (!CN) {
3127  const BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N);
3128  if (!BV)
3129  return false;
3130 
3131  // Only interested in constant splats, we don't care about undef
3132  // elements in identifying boolean constants and getConstantSplatNode
3133  // returns NULL if all ops are undef;
3134  CN = BV->getConstantSplatNode();
3135  if (!CN)
3136  return false;
3137  }
3138 
3139  if (getBooleanContents(N->getValueType(0)) == UndefinedBooleanContent)
3140  return !CN->getAPIntValue()[0];
3141 
3142  return CN->isZero();
3143 }
3144 
3146  bool SExt) const {
3147  if (VT == MVT::i1)
3148  return N->isOne();
3149 
3151  switch (Cnt) {
3153  // An extended value of 1 is always true, unless its original type is i1,
3154  // in which case it will be sign extended to -1.
3155  return (N->isOne() && !SExt) || (SExt && (N->getValueType(0) != MVT::i1));
3158  return N->isAllOnes() && SExt;
3159  }
3160  llvm_unreachable("Unexpected enumeration.");
3161 }
3162 
3163 /// This helper function of SimplifySetCC tries to optimize the comparison when
3164 /// either operand of the SetCC node is a bitwise-and instruction.
3165 SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1,
3166  ISD::CondCode Cond, const SDLoc &DL,
3167  DAGCombinerInfo &DCI) const {
3168  // Match these patterns in any of their permutations:
3169  // (X & Y) == Y
3170  // (X & Y) != Y
3171  if (N1.getOpcode() == ISD::AND && N0.getOpcode() != ISD::AND)
3172  std::swap(N0, N1);
3173 
3174  EVT OpVT = N0.getValueType();
3175  if (N0.getOpcode() != ISD::AND || !OpVT.isInteger() ||
3176  (Cond != ISD::SETEQ && Cond != ISD::SETNE))
3177  return SDValue();
3178 
3179  SDValue X, Y;
3180  if (N0.getOperand(0) == N1) {
3181  X = N0.getOperand(1);
3182  Y = N0.getOperand(0);
3183  } else if (N0.getOperand(1) == N1) {
3184  X = N0.getOperand(0);
3185  Y = N0.getOperand(1);
3186  } else {
3187  return SDValue();
3188  }
3189 
3190  SelectionDAG &DAG = DCI.DAG;
3191  SDValue Zero = DAG.getConstant(0, DL, OpVT);
3192  if (DAG.isKnownToBeAPowerOfTwo(Y)) {
3193  // Simplify X & Y == Y to X & Y != 0 if Y has exactly one bit set.
3194  // Note that where Y is variable and is known to have at most one bit set
3195  // (for example, if it is Z & 1) we cannot do this; the expressions are not
3196  // equivalent when Y == 0.
3197  assert(OpVT.isInteger());
3198  Cond = ISD::getSetCCInverse(Cond, OpVT);
3199  if (DCI.isBeforeLegalizeOps() ||
3201  return DAG.getSetCC(DL, VT, N0, Zero, Cond);
3202  } else if (N0.hasOneUse() && hasAndNotCompare(Y)) {
3203  // If the target supports an 'and-not' or 'and-complement' logic operation,
3204  // try to use that to make a comparison operation more efficient.
3205  // But don't do this transform if the mask is a single bit because there are
3206  // more efficient ways to deal with that case (for example, 'bt' on x86 or
3207  // 'rlwinm' on PPC).
3208 
3209  // Bail out if the compare operand that we want to turn into a zero is
3210  // already a zero (otherwise, infinite loop).
3211  auto *YConst = dyn_cast<ConstantSDNode>(Y);
3212  if (YConst && YConst->isZero())
3213  return SDValue();
3214 
3215  // Transform this into: ~X & Y == 0.
3216  SDValue NotX = DAG.getNOT(SDLoc(X), X, OpVT);
3217  SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, NotX, Y);
3218  return DAG.getSetCC(DL, VT, NewAnd, Zero, Cond);
3219  }
3220 
3221  return SDValue();
3222 }
3223 
3224 /// There are multiple IR patterns that could be checking whether certain
3225 /// truncation of a signed number would be lossy or not. The pattern which is
3226 /// best at IR level, may not lower optimally. Thus, we want to unfold it.
3227 /// We are looking for the following pattern: (KeptBits is a constant)
3228 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
3229 /// KeptBits won't be bitwidth(x), that will be constant-folded to true/false.
3230 /// KeptBits also can't be 1, that would have been folded to %x dstcond 0
3231 /// We will unfold it into the natural trunc+sext pattern:
3232 /// ((%x << C) a>> C) dstcond %x
3233 /// Where C = bitwidth(x) - KeptBits and C u< bitwidth(x)
3234 SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck(
3235  EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI,
3236  const SDLoc &DL) const {
3237  // We must be comparing with a constant.
3238  ConstantSDNode *C1;
3239  if (!(C1 = dyn_cast<ConstantSDNode>(N1)))
3240  return SDValue();
3241 
3242  // N0 should be: add %x, (1 << (KeptBits-1))
3243  if (N0->getOpcode() != ISD::ADD)
3244  return SDValue();
3245 
3246  // And we must be 'add'ing a constant.
3247  ConstantSDNode *C01;
3248  if (!(C01 = dyn_cast<ConstantSDNode>(N0->getOperand(1))))
3249  return SDValue();
3250 
3251  SDValue X = N0->getOperand(0);
3252  EVT XVT = X.getValueType();
3253 
3254  // Validate constants ...
3255 
3256  APInt I1 = C1->getAPIntValue();
3257 
3258  ISD::CondCode NewCond;
3259  if (Cond == ISD::CondCode::SETULT) {
3260  NewCond = ISD::CondCode::SETEQ;
3261  } else if (Cond == ISD::CondCode::SETULE) {
3262  NewCond = ISD::CondCode::SETEQ;
3263  // But need to 'canonicalize' the constant.
3264  I1 += 1;
3265  } else if (Cond == ISD::CondCode::SETUGT) {
3266  NewCond = ISD::CondCode::SETNE;
3267  // But need to 'canonicalize' the constant.
3268  I1 += 1;
3269  } else if (Cond == ISD::CondCode::SETUGE) {
3270  NewCond = ISD::CondCode::SETNE;
3271  } else
3272  return SDValue();
3273 
3274  APInt I01 = C01->getAPIntValue();
3275 
3276  auto checkConstants = [&I1, &I01]() -> bool {
3277  // Both of them must be power-of-two, and the constant from setcc is bigger.
3278  return I1.ugt(I01) && I1.isPowerOf2() && I01.isPowerOf2();
3279  };
3280 
3281  if (checkConstants()) {
3282  // Great, e.g. got icmp ult i16 (add i16 %x, 128), 256
3283  } else {
3284  // What if we invert constants? (and the target predicate)
3285  I1.negate();
3286  I01.negate();
3287  assert(XVT.isInteger());
3288  NewCond = getSetCCInverse(NewCond, XVT);
3289  if (!checkConstants())
3290  return SDValue();
3291  // Great, e.g. got icmp uge i16 (add i16 %x, -128), -256
3292  }
3293 
3294  // They are power-of-two, so which bit is set?
3295  const unsigned KeptBits = I1.logBase2();
3296  const unsigned KeptBitsMinusOne = I01.logBase2();
3297 
3298  // Magic!
3299  if (KeptBits != (KeptBitsMinusOne + 1))
3300  return SDValue();
3301  assert(KeptBits > 0 && KeptBits < XVT.getSizeInBits() && "unreachable");
3302 
3303  // We don't want to do this in every single case.
3304  SelectionDAG &DAG = DCI.DAG;
3306  XVT, KeptBits))
3307  return SDValue();
3308 
3309  const unsigned MaskedBits = XVT.getSizeInBits() - KeptBits;
3310  assert(MaskedBits > 0 && MaskedBits < XVT.getSizeInBits() && "unreachable");
3311 
3312  // Unfold into: ((%x << C) a>> C) cond %x
3313  // Where 'cond' will be either 'eq' or 'ne'.
3314  SDValue ShiftAmt = DAG.getConstant(MaskedBits, DL, XVT);
3315  SDValue T0 = DAG.getNode(ISD::SHL, DL, XVT, X, ShiftAmt);
3316  SDValue T1 = DAG.getNode(ISD::SRA, DL, XVT, T0, ShiftAmt);
3317  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, X, NewCond);
3318 
3319  return T2;
3320 }
3321 
3322 // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
3323 SDValue TargetLowering::optimizeSetCCByHoistingAndByConstFromLogicalShift(
3324  EVT SCCVT, SDValue N0, SDValue N1C, ISD::CondCode Cond,
3325  DAGCombinerInfo &DCI, const SDLoc &DL) const {
3326  assert(isConstOrConstSplat(N1C) &&
3327  isConstOrConstSplat(N1C)->getAPIntValue().isZero() &&
3328  "Should be a comparison with 0.");
3329  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3330  "Valid only for [in]equality comparisons.");
3331 
3332  unsigned NewShiftOpcode;
3333  SDValue X, C, Y;
3334 
3335  SelectionDAG &DAG = DCI.DAG;
3336  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3337 
3338  // Look for '(C l>>/<< Y)'.
3339  auto Match = [&NewShiftOpcode, &X, &C, &Y, &TLI, &DAG](SDValue V) {
3340  // The shift should be one-use.
3341  if (!V.hasOneUse())
3342  return false;
3343  unsigned OldShiftOpcode = V.getOpcode();
3344  switch (OldShiftOpcode) {
3345  case ISD::SHL:
3346  NewShiftOpcode = ISD::SRL;
3347  break;
3348  case ISD::SRL:
3349  NewShiftOpcode = ISD::SHL;
3350  break;
3351  default:
3352  return false; // must be a logical shift.
3353  }
3354  // We should be shifting a constant.
3355  // FIXME: best to use isConstantOrConstantVector().
3356  C = V.getOperand(0);
3357  ConstantSDNode *CC =
3358  isConstOrConstSplat(C, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3359  if (!CC)
3360  return false;
3361  Y = V.getOperand(1);
3362 
3363  ConstantSDNode *XC =
3364  isConstOrConstSplat(X, /*AllowUndefs=*/true, /*AllowTruncation=*/true);
3365  return TLI.shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
3366  X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG);
3367  };
3368 
3369  // LHS of comparison should be an one-use 'and'.
3370  if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
3371  return SDValue();
3372 
3373  X = N0.getOperand(0);
3374  SDValue Mask = N0.getOperand(1);
3375 
3376  // 'and' is commutative!
3377  if (!Match(Mask)) {
3378  std::swap(X, Mask);
3379  if (!Match(Mask))
3380  return SDValue();
3381  }
3382 
3383  EVT VT = X.getValueType();
3384 
3385  // Produce:
3386  // ((X 'OppositeShiftOpcode' Y) & C) Cond 0
3387  SDValue T0 = DAG.getNode(NewShiftOpcode, DL, VT, X, Y);
3388  SDValue T1 = DAG.getNode(ISD::AND, DL, VT, T0, C);
3389  SDValue T2 = DAG.getSetCC(DL, SCCVT, T1, N1C, Cond);
3390  return T2;
3391 }
3392 
3393 /// Try to fold an equality comparison with a {add/sub/xor} binary operation as
3394 /// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to
3395 /// handle the commuted versions of these patterns.
3396 SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1,
3397  ISD::CondCode Cond, const SDLoc &DL,
3398  DAGCombinerInfo &DCI) const {
3399  unsigned BOpcode = N0.getOpcode();
3400  assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) &&
3401  "Unexpected binop");
3402  assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode");
3403 
3404  // (X + Y) == X --> Y == 0
3405  // (X - Y) == X --> Y == 0
3406  // (X ^ Y) == X --> Y == 0
3407  SelectionDAG &DAG = DCI.DAG;
3408  EVT OpVT = N0.getValueType();
3409  SDValue X = N0.getOperand(0);
3410  SDValue Y = N0.getOperand(1);
3411  if (X == N1)
3412  return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond);
3413 
3414  if (Y != N1)
3415  return SDValue();
3416 
3417  // (X + Y) == Y --> X == 0
3418  // (X ^ Y) == Y --> X == 0
3419  if (BOpcode == ISD::ADD || BOpcode == ISD::XOR)
3420  return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond);
3421 
3422  // The shift would not be valid if the operands are boolean (i1).
3423  if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1)
3424  return SDValue();
3425 
3426  // (X - Y) == Y --> X == Y << 1
3427  EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(),
3428  !DCI.isBeforeLegalize());
3429  SDValue One = DAG.getConstant(1, DL, ShiftVT);
3430  SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One);
3431  if (!DCI.isCalledByLegalizer())
3432  DCI.AddToWorklist(YShl1.getNode());
3433  return DAG.getSetCC(DL, VT, X, YShl1, Cond);
3434 }
3435 
3437  SDValue N0, const APInt &C1,
3438  ISD::CondCode Cond, const SDLoc &dl,
3439  SelectionDAG &DAG) {
3440  // Look through truncs that don't change the value of a ctpop.
3441  // FIXME: Add vector support? Need to be careful with setcc result type below.
3442  SDValue CTPOP = N0;
3443  if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() && !VT.isVector() &&
3445  CTPOP = N0.getOperand(0);
3446 
3447  if (CTPOP.getOpcode() != ISD::CTPOP || !CTPOP.hasOneUse())
3448  return SDValue();
3449 
3450  EVT CTVT = CTPOP.getValueType();
3451  SDValue CTOp = CTPOP.getOperand(0);
3452 
3453  // If this is a vector CTPOP, keep the CTPOP if it is legal.
3454  // TODO: Should we check if CTPOP is legal(or custom) for scalars?
3455  if (VT.isVector() && TLI.isOperationLegal(ISD::CTPOP, CTVT))
3456  return SDValue();
3457 
3458  // (ctpop x) u< 2 -> (x & x-1) == 0
3459  // (ctpop x) u> 1 -> (x & x-1) != 0
3460  if (Cond == ISD::SETULT || Cond == ISD::SETUGT) {
3461  unsigned CostLimit = TLI.getCustomCtpopCost(CTVT, Cond);
3462  if (C1.ugt(CostLimit + (Cond == ISD::SETULT)))
3463  return SDValue();
3464  if (C1 == 0 && (Cond == ISD::SETULT))
3465  return SDValue(); // This is handled elsewhere.
3466 
3467  unsigned Passes = C1.getLimitedValue() - (Cond == ISD::SETULT);
3468 
3469  SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3470  SDValue Result = CTOp;
3471  for (unsigned i = 0; i < Passes; i++) {
3472  SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, Result, NegOne);
3473  Result = DAG.getNode(ISD::AND, dl, CTVT, Result, Add);
3474  }
3476  return DAG.getSetCC(dl, VT, Result, DAG.getConstant(0, dl, CTVT), CC);
3477  }
3478 
3479  // If ctpop is not supported, expand a power-of-2 comparison based on it.
3480  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && C1 == 1) {
3481  // For scalars, keep CTPOP if it is legal or custom.
3482  if (!VT.isVector() && TLI.isOperationLegalOrCustom(ISD::CTPOP, CTVT))
3483  return SDValue();
3484  // This is based on X86's custom lowering for CTPOP which produces more
3485  // instructions than the expansion here.
3486 
3487  // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0)
3488  // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0)
3489  SDValue Zero = DAG.getConstant(0, dl, CTVT);
3490  SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT);
3491  assert(CTVT.isInteger());
3492  ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, CTVT);
3493  SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne);
3494  SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add);
3495  SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond);
3496  SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond);
3497  unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR;
3498  return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS);
3499  }
3500 
3501  return SDValue();
3502 }
3503 
3504 /// Try to simplify a setcc built with the specified operands and cc. If it is
3505 /// unable to simplify it, return a null SDValue.
3507  ISD::CondCode Cond, bool foldBooleans,
3508  DAGCombinerInfo &DCI,
3509  const SDLoc &dl) const {
3510  SelectionDAG &DAG = DCI.DAG;
3511  const DataLayout &Layout = DAG.getDataLayout();
3512  EVT OpVT = N0.getValueType();
3513 
3514  // Constant fold or commute setcc.
3515  if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl))
3516  return Fold;
3517 
3518  // Ensure that the constant occurs on the RHS and fold constant comparisons.
3519  // TODO: Handle non-splat vector constants. All undef causes trouble.
3520  // FIXME: We can't yet fold constant scalable vector splats, so avoid an
3521  // infinite loop here when we encounter one.
3523  if (isConstOrConstSplat(N0) &&
3524  (!OpVT.isScalableVector() || !isConstOrConstSplat(N1)) &&
3525  (DCI.isBeforeLegalizeOps() ||
3526  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())))
3527  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3528 
3529  // If we have a subtract with the same 2 non-constant operands as this setcc
3530  // -- but in reverse order -- then try to commute the operands of this setcc
3531  // to match. A matching pair of setcc (cmp) and sub may be combined into 1
3532  // instruction on some targets.
3533  if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) &&
3534  (DCI.isBeforeLegalizeOps() ||
3535  isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) &&
3536  DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N1, N0}) &&
3537  !DAG.doesNodeExist(ISD::SUB, DAG.getVTList(OpVT), {N0, N1}))
3538  return DAG.getSetCC(dl, VT, N1, N0, SwappedCC);
3539 
3540  if (auto *N1C = isConstOrConstSplat(N1)) {
3541  const APInt &C1 = N1C->getAPIntValue();
3542 
3543  // Optimize some CTPOP cases.
3544  if (SDValue V = simplifySetCCWithCTPOP(*this, VT, N0, C1, Cond, dl, DAG))
3545  return V;
3546 
3547  // If the LHS is '(srl (ctlz x), 5)', the RHS is 0/1, and this is an
3548  // equality comparison, then we're just comparing whether X itself is
3549  // zero.
3550  if (N0.getOpcode() == ISD::SRL && (C1.isZero() || C1.isOne()) &&
3551  N0.getOperand(0).getOpcode() == ISD::CTLZ &&
3553  if (ConstantSDNode *ShAmt = isConstOrConstSplat(N0.getOperand(1))) {
3554  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3555  ShAmt->getAPIntValue() == Log2_32(N0.getScalarValueSizeInBits())) {
3556  if ((C1 == 0) == (Cond == ISD::SETEQ)) {
3557  // (srl (ctlz x), 5) == 0 -> X != 0
3558  // (srl (ctlz x), 5) != 1 -> X != 0
3559  Cond = ISD::SETNE;
3560  } else {
3561  // (srl (ctlz x), 5) != 0 -> X == 0
3562  // (srl (ctlz x), 5) == 1 -> X == 0
3563  Cond = ISD::SETEQ;
3564  }
3565  SDValue Zero = DAG.getConstant(0, dl, N0.getValueType());
3566  return DAG.getSetCC(dl, VT, N0.getOperand(0).getOperand(0), Zero,
3567  Cond);
3568  }
3569  }
3570  }
3571  }
3572 
3573  // FIXME: Support vectors.
3574  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
3575  const APInt &C1 = N1C->getAPIntValue();
3576 
3577  // (zext x) == C --> x == (trunc C)
3578  // (sext x) == C --> x == (trunc C)
3579  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3580  DCI.isBeforeLegalize() && N0->hasOneUse()) {
3581  unsigned MinBits = N0.getValueSizeInBits();
3582  SDValue PreExt;
3583  bool Signed = false;
3584  if (N0->getOpcode() == ISD::ZERO_EXTEND) {
3585  // ZExt
3586  MinBits = N0->getOperand(0).getValueSizeInBits();
3587  PreExt = N0->getOperand(0);
3588  } else if (N0->getOpcode() == ISD::AND) {
3589  // DAGCombine turns costly ZExts into ANDs
3590  if (auto *C = dyn_cast<ConstantSDNode>(N0->getOperand(1)))
3591  if ((C->getAPIntValue()+1).isPowerOf2()) {
3592  MinBits = C->getAPIntValue().countTrailingOnes();
3593  PreExt = N0->getOperand(0);
3594  }
3595  } else if (N0->getOpcode() == ISD::SIGN_EXTEND) {
3596  // SExt
3597  MinBits = N0->getOperand(0).getValueSizeInBits();
3598  PreExt = N0->getOperand(0);
3599  Signed = true;
3600  } else if (auto *LN0 = dyn_cast<LoadSDNode>(N0)) {
3601  // ZEXTLOAD / SEXTLOAD
3602  if (LN0->getExtensionType() == ISD::ZEXTLOAD) {
3603  MinBits = LN0->getMemoryVT().getSizeInBits();
3604  PreExt = N0;
3605  } else if (LN0->getExtensionType() == ISD::SEXTLOAD) {
3606  Signed = true;
3607  MinBits = LN0->getMemoryVT().getSizeInBits();
3608  PreExt = N0;
3609  }
3610  }
3611 
3612  // Figure out how many bits we need to preserve this constant.
3613  unsigned ReqdBits = Signed ?
3614  C1.getBitWidth() - C1.getNumSignBits() + 1 :
3615  C1.getActiveBits();
3616 
3617  // Make sure we're not losing bits from the constant.
3618  if (MinBits > 0 &&
3619  MinBits < C1.getBitWidth() &&
3620  MinBits >= ReqdBits) {
3621  EVT MinVT = EVT::getIntegerVT(*DAG.getContext(), MinBits);
3622  if (isTypeDesirableForOp(ISD::SETCC, MinVT)) {
3623  // Will get folded away.
3624  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, MinVT, PreExt);
3625  if (MinBits == 1 && C1 == 1)
3626  // Invert the condition.
3627  return DAG.getSetCC(dl, VT, Trunc, DAG.getConstant(0, dl, MVT::i1),
3629  SDValue C = DAG.getConstant(C1.trunc(MinBits), dl, MinVT);
3630  return DAG.getSetCC(dl, VT, Trunc, C, Cond);
3631  }
3632 
3633  // If truncating the setcc operands is not desirable, we can still
3634  // simplify the expression in some cases:
3635  // setcc ([sz]ext (setcc x, y, cc)), 0, setne) -> setcc (x, y, cc)
3636  // setcc ([sz]ext (setcc x, y, cc)), 0, seteq) -> setcc (x, y, inv(cc))
3637  // setcc (zext (setcc x, y, cc)), 1, setne) -> setcc (x, y, inv(cc))
3638  // setcc (zext (setcc x, y, cc)), 1, seteq) -> setcc (x, y, cc)
3639  // setcc (sext (setcc x, y, cc)), -1, setne) -> setcc (x, y, inv(cc))
3640  // setcc (sext (setcc x, y, cc)), -1, seteq) -> setcc (x, y, cc)
3641  SDValue TopSetCC = N0->getOperand(0);
3642  unsigned N0Opc = N0->getOpcode();
3643  bool SExt = (N0Opc == ISD::SIGN_EXTEND);
3644  if (TopSetCC.getValueType() == MVT::i1 && VT == MVT::i1 &&
3645  TopSetCC.getOpcode() == ISD::SETCC &&
3646  (N0Opc == ISD::ZERO_EXTEND || N0Opc == ISD::SIGN_EXTEND) &&
3647  (isConstFalseVal(N1C) ||
3648  isExtendedTrueVal(N1C, N0->getValueType(0), SExt))) {
3649 
3650  bool Inverse = (N1C->isZero() && Cond == ISD::SETEQ) ||
3651  (!N1C->isZero() && Cond == ISD::SETNE);
3652 
3653  if (!Inverse)
3654  return TopSetCC;
3655 
3657  cast<CondCodeSDNode>(TopSetCC.getOperand(2))->get(),
3658  TopSetCC.getOperand(0).getValueType());
3659  return DAG.getSetCC(dl, VT, TopSetCC.getOperand(0),
3660  TopSetCC.getOperand(1),
3661  InvCond);
3662  }
3663  }
3664  }
3665 
3666  // If the LHS is '(and load, const)', the RHS is 0, the test is for
3667  // equality or unsigned, and all 1 bits of the const are in the same
3668  // partial word, see if we can shorten the load.
3669  if (DCI.isBeforeLegalize() &&
3671  N0.getOpcode() == ISD::AND && C1 == 0 &&
3672  N0.getNode()->hasOneUse() &&
3673  isa<LoadSDNode>(N0.getOperand(0)) &&
3674  N0.getOperand(0).getNode()->hasOneUse() &&
3675  isa<ConstantSDNode>(N0.getOperand(1))) {
3676  LoadSDNode *Lod = cast<LoadSDNode>(N0.getOperand(0));
3677  APInt bestMask;
3678  unsigned bestWidth = 0, bestOffset = 0;
3679  if (Lod->isSimple() && Lod->isUnindexed()) {
3680  unsigned origWidth = N0.getValueSizeInBits();
3681  unsigned maskWidth = origWidth;
3682  // We can narrow (e.g.) 16-bit extending loads on 32-bit target to
3683  // 8 bits, but have to be careful...
3684  if (Lod->getExtensionType() != ISD::NON_EXTLOAD)
3685  origWidth = Lod->getMemoryVT().getSizeInBits();
3686  const APInt &Mask = N0.getConstantOperandAPInt(1);
3687  for (unsigned width = origWidth / 2; width>=8; width /= 2) {
3688  APInt newMask = APInt::getLowBitsSet(maskWidth, width);
3689  for (unsigned offset=0; offset<origWidth/width; offset++) {
3690  if (Mask.isSubsetOf(newMask)) {
3691  if (Layout.isLittleEndian())
3692  bestOffset = (uint64_t)offset * (width/8);
3693  else
3694  bestOffset = (origWidth/width - offset - 1) * (width/8);
3695  bestMask = Mask.lshr(offset * (width/8) * 8);
3696  bestWidth = width;
3697  break;
3698  }
3699  newMask <<= width;
3700  }
3701  }
3702  }
3703  if (bestWidth) {
3704  EVT newVT = EVT::getIntegerVT(*DAG.getContext(), bestWidth);
3705  if (newVT.isRound() &&
3706  shouldReduceLoadWidth(Lod, ISD::NON_EXTLOAD, newVT)) {
3707  SDValue Ptr = Lod->getBasePtr();
3708  if (bestOffset != 0)
3709  Ptr =
3710  DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(bestOffset), dl);
3711  SDValue NewLoad =
3712  DAG.getLoad(newVT, dl, Lod->getChain(), Ptr,
3713  Lod->getPointerInfo().getWithOffset(bestOffset),
3714  Lod->getOriginalAlign());
3715  return DAG.getSetCC(dl, VT,
3716  DAG.getNode(ISD::AND, dl, newVT, NewLoad,
3717  DAG.getConstant(bestMask.trunc(bestWidth),
3718  dl, newVT)),
3719  DAG.getConstant(0LL, dl, newVT), Cond);
3720  }
3721  }
3722  }
3723 
3724  // If the LHS is a ZERO_EXTEND, perform the comparison on the input.
3725  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
3726  unsigned InSize = N0.getOperand(0).getValueSizeInBits();
3727 
3728  // If the comparison constant has bits in the upper part, the
3729  // zero-extended value could never match.
3730  if (C1.intersects(APInt::getHighBitsSet(C1.getBitWidth(),
3731  C1.getBitWidth() - InSize))) {
3732  switch (Cond) {
3733  case ISD::SETUGT:
3734  case ISD::SETUGE:
3735  case ISD::SETEQ:
3736  return DAG.getConstant(0, dl, VT);
3737  case ISD::SETULT:
3738  case ISD::SETULE:
3739  case ISD::SETNE:
3740  return DAG.getConstant(1, dl, VT);
3741  case ISD::SETGT:
3742  case ISD::SETGE:
3743  // True if the sign bit of C1 is set.
3744  return DAG.getConstant(C1.isNegative(), dl, VT);
3745  case ISD::SETLT:
3746  case ISD::SETLE:
3747  // True if the sign bit of C1 isn't set.
3748  return DAG.getConstant(C1.isNonNegative(), dl, VT);
3749  default:
3750  break;
3751  }
3752  }
3753 
3754  // Otherwise, we can perform the comparison with the low bits.
3755  switch (Cond) {
3756  case ISD::SETEQ:
3757  case ISD::SETNE:
3758  case ISD::SETUGT:
3759  case ISD::SETUGE:
3760  case ISD::SETULT:
3761  case ISD::SETULE: {
3762  EVT newVT = N0.getOperand(0).getValueType();
3763  if (DCI.isBeforeLegalizeOps() ||
3764  (isOperationLegal(ISD::SETCC, newVT) &&
3765  isCondCodeLegal(Cond, newVT.getSimpleVT()))) {
3766  EVT NewSetCCVT = getSetCCResultType(Layout, *DAG.getContext(), newVT);
3767  SDValue NewConst = DAG.getConstant(C1.trunc(InSize), dl, newVT);
3768 
3769  SDValue NewSetCC = DAG.getSetCC(dl, NewSetCCVT, N0.getOperand(0),
3770  NewConst, Cond);
3771  return DAG.getBoolExtOrTrunc(NewSetCC, dl, VT, N0.getValueType());
3772  }
3773  break;
3774  }
3775  default:
3776  break; // todo, be more careful with signed comparisons
3777  }
3778  } else if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
3779  (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3780  !isSExtCheaperThanZExt(cast<VTSDNode>(N0.getOperand(1))->getVT(),
3781  OpVT)) {
3782  EVT ExtSrcTy = cast<VTSDNode>(N0.getOperand(1))->getVT();
3783  unsigned ExtSrcTyBits = ExtSrcTy.getSizeInBits();
3784  EVT ExtDstTy = N0.getValueType();
3785  unsigned ExtDstTyBits = ExtDstTy.getSizeInBits();
3786 
3787  // If the constant doesn't fit into the number of bits for the source of
3788  // the sign extension, it is impossible for both sides to be equal.
3789  if (C1.getMinSignedBits() > ExtSrcTyBits)
3790  return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
3791 
3792  assert(ExtDstTy == N0.getOperand(0).getValueType() &&
3793  ExtDstTy != ExtSrcTy && "Unexpected types!");
3794  APInt Imm = APInt::getLowBitsSet(ExtDstTyBits, ExtSrcTyBits);
3795  SDValue ZextOp = DAG.getNode(ISD::AND, dl, ExtDstTy, N0.getOperand(0),
3796  DAG.getConstant(Imm, dl, ExtDstTy));
3797  if (!DCI.isCalledByLegalizer())
3798  DCI.AddToWorklist(ZextOp.getNode());
3799  // Otherwise, make this a use of a zext.
3800  return DAG.getSetCC(dl, VT, ZextOp,
3801  DAG.getConstant(C1 & Imm, dl, ExtDstTy), Cond);
3802  } else if ((N1C->isZero() || N1C->isOne()) &&
3803  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3804  // SETCC (SETCC), [0|1], [EQ|NE] -> SETCC
3805  if (N0.getOpcode() == ISD::SETCC &&
3806  isTypeLegal(VT) && VT.bitsLE(N0.getValueType()) &&
3807  (N0.getValueType() == MVT::i1 ||
3810  bool TrueWhenTrue = (Cond == ISD::SETEQ) ^ (!N1C->isOne());
3811  if (TrueWhenTrue)
3812  return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
3813  // Invert the condition.
3814  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
3815  CC = ISD::getSetCCInverse(CC, N0.getOperand(0).getValueType());
3816  if (DCI.isBeforeLegalizeOps() ||
3818  return DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
3819  }
3820 
3821  if ((N0.getOpcode() == ISD::XOR ||
3822  (N0.getOpcode() == ISD::AND &&
3823  N0.getOperand(0).getOpcode() == ISD::XOR &&
3824  N0.getOperand(1) == N0.getOperand(0).getOperand(1))) &&
3825  isOneConstant(N0.getOperand(1))) {
3826  // If this is (X^1) == 0/1, swap the RHS and eliminate the xor. We
3827  // can only do this if the top bits are known zero.
3828  unsigned BitWidth = N0.getValueSizeInBits();
3829  if (DAG.MaskedValueIsZero(N0,
3831  BitWidth-1))) {
3832  // Okay, get the un-inverted input value.
3833  SDValue Val;
3834  if (N0.getOpcode() == ISD::XOR) {
3835  Val = N0.getOperand(0);
3836  } else {
3837  assert(N0.getOpcode() == ISD::AND &&
3838  N0.getOperand(0).getOpcode() == ISD::XOR);
3839  // ((X^1)&1)^1 -> X & 1
3840  Val = DAG.getNode(ISD::AND, dl, N0.getValueType(),
3841  N0.getOperand(0).getOperand(0),
3842  N0.getOperand(1));
3843  }
3844 
3845  return DAG.getSetCC(dl, VT, Val, N1,
3847  }
3848  } else if (N1C->isOne()) {
3849  SDValue Op0 = N0;
3850  if (Op0.getOpcode() == ISD::TRUNCATE)
3851  Op0 = Op0.getOperand(0);
3852 
3853  if ((Op0.getOpcode() == ISD::XOR) &&
3854  Op0.getOperand(0).getOpcode() == ISD::SETCC &&
3855  Op0.getOperand(1).getOpcode() == ISD::SETCC) {
3856  SDValue XorLHS = Op0.getOperand(0);
3857  SDValue XorRHS = Op0.getOperand(1);
3858  // Ensure that the input setccs return an i1 type or 0/1 value.
3859  if (Op0.getValueType() == MVT::i1 ||
3860  (getBooleanContents(XorLHS.getOperand(0).getValueType()) ==
3864  // (xor (setcc), (setcc)) == / != 1 -> (setcc) != / == (setcc)
3866  return DAG.getSetCC(dl, VT, XorLHS, XorRHS, Cond);
3867  }
3868  }
3869  if (Op0.getOpcode() == ISD::AND && isOneConstant(Op0.getOperand(1))) {
3870  // If this is (X&1) == / != 1, normalize it to (X&1) != / == 0.
3871  if (Op0.getValueType().bitsGT(VT))
3872  Op0 = DAG.getNode(ISD::AND, dl, VT,
3873  DAG.getNode(ISD::TRUNCATE, dl, VT, Op0.getOperand(0)),
3874  DAG.getConstant(1, dl, VT));
3875  else if (Op0.getValueType().bitsLT(VT))
3876  Op0 = DAG.getNode(ISD::AND, dl, VT,
3877  DAG.getNode(ISD::ANY_EXTEND, dl, VT, Op0.getOperand(0)),
3878  DAG.getConstant(1, dl, VT));
3879 
3880  return DAG.getSetCC(dl, VT, Op0,
3881  DAG.getConstant(0, dl, Op0.getValueType()),
3883  }
3884  if (Op0.getOpcode() == ISD::AssertZext &&
3885  cast<VTSDNode>(Op0.getOperand(1))->getVT() == MVT::i1)
3886  return DAG.getSetCC(dl, VT, Op0,
3887  DAG.getConstant(0, dl, Op0.getValueType()),
3889  }
3890  }
3891 
3892  // Given:
3893  // icmp eq/ne (urem %x, %y), 0
3894  // Iff %x has 0 or 1 bits set, and %y has at least 2 bits set, omit 'urem':
3895  // icmp eq/ne %x, 0
3896  if (N0.getOpcode() == ISD::UREM && N1C->isZero() &&
3897  (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
3898  KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0));
3899  KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1));
3900  if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2)
3901  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond);
3902  }
3903 
3904  // Fold set_cc seteq (ashr X, BW-1), -1 -> set_cc setlt X, 0
3905  // and set_cc setne (ashr X, BW-1), -1 -> set_cc setge X, 0
3906  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
3907  N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
3908  N0.getConstantOperandAPInt(1) == OpVT.getScalarSizeInBits() - 1 &&
3909  N1C && N1C->isAllOnes()) {
3910  return DAG.getSetCC(dl, VT, N0.getOperand(0),
3911  DAG.getConstant(0, dl, OpVT),
3913  }
3914 
3915  if (SDValue V =
3916  optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl))
3917  return V;
3918  }
3919 
3920  // These simplifications apply to splat vectors as well.
3921  // TODO: Handle more splat vector cases.
3922  if (auto *N1C = isConstOrConstSplat(N1)) {
3923  const APInt &C1 = N1C->getAPIntValue();
3924 
3925  APInt MinVal, MaxVal;
3926  unsigned OperandBitSize = N1C->getValueType(0).getScalarSizeInBits();
3927  if (ISD::isSignedIntSetCC(Cond)) {
3928  MinVal = APInt::getSignedMinValue(OperandBitSize);
3929  MaxVal = APInt::getSignedMaxValue(OperandBitSize);
3930  } else {
3931  MinVal = APInt::getMinValue(OperandBitSize);
3932  MaxVal = APInt::getMaxValue(OperandBitSize);
3933  }
3934 
3935  // Canonicalize GE/LE comparisons to use GT/LT comparisons.
3936  if (Cond == ISD::SETGE || Cond == ISD::SETUGE) {
3937  // X >= MIN --> true
3938  if (C1 == MinVal)
3939  return DAG.getBoolConstant(true, dl, VT, OpVT);
3940 
3941  if (!VT.isVector()) { // TODO: Support this for vectors.
3942  // X >= C0 --> X > (C0 - 1)
3943  APInt C = C1 - 1;
3945  if ((DCI.isBeforeLegalizeOps() ||
3946  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3947  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3948  isLegalICmpImmediate(C.getSExtValue())))) {
3949  return DAG.getSetCC(dl, VT, N0,
3950  DAG.getConstant(C, dl, N1.getValueType()),
3951  NewCC);
3952  }
3953  }
3954  }
3955 
3956  if (Cond == ISD::SETLE || Cond == ISD::SETULE) {
3957  // X <= MAX --> true
3958  if (C1 == MaxVal)
3959  return DAG.getBoolConstant(true, dl, VT, OpVT);
3960 
3961  // X <= C0 --> X < (C0 + 1)
3962  if (!VT.isVector()) { // TODO: Support this for vectors.
3963  APInt C = C1 + 1;
3965  if ((DCI.isBeforeLegalizeOps() ||
3966  isCondCodeLegal(NewCC, VT.getSimpleVT())) &&
3967  (!N1C->isOpaque() || (C.getBitWidth() <= 64 &&
3968  isLegalICmpImmediate(C.getSExtValue())))) {
3969  return DAG.getSetCC(dl, VT, N0,
3970  DAG.getConstant(C, dl, N1.getValueType()),
3971  NewCC);
3972  }
3973  }
3974  }
3975 
3976  if (Cond == ISD::SETLT || Cond == ISD::SETULT) {
3977  if (C1 == MinVal)
3978  return DAG.getBoolConstant(false, dl, VT, OpVT); // X < MIN --> false
3979 
3980  // TODO: Support this for vectors after legalize ops.
3981  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
3982  // Canonicalize setlt X, Max --> setne X, Max
3983  if (C1 == MaxVal)
3984  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
3985 
3986  // If we have setult X, 1, turn it into seteq X, 0
3987  if (C1 == MinVal+1)
3988  return DAG.getSetCC(dl, VT, N0,
3989  DAG.getConstant(MinVal, dl, N0.getValueType()),
3990  ISD::SETEQ);
3991  }
3992  }
3993 
3994  if (Cond == ISD::SETGT || Cond == ISD::SETUGT) {
3995  if (C1 == MaxVal)
3996  return DAG.getBoolConstant(false, dl, VT, OpVT); // X > MAX --> false
3997 
3998  // TODO: Support this for vectors after legalize ops.
3999  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4000  // Canonicalize setgt X, Min --> setne X, Min
4001  if (C1 == MinVal)
4002  return DAG.getSetCC(dl, VT, N0, N1, ISD::SETNE);
4003 
4004  // If we have setugt X, Max-1, turn it into seteq X, Max
4005  if (C1 == MaxVal-1)
4006  return DAG.getSetCC(dl, VT, N0,
4007  DAG.getConstant(MaxVal, dl, N0.getValueType()),
4008  ISD::SETEQ);
4009  }
4010  }
4011 
4012  if (Cond == ISD::SETEQ || Cond == ISD::SETNE) {
4013  // (X & (C l>>/<< Y)) ==/!= 0 --> ((X <</l>> Y) & C) ==/!= 0
4014  if (C1.isZero())
4015  if (SDValue CC = optimizeSetCCByHoistingAndByConstFromLogicalShift(
4016  VT, N0, N1, Cond, DCI, dl))
4017  return CC;
4018 
4019  // For all/any comparisons, replace or(x,shl(y,bw/2)) with and/or(x,y).
4020  // For example, when high 32-bits of i64 X are known clear:
4021  // all bits clear: (X | (Y<<32)) == 0 --> (X | Y) == 0
4022  // all bits set: (X | (Y<<32)) == -1 --> (X & Y) == -1
4023  bool CmpZero = N1C->getAPIntValue().isZero();
4024  bool CmpNegOne = N1C->getAPIntValue().isAllOnes();
4025  if ((CmpZero || CmpNegOne) && N0.hasOneUse()) {
4026  // Match or(lo,shl(hi,bw/2)) pattern.
4027  auto IsConcat = [&](SDValue V, SDValue &Lo, SDValue &Hi) {
4028  unsigned EltBits = V.getScalarValueSizeInBits();
4029  if (V.getOpcode() != ISD::OR || (EltBits % 2) != 0)
4030  return false;
4031  SDValue LHS = V.getOperand(0);
4032  SDValue RHS = V.getOperand(1);
4033  APInt HiBits = APInt::getHighBitsSet(EltBits, EltBits / 2);
4034  // Unshifted element must have zero upperbits.
4035  if (RHS.getOpcode() == ISD::SHL &&
4036  isa<ConstantSDNode>(RHS.getOperand(1)) &&
4037  RHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4038  DAG.MaskedValueIsZero(LHS, HiBits)) {
4039  Lo = LHS;
4040  Hi = RHS.getOperand(0);
4041  return true;
4042  }
4043  if (LHS.getOpcode() == ISD::SHL &&
4044  isa<ConstantSDNode>(LHS.getOperand(1)) &&
4045  LHS.getConstantOperandAPInt(1) == (EltBits / 2) &&
4046  DAG.MaskedValueIsZero(RHS, HiBits)) {
4047  Lo = RHS;
4048  Hi = LHS.getOperand(0);
4049  return true;
4050  }
4051  return false;
4052  };
4053 
4054  auto MergeConcat = [&](SDValue Lo, SDValue Hi) {
4055  unsigned EltBits = N0.getScalarValueSizeInBits();
4056  unsigned HalfBits = EltBits / 2;
4057  APInt HiBits = APInt::getHighBitsSet(EltBits, HalfBits);
4058  SDValue LoBits = DAG.getConstant(~HiBits, dl, OpVT);
4059  SDValue HiMask = DAG.getNode(ISD::AND, dl, OpVT, Hi, LoBits);
4060  SDValue NewN0 =
4061  DAG.getNode(CmpZero ? ISD::OR : ISD::AND, dl, OpVT, Lo, HiMask);
4062  SDValue NewN1 = CmpZero ? DAG.getConstant(0, dl, OpVT) : LoBits;
4063  return DAG.getSetCC(dl, VT, NewN0, NewN1, Cond);
4064  };
4065 
4066  SDValue Lo, Hi;
4067  if (IsConcat(N0, Lo, Hi))
4068  return MergeConcat(Lo, Hi);
4069 
4070  if (N0.getOpcode() == ISD::AND || N0.getOpcode() == ISD::OR) {
4071  SDValue Lo0, Lo1, Hi0, Hi1;
4072  if (IsConcat(N0.getOperand(0), Lo0, Hi0) &&
4073  IsConcat(N0.getOperand(1), Lo1, Hi1)) {
4074  return MergeConcat(DAG.getNode(N0.getOpcode(), dl, OpVT, Lo0, Lo1),
4075  DAG.getNode(N0.getOpcode(), dl, OpVT, Hi0, Hi1));
4076  }
4077  }
4078  }
4079  }
4080 
4081  // If we have "setcc X, C0", check to see if we can shrink the immediate
4082  // by changing cc.
4083  // TODO: Support this for vectors after legalize ops.
4084  if (!VT.isVector() || DCI.isBeforeLegalizeOps()) {
4085  // SETUGT X, SINTMAX -> SETLT X, 0
4086  // SETUGE X, SINTMIN -> SETLT X, 0
4087  if ((Cond == ISD::SETUGT && C1.isMaxSignedValue()) ||
4088  (Cond == ISD::SETUGE && C1.isMinSignedValue()))
4089  return DAG.getSetCC(dl, VT, N0,
4090  DAG.getConstant(0, dl, N1.getValueType()),
4091  ISD::SETLT);
4092 
4093  // SETULT X, SINTMIN -> SETGT X, -1
4094  // SETULE X, SINTMAX -> SETGT X, -1
4095  if ((Cond == ISD::SETULT && C1.isMinSignedValue()) ||
4096  (Cond == ISD::SETULE && C1.isMaxSignedValue()))
4097  return DAG.getSetCC(dl, VT, N0,
4098  DAG.getAllOnesConstant(dl, N1.getValueType()),
4099  ISD::SETGT);
4100  }
4101  }
4102 
4103  // Back to non-vector simplifications.
4104  // TODO: Can we do these for vector splats?
4105  if (auto *N1C = dyn_cast<ConstantSDNode>(N1.getNode())) {
4106  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4107  const APInt &C1 = N1C->getAPIntValue();
4108  EVT ShValTy = N0.getValueType();
4109 
4110  // Fold bit comparisons when we can. This will result in an
4111  // incorrect value when boolean false is negative one, unless
4112  // the bitsize is 1 in which case the false value is the same
4113  // in practice regardless of the representation.
4114  if ((VT.getSizeInBits() == 1 ||
4116  (Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4117  (VT == ShValTy || (isTypeLegal(VT) && VT.bitsLE(ShValTy))) &&
4118  N0.getOpcode() == ISD::AND) {
4119  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4120  EVT ShiftTy =
4121  getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4122  if (Cond == ISD::SETNE && C1 == 0) {// (X & 8) != 0 --> (X & 8) >> 3
4123  // Perform the xform if the AND RHS is a single bit.
4124  unsigned ShCt = AndRHS->getAPIntValue().logBase2();
4125  if (AndRHS->getAPIntValue().isPowerOf2() &&
4126  !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4127  return DAG.getNode(ISD::TRUNCATE, dl, VT,
4128  DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4129  DAG.getConstant(ShCt, dl, ShiftTy)));
4130  }
4131  } else if (Cond == ISD::SETEQ && C1 == AndRHS->getAPIntValue()) {
4132  // (X & 8) == 8 --> (X & 8) >> 3
4133  // Perform the xform if C1 is a single bit.
4134  unsigned ShCt = C1.logBase2();
4135  if (C1.isPowerOf2() &&
4136  !TLI.shouldAvoidTransformToShift(ShValTy, ShCt)) {
4137  return DAG.getNode(ISD::TRUNCATE, dl, VT,
4138  DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4139  DAG.getConstant(ShCt, dl, ShiftTy)));
4140  }
4141  }
4142  }
4143  }
4144 
4145  if (C1.getMinSignedBits() <= 64 &&
4146  !isLegalICmpImmediate(C1.getSExtValue())) {
4147  EVT ShiftTy = getShiftAmountTy(ShValTy, Layout, !DCI.isBeforeLegalize());
4148  // (X & -256) == 256 -> (X >> 8) == 1
4149  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4150  N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
4151  if (auto *AndRHS = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4152  const APInt &AndRHSC = AndRHS->getAPIntValue();
4153  if ((-AndRHSC).isPowerOf2() && (AndRHSC & C1) == C1) {
4154  unsigned ShiftBits = AndRHSC.countTrailingZeros();
4155  if (!TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4156  SDValue Shift =
4157  DAG.getNode(ISD::SRL, dl, ShValTy, N0.getOperand(0),
4158  DAG.getConstant(ShiftBits, dl, ShiftTy));
4159  SDValue CmpRHS = DAG.getConstant(C1.lshr(ShiftBits), dl, ShValTy);
4160  return DAG.getSetCC(dl, VT, Shift, CmpRHS, Cond);
4161  }
4162  }
4163  }
4164  } else if (Cond == ISD::SETULT || Cond == ISD::SETUGE ||
4165  Cond == ISD::SETULE || Cond == ISD::SETUGT) {
4166  bool AdjOne = (Cond == ISD::SETULE || Cond == ISD::SETUGT);
4167  // X < 0x100000000 -> (X >> 32) < 1
4168  // X >= 0x100000000 -> (X >> 32) >= 1
4169  // X <= 0x0ffffffff -> (X >> 32) < 1
4170  // X > 0x0ffffffff -> (X >> 32) >= 1
4171  unsigned ShiftBits;
4172  APInt NewC = C1;
4173  ISD::CondCode NewCond = Cond;
4174  if (AdjOne) {
4175  ShiftBits = C1.countTrailingOnes();
4176  NewC = NewC + 1;
4177  NewCond = (Cond == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4178  } else {
4179  ShiftBits = C1.countTrailingZeros();
4180  }
4181  NewC.lshrInPlace(ShiftBits);
4182  if (ShiftBits && NewC.getMinSignedBits() <= 64 &&
4184  !TLI.shouldAvoidTransformToShift(ShValTy, ShiftBits)) {
4185  SDValue Shift = DAG.getNode(ISD::SRL, dl, ShValTy, N0,
4186  DAG.getConstant(ShiftBits, dl, ShiftTy));
4187  SDValue CmpRHS = DAG.getConstant(NewC, dl, ShValTy);
4188  return DAG.getSetCC(dl, VT, Shift, CmpRHS, NewCond);
4189  }
4190  }
4191  }
4192  }
4193 
4194  if (!isa<ConstantFPSDNode>(N0) && isa<ConstantFPSDNode>(N1)) {
4195  auto *CFP = cast<ConstantFPSDNode>(N1);
4196  assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value");
4197 
4198  // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the
4199  // constant if knowing that the operand is non-nan is enough. We prefer to
4200  // have SETO(x,x) instead of SETO(x, 0.0) because this avoids having to
4201  // materialize 0.0.
4202  if (Cond == ISD::SETO || Cond == ISD::SETUO)
4203  return DAG.getSetCC(dl, VT, N0, N0, Cond);
4204 
4205  // setcc (fneg x), C -> setcc swap(pred) x, -C
4206  if (N0.getOpcode() == ISD::FNEG) {
4208  if (DCI.isBeforeLegalizeOps() ||
4209  isCondCodeLegal(SwapCond, N0.getSimpleValueType())) {
4210  SDValue NegN1 = DAG.getNode(ISD::FNEG, dl, N0.getValueType(), N1);
4211  return DAG.getSetCC(dl, VT, N0.getOperand(0), NegN1, SwapCond);
4212  }
4213  }
4214 
4215  // If the condition is not legal, see if we can find an equivalent one
4216  // which is legal.
4217  if (!isCondCodeLegal(Cond, N0.getSimpleValueType())) {
4218  // If the comparison was an awkward floating-point == or != and one of
4219  // the comparison operands is infinity or negative infinity, convert the
4220  // condition to a less-awkward <= or >=.
4221  if (CFP->getValueAPF().isInfinity()) {
4222  bool IsNegInf = CFP->getValueAPF().isNegative();
4224  switch (Cond) {
4225  case ISD::SETOEQ: NewCond = IsNegInf ? ISD::SETOLE : ISD::SETOGE; break;
4226  case ISD::SETUEQ: NewCond = IsNegInf ? ISD::SETULE : ISD::SETUGE; break;
4227  case ISD::SETUNE: NewCond = IsNegInf ? ISD::SETUGT : ISD::SETULT; break;
4228  case ISD::SETONE: NewCond = IsNegInf ? ISD::SETOGT : ISD::SETOLT; break;
4229  default: break;
4230  }
4231  if (NewCond != ISD::SETCC_INVALID &&
4232  isCondCodeLegal(NewCond, N0.getSimpleValueType()))
4233  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4234  }
4235  }
4236  }
4237 
4238  if (N0 == N1) {
4239  // The sext(setcc()) => setcc() optimization relies on the appropriate
4240  // constant being emitted.
4241  assert(!N0.getValueType().isInteger() &&
4242  "Integer types should be handled by FoldSetCC");
4243 
4244  bool EqTrue = ISD::isTrueWhenEqual(Cond);
4245  unsigned UOF = ISD::getUnorderedFlavor(Cond);
4246  if (UOF == 2) // FP operators that are undefined on NaNs.
4247  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4248  if (UOF == unsigned(EqTrue))
4249  return DAG.getBoolConstant(EqTrue, dl, VT, OpVT);
4250  // Otherwise, we can't fold it. However, we can simplify it to SETUO/SETO
4251  // if it is not already.
4252  ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO;
4253  if (NewCond != Cond &&
4254  (DCI.isBeforeLegalizeOps() ||
4255  isCondCodeLegal(NewCond, N0.getSimpleValueType())))
4256  return DAG.getSetCC(dl, VT, N0, N1, NewCond);
4257  }
4258 
4259  if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) &&
4260  N0.getValueType().isInteger()) {
4261  if (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB ||
4262  N0.getOpcode() == ISD::XOR) {
4263  // Simplify (X+Y) == (X+Z) --> Y == Z
4264  if (N0.getOpcode() == N1.getOpcode()) {
4265  if (N0.getOperand(0) == N1.getOperand(0))
4266  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(1), Cond);
4267  if (N0.getOperand(1) == N1.getOperand(1))
4268  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(0), Cond);
4269  if (isCommutativeBinOp(N0.getOpcode())) {
4270  // If X op Y == Y op X, try other combinations.
4271  if (N0.getOperand(0) == N1.getOperand(1))
4272  return DAG.getSetCC(dl, VT, N0.getOperand(1), N1.getOperand(0),
4273  Cond);
4274  if (N0.getOperand(1) == N1.getOperand(0))
4275  return DAG.getSetCC(dl, VT, N0.getOperand(0), N1.getOperand(1),
4276  Cond);
4277  }
4278  }
4279 
4280  // If RHS is a legal immediate value for a compare instruction, we need
4281  // to be careful about increasing register pressure needlessly.
4282  bool LegalRHSImm = false;
4283 
4284  if (auto *RHSC = dyn_cast<ConstantSDNode>(N1)) {
4285  if (auto *LHSR = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
4286  // Turn (X+C1) == C2 --> X == C2-C1
4287  if (N0.getOpcode() == ISD::ADD && N0.getNode()->hasOneUse()) {
4288  return DAG.getSetCC(dl, VT, N0.getOperand(0),
4289  DAG.getConstant(RHSC->getAPIntValue()-
4290  LHSR->getAPIntValue(),
4291  dl, N0.getValueType()), Cond);
4292  }
4293 
4294  // Turn (X^C1) == C2 into X == C1^C2 iff X&~C1 = 0.
4295  if (N0.getOpcode() == ISD::XOR)
4296  // If we know that all of the inverted bits are zero, don't bother
4297  // performing the inversion.
4298  if (DAG.MaskedValueIsZero(N0.getOperand(0), ~LHSR->getAPIntValue()))
4299  return
4300  DAG.getSetCC(dl, VT, N0.getOperand(0),
4301  DAG.getConstant(LHSR->getAPIntValue() ^
4302  RHSC->getAPIntValue(),
4303  dl, N0.getValueType()),
4304  Cond);
4305  }
4306 
4307  // Turn (C1-X) == C2 --> X == C1-C2
4308  if (auto *SUBC = dyn_cast<ConstantSDNode>(N0.getOperand(0))) {
4309  if (N0.getOpcode() == ISD::SUB && N0.getNode()->hasOneUse()) {
4310  return
4311  DAG.getSetCC(dl, VT, N0.getOperand(1),
4312  DAG.getConstant(SUBC->getAPIntValue() -
4313  RHSC->getAPIntValue(),
4314  dl, N0.getValueType()),
4315  Cond);
4316  }
4317  }
4318 
4319  // Could RHSC fold directly into a compare?
4320  if (RHSC->getValueType(0).getSizeInBits() <= 64)
4321  LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue());
4322  }
4323 
4324  // (X+Y) == X --> Y == 0 and similar folds.
4325  // Don't do this if X is an immediate that can fold into a cmp
4326  // instruction and X+Y has other uses. It could be an induction variable
4327  // chain, and the transform would increase register pressure.
4328  if (!LegalRHSImm || N0.hasOneUse())
4329  if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI))
4330  return V;
4331  }
4332 
4333  if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB ||
4334  N1.getOpcode() == ISD::XOR)
4335  if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI))
4336  return V;
4337 
4338  if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI))
4339  return V;
4340  }
4341 
4342  // Fold remainder of division by a constant.
4343  if ((N0.getOpcode() == ISD::UREM || N0.getOpcode() == ISD::SREM) &&
4344  N0.hasOneUse() && (Cond == ISD::SETEQ || Cond == ISD::SETNE)) {
4346 
4347  // When division is cheap or optimizing for minimum size,
4348  // fall through to DIVREM creation by skipping this fold.
4349  if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttr(Attribute::MinSize)) {
4350  if (N0.getOpcode() == ISD::UREM) {
4351  if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl))
4352  return Folded;
4353  } else if (N0.getOpcode() == ISD::SREM) {
4354  if (SDValue Folded = buildSREMEqFold(VT, N0, N1, Cond, DCI, dl))
4355  return Folded;
4356  }
4357  }
4358  }
4359 
4360  // Fold away ALL boolean setcc's.
4361  if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) {
4362  SDValue Temp;
4363  switch (Cond) {
4364  default: llvm_unreachable("Unknown integer setcc!");
4365  case ISD::SETEQ: // X == Y -> ~(X^Y)
4366  Temp = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4367  N0 = DAG.getNOT(dl, Temp, OpVT);
4368  if (!DCI.isCalledByLegalizer())
4369  DCI.AddToWorklist(Temp.getNode());
4370  break;
4371  case ISD::SETNE: // X != Y --> (X^Y)
4372  N0 = DAG.getNode(ISD::XOR, dl, OpVT, N0, N1);
4373  break;
4374  case ISD::SETGT: // X >s Y --> X == 0 & Y == 1 --> ~X & Y
4375  case ISD::SETULT: // X <u Y --> X == 0 & Y == 1 --> ~X & Y
4376  Temp = DAG.getNOT(dl, N0, OpVT);
4377  N0 = DAG.getNode(ISD::AND, dl, OpVT, N1, Temp);
4378  if (!DCI.isCalledByLegalizer())
4379  DCI.AddToWorklist(Temp.getNode());
4380  break;
4381  case ISD::SETLT: // X <s Y --> X == 1 & Y == 0 --> ~Y & X
4382  case ISD::SETUGT: // X >u Y --> X == 1 & Y == 0 --> ~Y & X
4383  Temp = DAG.getNOT(dl, N1, OpVT);
4384  N0 = DAG.getNode(ISD::AND, dl, OpVT, N0, Temp);
4385  if (!DCI.isCalledByLegalizer())
4386  DCI.AddToWorklist(Temp.getNode());
4387  break;
4388  case ISD::SETULE: // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
4389  case ISD::SETGE: // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
4390  Temp = DAG.getNOT(dl, N0, OpVT);
4391  N0 = DAG.getNode(ISD::OR, dl, OpVT, N1, Temp);
4392  if (!DCI.isCalledByLegalizer())
4393  DCI.AddToWorklist(Temp.getNode());
4394  break;
4395  case ISD::SETUGE: // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
4396  case ISD::SETLE: // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
4397  Temp = DAG.getNOT(dl, N1, OpVT);
4398  N0 = DAG.getNode(ISD::OR, dl, OpVT, N0, Temp);
4399  break;
4400  }
4401  if (VT.getScalarType() != MVT::i1) {
4402  if (!DCI.isCalledByLegalizer())
4403  DCI.AddToWorklist(N0.getNode());
4404  // FIXME: If running after legalize, we probably can't do this.
4406  N0 = DAG.getNode(ExtendCode, dl, VT, N0);
4407  }
4408  return N0;
4409  }
4410 
4411  // Could not fold it.
4412  return SDValue();
4413 }
4414 
4415 /// Returns true (and the GlobalValue and the offset) if the node is a
4416 /// GlobalAddress + offset.
4418  int64_t &Offset) const {
4419 
4420  SDNode *N = unwrapAddress(SDValue(WN, 0)).getNode();
4421 
4422  if (auto *GASD = dyn_cast<GlobalAddressSDNode>(N)) {
4423  GA = GASD->getGlobal();
4424  Offset += GASD->getOffset();
4425  return true;
4426  }
4427 
4428  if (N->getOpcode() == ISD::ADD) {
4429  SDValue N1 = N->getOperand(0);
4430  SDValue N2 = N->getOperand(1);
4431  if (isGAPlusOffset(N1.getNode(), GA, Offset)) {
4432  if (auto *V = dyn_cast<ConstantSDNode>(N2)) {
4433  Offset += V->getSExtValue();
4434  return true;
4435  }
4436  } else if (isGAPlusOffset(N2.getNode(), GA, Offset)) {
4437  if (auto *V = dyn_cast<ConstantSDNode>(N1)) {
4438  Offset += V->getSExtValue();
4439  return true;
4440  }
4441  }
4442  }
4443 
4444  return false;
4445 }
4446