DAGCombiner.cpp

Go to the documentation of this file.

//===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass combines dag nodes to form fewer, simpler DAG nodes.  It can be run
// both before and after the DAG is legalized.
//
// This pass is not a substitute for the LLVM IR instcombine pass. This pass is
// primarily intended to handle simplification opportunities that are implicit
// in the LLVM IR and exposed by the various codegen lowering phases.
//
//===----------------------------------------------------------------------===//
 
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/APSInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/IntervalMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/ByteProvider.h"
#include "llvm/CodeGen/DAGCombine.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Metadata.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <functional>
#include <iterator>
#include <optional>
#include <string>
#include <tuple>
#include <utility>
#include <variant>
 
#include "MatchContext.h"
#include "SDNodeDbgValue.h"
 
using namespace llvm;
using namespace llvm::SDPatternMatch;
 
#define DEBUG_TYPE "dagcombine"
 
STATISTIC(NodesCombined   , "Number of dag nodes combined");
STATISTIC(PreIndexedNodes , "Number of pre-indexed nodes created");
STATISTIC(PostIndexedNodes, "Number of post-indexed nodes created");
STATISTIC(OpsNarrowed     , "Number of load/op/store narrowed");
STATISTIC(LdStFP2Int      , "Number of fp load/store pairs transformed to int");
STATISTIC(SlicedLoads, "Number of load sliced");
STATISTIC(NumFPLogicOpsConv, "Number of logic ops converted to fp ops");
 
DEBUG_COUNTER(DAGCombineCounter, "dagcombine",
              "Controls whether a DAG combine is performed for a node");
 
static cl::opt<bool>
CombinerGlobalAA("combiner-global-alias-analysis", cl::Hidden,
                 cl::desc("Enable DAG combiner's use of IR alias analysis"));
 
static cl::opt<bool>
UseTBAA("combiner-use-tbaa", cl::Hidden, cl::init(true),
        cl::desc("Enable DAG combiner's use of TBAA"));
 
#ifndef NDEBUG
static cl::opt<std::string>
CombinerAAOnlyFunc("combiner-aa-only-func", cl::Hidden,
                   cl::desc("Only use DAG-combiner alias analysis in this"
                            " function"));
#endif
 
/// Hidden option to stress test load slicing, i.e., when this option
/// is enabled, load slicing bypasses most of its profitability guards.
static cl::opt<bool>
StressLoadSlicing("combiner-stress-load-slicing", cl::Hidden,
                  cl::desc("Bypass the profitability model of load slicing"),
                  cl::init(false));
 
static cl::opt<bool>
  MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true),
                    cl::desc("DAG combiner may split indexing from loads"));
 
static cl::opt<bool>
    EnableStoreMerging("combiner-store-merging", cl::Hidden, cl::init(true),
                       cl::desc("DAG combiner enable merging multiple stores "
                                "into a wider store"));
 
static cl::opt<unsigned> TokenFactorInlineLimit(
    "combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048),
    cl::desc("Limit the number of operands to inline for Token Factors"));
 
static cl::opt<unsigned> StoreMergeDependenceLimit(
    "combiner-store-merge-dependence-limit", cl::Hidden, cl::init(10),
    cl::desc("Limit the number of times for the same StoreNode and RootNode "
             "to bail out in store merging dependence check"));
 
static cl::opt<bool> EnableReduceLoadOpStoreWidth(
    "combiner-reduce-load-op-store-width", cl::Hidden, cl::init(true),
    cl::desc("DAG combiner enable reducing the width of load/op/store "
             "sequence"));
static cl::opt<bool> ReduceLoadOpStoreWidthForceNarrowingProfitable(
    "combiner-reduce-load-op-store-width-force-narrowing-profitable",
    cl::Hidden, cl::init(false),
    cl::desc("DAG combiner force override the narrowing profitable check when "
             "reducing the width of load/op/store sequences"));
 
static cl::opt<bool> EnableShrinkLoadReplaceStoreWithStore(
    "combiner-shrink-load-replace-store-with-store", cl::Hidden, cl::init(true),
    cl::desc("DAG combiner enable load/<replace bytes>/store with "
             "a narrower store"));
 
static cl::opt<bool> EnableTopologicalSorting(
    "combiner-topological-sorting", cl::Hidden, cl::init(false),
    cl::desc("DAG combiner nodes consistently processed in topological order"));
 
static cl::opt<bool> DisableCombines("combiner-disabled", cl::Hidden,
                                     cl::init(false),
                                     cl::desc("Disable the DAG combiner"));
 
namespace {
 
  class DAGCombiner {
    SelectionDAG &DAG;
    const TargetLowering &TLI;
    const SelectionDAGTargetInfo *STI;
    CombineLevel Level = BeforeLegalizeTypes;
    CodeGenOptLevel OptLevel;
    bool LegalDAG = false;
    bool LegalOperations = false;
    bool LegalTypes = false;
    bool ForCodeSize;
    bool DisableGenericCombines;
 
    /// Worklist of all of the nodes that need to be simplified.
    ///
    /// This must behave as a stack -- new nodes to process are pushed onto the
    /// back and when processing we pop off of the back.
    ///
    /// The worklist will not contain duplicates but may contain null entries
    /// due to nodes being deleted from the underlying DAG. For fast lookup and
    /// deduplication, the index of the node in this vector is stored in the
    /// node in SDNode::CombinerWorklistIndex.
    SmallVector<SDNode *, 64> Worklist;
 
    /// This records all nodes attempted to be added to the worklist since we
    /// considered a new worklist entry. As we keep do not add duplicate nodes
    /// in the worklist, this is different from the tail of the worklist.
    SmallSetVector<SDNode *, 32> PruningList;
 
    /// Map from candidate StoreNode to the pair of RootNode and count.
    /// The count is used to track how many times we have seen the StoreNode
    /// with the same RootNode bail out in dependence check. If we have seen
    /// the bail out for the same pair many times over a limit, we won't
    /// consider the StoreNode with the same RootNode as store merging
    /// candidate again.
    DenseMap<SDNode *, std::pair<SDNode *, unsigned>> StoreRootCountMap;
 
    // BatchAA - Used for DAG load/store alias analysis.
    BatchAAResults *BatchAA;
 
    /// This caches all chains that have already been processed in
    /// DAGCombiner::getStoreMergeCandidates() and found to have no mergeable
    /// stores candidates.
    SmallPtrSet<SDNode *, 4> ChainsWithoutMergeableStores;
 
    /// When an instruction is simplified, add all users of the instruction to
    /// the work lists because they might get more simplified now.
    void AddUsersToWorklist(SDNode *N) {
      for (SDNode *Node : N->users())
        AddToWorklist(Node);
    }
 
    /// Convenient shorthand to add a node and all of its user to the worklist.
    void AddToWorklistWithUsers(SDNode *N) {
      AddUsersToWorklist(N);
      AddToWorklist(N);
    }
 
    // Prune potentially dangling nodes. This is called after
    // any visit to a node, but should also be called during a visit after any
    // failed combine which may have created a DAG node.
    void clearAddedDanglingWorklistEntries() {
      // Check any nodes added to the worklist to see if they are prunable.
      while (!PruningList.empty()) {
        auto *N = PruningList.pop_back_val();
        if (N->use_empty())
          recursivelyDeleteUnusedNodes(N);
      }
    }
 
    SDNode *getNextWorklistEntry() {
      // Before we do any work, remove nodes that are not in use.
      clearAddedDanglingWorklistEntries();
      SDNode *N = nullptr;
      // The Worklist holds the SDNodes in order, but it may contain null
      // entries.
      while (!N && !Worklist.empty()) {
        N = Worklist.pop_back_val();
      }
 
      if (N) {
        assert(N->getCombinerWorklistIndex() >= 0 &&
               "Found a worklist entry without a corresponding map entry!");
        // Set to -2 to indicate that we combined the node.
        N->setCombinerWorklistIndex(-2);
      }
      return N;
    }
 
    /// Call the node-specific routine that folds each particular type of node.
    SDValue visit(SDNode *N);
 
  public:
    DAGCombiner(SelectionDAG &D, BatchAAResults *BatchAA, CodeGenOptLevel OL)
        : DAG(D), TLI(D.getTargetLoweringInfo()),
          STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL),
          BatchAA(BatchAA) {
      ForCodeSize = DAG.shouldOptForSize();
      DisableGenericCombines =
          DisableCombines || (STI && STI->disableGenericCombines(OptLevel));
    }
 
    void ConsiderForPruning(SDNode *N) {
      // Mark this for potential pruning.
      PruningList.insert(N);
    }
 
    /// Add to the worklist making sure its instance is at the back (next to be
    /// processed.)
    void AddToWorklist(SDNode *N, bool IsCandidateForPruning = true,
                       bool SkipIfCombinedBefore = false) {
      assert(N->getOpcode() != ISD::DELETED_NODE &&
             "Deleted Node added to Worklist");
 
      // Skip handle nodes as they can't usefully be combined and confuse the
      // zero-use deletion strategy.
      if (N->getOpcode() == ISD::HANDLENODE)
        return;
 
      if (SkipIfCombinedBefore && N->getCombinerWorklistIndex() == -2)
        return;
 
      if (IsCandidateForPruning)
        ConsiderForPruning(N);
 
      if (N->getCombinerWorklistIndex() < 0) {
        N->setCombinerWorklistIndex(Worklist.size());
        Worklist.push_back(N);
      }
    }
 
    /// Remove all instances of N from the worklist.
    void removeFromWorklist(SDNode *N) {
      PruningList.remove(N);
      StoreRootCountMap.erase(N);
 
      int WorklistIndex = N->getCombinerWorklistIndex();
      // If not in the worklist, the index might be -1 or -2 (was combined
      // before). As the node gets deleted anyway, there's no need to update
      // the index.
      if (WorklistIndex < 0)
        return; // Not in the worklist.
 
      // Null out the entry rather than erasing it to avoid a linear operation.
      Worklist[WorklistIndex] = nullptr;
      N->setCombinerWorklistIndex(-1);
    }
 
    void deleteAndRecombine(SDNode *N);
    bool recursivelyDeleteUnusedNodes(SDNode *N);
 
    /// Replaces all uses of the results of one DAG node with new values.
    SDValue CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
                      bool AddTo = true);
 
    /// Replaces all uses of the results of one DAG node with new values.
    SDValue CombineTo(SDNode *N, SDValue Res, bool AddTo = true) {
      return CombineTo(N, &Res, 1, AddTo);
    }
 
    /// Replaces all uses of the results of one DAG node with new values.
    SDValue CombineTo(SDNode *N, SDValue Res0, SDValue Res1,
                      bool AddTo = true) {
      SDValue To[] = { Res0, Res1 };
      return CombineTo(N, To, 2, AddTo);
    }
 
    SDValue CombineTo(SDNode *N, SmallVectorImpl<SDValue> *To,
                      bool AddTo = true) {
      return CombineTo(N, To->data(), To->size(), AddTo);
    }
 
    void CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO);
 
  private:
    /// Check the specified integer node value to see if it can be simplified or
    /// if things it uses can be simplified by bit propagation.
    /// If so, return true.
    bool SimplifyDemandedBits(SDValue Op) {
      unsigned BitWidth = Op.getScalarValueSizeInBits();
      APInt DemandedBits = APInt::getAllOnes(BitWidth);
      return SimplifyDemandedBits(Op, DemandedBits);
    }
 
    bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) {
      EVT VT = Op.getValueType();
      APInt DemandedElts = VT.isFixedLengthVector()
                               ? APInt::getAllOnes(VT.getVectorNumElements())
                               : APInt(1, 1);
      return SimplifyDemandedBits(Op, DemandedBits, DemandedElts, false);
    }
 
    /// Check the specified vector node value to see if it can be simplified or
    /// if things it uses can be simplified as it only uses some of the
    /// elements. If so, return true.
    bool SimplifyDemandedVectorElts(SDValue Op) {
      // TODO: For now just pretend it cannot be simplified.
      if (Op.getValueType().isScalableVector())
        return false;
 
      unsigned NumElts = Op.getValueType().getVectorNumElements();
      APInt DemandedElts = APInt::getAllOnes(NumElts);
      return SimplifyDemandedVectorElts(Op, DemandedElts);
    }
 
    bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                              const APInt &DemandedElts,
                              bool AssumeSingleUse = false);
    bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts,
                                    bool AssumeSingleUse = false);
 
    bool CombineToPreIndexedLoadStore(SDNode *N);
    bool CombineToPostIndexedLoadStore(SDNode *N);
    SDValue SplitIndexingFromLoad(LoadSDNode *LD);
    bool SliceUpLoad(SDNode *N);
 
    // Looks up the chain to find a unique (unaliased) store feeding the passed
    // load. If no such store is found, returns a nullptr.
    // Note: This will look past a CALLSEQ_START if the load is chained to it so
    //       so that it can find stack stores for byval params.
    StoreSDNode *getUniqueStoreFeeding(LoadSDNode *LD, int64_t &Offset);
    // Scalars have size 0 to distinguish from singleton vectors.
    SDValue ForwardStoreValueToDirectLoad(LoadSDNode *LD);
    bool getTruncatedStoreValue(StoreSDNode *ST, SDValue &Val);
    bool extendLoadedValueToExtension(LoadSDNode *LD, SDValue &Val);
 
    void ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad);
    SDValue PromoteOperand(SDValue Op, EVT PVT, bool &Replace);
    SDValue SExtPromoteOperand(SDValue Op, EVT PVT);
    SDValue ZExtPromoteOperand(SDValue Op, EVT PVT);
    SDValue PromoteIntBinOp(SDValue Op);
    SDValue PromoteIntShiftOp(SDValue Op);
    SDValue PromoteExtend(SDValue Op);
    bool PromoteLoad(SDValue Op);
 
    SDValue foldShiftToAvg(SDNode *N, const SDLoc &DL);
    // Fold `a bitwiseop (~b +/- c)` -> `a bitwiseop ~(b -/+ c)`
    SDValue foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT);
 
    SDValue combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
                                SDValue RHS, SDValue True, SDValue False,
                                ISD::CondCode CC);
 
    /// Call the node-specific routine that knows how to fold each
    /// particular type of node. If that doesn't do anything, try the
    /// target-specific DAG combines.
    SDValue combine(SDNode *N);
 
    // Visitation implementation - Implement dag node combining for different
    // node types.  The semantics are as follows:
    // Return Value:
    //   SDValue.getNode() == 0 - No change was made
    //   SDValue.getNode() == N - N was replaced, is dead and has been handled.
    //   otherwise              - N should be replaced by the returned Operand.
    //
    SDValue visitTokenFactor(SDNode *N);
    SDValue visitMERGE_VALUES(SDNode *N);
    SDValue visitADD(SDNode *N);
    SDValue visitADDLike(SDNode *N);
    SDValue visitADDLikeCommutative(SDValue N0, SDValue N1,
                                    SDNode *LocReference);
    SDValue visitPTRADD(SDNode *N);
    SDValue visitSUB(SDNode *N);
    SDValue visitADDSAT(SDNode *N);
    SDValue visitSUBSAT(SDNode *N);
    SDValue visitADDC(SDNode *N);
    SDValue visitADDO(SDNode *N);
    SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N);
    SDValue visitSUBC(SDNode *N);
    SDValue visitSUBO(SDNode *N);
    SDValue visitADDE(SDNode *N);
    SDValue visitUADDO_CARRY(SDNode *N);
    SDValue visitSADDO_CARRY(SDNode *N);
    SDValue visitUADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
                                 SDNode *N);
    SDValue visitSADDO_CARRYLike(SDValue N0, SDValue N1, SDValue CarryIn,
                                 SDNode *N);
    SDValue visitSUBE(SDNode *N);
    SDValue visitUSUBO_CARRY(SDNode *N);
    SDValue visitSSUBO_CARRY(SDNode *N);
    SDValue visitMUL(SDNode *N);
    SDValue visitMULFIX(SDNode *N);
    SDValue useDivRem(SDNode *N);
    SDValue visitSDIV(SDNode *N);
    SDValue visitSDIVLike(SDValue N0, SDValue N1, SDNode *N);
    SDValue visitUDIV(SDNode *N);
    SDValue visitUDIVLike(SDValue N0, SDValue N1, SDNode *N);
    SDValue visitREM(SDNode *N);
    SDValue visitMULHU(SDNode *N);
    SDValue visitMULHS(SDNode *N);
    SDValue visitAVG(SDNode *N);
    SDValue visitABD(SDNode *N);
    SDValue visitSMUL_LOHI(SDNode *N);
    SDValue visitUMUL_LOHI(SDNode *N);
    SDValue visitMULO(SDNode *N);
    SDValue visitIMINMAX(SDNode *N);
    SDValue visitAND(SDNode *N);
    SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N);
    SDValue visitOR(SDNode *N);
    SDValue visitORLike(SDValue N0, SDValue N1, const SDLoc &DL);
    SDValue visitXOR(SDNode *N);
    SDValue SimplifyVCastOp(SDNode *N, const SDLoc &DL);
    SDValue SimplifyVBinOp(SDNode *N, const SDLoc &DL);
    SDValue visitSHL(SDNode *N);
    SDValue visitSRA(SDNode *N);
    SDValue visitSRL(SDNode *N);
    SDValue visitFunnelShift(SDNode *N);
    SDValue visitSHLSAT(SDNode *N);
    SDValue visitRotate(SDNode *N);
    SDValue visitABS(SDNode *N);
    SDValue visitABS_MIN_POISON(SDNode *N);
    SDValue visitCLMUL(SDNode *N);
    SDValue visitBSWAP(SDNode *N);
    SDValue visitBITREVERSE(SDNode *N);
    SDValue visitCTLZ(SDNode *N);
    SDValue visitCTLZ_ZERO_POISON(SDNode *N);
    SDValue visitCTTZ(SDNode *N);
    SDValue visitCTTZ_ZERO_POISON(SDNode *N);
    SDValue visitCTPOP(SDNode *N);
    SDValue visitSELECT(SDNode *N);
    SDValue visitVSELECT(SDNode *N);
    SDValue visitVP_SELECT(SDNode *N);
    SDValue visitSELECT_CC(SDNode *N);
    SDValue visitSETCC(SDNode *N);
    SDValue visitSETCCCARRY(SDNode *N);
    SDValue visitSIGN_EXTEND(SDNode *N);
    SDValue visitZERO_EXTEND(SDNode *N);
    SDValue visitANY_EXTEND(SDNode *N);
    SDValue visitAssertExt(SDNode *N);
    SDValue visitAssertAlign(SDNode *N);
    SDValue visitIS_FPCLASS(SDNode *N);
    SDValue visitSIGN_EXTEND_INREG(SDNode *N);
    SDValue visitEXTEND_VECTOR_INREG(SDNode *N);
    SDValue visitTRUNCATE(SDNode *N);
    SDValue visitTRUNCATE_USAT_U(SDNode *N);
    SDValue visitBITCAST(SDNode *N);
    SDValue visitFREEZE(SDNode *N);
    SDValue visitBUILD_PAIR(SDNode *N);
    SDValue visitFADD(SDNode *N);
    SDValue visitSTRICT_FADD(SDNode *N);
    SDValue visitFSUB(SDNode *N);
    SDValue visitFMUL(SDNode *N);
    SDValue visitFMA(SDNode *N);
    SDValue visitFMAD(SDNode *N);
    SDValue visitFMULADD(SDNode *N);
    SDValue visitFDIV(SDNode *N);
    SDValue visitFREM(SDNode *N);
    SDValue visitFSQRT(SDNode *N);
    SDValue visitFCOPYSIGN(SDNode *N);
    SDValue visitFPOW(SDNode *N);
    SDValue visitFCANONICALIZE(SDNode *N);
    SDValue visitSINT_TO_FP(SDNode *N);
    SDValue visitUINT_TO_FP(SDNode *N);
    SDValue visitFP_TO_SINT(SDNode *N);
    SDValue visitFP_TO_UINT(SDNode *N);
    SDValue visitXROUND(SDNode *N);
    SDValue visitFP_ROUND(SDNode *N);
    SDValue visitFP_EXTEND(SDNode *N);
    SDValue visitFNEG(SDNode *N);
    SDValue visitFABS(SDNode *N);
    SDValue visitFCEIL(SDNode *N);
    SDValue visitFTRUNC(SDNode *N);
    SDValue visitFFREXP(SDNode *N);
    SDValue visitFFLOOR(SDNode *N);
    SDValue visitFMinMax(SDNode *N);
    SDValue visitBRCOND(SDNode *N);
    SDValue visitBR_CC(SDNode *N);
    SDValue visitLOAD(SDNode *N);
 
    SDValue replaceStoreChain(StoreSDNode *ST, SDValue BetterChain);
    SDValue replaceStoreOfFPConstant(StoreSDNode *ST);
    SDValue replaceStoreOfInsertLoad(StoreSDNode *ST);
 
    bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(SDNode *N);
    SDValue combineStoreConcatTruncVector(StoreSDNode *N);
    SDValue visitSTORE(SDNode *N);
    SDValue visitATOMIC_STORE(SDNode *N);
    SDValue visitLIFETIME_END(SDNode *N);
    SDValue visitINSERT_VECTOR_ELT(SDNode *N);
    SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
    SDValue visitBUILD_VECTOR(SDNode *N);
    SDValue visitCONCAT_VECTORS(SDNode *N);
    SDValue visitVECTOR_INTERLEAVE(SDNode *N);
    SDValue visitEXTRACT_SUBVECTOR(SDNode *N);
    SDValue visitVECTOR_SHUFFLE(SDNode *N);
    SDValue visitSCALAR_TO_VECTOR(SDNode *N);
    SDValue visitINSERT_SUBVECTOR(SDNode *N);
    SDValue visitVECTOR_COMPRESS(SDNode *N);
    SDValue visitMLOAD(SDNode *N);
    SDValue visitMSTORE(SDNode *N);
    SDValue visitMGATHER(SDNode *N);
    SDValue visitMSCATTER(SDNode *N);
    SDValue visitMHISTOGRAM(SDNode *N);
    SDValue visitPARTIAL_REDUCE_MLA(SDNode *N);
    SDValue visitVPGATHER(SDNode *N);
    SDValue visitVPSCATTER(SDNode *N);
    SDValue visitVP_STRIDED_LOAD(SDNode *N);
    SDValue visitVP_STRIDED_STORE(SDNode *N);
    SDValue visitFP_TO_FP16(SDNode *N);
    SDValue visitFP16_TO_FP(SDNode *N);
    SDValue visitFP_TO_BF16(SDNode *N);
    SDValue visitBF16_TO_FP(SDNode *N);
    SDValue visitVECREDUCE(SDNode *N);
    SDValue visitVPOp(SDNode *N);
    SDValue visitGET_FPENV_MEM(SDNode *N);
    SDValue visitSET_FPENV_MEM(SDNode *N);
 
    SDValue visitFADDForFMACombine(SDNode *N);
    SDValue visitFSUBForFMACombine(SDNode *N);
    SDValue visitFMULForFMADistributiveCombine(SDNode *N);
 
    SDValue XformToShuffleWithZero(SDNode *N);
    bool reassociationCanBreakAddressingModePattern(unsigned Opc,
                                                    const SDLoc &DL,
                                                    SDNode *N,
                                                    SDValue N0,
                                                    SDValue N1);
    SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0,
                                      SDValue N1, SDNodeFlags Flags);
    SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
                           SDValue N1, SDNodeFlags Flags);
    SDValue reassociateReduction(unsigned RedOpc, unsigned Opc, const SDLoc &DL,
                                 EVT VT, SDValue N0, SDValue N1,
                                 SDNodeFlags Flags = SDNodeFlags());
 
    SDValue visitShiftByConstant(SDNode *N);
 
    SDValue foldSelectOfConstants(SDNode *N);
    SDValue foldVSelectOfConstants(SDNode *N);
    SDValue foldBinOpIntoSelect(SDNode *BO);
    bool SimplifySelectOps(SDNode *SELECT, SDValue LHS, SDValue RHS);
    SDValue hoistLogicOpWithSameOpcodeHands(SDNode *N);
    SDValue SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2);
    SDValue SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1,
                             SDValue N2, SDValue N3, ISD::CondCode CC,
                             bool NotExtCompare = false);
    SDValue convertSelectOfFPConstantsToLoadOffset(
        const SDLoc &DL, SDValue N0, SDValue N1, SDValue N2, SDValue N3,
        ISD::CondCode CC);
    SDValue foldSignChangeInBitcast(SDNode *N);
    SDValue foldSelectCCToShiftAnd(const SDLoc &DL, SDValue N0, SDValue N1,
                                   SDValue N2, SDValue N3, ISD::CondCode CC);
    SDValue foldSelectOfBinops(SDNode *N);
    SDValue foldSextSetcc(SDNode *N);
    SDValue foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
                              const SDLoc &DL);
    SDValue foldSubToUSubSat(EVT DstVT, SDNode *N, const SDLoc &DL);
    SDValue foldABSToABD(SDNode *N, const SDLoc &DL);
    SDValue foldSelectToABD(SDValue LHS, SDValue RHS, SDValue True,
                            SDValue False, ISD::CondCode CC, const SDLoc &DL);
    SDValue foldSelectToUMin(SDValue LHS, SDValue RHS, SDValue True,
                             SDValue False, ISD::CondCode CC, const SDLoc &DL);
    SDValue unfoldMaskedMerge(SDNode *N);
    SDValue unfoldExtremeBitClearingToShifts(SDNode *N);
    SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond,
                          const SDLoc &DL, bool foldBooleans);
    SDValue rebuildSetCC(SDValue N);
 
    bool isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
                           SDValue &CC, bool MatchStrict = false) const;
    bool isOneUseSetCC(SDValue N) const;
 
    SDValue foldAddToAvg(SDNode *N, const SDLoc &DL);
    SDValue foldSubToAvg(SDNode *N, const SDLoc &DL);
 
    SDValue foldCTLZToCTLS(SDValue Src, const SDLoc &DL);
 
    SDValue SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
                                         unsigned HiOp);
    SDValue CombineConsecutiveLoads(SDNode *N, EVT VT);
    SDValue foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
                                 const TargetLowering &TLI);
    SDValue foldPartialReduceMLAMulOp(SDNode *N);
    SDValue foldPartialReduceAdd(SDNode *N);
 
    SDValue CombineExtLoad(SDNode *N);
    SDValue CombineZExtLogicopShiftLoad(SDNode *N);
    SDValue combineRepeatedFPDivisors(SDNode *N);
    SDValue combineFMulOrFDivWithIntPow2(SDNode *N);
    SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf);
    SDValue mergeInsertEltWithShuffle(SDNode *N, unsigned InsIndex);
    SDValue combineInsertEltToShuffle(SDNode *N, unsigned InsIndex);
    SDValue combineInsertEltToLoad(SDNode *N, unsigned InsIndex);
    SDValue foldExtractSubvectorFromConcatVectors(EVT VT, SDValue V,
                                                  uint64_t ExtIdx,
                                                  const SDLoc &DL);
    SDValue BuildSDIV(SDNode *N);
    SDValue BuildSDIVPow2(SDNode *N);
    SDValue BuildUDIV(SDNode *N);
    SDValue BuildSREMPow2(SDNode *N);
    SDValue buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N);
    SDValue BuildLogBase2(SDValue V, const SDLoc &DL,
                          bool KnownNeverZero = false,
                          bool InexpensiveOnly = false,
                          std::optional<EVT> OutVT = std::nullopt);
    SDValue BuildDivEstimate(SDValue N, SDValue Op, SDNodeFlags Flags);
    SDValue buildRsqrtEstimate(SDValue Op, SDNodeFlags Flags);
    SDValue buildSqrtEstimate(SDValue Op, SDNodeFlags Flags);
    SDValue buildSqrtEstimateImpl(SDValue Op, bool Recip, SDNodeFlags Flags);
    SDValue buildSqrtNROneConst(SDValue Arg, SDValue Est, unsigned Iterations,
                                bool Reciprocal);
    SDValue buildSqrtNRTwoConst(SDValue Arg, SDValue Est, unsigned Iterations,
                                bool Reciprocal);
    SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
                               bool DemandHighBits = true);
    SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
    SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
                              SDValue InnerPos, SDValue InnerNeg, bool FromAdd,
                              bool HasPos, unsigned PosOpcode,
                              unsigned NegOpcode, const SDLoc &DL);
    SDValue MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos, SDValue Neg,
                              SDValue InnerPos, SDValue InnerNeg, bool FromAdd,
                              bool HasPos, unsigned PosOpcode,
                              unsigned NegOpcode, const SDLoc &DL);
    SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL,
                        bool FromAdd);
    SDValue MatchLoadCombine(SDNode *N);
    SDValue mergeTruncStores(StoreSDNode *N);
    SDValue reduceLoadWidth(SDNode *N);
    SDValue ReduceLoadOpStoreWidth(SDNode *N);
    SDValue splitMergedValStore(StoreSDNode *ST);
    SDValue TransformFPLoadStorePair(SDNode *N);
    SDValue convertBuildVecExtToExt(SDNode *N);
    SDValue convertBuildVecZextToBuildVecWithZeros(SDNode *N);
    SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
    SDValue reduceBuildVecTruncToBitCast(SDNode *N);
    SDValue reduceBuildVecToShuffle(SDNode *N);
    SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N,
                                  ArrayRef<int> VectorMask, SDValue VecIn1,
                                  SDValue VecIn2, unsigned LeftIdx,
                                  bool DidSplitVec);
    SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast);
 
    /// Walk up chain skipping non-aliasing memory nodes,
    /// looking for aliasing nodes and adding them to the Aliases vector.
    void GatherAllAliases(SDNode *N, SDValue OriginalChain,
                          SmallVectorImpl<SDValue> &Aliases);
 
    /// Return true if there is any possibility that the two addresses overlap.
    bool mayAlias(SDNode *Op0, SDNode *Op1) const;
 
    /// Walk up chain skipping non-aliasing memory nodes, looking for a better
    /// chain (aliasing node.)
    SDValue FindBetterChain(SDNode *N, SDValue Chain);
 
    /// Try to replace a store and any possibly adjacent stores on
    /// consecutive chains with better chains. Return true only if St is
    /// replaced.
    ///
    /// Notice that other chains may still be replaced even if the function
    /// returns false.
    bool findBetterNeighborChains(StoreSDNode *St);
 
    // Helper for findBetterNeighborChains. Walk up store chain add additional
    // chained stores that do not overlap and can be parallelized.
    bool parallelizeChainedStores(StoreSDNode *St);
 
    /// Holds a pointer to an LSBaseSDNode as well as information on where it
    /// is located in a sequence of memory operations connected by a chain.
    struct MemOpLink {
      // Ptr to the mem node.
      LSBaseSDNode *MemNode;
 
      // Offset from the base ptr.
      int64_t OffsetFromBase;
 
      MemOpLink(LSBaseSDNode *N, int64_t Offset)
          : MemNode(N), OffsetFromBase(Offset) {}
    };
 
    // Classify the origin of a stored value.
    enum class StoreSource { Unknown, Constant, Extract, Load };
    StoreSource getStoreSource(SDValue StoreVal) {
      switch (StoreVal.getOpcode()) {
      case ISD::Constant:
      case ISD::ConstantFP:
        return StoreSource::Constant;
      case ISD::BUILD_VECTOR:
        if (ISD::isBuildVectorOfConstantSDNodes(StoreVal.getNode()) ||
            ISD::isBuildVectorOfConstantFPSDNodes(StoreVal.getNode()))
          return StoreSource::Constant;
        return StoreSource::Unknown;
      case ISD::EXTRACT_VECTOR_ELT:
      case ISD::EXTRACT_SUBVECTOR:
        return StoreSource::Extract;
      case ISD::LOAD:
        return StoreSource::Load;
      default:
        return StoreSource::Unknown;
      }
    }
 
    /// This is a helper function for visitMUL to check the profitability
    /// of folding (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2).
    /// MulNode is the original multiply, AddNode is (add x, c1),
    /// and ConstNode is c2.
    bool isMulAddWithConstProfitable(SDNode *MulNode, SDValue AddNode,
                                     SDValue ConstNode);
 
    /// This is a helper function for visitAND and visitZERO_EXTEND.  Returns
    /// true if the (and (load x) c) pattern matches an extload.  ExtVT returns
    /// the type of the loaded value to be extended.
    bool isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
                          EVT LoadResultTy, EVT &ExtVT);
 
    /// Helper function to calculate whether the given Load/Store can have its
    /// width reduced to ExtVT.
    bool isLegalNarrowLdSt(LSBaseSDNode *LDSTN, ISD::LoadExtType ExtType,
                           EVT &MemVT, unsigned ShAmt = 0);
 
    /// Used by BackwardsPropagateMask to find suitable loads.
    bool SearchForAndLoads(SDNode *N, SmallVectorImpl<LoadSDNode*> &Loads,
                           SmallPtrSetImpl<SDNode*> &NodesWithConsts,
                           ConstantSDNode *Mask, SDNode *&NodeToMask);
    /// Attempt to propagate a given AND node back to load leaves so that they
    /// can be combined into narrow loads.
    bool BackwardsPropagateMask(SDNode *N);
 
    /// Helper function for mergeConsecutiveStores which merges the component
    /// store chains.
    SDValue getMergeStoreChains(SmallVectorImpl<MemOpLink> &StoreNodes,
                                unsigned NumStores);
 
    /// Helper function for mergeConsecutiveStores which checks if all the store
    /// nodes have the same underlying object. We can still reuse the first
    /// store's pointer info if all the stores are from the same object.
    bool hasSameUnderlyingObj(ArrayRef<MemOpLink> StoreNodes);
 
    /// This is a helper function for mergeConsecutiveStores. When the source
    /// elements of the consecutive stores are all constants or all extracted
    /// vector elements, try to merge them into one larger store introducing
    /// bitcasts if necessary.  \return True if a merged store was created.
    bool mergeStoresOfConstantsOrVecElts(SmallVectorImpl<MemOpLink> &StoreNodes,
                                         EVT MemVT, unsigned NumStores,
                                         bool IsConstantSrc, bool UseVector,
                                         bool UseTrunc);
 
    /// This is a helper function for mergeConsecutiveStores. Stores that
    /// potentially may be merged with St are placed in StoreNodes. On success,
    /// returns a chain predecessor to all store candidates.
    SDNode *getStoreMergeCandidates(StoreSDNode *St,
                                    SmallVectorImpl<MemOpLink> &StoreNodes);
 
    /// Helper function for mergeConsecutiveStores. Checks if candidate stores
    /// have indirect dependency through their operands. RootNode is the
    /// predecessor to all stores calculated by getStoreMergeCandidates and is
    /// used to prune the dependency check. \return True if safe to merge.
    bool checkMergeStoreCandidatesForDependencies(
        SmallVectorImpl<MemOpLink> &StoreNodes, unsigned NumStores,
        SDNode *RootNode);
 
    /// Helper function for tryStoreMergeOfLoads. Checks if the load/store
    /// chain has a call in it. \return True if a call is found.
    bool hasCallInLdStChain(StoreSDNode *St, LoadSDNode *Ld);
 
    /// This is a helper function for mergeConsecutiveStores. Given a list of
    /// store candidates, find the first N that are consecutive in memory.
    /// Returns 0 if there are not at least 2 consecutive stores to try merging.
    unsigned getConsecutiveStores(SmallVectorImpl<MemOpLink> &StoreNodes,
                                  int64_t ElementSizeBytes) const;
 
    /// This is a helper function for mergeConsecutiveStores. It is used for
    /// store chains that are composed entirely of constant values.
    bool tryStoreMergeOfConstants(SmallVectorImpl<MemOpLink> &StoreNodes,
                                  unsigned NumConsecutiveStores,
                                  EVT MemVT, SDNode *Root, bool AllowVectors);
 
    /// This is a helper function for mergeConsecutiveStores. It is used for
    /// store chains that are composed entirely of extracted vector elements.
    /// When extracting multiple vector elements, try to store them in one
    /// vector store rather than a sequence of scalar stores.
    bool tryStoreMergeOfExtracts(SmallVectorImpl<MemOpLink> &StoreNodes,
                                 unsigned NumConsecutiveStores, EVT MemVT,
                                 SDNode *Root);
 
    /// This is a helper function for mergeConsecutiveStores. It is used for
    /// store chains that are composed entirely of loaded values.
    bool tryStoreMergeOfLoads(SmallVectorImpl<MemOpLink> &StoreNodes,
                              unsigned NumConsecutiveStores, EVT MemVT,
                              SDNode *Root, bool AllowVectors,
                              bool IsNonTemporalStore, bool IsNonTemporalLoad);
 
    /// Merge consecutive store operations into a wide store.
    /// This optimization uses wide integers or vectors when possible.
    /// \return true if stores were merged.
    bool mergeConsecutiveStores(StoreSDNode *St);
 
    /// Try to transform a truncation where C is a constant:
    ///     (trunc (and X, C)) -> (and (trunc X), (trunc C))
    ///
    /// \p N needs to be a truncation and its first operand an AND. Other
    /// requirements are checked by the function (e.g. that trunc is
    /// single-use) and if missed an empty SDValue is returned.
    SDValue distributeTruncateThroughAnd(SDNode *N);
 
    /// Helper function to determine whether the target supports operation
    /// given by \p Opcode for type \p VT, that is, whether the operation
    /// is legal or custom before legalizing operations, and whether is
    /// legal (but not custom) after legalization.
    bool hasOperation(unsigned Opcode, EVT VT) {
      return TLI.isOperationLegalOrCustom(Opcode, VT, LegalOperations);
    }
 
    bool hasUMin(EVT VT) const {
      auto LK = TLI.getTypeConversion(*DAG.getContext(), VT);
      return (LK.first == TargetLoweringBase::TypeLegal ||
              LK.first == TargetLoweringBase::TypePromoteInteger) &&
             TLI.isOperationLegalOrCustom(ISD::UMIN, LK.second);
    }
 
  public:
    /// Runs the dag combiner on all nodes in the work list
    void Run(CombineLevel AtLevel);
 
    SelectionDAG &getDAG() const { return DAG; }
 
    /// Convenience wrapper around TargetLowering::getShiftAmountTy.
    EVT getShiftAmountTy(EVT LHSTy) {
      return TLI.getShiftAmountTy(LHSTy, DAG.getDataLayout());
    }
 
    /// This method returns true if we are running before type legalization or
    /// if the specified VT is legal.
    bool isTypeLegal(const EVT &VT) {
      if (!LegalTypes) return true;
      return TLI.isTypeLegal(VT);
    }
 
    /// Convenience wrapper around TargetLowering::getSetCCResultType
    EVT getSetCCResultType(EVT VT) const {
      return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
    }
 
    void ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
                         SDValue OrigLoad, SDValue ExtLoad,
                         ISD::NodeType ExtType);
  };
 
/// This class is a DAGUpdateListener that removes any deleted
/// nodes from the worklist.
class WorklistRemover : public SelectionDAG::DAGUpdateListener {
  DAGCombiner &DC;
 
public:
  explicit WorklistRemover(DAGCombiner &dc)
    : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
 
  void NodeDeleted(SDNode *N, SDNode *E) override {
    DC.removeFromWorklist(N);
  }
};
 
class WorklistInserter : public SelectionDAG::DAGUpdateListener {
  DAGCombiner &DC;
 
public:
  explicit WorklistInserter(DAGCombiner &dc)
      : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {}
 
  // FIXME: Ideally we could add N to the worklist, but this causes exponential
  //        compile time costs in large DAGs, e.g. Halide.
  void NodeInserted(SDNode *N) override { DC.ConsiderForPruning(N); }
};
 
} // end anonymous namespace
 
//===----------------------------------------------------------------------===//
//  TargetLowering::DAGCombinerInfo implementation
//===----------------------------------------------------------------------===//
 
void TargetLowering::DAGCombinerInfo::AddToWorklist(SDNode *N) {
  ((DAGCombiner*)DC)->AddToWorklist(N);
}
 
SDValue TargetLowering::DAGCombinerInfo::
CombineTo(SDNode *N, ArrayRef<SDValue> To, bool AddTo) {
  return ((DAGCombiner*)DC)->CombineTo(N, &To[0], To.size(), AddTo);
}
 
SDValue TargetLowering::DAGCombinerInfo::
CombineTo(SDNode *N, SDValue Res, bool AddTo) {
  return ((DAGCombiner*)DC)->CombineTo(N, Res, AddTo);
}
 
SDValue TargetLowering::DAGCombinerInfo::
CombineTo(SDNode *N, SDValue Res0, SDValue Res1, bool AddTo) {
  return ((DAGCombiner*)DC)->CombineTo(N, Res0, Res1, AddTo);
}
 
bool TargetLowering::DAGCombinerInfo::
recursivelyDeleteUnusedNodes(SDNode *N) {
  return ((DAGCombiner*)DC)->recursivelyDeleteUnusedNodes(N);
}
 
void TargetLowering::DAGCombinerInfo::
CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
  return ((DAGCombiner*)DC)->CommitTargetLoweringOpt(TLO);
}
 
//===----------------------------------------------------------------------===//
// Helper Functions
//===----------------------------------------------------------------------===//
 
void DAGCombiner::deleteAndRecombine(SDNode *N) {
  removeFromWorklist(N);
 
  // If the operands of this node are only used by the node, they will now be
  // dead. Make sure to re-visit them and recursively delete dead nodes.
  for (const SDValue &Op : N->ops())
    // For an operand generating multiple values, one of the values may
    // become dead allowing further simplification (e.g. split index
    // arithmetic from an indexed load).
    if (Op->hasOneUse() || Op->getNumValues() > 1)
      AddToWorklist(Op.getNode());
 
  DAG.DeleteNode(N);
}
 
// APInts must be the same size for most operations, this helper
// function zero extends the shorter of the pair so that they match.
// We provide an Offset so that we can create bitwidths that won't overflow.
static void zeroExtendToMatch(APInt &LHS, APInt &RHS, unsigned Offset = 0) {
  unsigned Bits = Offset + std::max(LHS.getBitWidth(), RHS.getBitWidth());
  LHS = LHS.zext(Bits);
  RHS = RHS.zext(Bits);
}
 
// Return true if this node is a setcc, or is a select_cc
// that selects between the target values used for true and false, making it
// equivalent to a setcc. Also, set the incoming LHS, RHS, and CC references to
// the appropriate nodes based on the type of node we are checking. This
// simplifies life a bit for the callers.
bool DAGCombiner::isSetCCEquivalent(SDValue N, SDValue &LHS, SDValue &RHS,
                                    SDValue &CC, bool MatchStrict) const {
  if (N.getOpcode() == ISD::SETCC) {
    LHS = N.getOperand(0);
    RHS = N.getOperand(1);
    CC  = N.getOperand(2);
    return true;
  }
 
  if (MatchStrict &&
      (N.getOpcode() == ISD::STRICT_FSETCC ||
       N.getOpcode() == ISD::STRICT_FSETCCS)) {
    LHS = N.getOperand(1);
    RHS = N.getOperand(2);
    CC  = N.getOperand(3);
    return true;
  }
 
  if (N.getOpcode() != ISD::SELECT_CC || !TLI.isConstTrueVal(N.getOperand(2)) ||
      !TLI.isConstFalseVal(N.getOperand(3)))
    return false;
 
  if (TLI.getBooleanContents(N.getValueType()) ==
      TargetLowering::UndefinedBooleanContent)
    return false;
 
  LHS = N.getOperand(0);
  RHS = N.getOperand(1);
  CC  = N.getOperand(4);
  return true;
}
 
/// Return true if this is a SetCC-equivalent operation with only one use.
/// If this is true, it allows the users to invert the operation for free when
/// it is profitable to do so.
bool DAGCombiner::isOneUseSetCC(SDValue N) const {
  SDValue N0, N1, N2;
  if (isSetCCEquivalent(N, N0, N1, N2) && N->hasOneUse())
    return true;
  return false;
}
 
static bool isConstantSplatVectorMaskForType(SDNode *N, EVT ScalarTy) {
  if (!ScalarTy.isSimple())
    return false;
 
  uint64_t MaskForTy = 0ULL;
  switch (ScalarTy.getSimpleVT().SimpleTy) {
  case MVT::i8:
    MaskForTy = 0xFFULL;
    break;
  case MVT::i16:
    MaskForTy = 0xFFFFULL;
    break;
  case MVT::i32:
    MaskForTy = 0xFFFFFFFFULL;
    break;
  default:
    return false;
    break;
  }
 
  APInt Val;
  if (ISD::isConstantSplatVector(N, Val))
    return Val.getLimitedValue() == MaskForTy;
 
  return false;
}
 
// Determines if it is a constant integer or a splat/build vector of constant
// integers (and undefs).
// Do not permit build vector implicit truncation unless AllowTruncation is set.
static bool isConstantOrConstantVector(SDValue N, bool NoOpaques = false,
                                       bool AllowTruncation = false) {
  if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N))
    return !(Const->isOpaque() && NoOpaques);
  if (N.getOpcode() != ISD::BUILD_VECTOR && N.getOpcode() != ISD::SPLAT_VECTOR)
    return false;
  unsigned BitWidth = N.getScalarValueSizeInBits();
  for (const SDValue &Op : N->op_values()) {
    if (Op.isUndef())
      continue;
    ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Op);
    if (!Const || (Const->isOpaque() && NoOpaques))
      return false;
    // When AllowTruncation is true, allow constants that have been promoted
    // during type legalization as long as the value fits in the target type.
    if ((AllowTruncation &&
         Const->getAPIntValue().getActiveBits() > BitWidth) ||
        (!AllowTruncation && Const->getAPIntValue().getBitWidth() != BitWidth))
      return false;
  }
  return true;
}
 
// Determines if a BUILD_VECTOR is composed of all-constants possibly mixed with
// undef's.
static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) {
  if (V.getOpcode() != ISD::BUILD_VECTOR)
    return false;
  return isConstantOrConstantVector(V, NoOpaques) ||
         ISD::isBuildVectorOfConstantFPSDNodes(V.getNode());
}
 
// Determine if this an indexed load with an opaque target constant index.
static bool canSplitIdx(LoadSDNode *LD) {
  return MaySplitLoadIndex &&
         (LD->getOperand(2).getOpcode() != ISD::TargetConstant ||
          !cast<ConstantSDNode>(LD->getOperand(2))->isOpaque());
}
 
bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc,
                                                             const SDLoc &DL,
                                                             SDNode *N,
                                                             SDValue N0,
                                                             SDValue N1) {
  // Currently this only tries to ensure we don't undo the GEP splits done by
  // CodeGenPrepare when shouldConsiderGEPOffsetSplit is true. To ensure this,
  // we check if the following transformation would be problematic:
  // (load/store (add, (add, x, offset1), offset2)) ->
  // (load/store (add, x, offset1+offset2)).
 
  // (load/store (add, (add, x, y), offset2)) ->
  // (load/store (add, (add, x, offset2), y)).
 
  if (!N0.isAnyAdd())
    return false;
 
  // Check for vscale addressing modes.
  // (load/store (add/sub (add x, y), vscale))
  // (load/store (add/sub (add x, y), (lsl vscale, C)))
  // (load/store (add/sub (add x, y), (mul vscale, C)))
  if ((N1.getOpcode() == ISD::VSCALE ||
       ((N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::MUL) &&
        N1.getOperand(0).getOpcode() == ISD::VSCALE &&
        isa<ConstantSDNode>(N1.getOperand(1)))) &&
      N1.getValueType().getFixedSizeInBits() <= 64) {
    int64_t ScalableOffset = N1.getOpcode() == ISD::VSCALE
                                 ? N1.getConstantOperandVal(0)
                                 : (N1.getOperand(0).getConstantOperandVal(0) *
                                    (N1.getOpcode() == ISD::SHL
                                         ? (1LL << N1.getConstantOperandVal(1))
                                         : N1.getConstantOperandVal(1)));
    if (Opc == ISD::SUB)
      ScalableOffset = -ScalableOffset;
    if (all_of(N->users(), [&](SDNode *Node) {
          if (auto *LoadStore = dyn_cast<MemSDNode>(Node);
              LoadStore && LoadStore->hasUniqueMemOperand() &&
              LoadStore->getBasePtr().getNode() == N) {
            TargetLoweringBase::AddrMode AM;
            AM.HasBaseReg = true;
            AM.ScalableOffset = ScalableOffset;
            EVT VT = LoadStore->getMemoryVT();
            unsigned AS = LoadStore->getAddressSpace();
            Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
            return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy,
                                             AS);
          }
          return false;
        }))
      return true;
  }
 
  if (Opc != ISD::ADD && Opc != ISD::PTRADD)
    return false;
 
  auto *C2 = dyn_cast<ConstantSDNode>(N1);
  if (!C2)
    return false;
 
  const APInt &C2APIntVal = C2->getAPIntValue();
  if (C2APIntVal.getSignificantBits() > 64)
    return false;
 
  if (auto *C1 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
    if (N0.hasOneUse())
      return false;
 
    const APInt &C1APIntVal = C1->getAPIntValue();
    const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal;
    if (CombinedValueIntVal.getSignificantBits() > 64)
      return false;
    const int64_t CombinedValue = CombinedValueIntVal.getSExtValue();
 
    for (SDNode *Node : N->users()) {
      if (auto *LoadStore = dyn_cast<MemSDNode>(Node)) {
        if (!LoadStore->hasUniqueMemOperand())
          continue;
        // Is x[offset2] already not a legal addressing mode? If so then
        // reassociating the constants breaks nothing (we test offset2 because
        // that's the one we hope to fold into the load or store).
        TargetLoweringBase::AddrMode AM;
        AM.HasBaseReg = true;
        AM.BaseOffs = C2APIntVal.getSExtValue();
        EVT VT = LoadStore->getMemoryVT();
        unsigned AS = LoadStore->getAddressSpace();
        Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
        if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
          continue;
 
        // Would x[offset1+offset2] still be a legal addressing mode?
        AM.BaseOffs = CombinedValue;
        if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
          return true;
      }
    }
  } else {
    if (auto *GA = dyn_cast<GlobalAddressSDNode>(N0.getOperand(1)))
      if (GA->getOpcode() == ISD::GlobalAddress && TLI.isOffsetFoldingLegal(GA))
        return false;
 
    for (SDNode *Node : N->users()) {
      auto *LoadStore = dyn_cast<MemSDNode>(Node);
      if (!LoadStore || !LoadStore->hasUniqueMemOperand())
        return false;
 
      // Is x[offset2] a legal addressing mode? If so then
      // reassociating the constants breaks address pattern
      TargetLoweringBase::AddrMode AM;
      AM.HasBaseReg = true;
      AM.BaseOffs = C2APIntVal.getSExtValue();
      EVT VT = LoadStore->getMemoryVT();
      unsigned AS = LoadStore->getAddressSpace();
      Type *AccessTy = VT.getTypeForEVT(*DAG.getContext());
      if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS))
        return false;
    }
    return true;
  }
 
  return false;
}
 
/// Helper for DAGCombiner::reassociateOps. Try to reassociate (Opc N0, N1) if
/// \p N0 is the same kind of operation as \p Opc.
SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL,
                                               SDValue N0, SDValue N1,
                                               SDNodeFlags Flags) {
  EVT VT = N0.getValueType();
 
  if (N0.getOpcode() != Opc)
    return SDValue();
 
  SDValue N00 = N0.getOperand(0);
  SDValue N01 = N0.getOperand(1);
 
  if (DAG.isConstantIntBuildVectorOrConstantInt(N01)) {
    SDNodeFlags NewFlags;
    if (N0.getOpcode() == ISD::ADD && N0->getFlags().hasNoUnsignedWrap() &&
        Flags.hasNoUnsignedWrap())
      NewFlags |= SDNodeFlags::NoUnsignedWrap;
 
    if (DAG.isConstantIntBuildVectorOrConstantInt(N1)) {
      // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2))
      if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, {N01, N1})) {
        NewFlags.setDisjoint(Flags.hasDisjoint() &&
                             N0->getFlags().hasDisjoint());
        return DAG.getNode(Opc, DL, VT, N00, OpNode, NewFlags);
      }
      return SDValue();
    }
    if (TLI.isReassocProfitable(DAG, N0, N1)) {
      // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1)
      //              iff (op x, c1) has one use
      SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1, NewFlags);
      return DAG.getNode(Opc, DL, VT, OpNode, N01, NewFlags);
    }
  }
 
  // Check for repeated operand logic simplifications.
  if (Opc == ISD::AND || Opc == ISD::OR) {
    // (N00 & N01) & N00 --> N00 & N01
    // (N00 & N01) & N01 --> N00 & N01
    // (N00 | N01) | N00 --> N00 | N01
    // (N00 | N01) | N01 --> N00 | N01
    if (N1 == N00 || N1 == N01)
      return N0;
  }
  if (Opc == ISD::XOR) {
    // (N00 ^ N01) ^ N00 --> N01
    if (N1 == N00)
      return N01;
    // (N00 ^ N01) ^ N01 --> N00
    if (N1 == N01)
      return N00;
  }
 
  if (TLI.isReassocProfitable(DAG, N0, N1)) {
    if (N1 != N01) {
      // Reassociate if (op N00, N1) already exist
      if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N00, N1})) {
        // if Op (Op N00, N1), N01 already exist
        // we need to stop reassciate to avoid dead loop
        if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N01}))
          return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N01);
      }
    }
 
    if (N1 != N00) {
      // Reassociate if (op N01, N1) already exist
      if (SDNode *NE = DAG.getNodeIfExists(Opc, DAG.getVTList(VT), {N01, N1})) {
        // if Op (Op N01, N1), N00 already exist
        // we need to stop reassciate to avoid dead loop
        if (!DAG.doesNodeExist(Opc, DAG.getVTList(VT), {SDValue(NE, 0), N00}))
          return DAG.getNode(Opc, DL, VT, SDValue(NE, 0), N00);
      }
    }
 
    // Reassociate the operands from (OR/AND (OR/AND(N00, N001)), N1) to (OR/AND
    // (OR/AND(N00, N1)), N01) when N00 and N1 are comparisons with the same
    // predicate or to (OR/AND (OR/AND(N1, N01)), N00) when N01 and N1 are
    // comparisons with the same predicate. This enables optimizations as the
    // following one:
    // CMP(A,C)||CMP(B,C) => CMP(MIN/MAX(A,B), C)
    // CMP(A,C)&&CMP(B,C) => CMP(MIN/MAX(A,B), C)
    if (Opc == ISD::AND || Opc == ISD::OR) {
      if (N1->getOpcode() == ISD::SETCC && N00->getOpcode() == ISD::SETCC &&
          N01->getOpcode() == ISD::SETCC) {
        ISD::CondCode CC1 = cast<CondCodeSDNode>(N1.getOperand(2))->get();
        ISD::CondCode CC00 = cast<CondCodeSDNode>(N00.getOperand(2))->get();
        ISD::CondCode CC01 = cast<CondCodeSDNode>(N01.getOperand(2))->get();
        if (CC1 == CC00 && CC1 != CC01) {
          SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N00, N1, Flags);
          return DAG.getNode(Opc, DL, VT, OpNode, N01, Flags);
        }
        if (CC1 == CC01 && CC1 != CC00) {
          SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N01, N1, Flags);
          return DAG.getNode(Opc, DL, VT, OpNode, N00, Flags);
        }
      }
    }
  }
 
  return SDValue();
}
 
/// Try to reassociate commutative (Opc N0, N1) if either \p N0 or \p N1 is the
/// same kind of operation as \p Opc.
SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0,
                                    SDValue N1, SDNodeFlags Flags) {
  assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative.");
 
  // Floating-point reassociation is not allowed without loose FP math.
  if (N0.getValueType().isFloatingPoint() ||
      N1.getValueType().isFloatingPoint())
    if (!Flags.hasAllowReassociation() || !Flags.hasNoSignedZeros())
      return SDValue();
 
  if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1, Flags))
    return Combined;
  if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0, Flags))
    return Combined;
  return SDValue();
}
 
// Try to fold Opc(vecreduce(x), vecreduce(y)) -> vecreduce(Opc(x, y))
// Note that we only expect Flags to be passed from FP operations. For integer
// operations they need to be dropped.
SDValue DAGCombiner::reassociateReduction(unsigned RedOpc, unsigned Opc,
                                          const SDLoc &DL, EVT VT, SDValue N0,
                                          SDValue N1, SDNodeFlags Flags) {
  if (N0.getOpcode() == RedOpc && N1.getOpcode() == RedOpc &&
      N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType() &&
      N0->hasOneUse() && N1->hasOneUse() &&
      TLI.isOperationLegalOrCustom(Opc, N0.getOperand(0).getValueType()) &&
      TLI.shouldReassociateReduction(RedOpc, N0.getOperand(0).getValueType())) {
    SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
    return DAG.getNode(RedOpc, DL, VT,
                       DAG.getNode(Opc, DL, N0.getOperand(0).getValueType(),
                                   N0.getOperand(0), N1.getOperand(0)));
  }
 
  // Reassociate op(op(vecreduce(a), b), op(vecreduce(c), d)) into
  // op(vecreduce(op(a, c)), op(b, d)), to combine the reductions into a
  // single node.
  SDValue A, B, C, D, RedA, RedB;
  if (sd_match(N0, m_OneUse(m_c_BinOp(
                       Opc,
                       m_AllOf(m_OneUse(m_UnaryOp(RedOpc, m_Value(A))),
                               m_Value(RedA)),
                       m_Value(B)))) &&
      sd_match(N1, m_OneUse(m_c_BinOp(
                       Opc,
                       m_AllOf(m_OneUse(m_UnaryOp(RedOpc, m_Value(C))),
                               m_Value(RedB)),
                       m_Value(D)))) &&
      !sd_match(B, m_UnaryOp(RedOpc, m_Value())) &&
      !sd_match(D, m_UnaryOp(RedOpc, m_Value())) &&
      A.getValueType() == C.getValueType() &&
      hasOperation(Opc, A.getValueType()) &&
      TLI.shouldReassociateReduction(RedOpc, VT)) {
    if ((Opc == ISD::FADD || Opc == ISD::FMUL) &&
        (!N0->getFlags().hasAllowReassociation() ||
         !N1->getFlags().hasAllowReassociation() ||
         !RedA->getFlags().hasAllowReassociation() ||
         !RedB->getFlags().hasAllowReassociation()))
      return SDValue();
    SelectionDAG::FlagInserter FlagsInserter(
        DAG, Flags & N0->getFlags() & N1->getFlags() & RedA->getFlags() &
                 RedB->getFlags());
    SDValue Op = DAG.getNode(Opc, DL, A.getValueType(), A, C);
    SDValue Red = DAG.getNode(RedOpc, DL, VT, Op);
    SDValue Op2 = DAG.getNode(Opc, DL, VT, B, D);
    return DAG.getNode(Opc, DL, VT, Red, Op2);
  }
  return SDValue();
}
 
SDValue DAGCombiner::CombineTo(SDNode *N, const SDValue *To, unsigned NumTo,
                               bool AddTo) {
  assert(N->getNumValues() == NumTo && "Broken CombineTo call!");
  ++NodesCombined;
  LLVM_DEBUG(dbgs() << "\nReplacing.1 "; N->dump(&DAG); dbgs() << "\nWith: ";
             To[0].dump(&DAG);
             dbgs() << " and " << NumTo - 1 << " other values\n");
  for (unsigned i = 0, e = NumTo; i != e; ++i)
    assert((!To[i].getNode() ||
            N->getValueType(i) == To[i].getValueType()) &&
           "Cannot combine value to value of different type!");
 
  WorklistRemover DeadNodes(*this);
  DAG.ReplaceAllUsesWith(N, To);
  if (AddTo) {
    // Push the new nodes and any users onto the worklist
    for (unsigned i = 0, e = NumTo; i != e; ++i) {
      if (To[i].getNode())
        AddToWorklistWithUsers(To[i].getNode());
    }
  }
 
  // Finally, if the node is now dead, remove it from the graph.  The node
  // may not be dead if the replacement process recursively simplified to
  // something else needing this node.
  if (N->use_empty())
    deleteAndRecombine(N);
  return SDValue(N, 0);
}
 
void DAGCombiner::
CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) {
  // Replace the old value with the new one.
  ++NodesCombined;
  LLVM_DEBUG(dbgs() << "\nReplacing.2 "; TLO.Old.dump(&DAG);
             dbgs() << "\nWith: "; TLO.New.dump(&DAG); dbgs() << '\n');
 
  // Replace all uses.
  DAG.ReplaceAllUsesOfValueWith(TLO.Old, TLO.New);
 
  // Push the new node and any (possibly new) users onto the worklist.
  AddToWorklistWithUsers(TLO.New.getNode());
 
  // Finally, if the node is now dead, remove it from the graph.
  recursivelyDeleteUnusedNodes(TLO.Old.getNode());
}
 
/// Check the specified integer node value to see if it can be simplified or if
/// things it uses can be simplified by bit propagation. If so, return true.
bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits,
                                       const APInt &DemandedElts,
                                       bool AssumeSingleUse) {
  TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
  KnownBits Known;
  if (!TLI.SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO, 0,
                                AssumeSingleUse))
    return false;
 
  // Revisit the node.
  AddToWorklist(Op.getNode());
 
  CommitTargetLoweringOpt(TLO);
  return true;
}
 
/// Check the specified vector node value to see if it can be simplified or
/// if things it uses can be simplified as it only uses some of the elements.
/// If so, return true.
bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op,
                                             const APInt &DemandedElts,
                                             bool AssumeSingleUse) {
  TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations);
  APInt KnownUndef, KnownZero;
  if (!TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero,
                                      TLO, 0, AssumeSingleUse))
    return false;
 
  // Revisit the node.
  AddToWorklist(Op.getNode());
 
  CommitTargetLoweringOpt(TLO);
  return true;
}
 
void DAGCombiner::ReplaceLoadWithPromotedLoad(SDNode *Load, SDNode *ExtLoad) {
  SDLoc DL(Load);
  EVT VT = Load->getValueType(0);
  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, SDValue(ExtLoad, 0));
 
  LLVM_DEBUG(dbgs() << "\nReplacing.9 "; Load->dump(&DAG); dbgs() << "\nWith: ";
             Trunc.dump(&DAG); dbgs() << '\n');
 
  DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), Trunc);
  DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), SDValue(ExtLoad, 1));
 
  AddToWorklist(Trunc.getNode());
  recursivelyDeleteUnusedNodes(Load);
}
 
SDValue DAGCombiner::PromoteOperand(SDValue Op, EVT PVT, bool &Replace) {
  Replace = false;
  SDLoc DL(Op);
  if (ISD::isUNINDEXEDLoad(Op.getNode())) {
    LoadSDNode *LD = cast<LoadSDNode>(Op);
    EVT MemVT = LD->getMemoryVT();
    ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
                                                      : LD->getExtensionType();
    Replace = true;
    return DAG.getExtLoad(ExtType, DL, PVT,
                          LD->getChain(), LD->getBasePtr(),
                          MemVT, LD->getMemOperand());
  }
 
  unsigned Opc = Op.getOpcode();
  switch (Opc) {
  default: break;
  case ISD::AssertSext:
    if (SDValue Op0 = SExtPromoteOperand(Op.getOperand(0), PVT))
      return DAG.getNode(ISD::AssertSext, DL, PVT, Op0, Op.getOperand(1));
    break;
  case ISD::AssertZext:
    if (SDValue Op0 = ZExtPromoteOperand(Op.getOperand(0), PVT))
      return DAG.getNode(ISD::AssertZext, DL, PVT, Op0, Op.getOperand(1));
    break;
  case ISD::Constant: {
    unsigned ExtOpc =
      Op.getValueType().isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
    return DAG.getNode(ExtOpc, DL, PVT, Op);
  }
  }
 
  if (!TLI.isOperationLegal(ISD::ANY_EXTEND, PVT))
    return SDValue();
  return DAG.getNode(ISD::ANY_EXTEND, DL, PVT, Op);
}
 
SDValue DAGCombiner::SExtPromoteOperand(SDValue Op, EVT PVT) {
  if (!TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG, PVT))
    return SDValue();
  EVT OldVT = Op.getValueType();
  SDLoc DL(Op);
  bool Replace = false;
  SDValue NewOp = PromoteOperand(Op, PVT, Replace);
  if (!NewOp.getNode())
    return SDValue();
  AddToWorklist(NewOp.getNode());
 
  if (Replace)
    ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
  return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, NewOp.getValueType(), NewOp,
                     DAG.getValueType(OldVT));
}
 
SDValue DAGCombiner::ZExtPromoteOperand(SDValue Op, EVT PVT) {
  EVT OldVT = Op.getValueType();
  SDLoc DL(Op);
  bool Replace = false;
  SDValue NewOp = PromoteOperand(Op, PVT, Replace);
  if (!NewOp.getNode())
    return SDValue();
  AddToWorklist(NewOp.getNode());
 
  if (Replace)
    ReplaceLoadWithPromotedLoad(Op.getNode(), NewOp.getNode());
  return DAG.getZeroExtendInReg(NewOp, DL, OldVT);
}
 
/// Promote the specified integer binary operation if the target indicates it is
/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
/// i32 since i16 instructions are longer.
SDValue DAGCombiner::PromoteIntBinOp(SDValue Op) {
  if (!LegalOperations)
    return SDValue();
 
  EVT VT = Op.getValueType();
  if (VT.isVector() || !VT.isInteger())
    return SDValue();
 
  // If operation type is 'undesirable', e.g. i16 on x86, consider
  // promoting it.
  unsigned Opc = Op.getOpcode();
  if (TLI.isTypeDesirableForOp(Opc, VT))
    return SDValue();
 
  EVT PVT = VT;
  // Consult target whether it is a good idea to promote this operation and
  // what's the right type to promote it to.
  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    assert(PVT != VT && "Don't know what type to promote to!");
 
    LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
 
    bool Replace0 = false;
    SDValue N0 = Op.getOperand(0);
    SDValue NN0 = PromoteOperand(N0, PVT, Replace0);
 
    bool Replace1 = false;
    SDValue N1 = Op.getOperand(1);
    SDValue NN1 = PromoteOperand(N1, PVT, Replace1);
    SDLoc DL(Op);
 
    SDValue RV =
        DAG.getNode(ISD::TRUNCATE, DL, VT, DAG.getNode(Opc, DL, PVT, NN0, NN1));
 
    // We are always replacing N0/N1's use in N and only need additional
    // replacements if there are additional uses.
    // Note: We are checking uses of the *nodes* (SDNode) rather than values
    //       (SDValue) here because the node may reference multiple values
    //       (for example, the chain value of a load node).
    Replace0 &= !N0->hasOneUse();
    Replace1 &= (N0 != N1) && !N1->hasOneUse();
 
    // Combine Op here so it is preserved past replacements.
    CombineTo(Op.getNode(), RV);
 
    // If operands have a use ordering, make sure we deal with
    // predecessor first.
    if (Replace0 && Replace1 && N0->isPredecessorOf(N1.getNode())) {
      std::swap(N0, N1);
      std::swap(NN0, NN1);
    }
 
    if (Replace0) {
      AddToWorklist(NN0.getNode());
      ReplaceLoadWithPromotedLoad(N0.getNode(), NN0.getNode());
    }
    if (Replace1) {
      AddToWorklist(NN1.getNode());
      ReplaceLoadWithPromotedLoad(N1.getNode(), NN1.getNode());
    }
    return Op;
  }
  return SDValue();
}
 
/// Promote the specified integer shift operation if the target indicates it is
/// beneficial. e.g. On x86, it's usually better to promote i16 operations to
/// i32 since i16 instructions are longer.
SDValue DAGCombiner::PromoteIntShiftOp(SDValue Op) {
  if (!LegalOperations)
    return SDValue();
 
  EVT VT = Op.getValueType();
  if (VT.isVector() || !VT.isInteger())
    return SDValue();
 
  // If operation type is 'undesirable', e.g. i16 on x86, consider
  // promoting it.
  unsigned Opc = Op.getOpcode();
  if (TLI.isTypeDesirableForOp(Opc, VT))
    return SDValue();
 
  EVT PVT = VT;
  // Consult target whether it is a good idea to promote this operation and
  // what's the right type to promote it to.
  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    assert(PVT != VT && "Don't know what type to promote to!");
 
    LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
 
    SDNodeFlags TruncFlags;
    bool Replace = false;
    SDValue N0 = Op.getOperand(0);
    if (Opc == ISD::SRA) {
      N0 = SExtPromoteOperand(N0, PVT);
    } else if (Opc == ISD::SRL) {
      N0 = ZExtPromoteOperand(N0, PVT);
    } else {
      if (Op->getFlags().hasNoUnsignedWrap()) {
        N0 = ZExtPromoteOperand(N0, PVT);
        TruncFlags = SDNodeFlags::NoUnsignedWrap;
      } else if (Op->getFlags().hasNoSignedWrap()) {
        N0 = SExtPromoteOperand(N0, PVT);
        TruncFlags = SDNodeFlags::NoSignedWrap;
      } else {
        N0 = PromoteOperand(N0, PVT, Replace);
      }
    }
 
    if (!N0.getNode())
      return SDValue();
 
    SDLoc DL(Op);
    SDValue N1 = Op.getOperand(1);
    SDValue RV = DAG.getNode(ISD::TRUNCATE, DL, VT,
                             DAG.getNode(Opc, DL, PVT, N0, N1), TruncFlags);
 
    if (Replace)
      ReplaceLoadWithPromotedLoad(Op.getOperand(0).getNode(), N0.getNode());
 
    // Deal with Op being deleted.
    if (Op && Op.getOpcode() != ISD::DELETED_NODE)
      return RV;
  }
  return SDValue();
}
 
SDValue DAGCombiner::PromoteExtend(SDValue Op) {
  if (!LegalOperations)
    return SDValue();
 
  EVT VT = Op.getValueType();
  if (VT.isVector() || !VT.isInteger())
    return SDValue();
 
  // If operation type is 'undesirable', e.g. i16 on x86, consider
  // promoting it.
  unsigned Opc = Op.getOpcode();
  if (TLI.isTypeDesirableForOp(Opc, VT))
    return SDValue();
 
  EVT PVT = VT;
  // Consult target whether it is a good idea to promote this operation and
  // what's the right type to promote it to.
  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    assert(PVT != VT && "Don't know what type to promote to!");
    // fold (aext (aext x)) -> (aext x)
    // fold (aext (zext x)) -> (zext x)
    // fold (aext (sext x)) -> (sext x)
    LLVM_DEBUG(dbgs() << "\nPromoting "; Op.dump(&DAG));
    return DAG.getNode(Op.getOpcode(), SDLoc(Op), VT, Op.getOperand(0));
  }
  return SDValue();
}
 
bool DAGCombiner::PromoteLoad(SDValue Op) {
  if (!LegalOperations)
    return false;
 
  if (!ISD::isUNINDEXEDLoad(Op.getNode()))
    return false;
 
  EVT VT = Op.getValueType();
  if (VT.isVector() || !VT.isInteger())
    return false;
 
  // If operation type is 'undesirable', e.g. i16 on x86, consider
  // promoting it.
  unsigned Opc = Op.getOpcode();
  if (TLI.isTypeDesirableForOp(Opc, VT))
    return false;
 
  EVT PVT = VT;
  // Consult target whether it is a good idea to promote this operation and
  // what's the right type to promote it to.
  if (TLI.IsDesirableToPromoteOp(Op, PVT)) {
    assert(PVT != VT && "Don't know what type to promote to!");
 
    SDLoc DL(Op);
    SDNode *N = Op.getNode();
    LoadSDNode *LD = cast<LoadSDNode>(N);
    EVT MemVT = LD->getMemoryVT();
    ISD::LoadExtType ExtType = ISD::isNON_EXTLoad(LD) ? ISD::EXTLOAD
                                                      : LD->getExtensionType();
    SDValue NewLD = DAG.getExtLoad(ExtType, DL, PVT,
                                   LD->getChain(), LD->getBasePtr(),
                                   MemVT, LD->getMemOperand());
    SDValue Result = DAG.getNode(ISD::TRUNCATE, DL, VT, NewLD);
 
    LLVM_DEBUG(dbgs() << "\nPromoting "; N->dump(&DAG); dbgs() << "\nTo: ";
               Result.dump(&DAG); dbgs() << '\n');
 
    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 1), NewLD.getValue(1));
 
    AddToWorklist(Result.getNode());
    recursivelyDeleteUnusedNodes(N);
    return true;
  }
 
  return false;
}
 
/// Recursively delete a node which has no uses and any operands for
/// which it is the only use.
///
/// Note that this both deletes the nodes and removes them from the worklist.
/// It also adds any nodes who have had a user deleted to the worklist as they
/// may now have only one use and subject to other combines.
bool DAGCombiner::recursivelyDeleteUnusedNodes(SDNode *N) {
  if (!N->use_empty())
    return false;
 
  SmallSetVector<SDNode *, 16> Nodes;
  Nodes.insert(N);
  do {
    N = Nodes.pop_back_val();
    if (!N)
      continue;
 
    if (N->use_empty()) {
      for (const SDValue &ChildN : N->op_values())
        Nodes.insert(ChildN.getNode());
 
      removeFromWorklist(N);
      DAG.DeleteNode(N);
    } else {
      AddToWorklist(N);
    }
  } while (!Nodes.empty());
  return true;
}
 
//===----------------------------------------------------------------------===//
//  Main DAG Combiner implementation
//===----------------------------------------------------------------------===//
 
void DAGCombiner::Run(CombineLevel AtLevel) {
  // set the instance variables, so that the various visit routines may use it.
  Level = AtLevel;
  LegalDAG = Level >= AfterLegalizeDAG;
  LegalOperations = Level >= AfterLegalizeVectorOps;
  LegalTypes = Level >= AfterLegalizeTypes;
 
  bool UseTopologicalSorting = EnableTopologicalSorting.getNumOccurrences() > 0
                                   ? EnableTopologicalSorting
                                   : TLI.useTopologicalSorting();
 
  WorklistInserter AddNodes(*this);
 
  if (UseTopologicalSorting)
    DAG.AssignTopologicalOrder();
 
  // Add all the dag nodes to the worklist.
  //
  // Note: All nodes are not added to PruningList here, this is because the only
  // nodes which can be deleted are those which have no uses and all other nodes
  // which would otherwise be added to the worklist by the first call to
  // getNextWorklistEntry are already present in it.
  if (UseTopologicalSorting) {
    for (SDNode &Node : reverse(DAG.allnodes()))
      AddToWorklist(&Node, /* IsCandidateForPruning */ Node.use_empty());
  } else {
    for (SDNode &Node : DAG.allnodes())
      AddToWorklist(&Node, /* IsCandidateForPruning */ Node.use_empty());
  }
 
  // Create a dummy node (which is not added to allnodes), that adds a reference
  // to the root node, preventing it from being deleted, and tracking any
  // changes of the root.
  HandleSDNode Dummy(DAG.getRoot());
 
  // While we have a valid worklist entry node, try to combine it.
  while (SDNode *N = getNextWorklistEntry()) {
    // If N has no uses, it is dead.  Make sure to revisit all N's operands once
    // N is deleted from the DAG, since they too may now be dead or may have a
    // reduced number of uses, allowing other xforms.
    if (recursivelyDeleteUnusedNodes(N))
      continue;
 
    WorklistRemover DeadNodes(*this);
 
    // If this combine is running after legalizing the DAG, re-legalize any
    // nodes pulled off the worklist.
    if (LegalDAG) {
      SmallSetVector<SDNode *, 16> UpdatedNodes;
      bool NIsValid = DAG.LegalizeOp(N, UpdatedNodes);
 
      for (SDNode *LN : UpdatedNodes)
        AddToWorklistWithUsers(LN);
 
      if (!NIsValid)
        continue;
    }
 
    LLVM_DEBUG(dbgs() << "\nCombining: "; N->dump(&DAG));
 
    // Add any operands of the new node which have not yet been combined to the
    // worklist as well. getNextWorklistEntry flags nodes that have been
    // combined before. Because the worklist uniques things already, this won't
    // repeatedly process the same operand.
    for (const SDValue &ChildN : N->op_values())
      AddToWorklist(ChildN.getNode(), /*IsCandidateForPruning=*/true,
                    /*SkipIfCombinedBefore=*/true);
 
    SDValue RV = combine(N);
 
    if (!RV.getNode())
      continue;
 
    ++NodesCombined;
 
    // Invalidate cached info.
    ChainsWithoutMergeableStores.clear();
 
    // If we get back the same node we passed in, rather than a new node or
    // zero, we know that the node must have defined multiple values and
    // CombineTo was used.  Since CombineTo takes care of the worklist
    // mechanics for us, we have no work to do in this case.
    if (RV.getNode() == N)
      continue;
 
    assert(N->getOpcode() != ISD::DELETED_NODE &&
           RV.getOpcode() != ISD::DELETED_NODE &&
           "Node was deleted but visit returned new node!");
 
    LLVM_DEBUG(dbgs() << " ... into: "; RV.dump(&DAG));
 
    if (N->getNumValues() == RV->getNumValues())
      DAG.ReplaceAllUsesWith(N, RV.getNode());
    else {
      assert(N->getValueType(0) == RV.getValueType() &&
             N->getNumValues() == 1 && "Type mismatch");
      DAG.ReplaceAllUsesWith(N, &RV);
    }
 
    // Push the new node and any users onto the worklist.  Omit this if the
    // new node is the EntryToken (e.g. if a store managed to get optimized
    // out), because re-visiting the EntryToken and its users will not uncover
    // any additional opportunities, but there may be a large number of such
    // users, potentially causing compile time explosion.
    if (RV.getOpcode() != ISD::EntryToken)
      AddToWorklistWithUsers(RV.getNode());
 
    // Finally, if the node is now dead, remove it from the graph.  The node
    // may not be dead if the replacement process recursively simplified to
    // something else needing this node. This will also take care of adding any
    // operands which have lost a user to the worklist.
    recursivelyDeleteUnusedNodes(N);
  }
 
  // If the root changed (e.g. it was a dead load, update the root).
  DAG.setRoot(Dummy.getValue());
  DAG.RemoveDeadNodes();
}
 
SDValue DAGCombiner::visit(SDNode *N) {
  // clang-format off
  switch (N->getOpcode()) {
  default: break;
  case ISD::TokenFactor:        return visitTokenFactor(N);
  case ISD::MERGE_VALUES:       return visitMERGE_VALUES(N);
  case ISD::ADD:                return visitADD(N);
  case ISD::PTRADD:             return visitPTRADD(N);
  case ISD::SUB:                return visitSUB(N);
  case ISD::SADDSAT:
  case ISD::UADDSAT:            return visitADDSAT(N);
  case ISD::SSUBSAT:
  case ISD::USUBSAT:            return visitSUBSAT(N);
  case ISD::ADDC:               return visitADDC(N);
  case ISD::SADDO:
  case ISD::UADDO:              return visitADDO(N);
  case ISD::SUBC:               return visitSUBC(N);
  case ISD::SSUBO:
  case ISD::USUBO:              return visitSUBO(N);
  case ISD::ADDE:               return visitADDE(N);
  case ISD::UADDO_CARRY:        return visitUADDO_CARRY(N);
  case ISD::SADDO_CARRY:        return visitSADDO_CARRY(N);
  case ISD::SUBE:               return visitSUBE(N);
  case ISD::USUBO_CARRY:        return visitUSUBO_CARRY(N);
  case ISD::SSUBO_CARRY:        return visitSSUBO_CARRY(N);
  case ISD::SMULFIX:
  case ISD::SMULFIXSAT:
  case ISD::UMULFIX:
  case ISD::UMULFIXSAT:         return visitMULFIX(N);
  case ISD::MUL:                return visitMUL(N);
  case ISD::SDIV:               return visitSDIV(N);
  case ISD::UDIV:               return visitUDIV(N);
  case ISD::SREM:
  case ISD::UREM:               return visitREM(N);
  case ISD::MULHU:              return visitMULHU(N);
  case ISD::MULHS:              return visitMULHS(N);
  case ISD::AVGFLOORS:
  case ISD::AVGFLOORU:
  case ISD::AVGCEILS:
  case ISD::AVGCEILU:           return visitAVG(N);
  case ISD::ABDS:
  case ISD::ABDU:               return visitABD(N);
  case ISD::SMUL_LOHI:          return visitSMUL_LOHI(N);
  case ISD::UMUL_LOHI:          return visitUMUL_LOHI(N);
  case ISD::SMULO:
  case ISD::UMULO:              return visitMULO(N);
  case ISD::SMIN:
  case ISD::SMAX:
  case ISD::UMIN:
  case ISD::UMAX:               return visitIMINMAX(N);
  case ISD::AND:                return visitAND(N);
  case ISD::OR:                 return visitOR(N);
  case ISD::XOR:                return visitXOR(N);
  case ISD::SHL:                return visitSHL(N);
  case ISD::SRA:                return visitSRA(N);
  case ISD::SRL:                return visitSRL(N);
  case ISD::ROTR:
  case ISD::ROTL:               return visitRotate(N);
  case ISD::FSHL:
  case ISD::FSHR:               return visitFunnelShift(N);
  case ISD::SSHLSAT:
  case ISD::USHLSAT:            return visitSHLSAT(N);
  case ISD::ABS:                return visitABS(N);
  case ISD::ABS_MIN_POISON:     return visitABS_MIN_POISON(N);
  case ISD::CLMUL:
  case ISD::CLMULR:
  case ISD::CLMULH:             return visitCLMUL(N);
  case ISD::BSWAP:              return visitBSWAP(N);
  case ISD::BITREVERSE:         return visitBITREVERSE(N);
  case ISD::CTLZ:               return visitCTLZ(N);
  case ISD::CTLZ_ZERO_POISON:   return visitCTLZ_ZERO_POISON(N);
  case ISD::CTTZ:               return visitCTTZ(N);
  case ISD::CTTZ_ZERO_POISON:   return visitCTTZ_ZERO_POISON(N);
  case ISD::CTPOP:              return visitCTPOP(N);
  case ISD::SELECT:             return visitSELECT(N);
  case ISD::VSELECT:            return visitVSELECT(N);
  case ISD::SELECT_CC:          return visitSELECT_CC(N);
  case ISD::SETCC:              return visitSETCC(N);
  case ISD::SETCCCARRY:         return visitSETCCCARRY(N);
  case ISD::SIGN_EXTEND:        return visitSIGN_EXTEND(N);
  case ISD::ZERO_EXTEND:        return visitZERO_EXTEND(N);
  case ISD::ANY_EXTEND:         return visitANY_EXTEND(N);
  case ISD::AssertSext:
  case ISD::AssertZext:         return visitAssertExt(N);
  case ISD::AssertAlign:        return visitAssertAlign(N);
  case ISD::IS_FPCLASS:         return visitIS_FPCLASS(N);
  case ISD::SIGN_EXTEND_INREG:  return visitSIGN_EXTEND_INREG(N);
  case ISD::SIGN_EXTEND_VECTOR_INREG:
  case ISD::ZERO_EXTEND_VECTOR_INREG:
  case ISD::ANY_EXTEND_VECTOR_INREG: return visitEXTEND_VECTOR_INREG(N);
  case ISD::TRUNCATE:           return visitTRUNCATE(N);
  case ISD::TRUNCATE_USAT_U:    return visitTRUNCATE_USAT_U(N);
  case ISD::BITCAST:            return visitBITCAST(N);
  case ISD::BUILD_PAIR:         return visitBUILD_PAIR(N);
  case ISD::FADD:               return visitFADD(N);
  case ISD::STRICT_FADD:        return visitSTRICT_FADD(N);
  case ISD::FSUB:               return visitFSUB(N);
  case ISD::FMUL:               return visitFMUL(N);
  case ISD::FMA:                return visitFMA(N);
  case ISD::FMAD:               return visitFMAD(N);
  case ISD::FMULADD:            return visitFMULADD(N);
  case ISD::FDIV:               return visitFDIV(N);
  case ISD::FREM:               return visitFREM(N);
  case ISD::FSQRT:              return visitFSQRT(N);
  case ISD::FCOPYSIGN:          return visitFCOPYSIGN(N);
  case ISD::FPOW:               return visitFPOW(N);
  case ISD::SINT_TO_FP:         return visitSINT_TO_FP(N);
  case ISD::UINT_TO_FP:         return visitUINT_TO_FP(N);
  case ISD::FP_TO_SINT:         return visitFP_TO_SINT(N);
  case ISD::FP_TO_UINT:         return visitFP_TO_UINT(N);
  case ISD::LROUND:
  case ISD::LLROUND:
  case ISD::LRINT:
  case ISD::LLRINT:             return visitXROUND(N);
  case ISD::FP_ROUND:           return visitFP_ROUND(N);
  case ISD::FP_EXTEND:          return visitFP_EXTEND(N);
  case ISD::FNEG:               return visitFNEG(N);
  case ISD::FABS:               return visitFABS(N);
  case ISD::FFLOOR:             return visitFFLOOR(N);
  case ISD::FMINNUM:
  case ISD::FMAXNUM:
  case ISD::FMINIMUM:
  case ISD::FMAXIMUM:
  case ISD::FMINIMUMNUM:
  case ISD::FMAXIMUMNUM:       return visitFMinMax(N);
  case ISD::FCEIL:              return visitFCEIL(N);
  case ISD::FTRUNC:             return visitFTRUNC(N);
  case ISD::FFREXP:             return visitFFREXP(N);
  case ISD::BRCOND:             return visitBRCOND(N);
  case ISD::BR_CC:              return visitBR_CC(N);
  case ISD::LOAD:               return visitLOAD(N);
  case ISD::STORE:              return visitSTORE(N);
  case ISD::ATOMIC_STORE:       return visitATOMIC_STORE(N);
  case ISD::INSERT_VECTOR_ELT:  return visitINSERT_VECTOR_ELT(N);
  case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
  case ISD::BUILD_VECTOR:       return visitBUILD_VECTOR(N);
  case ISD::CONCAT_VECTORS:     return visitCONCAT_VECTORS(N);
  case ISD::VECTOR_INTERLEAVE:  return visitVECTOR_INTERLEAVE(N);
  case ISD::EXTRACT_SUBVECTOR:  return visitEXTRACT_SUBVECTOR(N);
  case ISD::VECTOR_SHUFFLE:     return visitVECTOR_SHUFFLE(N);
  case ISD::SCALAR_TO_VECTOR:   return visitSCALAR_TO_VECTOR(N);
  case ISD::INSERT_SUBVECTOR:   return visitINSERT_SUBVECTOR(N);
  case ISD::MGATHER:            return visitMGATHER(N);
  case ISD::MLOAD:              return visitMLOAD(N);
  case ISD::MSCATTER:           return visitMSCATTER(N);
  case ISD::MSTORE:             return visitMSTORE(N);
  case ISD::EXPERIMENTAL_VECTOR_HISTOGRAM: return visitMHISTOGRAM(N);
  case ISD::PARTIAL_REDUCE_SMLA:
  case ISD::PARTIAL_REDUCE_UMLA:
  case ISD::PARTIAL_REDUCE_SUMLA:
  case ISD::PARTIAL_REDUCE_FMLA:
                                return visitPARTIAL_REDUCE_MLA(N);
  case ISD::VECTOR_COMPRESS:    return visitVECTOR_COMPRESS(N);
  case ISD::LIFETIME_END:       return visitLIFETIME_END(N);
  case ISD::FP_TO_FP16:         return visitFP_TO_FP16(N);
  case ISD::FP16_TO_FP:         return visitFP16_TO_FP(N);
  case ISD::FP_TO_BF16:         return visitFP_TO_BF16(N);
  case ISD::BF16_TO_FP:         return visitBF16_TO_FP(N);
  case ISD::FREEZE:             return visitFREEZE(N);
  case ISD::GET_FPENV_MEM:      return visitGET_FPENV_MEM(N);
  case ISD::SET_FPENV_MEM:      return visitSET_FPENV_MEM(N);
  case ISD::FCANONICALIZE:      return visitFCANONICALIZE(N);
  case ISD::VECREDUCE_FADD:
  case ISD::VECREDUCE_FMUL:
  case ISD::VECREDUCE_ADD:
  case ISD::VECREDUCE_MUL:
  case ISD::VECREDUCE_AND:
  case ISD::VECREDUCE_OR:
  case ISD::VECREDUCE_XOR:
  case ISD::VECREDUCE_SMAX:
  case ISD::VECREDUCE_SMIN:
  case ISD::VECREDUCE_UMAX:
  case ISD::VECREDUCE_UMIN:
  case ISD::VECREDUCE_FMAX:
  case ISD::VECREDUCE_FMIN:
  case ISD::VECREDUCE_FMAXIMUM:
  case ISD::VECREDUCE_FMINIMUM:     return visitVECREDUCE(N);
#define BEGIN_REGISTER_VP_SDNODE(SDOPC, ...) case ISD::SDOPC:
#include "llvm/IR/VPIntrinsics.def"
    return visitVPOp(N);
  }
  // clang-format on
  return SDValue();
}
 
SDValue DAGCombiner::combine(SDNode *N) {
  if (!DebugCounter::shouldExecute(DAGCombineCounter))
    return SDValue();
 
  SDValue RV;
  if (!DisableGenericCombines)
    RV = visit(N);
 
  // If nothing happened, try a target-specific DAG combine.
  if (!RV.getNode()) {
    assert(N->getOpcode() != ISD::DELETED_NODE &&
           "Node was deleted but visit returned NULL!");
 
    if (N->getOpcode() >= ISD::BUILTIN_OP_END ||
        TLI.hasTargetDAGCombine((ISD::NodeType)N->getOpcode())) {
 
      // Expose the DAG combiner to the target combiner impls.
      TargetLowering::DAGCombinerInfo
        DagCombineInfo(DAG, Level, false, this);
 
      RV = TLI.PerformDAGCombine(N, DagCombineInfo);
    }
  }
 
  // If nothing happened still, try promoting the operation.
  if (!RV.getNode()) {
    switch (N->getOpcode()) {
    default: break;
    case ISD::ADD:
    case ISD::SUB:
    case ISD::MUL:
    case ISD::AND:
    case ISD::OR:
    case ISD::XOR:
      RV = PromoteIntBinOp(SDValue(N, 0));
      break;
    case ISD::SHL:
    case ISD::SRA:
    case ISD::SRL:
      RV = PromoteIntShiftOp(SDValue(N, 0));
      break;
    case ISD::SIGN_EXTEND:
    case ISD::ZERO_EXTEND:
    case ISD::ANY_EXTEND:
      RV = PromoteExtend(SDValue(N, 0));
      break;
    case ISD::LOAD:
      if (PromoteLoad(SDValue(N, 0)))
        RV = SDValue(N, 0);
      break;
    }
  }
 
  // If N is a commutative binary node, try to eliminate it if the commuted
  // version is already present in the DAG.
  if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode())) {
    SDValue N0 = N->getOperand(0);
    SDValue N1 = N->getOperand(1);
 
    // Constant operands are canonicalized to RHS.
    if (N0 != N1 && (isa<ConstantSDNode>(N0) || !isa<ConstantSDNode>(N1))) {
      SDValue Ops[] = {N1, N0};
      SDNode *CSENode = DAG.getNodeIfExists(N->getOpcode(), N->getVTList(), Ops,
                                            N->getFlags());
      if (CSENode)
        return SDValue(CSENode, 0);
    }
  }
 
  return RV;
}
 
/// Given a node, return its input chain if it has one, otherwise return a null
/// sd operand.
static SDValue getInputChainForNode(SDNode *N) {
  if (unsigned NumOps = N->getNumOperands()) {
    if (N->getOperand(0).getValueType() == MVT::Other)
      return N->getOperand(0);
    if (N->getOperand(NumOps-1).getValueType() == MVT::Other)
      return N->getOperand(NumOps-1);
    for (unsigned i = 1; i < NumOps-1; ++i)
      if (N->getOperand(i).getValueType() == MVT::Other)
        return N->getOperand(i);
  }
  return SDValue();
}
 
SDValue DAGCombiner::visitFCANONICALIZE(SDNode *N) {
  SDValue Operand = N->getOperand(0);
  EVT VT = Operand.getValueType();
  SDLoc dl(N);
 
  // Canonicalize undef to quiet NaN.
  if (Operand.isUndef()) {
    APFloat CanonicalQNaN = APFloat::getQNaN(VT.getFltSemantics());
    return DAG.getConstantFP(CanonicalQNaN, dl, VT);
  }
  return SDValue();
}
 
SDValue DAGCombiner::visitTokenFactor(SDNode *N) {
  // If N has two operands, where one has an input chain equal to the other,
  // the 'other' chain is redundant.
  if (N->getNumOperands() == 2) {
    if (getInputChainForNode(N->getOperand(0).getNode()) == N->getOperand(1))
      return N->getOperand(0);
    if (getInputChainForNode(N->getOperand(1).getNode()) == N->getOperand(0))
      return N->getOperand(1);
  }
 
  // Don't simplify token factors if optnone.
  if (OptLevel == CodeGenOptLevel::None)
    return SDValue();
 
  // Don't simplify the token factor if the node itself has too many operands.
  if (N->getNumOperands() > TokenFactorInlineLimit)
    return SDValue();
 
  // If the sole user is a token factor, we should make sure we have a
  // chance to merge them together. This prevents TF chains from inhibiting
  // optimizations.
  if (N->hasOneUse() && N->user_begin()->getOpcode() == ISD::TokenFactor)
    AddToWorklist(*(N->user_begin()));
 
  SmallVector<SDNode *, 8> TFs;     // List of token factors to visit.
  SmallVector<SDValue, 8> Ops;      // Ops for replacing token factor.
  SmallPtrSet<SDNode*, 16> SeenOps;
  bool Changed = false;             // If we should replace this token factor.
 
  // Start out with this token factor.
  TFs.push_back(N);
 
  // Iterate through token factors.  The TFs grows when new token factors are
  // encountered.
  for (unsigned i = 0; i < TFs.size(); ++i) {
    // Limit number of nodes to inline, to avoid quadratic compile times.
    // We have to add the outstanding Token Factors to Ops, otherwise we might
    // drop Ops from the resulting Token Factors.
    if (Ops.size() > TokenFactorInlineLimit) {
      for (unsigned j = i; j < TFs.size(); j++)
        Ops.emplace_back(TFs[j], 0);
      // Drop unprocessed Token Factors from TFs, so we do not add them to the
      // combiner worklist later.
      TFs.resize(i);
      break;
    }
 
    SDNode *TF = TFs[i];
    // Check each of the operands.
    for (const SDValue &Op : TF->op_values()) {
      switch (Op.getOpcode()) {
      case ISD::EntryToken:
        // Entry tokens don't need to be added to the list. They are
        // redundant.
        Changed = true;
        break;
 
      case ISD::TokenFactor:
        if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) {
          // Queue up for processing.
          TFs.push_back(Op.getNode());
          Changed = true;
          break;
        }
        [[fallthrough]];
 
      default:
        // Only add if it isn't already in the list.
        if (SeenOps.insert(Op.getNode()).second)
          Ops.push_back(Op);
        else
          Changed = true;
        break;
      }
    }
  }
 
  // Re-visit inlined Token Factors, to clean them up in case they have been
  // removed. Skip the first Token Factor, as this is the current node.
  for (unsigned i = 1, e = TFs.size(); i < e; i++)
    AddToWorklist(TFs[i]);
 
  // Remove Nodes that are chained to another node in the list. Do so
  // by walking up chains breath-first stopping when we've seen
  // another operand. In general we must climb to the EntryNode, but we can exit
  // early if we find all remaining work is associated with just one operand as
  // no further pruning is possible.
 
  // List of nodes to search through and original Ops from which they originate.
  SmallVector<std::pair<SDNode *, unsigned>, 8> Worklist;
  SmallVector<unsigned, 8> OpWorkCount; // Count of work for each Op.
  SmallPtrSet<SDNode *, 16> SeenChains;
  bool DidPruneOps = false;
 
  unsigned NumLeftToConsider = 0;
  for (const SDValue &Op : Ops) {
    Worklist.push_back(std::make_pair(Op.getNode(), NumLeftToConsider++));
    OpWorkCount.push_back(1);
  }
 
  auto AddToWorklist = [&](unsigned CurIdx, SDNode *Op, unsigned OpNumber) {
    // If this is an Op, we can remove the op from the list. Remark any
    // search associated with it as from the current OpNumber.
    if (SeenOps.contains(Op)) {
      Changed = true;
      DidPruneOps = true;
      unsigned OrigOpNumber = 0;
      while (OrigOpNumber < Ops.size() && Ops[OrigOpNumber].getNode() != Op)
        OrigOpNumber++;
      assert((OrigOpNumber != Ops.size()) &&
             "expected to find TokenFactor Operand");
      // Re-mark worklist from OrigOpNumber to OpNumber
      for (unsigned i = CurIdx + 1; i < Worklist.size(); ++i) {
        if (Worklist[i].second == OrigOpNumber) {
          Worklist[i].second = OpNumber;
        }
      }
      OpWorkCount[OpNumber] += OpWorkCount[OrigOpNumber];
      OpWorkCount[OrigOpNumber] = 0;
      NumLeftToConsider--;
    }
    // Add if it's a new chain
    if (SeenChains.insert(Op).second) {
      OpWorkCount[OpNumber]++;
      Worklist.push_back(std::make_pair(Op, OpNumber));
    }
  };
 
  for (unsigned i = 0; i < Worklist.size() && i < 1024; ++i) {
    // We need at least be consider at least 2 Ops to prune.
    if (NumLeftToConsider <= 1)
      break;
    auto CurNode = Worklist[i].first;
    auto CurOpNumber = Worklist[i].second;
    assert((OpWorkCount[CurOpNumber] > 0) &&
           "Node should not appear in worklist");
    switch (CurNode->getOpcode()) {
    case ISD::EntryToken:
      // Hitting EntryToken is the only way for the search to terminate without
      // hitting
      // another operand's search. Prevent us from marking this operand
      // considered.
      NumLeftToConsider++;
      break;
    case ISD::TokenFactor:
      for (const SDValue &Op : CurNode->op_values())
        AddToWorklist(i, Op.getNode(), CurOpNumber);
      break;
    case ISD::LIFETIME_START:
    case ISD::LIFETIME_END:
    case ISD::CopyFromReg:
    case ISD::CopyToReg:
      AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber);
      break;
    default:
      if (auto *MemNode = dyn_cast<MemSDNode>(CurNode))
        AddToWorklist(i, MemNode->getChain().getNode(), CurOpNumber);
      break;
    }
    OpWorkCount[CurOpNumber]--;
    if (OpWorkCount[CurOpNumber] == 0)
      NumLeftToConsider--;
  }
 
  // If we've changed things around then replace token factor.
  if (Changed) {
    SDValue Result;
    if (Ops.empty()) {
      // The entry token is the only possible outcome.
      Result = DAG.getEntryNode();
    } else {
      if (DidPruneOps) {
        SmallVector<SDValue, 8> PrunedOps;
        //
        for (const SDValue &Op : Ops) {
          if (SeenChains.count(Op.getNode()) == 0)
            PrunedOps.push_back(Op);
        }
        Result = DAG.getTokenFactor(SDLoc(N), PrunedOps);
      } else {
        Result = DAG.getTokenFactor(SDLoc(N), Ops);
      }
    }
    return Result;
  }
  return SDValue();
}
 
/// MERGE_VALUES can always be eliminated.
SDValue DAGCombiner::visitMERGE_VALUES(SDNode *N) {
  WorklistRemover DeadNodes(*this);
  // Replacing results may cause a different MERGE_VALUES to suddenly
  // be CSE'd with N, and carry its uses with it. Iterate until no
  // uses remain, to ensure that the node can be safely deleted.
  // First add the users of this node to the work list so that they
  // can be tried again once they have new operands.
  AddUsersToWorklist(N);
  do {
    // Do as a single replacement to avoid rewalking use lists.
    SmallVector<SDValue, 8> Ops(N->ops());
    DAG.ReplaceAllUsesWith(N, Ops.data());
  } while (!N->use_empty());
  deleteAndRecombine(N);
  return SDValue(N, 0);   // Return N so it doesn't get rechecked!
}
 
/// If \p N is a ConstantSDNode with isOpaque() == false return it casted to a
/// ConstantSDNode pointer else nullptr.
static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) {
  ConstantSDNode *Const = dyn_cast<ConstantSDNode>(N);
  return Const != nullptr && !Const->isOpaque() ? Const : nullptr;
}
 
// isTruncateOf - If N is a truncate of some other value, return true, record
// the value being truncated in Op and which of Op's bits are zero/one in Known.
// This function computes KnownBits to avoid a duplicated call to
// computeKnownBits in the caller.
static bool isTruncateOf(SelectionDAG &DAG, SDValue N, SDValue &Op,
                         KnownBits &Known) {
  if (N->getOpcode() == ISD::TRUNCATE) {
    Op = N->getOperand(0);
    Known = DAG.computeKnownBits(Op);
    if (N->getFlags().hasNoUnsignedWrap())
      Known.Zero.setBitsFrom(N.getScalarValueSizeInBits());
    return true;
  }
 
  if (N.getValueType().getScalarType() != MVT::i1 ||
      !sd_match(
          N, m_c_SetCC(m_Value(Op), m_Zero(), m_SpecificCondCode(ISD::SETNE))))
    return false;
 
  Known = DAG.computeKnownBits(Op);
  return (Known.Zero | 1).isAllOnes();
}
 
/// Return true if 'Use' is a load or a store that uses N as its base pointer
/// and that N may be folded in the load / store addressing mode.
static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, SelectionDAG &DAG,
                                    const TargetLowering &TLI) {
  EVT VT;
  unsigned AS;
 
  if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Use)) {
    if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
      return false;
    VT = LD->getMemoryVT();
    AS = LD->getAddressSpace();
  } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Use)) {
    if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
      return false;
    VT = ST->getMemoryVT();
    AS = ST->getAddressSpace();
  } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(Use)) {
    if (LD->isIndexed() || LD->getBasePtr().getNode() != N)
      return false;
    VT = LD->getMemoryVT();
    AS = LD->getAddressSpace();
  } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(Use)) {
    if (ST->isIndexed() || ST->getBasePtr().getNode() != N)
      return false;
    VT = ST->getMemoryVT();
    AS = ST->getAddressSpace();
  } else {
    return false;
  }
 
  TargetLowering::AddrMode AM;
  if (N->isAnyAdd()) {
    AM.HasBaseReg = true;
    ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
    if (Offset)
      // [reg +/- imm]
      AM.BaseOffs = Offset->getSExtValue();
    else
      // [reg +/- reg]
      AM.Scale = 1;
  } else if (N->getOpcode() == ISD::SUB) {
    AM.HasBaseReg = true;
    ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
    if (Offset)
      // [reg +/- imm]
      AM.BaseOffs = -Offset->getSExtValue();
    else
      // [reg +/- reg]
      AM.Scale = 1;
  } else {
    return false;
  }
 
  return TLI.isLegalAddressingMode(DAG.getDataLayout(), AM,
                                   VT.getTypeForEVT(*DAG.getContext()), AS);
}
 
/// This inverts a canonicalization in IR that replaces a variable select arm
/// with an identity constant. Codegen improves if we re-use the variable
/// operand rather than load a constant. This can also be converted into a
/// masked vector operation if the target supports it.
static SDValue foldSelectWithIdentityConstant(SDNode *N, SelectionDAG &DAG,
                                              bool ShouldCommuteOperands) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
 
  // Match a select as operand 1. The identity constant that we are looking for
  // is only valid as operand 1 of a non-commutative binop.
  if (ShouldCommuteOperands)
    std::swap(N0, N1);
 
  SDValue Cond, TVal, FVal;
  if (!sd_match(N1, m_OneUse(m_SelectLike(m_Value(Cond), m_Value(TVal),
                                          m_Value(FVal)))))
    return SDValue();
 
  // We can't hoist all instructions because of immediate UB (not speculatable).
  // For example div/rem by zero.
  if (!DAG.isSafeToSpeculativelyExecuteNode(N))
    return SDValue();
 
  unsigned SelOpcode = N1.getOpcode();
  unsigned Opcode = N->getOpcode();
  EVT VT = N->getValueType(0);
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
  // This transform increases uses of N0, so freeze it to be safe.
  // binop N0, (vselect Cond, IDC, FVal) --> vselect Cond, N0, (binop N0, FVal)
  unsigned OpNo = ShouldCommuteOperands ? 0 : 1;
  if (DAG.isIdentityElement(Opcode, N->getFlags(), TVal, OpNo) &&
      TLI.shouldFoldSelectWithIdentityConstant(Opcode, VT, SelOpcode, N0,
                                               FVal)) {
    SDValue F0 = DAG.getFreeze(N0);
    SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, FVal, N->getFlags());
    return DAG.getSelect(SDLoc(N), VT, Cond, F0, NewBO);
  }
  // binop N0, (vselect Cond, TVal, IDC) --> vselect Cond, (binop N0, TVal), N0
  if (DAG.isIdentityElement(Opcode, N->getFlags(), FVal, OpNo) &&
      TLI.shouldFoldSelectWithIdentityConstant(Opcode, VT, SelOpcode, N0,
                                               TVal)) {
    SDValue F0 = DAG.getFreeze(N0);
    SDValue NewBO = DAG.getNode(Opcode, SDLoc(N), VT, F0, TVal, N->getFlags());
    return DAG.getSelect(SDLoc(N), VT, Cond, NewBO, F0);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) {
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 &&
         "Unexpected binary operator");
 
  if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, false))
    return Sel;
 
  if (TLI.isCommutativeBinOp(BO->getOpcode()))
    if (SDValue Sel = foldSelectWithIdentityConstant(BO, DAG, true))
      return Sel;
 
  // Don't do this unless the old select is going away. We want to eliminate the
  // binary operator, not replace a binop with a select.
  // TODO: Handle ISD::SELECT_CC.
  unsigned SelOpNo = 0;
  SDValue Sel = BO->getOperand(0);
  auto BinOpcode = BO->getOpcode();
  if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
    SelOpNo = 1;
    Sel = BO->getOperand(1);
 
    // Peek through trunc to shift amount type.
    if ((BinOpcode == ISD::SHL || BinOpcode == ISD::SRA ||
         BinOpcode == ISD::SRL) && Sel.hasOneUse()) {
      // This is valid when the truncated bits of x are already zero.
      SDValue Op;
      KnownBits Known;
      if (isTruncateOf(DAG, Sel, Op, Known) &&
          Known.countMaxActiveBits() < Sel.getScalarValueSizeInBits())
        Sel = Op;
    }
  }
 
  if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
    return SDValue();
 
  SDValue CT = Sel.getOperand(1);
  if (!isConstantOrConstantVector(CT, true) &&
      !DAG.isConstantFPBuildVectorOrConstantFP(CT))
    return SDValue();
 
  SDValue CF = Sel.getOperand(2);
  if (!isConstantOrConstantVector(CF, true) &&
      !DAG.isConstantFPBuildVectorOrConstantFP(CF))
    return SDValue();
 
  // Bail out if any constants are opaque because we can't constant fold those.
  // The exception is "and" and "or" with either 0 or -1 in which case we can
  // propagate non constant operands into select. I.e.:
  // and (select Cond, 0, -1), X --> select Cond, 0, X
  // or X, (select Cond, -1, 0) --> select Cond, -1, X
  bool CanFoldNonConst =
      (BinOpcode == ISD::AND || BinOpcode == ISD::OR) &&
      ((isNullOrNullSplat(CT) && isAllOnesOrAllOnesSplat(CF)) ||
       (isNullOrNullSplat(CF) && isAllOnesOrAllOnesSplat(CT)));
 
  SDValue CBO = BO->getOperand(SelOpNo ^ 1);
  if (!CanFoldNonConst &&
      !isConstantOrConstantVector(CBO, true) &&
      !DAG.isConstantFPBuildVectorOrConstantFP(CBO))
    return SDValue();
 
  SDLoc DL(Sel);
  SDValue NewCT, NewCF;
  EVT VT = BO->getValueType(0);
 
  if (CanFoldNonConst) {
    // If CBO is an opaque constant, we can't rely on getNode to constant fold.
    if ((BinOpcode == ISD::AND && isNullOrNullSplat(CT)) ||
        (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(CT)))
      NewCT = CT;
    else
      NewCT = CBO;
 
    if ((BinOpcode == ISD::AND && isNullOrNullSplat(CF)) ||
        (BinOpcode == ISD::OR && isAllOnesOrAllOnesSplat(CF)))
      NewCF = CF;
    else
      NewCF = CBO;
  } else {
    // We have a select-of-constants followed by a binary operator with a
    // constant. Eliminate the binop by pulling the constant math into the
    // select. Example: add (select Cond, CT, CF), CBO --> select Cond, CT +
    // CBO, CF + CBO
    NewCT = SelOpNo ? DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CBO, CT})
                    : DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CT, CBO});
    if (!NewCT)
      return SDValue();
 
    NewCF = SelOpNo ? DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CBO, CF})
                    : DAG.FoldConstantArithmetic(BinOpcode, DL, VT, {CF, CBO});
    if (!NewCF)
      return SDValue();
  }
 
  return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF, BO->getFlags());
}
 
static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, const SDLoc &DL,
                                         SelectionDAG &DAG) {
  assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
         "Expecting add or sub");
 
  // Match a constant operand and a zext operand for the math instruction:
  // add Z, C
  // sub C, Z
  bool IsAdd = N->getOpcode() == ISD::ADD;
  SDValue C = IsAdd ? N->getOperand(1) : N->getOperand(0);
  SDValue Z = IsAdd ? N->getOperand(0) : N->getOperand(1);
  auto *CN = dyn_cast<ConstantSDNode>(C);
  if (!CN || Z.getOpcode() != ISD::ZERO_EXTEND)
    return SDValue();
 
  // Match the zext operand as a setcc of a boolean.
  if (Z.getOperand(0).getValueType() != MVT::i1)
    return SDValue();
 
  // Match the compare as: setcc (X & 1), 0, eq.
  if (!sd_match(Z.getOperand(0), m_SetCC(m_And(m_Value(), m_One()), m_Zero(),
                                         m_SpecificCondCode(ISD::SETEQ))))
    return SDValue();
 
  // We are adding/subtracting a constant and an inverted low bit. Turn that
  // into a subtract/add of the low bit with incremented/decremented constant:
  // add (zext i1 (seteq (X & 1), 0)), C --> sub C+1, (zext (X & 1))
  // sub C, (zext i1 (seteq (X & 1), 0)) --> add C-1, (zext (X & 1))
  EVT VT = C.getValueType();
  SDValue LowBit = DAG.getZExtOrTrunc(Z.getOperand(0).getOperand(0), DL, VT);
  SDValue C1 = IsAdd ? DAG.getConstant(CN->getAPIntValue() + 1, DL, VT)
                     : DAG.getConstant(CN->getAPIntValue() - 1, DL, VT);
  return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, C1, LowBit);
}
 
// Attempt to form avgceil(A, B) from (A | B) - ((A ^ B) >> 1)
SDValue DAGCombiner::foldSubToAvg(SDNode *N, const SDLoc &DL) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N0.getValueType();
  SDValue A, B;
 
  if ((!LegalOperations || hasOperation(ISD::AVGCEILU, VT)) &&
      sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)),
                        m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_One())))) {
    return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B);
  }
  if ((!LegalOperations || hasOperation(ISD::AVGCEILS, VT)) &&
      sd_match(N, m_Sub(m_Or(m_Value(A), m_Value(B)),
                        m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), m_One())))) {
    return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B);
  }
  return SDValue();
}
 
/// Try to fold a pointer arithmetic node.
/// This needs to be done separately from normal addition, because pointer
/// addition is not commutative.
SDValue DAGCombiner::visitPTRADD(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT PtrVT = N0.getValueType();
  EVT IntVT = N1.getValueType();
  SDLoc DL(N);
 
  // This is already ensured by an assert in SelectionDAG::getNode(). Several
  // combines here depend on this assumption.
  assert(PtrVT == IntVT &&
         "PTRADD with different operand types is not supported");
 
  // fold (ptradd x, 0) -> x
  if (isNullConstant(N1))
    return N0;
 
  // fold (ptradd 0, x) -> x
  if (PtrVT == IntVT && isNullConstant(N0))
    return N1;
 
  if (N0.getOpcode() == ISD::PTRADD &&
      !reassociationCanBreakAddressingModePattern(ISD::PTRADD, DL, N, N0, N1)) {
    SDValue X = N0.getOperand(0);
    SDValue Y = N0.getOperand(1);
    SDValue Z = N1;
    bool N0OneUse = N0.hasOneUse();
    bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
    bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
 
    // (ptradd (ptradd x, y), z) -> (ptradd x, (add y, z)) if:
    //   * y is a constant and (ptradd x, y) has one use; or
    //   * y and z are both constants.
    if ((YIsConstant && N0OneUse) || (YIsConstant && ZIsConstant)) {
      // If both additions in the original were NUW, the new ones are as well.
      SDNodeFlags Flags =
          (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
      SDValue Add = DAG.getNode(ISD::ADD, DL, IntVT, {Y, Z}, Flags);
      AddToWorklist(Add.getNode());
      // We can't set InBounds even if both original ptradds were InBounds and
      // NUW: SDAG usually represents pointers as integers, therefore, the
      // matched pattern behaves as if it had implicit casts:
      //   (ptradd inbounds (inttoptr (ptrtoint (ptradd inbounds x, y))), z)
      // The outer inbounds ptradd might therefore rely on a provenance that x
      // does not have.
      return DAG.getMemBasePlusOffset(X, Add, DL, Flags);
    }
  }
 
  // The following combines can turn in-bounds pointer arithmetic out of bounds.
  // That is problematic for settings like AArch64's CPA, which checks that
  // intermediate results of pointer arithmetic remain in bounds. The target
  // therefore needs to opt-in to enable them.
  if (!TLI.canTransformPtrArithOutOfBounds(
          DAG.getMachineFunction().getFunction(), PtrVT))
    return SDValue();
 
  if (N0.getOpcode() == ISD::PTRADD && isa<ConstantSDNode>(N1)) {
    // Fold (ptradd (ptradd GA, v), c) -> (ptradd (ptradd GA, c) v) with
    // global address GA and constant c, such that c can be folded into GA.
    // TODO: Support constant vector splats.
    SDValue GAValue = N0.getOperand(0);
    if (const GlobalAddressSDNode *GA =
            dyn_cast<GlobalAddressSDNode>(GAValue)) {
      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
      if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
        // If both additions in the original were NUW, reassociation preserves
        // that.
        SDNodeFlags Flags =
            (N->getFlags() & N0->getFlags()) & SDNodeFlags::NoUnsignedWrap;
        // We can't set InBounds even if both original ptradds were InBounds and
        // NUW: SDAG usually represents pointers as integers, therefore, the
        // matched pattern behaves as if it had implicit casts:
        //   (ptradd inbounds (inttoptr (ptrtoint (ptradd inbounds GA, v))), c)
        // The outer inbounds ptradd might therefore rely on a provenance that
        // GA does not have.
        SDValue Inner = DAG.getMemBasePlusOffset(GAValue, N1, DL, Flags);
        AddToWorklist(Inner.getNode());
        return DAG.getMemBasePlusOffset(Inner, N0.getOperand(1), DL, Flags);
      }
    }
  }
 
  if (N1.getOpcode() == ISD::ADD && N1.hasOneUse()) {
    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant,
    //    y is not, and (add y, z) is used only once.
    // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant,
    //    z is not, and (add y, z) is used only once.
    // The goal is to move constant offsets to the outermost ptradd, to create
    // more opportunities to fold offsets into memory instructions.
    // Together with the another combine above, this also implements
    //   (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)).
    SDValue X = N0;
    SDValue Y = N1.getOperand(0);
    SDValue Z = N1.getOperand(1);
    bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y);
    bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z);
 
    // If both additions in the original were NUW, reassociation preserves that.
    SDNodeFlags CommonFlags = N->getFlags() & N1->getFlags();
    SDNodeFlags ReassocFlags = CommonFlags & SDNodeFlags::NoUnsignedWrap;
    if (CommonFlags.hasNoUnsignedWrap()) {
      // If both operations are NUW and the PTRADD is inbounds, the offests are
      // both non-negative, so the reassociated PTRADDs are also inbounds.
      ReassocFlags |= N->getFlags() & SDNodeFlags::InBounds;
    }
 
    if (ZIsConstant != YIsConstant) {
      if (YIsConstant)
        std::swap(Y, Z);
      SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, ReassocFlags);
      AddToWorklist(Inner.getNode());
      return DAG.getMemBasePlusOffset(Inner, Z, DL, ReassocFlags);
    }
  }
 
  // Transform (ptradd a, b) -> (or disjoint a, b) if it is equivalent and if
  // that transformation can't block an offset folding at any use of the ptradd.
  // This should be done late, after legalization, so that it doesn't block
  // other ptradd combines that could enable more offset folding.
  if (LegalOperations && DAG.haveNoCommonBitsSet(N0, N1)) {
    bool TransformCannotBreakAddrMode = none_of(N->users(), [&](SDNode *User) {
      return canFoldInAddressingMode(N, User, DAG, TLI);
    });
 
    if (TransformCannotBreakAddrMode)
      return DAG.getNode(ISD::OR, DL, PtrVT, N0, N1, SDNodeFlags::Disjoint);
  }
 
  return SDValue();
}
 
/// Try to fold a 'not' shifted sign-bit with add/sub with constant operand into
/// a shift and add with a different constant.
static SDValue foldAddSubOfSignBit(SDNode *N, const SDLoc &DL,
                                   SelectionDAG &DAG) {
  assert((N->getOpcode() == ISD::ADD || N->getOpcode() == ISD::SUB) &&
         "Expecting add or sub");
 
  // We need a constant operand for the add/sub, and the other operand is a
  // logical shift right: add (srl), C or sub C, (srl).
  bool IsAdd = N->getOpcode() == ISD::ADD;
  SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0);
  SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1);
  if (!DAG.isConstantIntBuildVectorOrConstantInt(ConstantOp) ||
      ShiftOp.getOpcode() != ISD::SRL)
    return SDValue();
 
  // The shift must be of a 'not' value.
  SDValue Not = ShiftOp.getOperand(0);
  if (!Not.hasOneUse() || !isBitwiseNot(Not))
    return SDValue();
 
  // The shift must be moving the sign bit to the least-significant-bit.
  EVT VT = ShiftOp.getValueType();
  SDValue ShAmt = ShiftOp.getOperand(1);
  ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
  if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1))
    return SDValue();
 
  // Eliminate the 'not' by adjusting the shift and add/sub constant:
  // add (srl (not X), 31), C --> add (sra X, 31), (C + 1)
  // sub C, (srl (not X), 31) --> add (srl X, 31), (C - 1)
  if (SDValue NewC = DAG.FoldConstantArithmetic(
          IsAdd ? ISD::ADD : ISD::SUB, DL, VT,
          {ConstantOp, DAG.getConstant(1, DL, VT)})) {
    SDValue NewShift = DAG.getNode(IsAdd ? ISD::SRA : ISD::SRL, DL, VT,
                                   Not.getOperand(0), ShAmt);
    return DAG.getNode(ISD::ADD, DL, VT, NewShift, NewC);
  }
 
  return SDValue();
}
 
static bool
areBitwiseNotOfEachother(SDValue Op0, SDValue Op1) {
  return (isBitwiseNot(Op0) && Op0.getOperand(0) == Op1) ||
         (isBitwiseNot(Op1) && Op1.getOperand(0) == Op0);
}
 
/// Try to fold a node that behaves like an ADD (note that N isn't necessarily
/// an ISD::ADD here, it could for example be an ISD::OR if we know that there
/// are no common bits set in the operands).
SDValue DAGCombiner::visitADDLike(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  SDLoc DL(N);
 
  // fold (add x, undef) -> undef
  if (N0.isUndef())
    return N0;
  if (N1.isUndef())
    return N1;
 
  // fold (add c1, c2) -> c1+c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
 
  if (areBitwiseNotOfEachother(N0, N1))
    return DAG.getConstant(APInt::getAllOnes(VT.getScalarSizeInBits()), DL, VT);
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (add x, 0) -> x, vector edition
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      return N0;
  }
 
  // fold (add x, 0) -> x
  if (isNullConstant(N1))
    return N0;
 
  if (N0.getOpcode() == ISD::SUB) {
    SDValue N00 = N0.getOperand(0);
    SDValue N01 = N0.getOperand(1);
 
    // fold ((A-c1)+c2) -> (A+(c2-c1))
    if (SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N1, N01}))
      return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub);
 
    // fold ((c1-A)+c2) -> (c1+c2)-A
    if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N00}))
      return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
  }
 
  // add (sext i1 X), 1 -> zext (not i1 X)
  // We don't transform this pattern:
  //   add (zext i1 X), -1 -> sext (not i1 X)
  // because most (?) targets generate better code for the zext form.
  if (N0.getOpcode() == ISD::SIGN_EXTEND && N0.hasOneUse() &&
      isOneOrOneSplat(N1)) {
    SDValue X = N0.getOperand(0);
    if ((!LegalOperations ||
         (TLI.isOperationLegal(ISD::XOR, X.getValueType()) &&
          TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) &&
        X.getScalarValueSizeInBits() == 1) {
      SDValue Not = DAG.getNOT(DL, X, X.getValueType());
      return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Not);
    }
  }
 
  // Fold (add (or x, c0), c1) -> (add x, (c0 + c1))
  // iff (or x, c0) is equivalent to (add x, c0).
  // Fold (add (xor x, c0), c1) -> (add x, (c0 + c1))
  // iff (xor x, c0) is equivalent to (add x, c0).
  if (DAG.isADDLike(N0)) {
    SDValue N01 = N0.getOperand(1);
    if (SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N1, N01}))
      return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Add);
  }
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // reassociate add
  if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N, N0, N1)) {
    if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags()))
      return RADD;
 
    // (X + Y) + X --> Y + (X + X)
    SDValue X, Y, InnerAdd;
    if (sd_match(
            N, m_Add(m_OneUse(m_Value(InnerAdd, m_Add(m_Value(X), m_Value(Y)))),
                     m_Deferred(X)))) {
      if (X != Y) {
        // Redistribute shared NUW flag.
        // TODO: If NSW+NUW occurs on both adds, that can be redistributed too.
        SDNodeFlags NewFlags =
            N->getFlags() & InnerAdd->getFlags() & SDNodeFlags::NoUnsignedWrap;
        SDValue X2 = DAG.getNode(ISD::ADD, DL, VT, X, X, NewFlags);
        return DAG.getNode(ISD::ADD, DL, VT, Y, X2, NewFlags);
      }
    }
 
    // Reassociate (add (or x, c), y) -> (add add(x, y), c)) if (or x, c) is
    // equivalent to (add x, c).
    // Reassociate (add (xor x, c), y) -> (add add(x, y), c)) if (xor x, c) is
    // equivalent to (add x, c).
    // Do this optimization only when adding c does not introduce instructions
    // for adding carries.
    auto ReassociateAddOr = [&](SDValue N0, SDValue N1) {
      if (DAG.isADDLike(N0) && N0.hasOneUse() &&
          isConstantOrConstantVector(N0.getOperand(1), /* NoOpaque */ true)) {
        // If N0's type does not split or is a sign mask, it does not introduce
        // add carry.
        auto TyActn = TLI.getTypeAction(*DAG.getContext(), N0.getValueType());
        bool NoAddCarry = TyActn == TargetLoweringBase::TypeLegal ||
                          TyActn == TargetLoweringBase::TypePromoteInteger ||
                          isMinSignedConstant(N0.getOperand(1));
        if (NoAddCarry)
          return DAG.getNode(
              ISD::ADD, DL, VT,
              DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)),
              N0.getOperand(1));
      }
      return SDValue();
    };
    if (SDValue Add = ReassociateAddOr(N0, N1))
      return Add;
    if (SDValue Add = ReassociateAddOr(N1, N0))
      return Add;
 
    // Fold add(vecreduce(x), vecreduce(y)) -> vecreduce(add(x, y))
    if (SDValue SD =
            reassociateReduction(ISD::VECREDUCE_ADD, ISD::ADD, DL, VT, N0, N1))
      return SD;
  }
 
  SDValue A, B, C, D;
 
  // fold ((0-A) + B) -> B-A
  if (sd_match(N0, m_Neg(m_Value(A))))
    return DAG.getNode(ISD::SUB, DL, VT, N1, A);
 
  // fold (A + (0-B)) -> A-B
  if (sd_match(N1, m_Neg(m_Value(B))))
    return DAG.getNode(ISD::SUB, DL, VT, N0, B);
 
  // fold (A+(B-A)) -> B
  if (sd_match(N1, m_Sub(m_Value(B), m_Specific(N0))))
    return B;
 
  // fold ((B-A)+A) -> B
  if (sd_match(N0, m_Sub(m_Value(B), m_Specific(N1))))
    return B;
 
  // fold ((A-B)+(C-A)) -> (C-B)
  if (sd_match(N0, m_Sub(m_Value(A), m_Value(B))) &&
      sd_match(N1, m_Sub(m_Value(C), m_Specific(A))))
    return DAG.getNode(ISD::SUB, DL, VT, C, B);
 
  // fold ((A-B)+(B-C)) -> (A-C)
  if (sd_match(N0, m_Sub(m_Value(A), m_Value(B))) &&
      sd_match(N1, m_Sub(m_Specific(B), m_Value(C))))
    return DAG.getNode(ISD::SUB, DL, VT, A, C);
 
  // fold (A+(B-(A+C))) to (B-C)
  // fold (A+(B-(C+A))) to (B-C)
  if (sd_match(N1, m_Sub(m_Value(B), m_Add(m_Specific(N0), m_Value(C)))))
    return DAG.getNode(ISD::SUB, DL, VT, B, C);
 
  // fold (A+((B-A)+or-C)) to (B+or-C)
  if (sd_match(N1,
               m_AnyOf(m_Add(m_Sub(m_Value(B), m_Specific(N0)), m_Value(C)),
                       m_Sub(m_Sub(m_Value(B), m_Specific(N0)), m_Value(C)))))
    return DAG.getNode(N1.getOpcode(), DL, VT, B, C);
 
  // fold (A-B)+(C-D) to (A+C)-(B+D) when A or C is constant
  if (sd_match(N0, m_OneUse(m_Sub(m_Value(A), m_Value(B)))) &&
      sd_match(N1, m_OneUse(m_Sub(m_Value(C), m_Value(D)))) &&
      (isConstantOrConstantVector(A) || isConstantOrConstantVector(C)))
    return DAG.getNode(ISD::SUB, DL, VT,
                       DAG.getNode(ISD::ADD, SDLoc(N0), VT, A, C),
                       DAG.getNode(ISD::ADD, SDLoc(N1), VT, B, D));
 
  // fold (add (umax X, C), -C) --> (usubsat X, C)
  if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) {
    auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) {
      return (!Max && !Op) ||
             (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue()));
    };
    if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT,
                                  /*AllowUndefs*/ true))
      return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0),
                         N0.getOperand(1));
  }
 
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  if (isOneOrOneSplat(N1)) {
    // fold (add (xor a, -1), 1) -> (sub 0, a)
    if (isBitwiseNot(N0))
      return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
                         N0.getOperand(0));
 
    // fold (add (add (xor a, -1), b), 1) -> (sub b, a)
    if (N0.getOpcode() == ISD::ADD) {
      SDValue A, Xor;
 
      if (isBitwiseNot(N0.getOperand(0))) {
        A = N0.getOperand(1);
        Xor = N0.getOperand(0);
      } else if (isBitwiseNot(N0.getOperand(1))) {
        A = N0.getOperand(0);
        Xor = N0.getOperand(1);
      }
 
      if (Xor)
        return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0));
    }
 
    // Look for:
    //   add (add x, y), 1
    // And if the target does not like this form then turn into:
    //   sub y, (xor x, -1)
    if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
        N0.hasOneUse() &&
        // Limit this to after legalization if the add has wrap flags
        (Level >= AfterLegalizeDAG || (!N->getFlags().hasNoUnsignedWrap() &&
                                       !N->getFlags().hasNoSignedWrap()))) {
      SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT);
      return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not);
    }
  }
 
  // (x - y) + -1  ->  add (xor y, -1), x
  if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
      isAllOnesOrAllOnesSplat(N1, /*AllowUndefs=*/true)) {
    SDValue Not = DAG.getNOT(DL, N0.getOperand(1), VT);
    return DAG.getNode(ISD::ADD, DL, VT, Not, N0.getOperand(0));
  }
 
  // Fold add(mul(add(A, CA), CM), CB) -> add(mul(A, CM), CM*CA+CB).
  // This can help if the inner add has multiple uses.
  APInt CM, CA;
  if (ConstantSDNode *CB = dyn_cast<ConstantSDNode>(N1)) {
    if (VT.getScalarSizeInBits() <= 64) {
      if (sd_match(N0, m_OneUse(m_Mul(m_Add(m_Value(A), m_ConstInt(CA)),
                                      m_ConstInt(CM)))) &&
          TLI.isLegalAddImmediate(
              (CA * CM + CB->getAPIntValue()).getSExtValue())) {
        SDNodeFlags Flags;
        // If all the inputs are nuw, the outputs can be nuw. If all the input
        // are _also_ nsw the outputs can be too.
        if (N->getFlags().hasNoUnsignedWrap() &&
            N0->getFlags().hasNoUnsignedWrap() &&
            N0.getOperand(0)->getFlags().hasNoUnsignedWrap()) {
          Flags |= SDNodeFlags::NoUnsignedWrap;
          if (N->getFlags().hasNoSignedWrap() &&
              N0->getFlags().hasNoSignedWrap() &&
              N0.getOperand(0)->getFlags().hasNoSignedWrap())
            Flags |= SDNodeFlags::NoSignedWrap;
        }
        SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A,
                                  DAG.getConstant(CM, DL, VT), Flags);
        return DAG.getNode(
            ISD::ADD, DL, VT, Mul,
            DAG.getConstant(CA * CM + CB->getAPIntValue(), DL, VT), Flags);
      }
      // Also look in case there is an intermediate add.
      if (sd_match(N0, m_OneUse(m_Add(
                           m_OneUse(m_Mul(m_Add(m_Value(A), m_ConstInt(CA)),
                                          m_ConstInt(CM))),
                           m_Value(B)))) &&
          TLI.isLegalAddImmediate(
              (CA * CM + CB->getAPIntValue()).getSExtValue())) {
        SDNodeFlags Flags;
        // If all the inputs are nuw, the outputs can be nuw. If all the input
        // are _also_ nsw the outputs can be too.
        SDValue OMul =
            N0.getOperand(0) == B ? N0.getOperand(1) : N0.getOperand(0);
        if (N->getFlags().hasNoUnsignedWrap() &&
            N0->getFlags().hasNoUnsignedWrap() &&
            OMul->getFlags().hasNoUnsignedWrap() &&
            OMul.getOperand(0)->getFlags().hasNoUnsignedWrap()) {
          Flags |= SDNodeFlags::NoUnsignedWrap;
          if (N->getFlags().hasNoSignedWrap() &&
              N0->getFlags().hasNoSignedWrap() &&
              OMul->getFlags().hasNoSignedWrap() &&
              OMul.getOperand(0)->getFlags().hasNoSignedWrap())
            Flags |= SDNodeFlags::NoSignedWrap;
        }
        SDValue Mul = DAG.getNode(ISD::MUL, SDLoc(N1), VT, A,
                                  DAG.getConstant(CM, DL, VT), Flags);
        SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N1), VT, Mul, B, Flags);
        return DAG.getNode(
            ISD::ADD, DL, VT, Add,
            DAG.getConstant(CA * CM + CB->getAPIntValue(), DL, VT), Flags);
      }
    }
  }
 
  if (SDValue Combined = visitADDLikeCommutative(N0, N1, N))
    return Combined;
 
  if (SDValue Combined = visitADDLikeCommutative(N1, N0, N))
    return Combined;
 
  return SDValue();
}
 
// Attempt to form avgfloor(A, B) from (A & B) + ((A ^ B) >> 1)
// Attempt to form avgfloor(A, B) from ((A >> 1) + (B >> 1)) + (A & B & 1)
// Attempt to form avgceil(A, B) from ((A >> 1) + (B >> 1)) + ((A | B) & 1)
SDValue DAGCombiner::foldAddToAvg(SDNode *N, const SDLoc &DL) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N0.getValueType();
  SDValue A, B;
 
  if ((!LegalOperations || hasOperation(ISD::AVGFLOORU, VT)) &&
      (sd_match(N,
                m_Add(m_And(m_Value(A), m_Value(B)),
                      m_Srl(m_Xor(m_Deferred(A), m_Deferred(B)), m_One()))) ||
       sd_match(N, m_ReassociatableAdd(
                       m_ReassociatableAnd(m_Value(A), m_Value(B), m_One()),
                       m_Srl(m_Deferred(A), m_One()),
                       m_Srl(m_Deferred(B), m_One()))))) {
    return DAG.getNode(ISD::AVGFLOORU, DL, VT, A, B);
  }
  if ((!LegalOperations || hasOperation(ISD::AVGFLOORS, VT)) &&
      (sd_match(N,
                m_Add(m_And(m_Value(A), m_Value(B)),
                      m_Sra(m_Xor(m_Deferred(A), m_Deferred(B)), m_One()))) ||
       sd_match(N, m_ReassociatableAdd(
                       m_ReassociatableAnd(m_Value(A), m_Value(B), m_One()),
                       m_Sra(m_Deferred(A), m_One()),
                       m_Sra(m_Deferred(B), m_One()))))) {
    return DAG.getNode(ISD::AVGFLOORS, DL, VT, A, B);
  }
 
  if ((!LegalOperations || hasOperation(ISD::AVGCEILU, VT)) &&
      sd_match(N,
               m_ReassociatableAdd(m_And(m_Or(m_Value(A), m_Value(B)), m_One()),
                                   m_Srl(m_Deferred(A), m_One()),
                                   m_Srl(m_Deferred(B), m_One())))) {
    return DAG.getNode(ISD::AVGCEILU, DL, VT, A, B);
  }
  if ((!LegalOperations || hasOperation(ISD::AVGCEILS, VT)) &&
      sd_match(N,
               m_ReassociatableAdd(m_And(m_Or(m_Value(A), m_Value(B)), m_One()),
                                   m_Sra(m_Deferred(A), m_One()),
                                   m_Sra(m_Deferred(B), m_One())))) {
    return DAG.getNode(ISD::AVGCEILS, DL, VT, A, B);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitADD(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  SDLoc DL(N);
 
  if (SDValue Combined = visitADDLike(N))
    return Combined;
 
  if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
    return V;
 
  if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
    return V;
 
  if (SDValue V = MatchRotate(N0, N1, SDLoc(N), /*FromAdd=*/true))
    return V;
 
  // Try to match AVGFLOOR fixedwidth pattern
  if (SDValue V = foldAddToAvg(N, DL))
    return V;
 
  // fold (a+b) -> (a|b) iff a and b share no bits.
  if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
      DAG.haveNoCommonBitsSet(N0, N1))
    return DAG.getNode(ISD::OR, DL, VT, N0, N1, SDNodeFlags::Disjoint);
 
  // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
  if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
    const APInt &C0 = N0->getConstantOperandAPInt(0);
    const APInt &C1 = N1->getConstantOperandAPInt(0);
    return DAG.getVScale(DL, VT, C0 + C1);
  }
 
  // fold a+vscale(c1)+vscale(c2) -> a+vscale(c1+c2)
  if (N0.getOpcode() == ISD::ADD &&
      N0.getOperand(1).getOpcode() == ISD::VSCALE &&
      N1.getOpcode() == ISD::VSCALE) {
    const APInt &VS0 = N0.getOperand(1)->getConstantOperandAPInt(0);
    const APInt &VS1 = N1->getConstantOperandAPInt(0);
    SDValue VS = DAG.getVScale(DL, VT, VS0 + VS1);
    return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), VS);
  }
 
  // Fold (add step_vector(c1), step_vector(c2)  to step_vector(c1+c2))
  if (N0.getOpcode() == ISD::STEP_VECTOR &&
      N1.getOpcode() == ISD::STEP_VECTOR) {
    const APInt &C0 = N0->getConstantOperandAPInt(0);
    const APInt &C1 = N1->getConstantOperandAPInt(0);
    APInt NewStep = C0 + C1;
    return DAG.getStepVector(DL, VT, NewStep);
  }
 
  // Fold a + step_vector(c1) + step_vector(c2) to a + step_vector(c1+c2)
  if (N0.getOpcode() == ISD::ADD &&
      N0.getOperand(1).getOpcode() == ISD::STEP_VECTOR &&
      N1.getOpcode() == ISD::STEP_VECTOR) {
    const APInt &SV0 = N0.getOperand(1)->getConstantOperandAPInt(0);
    const APInt &SV1 = N1->getConstantOperandAPInt(0);
    APInt NewStep = SV0 + SV1;
    SDValue SV = DAG.getStepVector(DL, VT, NewStep);
    return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), SV);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitADDSAT(SDNode *N) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  bool IsSigned = Opcode == ISD::SADDSAT;
  SDLoc DL(N);
 
  // fold (add_sat x, undef) -> -1
  if (N0.isUndef() || N1.isUndef())
    return DAG.getAllOnesConstant(DL, VT);
 
  // fold (add_sat c1, c2) -> c3
  if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(Opcode, DL, VT, N1, N0);
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (add_sat x, 0) -> x, vector edition
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      return N0;
  }
 
  // fold (add_sat x, 0) -> x
  if (isNullConstant(N1))
    return N0;
 
  // If it cannot overflow, transform into an add.
  if (DAG.willNotOverflowAdd(IsSigned, N0, N1))
    return DAG.getNode(ISD::ADD, DL, VT, N0, N1);
 
  return SDValue();
}
 
static SDValue getAsCarry(const TargetLowering &TLI, SDValue V,
                          bool ForceCarryReconstruction = false) {
  bool Masked = false;
 
  // First, peel away TRUNCATE/ZERO_EXTEND/AND nodes due to legalization.
  while (true) {
    if (ForceCarryReconstruction && V.getValueType() == MVT::i1)
      return V;
 
    if (V.getOpcode() == ISD::TRUNCATE || V.getOpcode() == ISD::ZERO_EXTEND) {
      V = V.getOperand(0);
      continue;
    }
 
    if (V.getOpcode() == ISD::AND && isOneConstant(V.getOperand(1))) {
      if (ForceCarryReconstruction)
        return V;
 
      Masked = true;
      V = V.getOperand(0);
      continue;
    }
 
    break;
  }
 
  // If this is not a carry, return.
  if (V.getResNo() != 1)
    return SDValue();
 
  if (V.getOpcode() != ISD::UADDO_CARRY && V.getOpcode() != ISD::USUBO_CARRY &&
      V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO)
    return SDValue();
 
  EVT VT = V->getValueType(0);
  if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT))
    return SDValue();
 
  // If the result is masked, then no matter what kind of bool it is we can
  // return. If it isn't, then we need to make sure the bool type is either 0 or
  // 1 and not other values.
  if (Masked ||
      TLI.getBooleanContents(V.getValueType()) ==
          TargetLoweringBase::ZeroOrOneBooleanContent)
    return V;
 
  return SDValue();
}
 
/// Given the operands of an add/sub operation, see if the 2nd operand is a
/// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert
/// the opcode and bypass the mask operation.
static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1,
                                 SelectionDAG &DAG, const SDLoc &DL) {
  if (N1.getOpcode() == ISD::ZERO_EXTEND)
    N1 = N1.getOperand(0);
 
  if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1)))
    return SDValue();
 
  EVT VT = N0.getValueType();
  SDValue N10 = N1.getOperand(0);
  if (N10.getValueType() != VT && N10.getOpcode() == ISD::TRUNCATE)
    N10 = N10.getOperand(0);
 
  if (N10.getValueType() != VT)
    return SDValue();
 
  if (DAG.ComputeNumSignBits(N10) != VT.getScalarSizeInBits())
    return SDValue();
 
  // add N0, (and (AssertSext X, i1), 1) --> sub N0, X
  // sub N0, (and (AssertSext X, i1), 1) --> add N0, X
  return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N10);
}
 
/// Helper for doing combines based on N0 and N1 being added to each other.
SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1,
                                             SDNode *LocReference) {
  EVT VT = N0.getValueType();
  SDLoc DL(LocReference);
 
  // fold (add x, shl(0 - y, n)) -> sub(x, shl(y, n))
  SDValue Y, N;
  if (sd_match(N1, m_Shl(m_Neg(m_Value(Y)), m_Value(N))))
    return DAG.getNode(ISD::SUB, DL, VT, N0,
                       DAG.getNode(ISD::SHL, DL, VT, Y, N));
 
  if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL))
    return V;
 
  // Look for:
  //   add (add x, 1), y
  // And if the target does not like this form then turn into:
  //   sub y, (xor x, -1)
  if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
      N0.hasOneUse() && isOneOrOneSplat(N0.getOperand(1)) &&
      // Limit this to after legalization if the add has wrap flags
      (Level >= AfterLegalizeDAG || (!N0->getFlags().hasNoUnsignedWrap() &&
                                     !N0->getFlags().hasNoSignedWrap()))) {
    SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT);
    return DAG.getNode(ISD::SUB, DL, VT, N1, Not);
  }
 
  if (N0.getOpcode() == ISD::SUB && N0.hasOneUse()) {
    // Hoist one-use subtraction by non-opaque constant:
    //   (x - C) + y  ->  (x + y) - C
    // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
    if (isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
      SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1);
      return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1));
    }
    // Hoist one-use subtraction from non-opaque constant:
    //   (C - x) + y  ->  (y - x) + C
    if (isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
      SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1));
      return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0));
    }
  }
 
  // add (mul x, C), x -> mul x, C+1
  if (N0.getOpcode() == ISD::MUL && N0.getOperand(0) == N1 &&
      isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true) &&
      N0.hasOneUse()) {
    SDValue NewC = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1),
                               DAG.getConstant(1, DL, VT));
    return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), NewC);
  }
 
  // If the target's bool is represented as 0/1, prefer to make this 'sub 0/1'
  // rather than 'add 0/-1' (the zext should get folded).
  // add (sext i1 Y), X --> sub X, (zext i1 Y)
  if (N0.getOpcode() == ISD::SIGN_EXTEND &&
      N0.getOperand(0).getScalarValueSizeInBits() == 1 &&
      TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) {
    SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
    return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt);
  }
 
  // add X, (sextinreg Y i1) -> sub X, (and Y 1)
  if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
    if (TN->getVT() == MVT::i1) {
      SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
                                 DAG.getConstant(1, DL, VT));
      return DAG.getNode(ISD::SUB, DL, VT, N0, ZExt);
    }
  }
 
  // (add X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry)
  if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1)) &&
      N1.getResNo() == 0)
    return DAG.getNode(ISD::UADDO_CARRY, DL, N1->getVTList(),
                       N0, N1.getOperand(0), N1.getOperand(2));
 
  // (add X, Carry) -> (uaddo_carry X, 0, Carry)
  if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT))
    if (SDValue Carry = getAsCarry(TLI, N1))
      return DAG.getNode(ISD::UADDO_CARRY, DL,
                         DAG.getVTList(VT, Carry.getValueType()), N0,
                         DAG.getConstant(0, DL, VT), Carry);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitADDC(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  SDLoc DL(N);
 
  // If the flag result is dead, turn this into an ADD.
  if (!N->hasAnyUseOfValue(1))
    return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
                     DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
 
  // canonicalize constant to RHS.
  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
  if (N0C && !N1C)
    return DAG.getNode(ISD::ADDC, DL, N->getVTList(), N1, N0);
 
  // fold (addc x, 0) -> x + no carry out
  if (isNullConstant(N1))
    return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE,
                                        DL, MVT::Glue));
 
  // If it cannot overflow, transform into an add.
  if (DAG.computeOverflowForUnsignedAdd(N0, N1) == SelectionDAG::OFK_Never)
    return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
                     DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
 
  return SDValue();
}
 
/**
 * Flips a boolean if it is cheaper to compute. If the Force parameters is set,
 * then the flip also occurs if computing the inverse is the same cost.
 * This function returns an empty SDValue in case it cannot flip the boolean
 * without increasing the cost of the computation. If you want to flip a boolean
 * no matter what, use DAG.getLogicalNOT.
 */
static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG,
                                  const TargetLowering &TLI,
                                  bool Force) {
  if (Force && isa<ConstantSDNode>(V))
    return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
 
  if (V.getOpcode() != ISD::XOR)
    return SDValue();
 
  if (DAG.isBoolConstant(V.getOperand(1)) == true)
    return V.getOperand(0);
  if (Force && isConstOrConstSplat(V.getOperand(1), false))
    return DAG.getLogicalNOT(SDLoc(V), V, V.getValueType());
  return SDValue();
}
 
SDValue DAGCombiner::visitADDO(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  bool IsSigned = (ISD::SADDO == N->getOpcode());
 
  EVT CarryVT = N->getValueType(1);
  SDLoc DL(N);
 
  // If the flag result is dead, turn this into an ADD.
  if (!N->hasAnyUseOfValue(1))
    return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
                     DAG.getUNDEF(CarryVT));
 
  // canonicalize constant to RHS.
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
 
  // fold (addo x, 0) -> x + no carry out
  if (isNullOrNullSplat(N1))
    return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
 
  // If it cannot overflow, transform into an add.
  if (DAG.willNotOverflowAdd(IsSigned, N0, N1))
    return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1),
                     DAG.getConstant(0, DL, CarryVT));
 
  if (IsSigned) {
    // fold (saddo (xor a, -1), 1) -> (ssub 0, a).
    if (isBitwiseNot(N0) && isOneOrOneSplat(N1))
      return DAG.getNode(ISD::SSUBO, DL, N->getVTList(),
                         DAG.getConstant(0, DL, VT), N0.getOperand(0));
  } else {
    // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry.
    if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) {
      SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(),
                                DAG.getConstant(0, DL, VT), N0.getOperand(0));
      return CombineTo(
          N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
    }
 
    if (SDValue Combined = visitUADDOLike(N0, N1, N))
      return Combined;
 
    if (SDValue Combined = visitUADDOLike(N1, N0, N))
      return Combined;
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) {
  EVT VT = N0.getValueType();
  if (VT.isVector())
    return SDValue();
 
  // (uaddo X, (uaddo_carry Y, 0, Carry)) -> (uaddo_carry X, Y, Carry)
  // If Y + 1 cannot overflow.
  if (N1.getOpcode() == ISD::UADDO_CARRY && isNullConstant(N1.getOperand(1))) {
    SDValue Y = N1.getOperand(0);
    SDValue One = DAG.getConstant(1, SDLoc(N), Y.getValueType());
    if (DAG.computeOverflowForUnsignedAdd(Y, One) == SelectionDAG::OFK_Never)
      return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0, Y,
                         N1.getOperand(2));
  }
 
  // (uaddo X, Carry) -> (uaddo_carry X, 0, Carry)
  if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT))
    if (SDValue Carry = getAsCarry(TLI, N1))
      return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(), N0,
                         DAG.getConstant(0, SDLoc(N), VT), Carry);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitADDE(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue CarryIn = N->getOperand(2);
 
  // canonicalize constant to RHS
  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
  if (N0C && !N1C)
    return DAG.getNode(ISD::ADDE, SDLoc(N), N->getVTList(),
                       N1, N0, CarryIn);
 
  // fold (adde x, y, false) -> (addc x, y)
  if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
    return DAG.getNode(ISD::ADDC, SDLoc(N), N->getVTList(), N0, N1);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitUADDO_CARRY(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue CarryIn = N->getOperand(2);
  SDLoc DL(N);
 
  // canonicalize constant to RHS
  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
  if (N0C && !N1C)
    return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn);
 
  // fold (uaddo_carry x, y, false) -> (uaddo x, y)
  if (isNullConstant(CarryIn)) {
    if (!LegalOperations ||
        TLI.isOperationLegalOrCustom(ISD::UADDO, N->getValueType(0)))
      return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1);
  }
 
  // fold (uaddo_carry 0, 0, X) -> (and (ext/trunc X), 1) and no carry.
  if (isNullConstant(N0) && isNullConstant(N1)) {
    EVT VT = N0.getValueType();
    EVT CarryVT = CarryIn.getValueType();
    SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT);
    AddToWorklist(CarryExt.getNode());
    return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt,
                                    DAG.getConstant(1, DL, VT)),
                     DAG.getConstant(0, DL, CarryVT));
  }
 
  if (SDValue Combined = visitUADDO_CARRYLike(N0, N1, CarryIn, N))
    return Combined;
 
  if (SDValue Combined = visitUADDO_CARRYLike(N1, N0, CarryIn, N))
    return Combined;
 
  // We want to avoid useless duplication.
  // TODO: This is done automatically for binary operations. As UADDO_CARRY is
  // not a binary operation, this is not really possible to leverage this
  // existing mechanism for it. However, if more operations require the same
  // deduplication logic, then it may be worth generalize.
  SDValue Ops[] = {N1, N0, CarryIn};
  SDNode *CSENode =
      DAG.getNodeIfExists(ISD::UADDO_CARRY, N->getVTList(), Ops, N->getFlags());
  if (CSENode)
    return SDValue(CSENode, 0);
 
  return SDValue();
}
 
/**
 * If we are facing some sort of diamond carry propagation pattern try to
 * break it up to generate something like:
 *   (uaddo_carry X, 0, (uaddo_carry A, B, Z):Carry)
 *
 * The end result is usually an increase in operation required, but because the
 * carry is now linearized, other transforms can kick in and optimize the DAG.
 *
 * Patterns typically look something like
 *                (uaddo A, B)
 *                /          \
 *             Carry         Sum
 *               |             \
 *               | (uaddo_carry *, 0, Z)
 *               |       /
 *                \   Carry
 *                 |   /
 * (uaddo_carry X, *, *)
 *
 * But numerous variation exist. Our goal is to identify A, B, X and Z and
 * produce a combine with a single path for carry propagation.
 */
static SDValue combineUADDO_CARRYDiamond(DAGCombiner &Combiner,
                                         SelectionDAG &DAG, SDValue X,
                                         SDValue Carry0, SDValue Carry1,
                                         SDNode *N) {
  if (Carry1.getResNo() != 1 || Carry0.getResNo() != 1)
    return SDValue();
  if (Carry1.getOpcode() != ISD::UADDO)
    return SDValue();
 
  SDValue Z;
 
  /**
   * First look for a suitable Z. It will present itself in the form of
   * (uaddo_carry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true
   */
  if (Carry0.getOpcode() == ISD::UADDO_CARRY &&
      isNullConstant(Carry0.getOperand(1))) {
    Z = Carry0.getOperand(2);
  } else if (Carry0.getOpcode() == ISD::UADDO &&
             isOneConstant(Carry0.getOperand(1))) {
    EVT VT = Carry0->getValueType(1);
    Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT);
  } else {
    // We couldn't find a suitable Z.
    return SDValue();
  }
 
 
  auto cancelDiamond = [&](SDValue A,SDValue B) {
    SDLoc DL(N);
    SDValue NewY =
        DAG.getNode(ISD::UADDO_CARRY, DL, Carry0->getVTList(), A, B, Z);
    Combiner.AddToWorklist(NewY.getNode());
    return DAG.getNode(ISD::UADDO_CARRY, DL, N->getVTList(), X,
                       DAG.getConstant(0, DL, X.getValueType()),
                       NewY.getValue(1));
  };
 
  /**
   *         (uaddo A, B)
   *              |
   *             Sum
   *              |
   * (uaddo_carry *, 0, Z)
   */
  if (Carry0.getOperand(0) == Carry1.getValue(0)) {
    return cancelDiamond(Carry1.getOperand(0), Carry1.getOperand(1));
  }
 
  /**
   * (uaddo_carry A, 0, Z)
   *         |
   *        Sum
   *         |
   *  (uaddo *, B)
   */
  if (Carry1.getOperand(0) == Carry0.getValue(0)) {
    return cancelDiamond(Carry0.getOperand(0), Carry1.getOperand(1));
  }
 
  if (Carry1.getOperand(1) == Carry0.getValue(0)) {
    return cancelDiamond(Carry1.getOperand(0), Carry0.getOperand(0));
  }
 
  return SDValue();
}
 
// If we are facing some sort of diamond carry/borrow in/out pattern try to
// match patterns like:
//
//          (uaddo A, B)            CarryIn
//            |  \                     |
//            |   \                    |
//    PartialSum   PartialCarryOutX   /
//            |        |             /
//            |    ____|____________/
//            |   /    |
//     (uaddo *, *)    \________
//       |  \                   \
//       |   \                   |
//       |    PartialCarryOutY   |
//       |        \              |
//       |         \            /
//   AddCarrySum    |    ______/
//                  |   /
//   CarryOut = (or *, *)
//
// And generate UADDO_CARRY (or USUBO_CARRY) with two result values:
//
//    {AddCarrySum, CarryOut} = (uaddo_carry A, B, CarryIn)
//
// Our goal is to identify A, B, and CarryIn and produce UADDO_CARRY/USUBO_CARRY
// with a single path for carry/borrow out propagation.
static SDValue combineCarryDiamond(SelectionDAG &DAG, const TargetLowering &TLI,
                                   SDValue N0, SDValue N1, SDNode *N) {
  SDValue Carry0 = getAsCarry(TLI, N0);
  if (!Carry0)
    return SDValue();
  SDValue Carry1 = getAsCarry(TLI, N1);
  if (!Carry1)
    return SDValue();
 
  unsigned Opcode = Carry0.getOpcode();
  if (Opcode != Carry1.getOpcode())
    return SDValue();
  if (Opcode != ISD::UADDO && Opcode != ISD::USUBO)
    return SDValue();
  // Guarantee identical type of CarryOut
  EVT CarryOutType = N->getValueType(0);
  if (CarryOutType != Carry0.getValue(1).getValueType() ||
      CarryOutType != Carry1.getValue(1).getValueType())
    return SDValue();
 
  // Canonicalize the add/sub of A and B (the top node in the above ASCII art)
  // as Carry0 and the add/sub of the carry in as Carry1 (the middle node).
  if (Carry1.getNode()->isOperandOf(Carry0.getNode()))
    std::swap(Carry0, Carry1);
 
  // Check if nodes are connected in expected way.
  if (Carry1.getOperand(0) != Carry0.getValue(0) &&
      Carry1.getOperand(1) != Carry0.getValue(0))
    return SDValue();
 
  // The carry in value must be on the righthand side for subtraction.
  unsigned CarryInOperandNum =
      Carry1.getOperand(0) == Carry0.getValue(0) ? 1 : 0;
  if (Opcode == ISD::USUBO && CarryInOperandNum != 1)
    return SDValue();
  SDValue CarryIn = Carry1.getOperand(CarryInOperandNum);
 
  unsigned NewOp = Opcode == ISD::UADDO ? ISD::UADDO_CARRY : ISD::USUBO_CARRY;
  if (!TLI.isOperationLegalOrCustom(NewOp, Carry0.getValue(0).getValueType()))
    return SDValue();
 
  // Verify that the carry/borrow in is plausibly a carry/borrow bit.
  CarryIn = getAsCarry(TLI, CarryIn, true);
  if (!CarryIn)
    return SDValue();
 
  SDLoc DL(N);
  CarryIn = DAG.getBoolExtOrTrunc(CarryIn, DL, Carry1->getValueType(1),
                                  Carry1->getValueType(0));
  SDValue Merged =
      DAG.getNode(NewOp, DL, Carry1->getVTList(), Carry0.getOperand(0),
                  Carry0.getOperand(1), CarryIn);
 
  // Please note that because we have proven that the result of the UADDO/USUBO
  // of A and B feeds into the UADDO/USUBO that does the carry/borrow in, we can
  // therefore prove that if the first UADDO/USUBO overflows, the second
  // UADDO/USUBO cannot. For example consider 8-bit numbers where 0xFF is the
  // maximum value.
  //
  //   0xFF + 0xFF == 0xFE with carry but 0xFE + 1 does not carry
  //   0x00 - 0xFF == 1 with a carry/borrow but 1 - 1 == 0 (no carry/borrow)
  //
  // This is important because it means that OR and XOR can be used to merge
  // carry flags; and that AND can return a constant zero.
  //
  // TODO: match other operations that can merge flags (ADD, etc)
  DAG.ReplaceAllUsesOfValueWith(Carry1.getValue(0), Merged.getValue(0));
  if (N->getOpcode() == ISD::AND)
    return DAG.getConstant(0, DL, CarryOutType);
  return Merged.getValue(1);
}
 
// Reconstruct a subtract-with-borrow chain from its canonicalized icmp form:
//   carry_out = or(icmp ult A, B, and(icmp eq A, B, carry_in))
// InstCombine folds usub.with.overflow chains into this, losing the
// USUBO_CARRY that lowers to sbb/sbcs.
static SDValue combineOrOfSetCCToUSUBOCarry(SDNode *N, SelectionDAG &DAG,
                                            const TargetLowering &TLI) {
  SDValue A, B, CarryIn;
  if (!sd_match(N, m_Or(m_SetCC(m_Value(A), m_Value(B),
                                m_SpecificCondCode(ISD::SETULT)),
                        m_And(m_c_SetCC(m_Deferred(A), m_Deferred(B),
                                        m_SpecificCondCode(ISD::SETEQ)),
                              m_Value(CarryIn)))))
    return SDValue();
 
  EVT IntVT = A.getValueType();
  // Skip vectors: USUBO_CARRY on a vector type has no legalization path and
  // would crash.
  if (IntVT.isVector() || !TLI.isOperationLegalOrCustom(
                              ISD::USUBO_CARRY, TLI.getLegalTypeToTransformTo(
                                                    *DAG.getContext(), IntVT)))
    return SDValue();
 
  SDLoc DL(N);
  SDVTList VTs = DAG.getVTList(IntVT, N->getValueType(0));
  return DAG.getNode(ISD::USUBO_CARRY, DL, VTs, A, B, CarryIn).getValue(1);
}
 
SDValue DAGCombiner::visitUADDO_CARRYLike(SDValue N0, SDValue N1,
                                          SDValue CarryIn, SDNode *N) {
  // fold (uaddo_carry (xor a, -1), b, c) -> (usubo_carry b, a, !c) and flip
  // carry.
  if (isBitwiseNot(N0))
    if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) {
      SDLoc DL(N);
      SDValue Sub = DAG.getNode(ISD::USUBO_CARRY, DL, N->getVTList(), N1,
                                N0.getOperand(0), NotC);
      return CombineTo(
          N, Sub, DAG.getLogicalNOT(DL, Sub.getValue(1), Sub->getValueType(1)));
    }
 
  // Iff the flag result is dead:
  // (uaddo_carry (add|uaddo X, Y), 0, Carry) -> (uaddo_carry X, Y, Carry)
  // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo
  // or the dependency between the instructions.
  if ((N0.getOpcode() == ISD::ADD ||
       (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 &&
        N0.getValue(1) != CarryIn)) &&
      isNullConstant(N1) && !N->hasAnyUseOfValue(1))
    return DAG.getNode(ISD::UADDO_CARRY, SDLoc(N), N->getVTList(),
                       N0.getOperand(0), N0.getOperand(1), CarryIn);
 
  /**
   * When one of the uaddo_carry argument is itself a carry, we may be facing
   * a diamond carry propagation. In which case we try to transform the DAG
   * to ensure linear carry propagation if that is possible.
   */
  if (auto Y = getAsCarry(TLI, N1)) {
    // Because both are carries, Y and Z can be swapped.
    if (auto R = combineUADDO_CARRYDiamond(*this, DAG, N0, Y, CarryIn, N))
      return R;
    if (auto R = combineUADDO_CARRYDiamond(*this, DAG, N0, CarryIn, Y, N))
      return R;
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSADDO_CARRYLike(SDValue N0, SDValue N1,
                                          SDValue CarryIn, SDNode *N) {
  // fold (saddo_carry (xor a, -1), b, c) -> (ssubo_carry b, a, !c)
  if (isBitwiseNot(N0)) {
    if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true))
      return DAG.getNode(ISD::SSUBO_CARRY, SDLoc(N), N->getVTList(), N1,
                         N0.getOperand(0), NotC);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSADDO_CARRY(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue CarryIn = N->getOperand(2);
  SDLoc DL(N);
 
  // canonicalize constant to RHS
  ConstantSDNode *N0C = dyn_cast<ConstantSDNode>(N0);
  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
  if (N0C && !N1C)
    return DAG.getNode(ISD::SADDO_CARRY, DL, N->getVTList(), N1, N0, CarryIn);
 
  // fold (saddo_carry x, y, false) -> (saddo x, y)
  if (isNullConstant(CarryIn)) {
    if (!LegalOperations ||
        TLI.isOperationLegalOrCustom(ISD::SADDO, N->getValueType(0)))
      return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, N1);
  }
 
  if (SDValue Combined = visitSADDO_CARRYLike(N0, N1, CarryIn, N))
    return Combined;
 
  if (SDValue Combined = visitSADDO_CARRYLike(N1, N0, CarryIn, N))
    return Combined;
 
  return SDValue();
}
 
// Attempt to create a USUBSAT(LHS, RHS) node with DstVT, performing a
// clamp/truncation if necessary.
static SDValue getTruncatedUSUBSAT(EVT DstVT, EVT SrcVT, SDValue LHS,
                                   SDValue RHS, SelectionDAG &DAG,
                                   const SDLoc &DL) {
  assert(DstVT.getScalarSizeInBits() <= SrcVT.getScalarSizeInBits() &&
         "Illegal truncation");
 
  if (DstVT == SrcVT)
    return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS);
 
  // If the LHS is zero-extended then we can perform the USUBSAT as DstVT by
  // clamping RHS.
  APInt UpperBits = APInt::getBitsSetFrom(SrcVT.getScalarSizeInBits(),
                                          DstVT.getScalarSizeInBits());
  if (!DAG.MaskedValueIsZero(LHS, UpperBits))
    return SDValue();
 
  SDValue SatLimit =
      DAG.getConstant(APInt::getLowBitsSet(SrcVT.getScalarSizeInBits(),
                                           DstVT.getScalarSizeInBits()),
                      DL, SrcVT);
  RHS = DAG.getNode(ISD::UMIN, DL, SrcVT, RHS, SatLimit);
  RHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, RHS);
  LHS = DAG.getNode(ISD::TRUNCATE, DL, DstVT, LHS);
  return DAG.getNode(ISD::USUBSAT, DL, DstVT, LHS, RHS);
}
 
// Try to find umax(a,b) - b or a - umin(a,b) patterns that may be converted to
// usubsat(a,b), optionally as a truncated type.
SDValue DAGCombiner::foldSubToUSubSat(EVT DstVT, SDNode *N, const SDLoc &DL) {
  if (N->getOpcode() != ISD::SUB ||
      !(!LegalOperations || hasOperation(ISD::USUBSAT, DstVT)))
    return SDValue();
 
  EVT SubVT = N->getValueType(0);
  SDValue Op0 = N->getOperand(0);
  SDValue Op1 = N->getOperand(1);
 
  // Try to find umax(a,b) - b or a - umin(a,b) patterns
  // they may be converted to usubsat(a,b).
  if (Op0.getOpcode() == ISD::UMAX && Op0.hasOneUse()) {
    SDValue MaxLHS = Op0.getOperand(0);
    SDValue MaxRHS = Op0.getOperand(1);
    if (MaxLHS == Op1)
      return getTruncatedUSUBSAT(DstVT, SubVT, MaxRHS, Op1, DAG, DL);
    if (MaxRHS == Op1)
      return getTruncatedUSUBSAT(DstVT, SubVT, MaxLHS, Op1, DAG, DL);
  }
 
  if (Op1.getOpcode() == ISD::UMIN && Op1.hasOneUse()) {
    SDValue MinLHS = Op1.getOperand(0);
    SDValue MinRHS = Op1.getOperand(1);
    if (MinLHS == Op0)
      return getTruncatedUSUBSAT(DstVT, SubVT, Op0, MinRHS, DAG, DL);
    if (MinRHS == Op0)
      return getTruncatedUSUBSAT(DstVT, SubVT, Op0, MinLHS, DAG, DL);
  }
 
  // sub(a,trunc(umin(zext(a),b))) -> usubsat(a,trunc(umin(b,SatLimit)))
  if (Op1.getOpcode() == ISD::TRUNCATE &&
      Op1.getOperand(0).getOpcode() == ISD::UMIN &&
      Op1.getOperand(0).hasOneUse()) {
    SDValue MinLHS = Op1.getOperand(0).getOperand(0);
    SDValue MinRHS = Op1.getOperand(0).getOperand(1);
    if (MinLHS.getOpcode() == ISD::ZERO_EXTEND && MinLHS.getOperand(0) == Op0)
      return getTruncatedUSUBSAT(DstVT, MinLHS.getValueType(), MinLHS, MinRHS,
                                 DAG, DL);
    if (MinRHS.getOpcode() == ISD::ZERO_EXTEND && MinRHS.getOperand(0) == Op0)
      return getTruncatedUSUBSAT(DstVT, MinLHS.getValueType(), MinRHS, MinLHS,
                                 DAG, DL);
  }
 
  return SDValue();
}
 
// Refinement of DAG/Type Legalisation (promotion) when CTLZ is used for
// counting leading ones. Broadly, it replaces the substraction with a left
// shift.
//
// * DAG Legalisation Pattern:
//
//     (sub (ctlz (zeroextend (not Src)))
//          BitWidthDiff)
//
//       if BitWidthDiff == BitWidth(Node) - BitWidth(Src)
//       -->
//
//     (ctlz_zero_poison (not (shl (anyextend Src)
//                                 BitWidthDiff)))
//
// * Type Legalisation Pattern:
//
//     (sub (ctlz (and (xor Src XorMask)
//                     AndMask))
//          BitWidthDiff)
//
//       if AndMask has only trailing ones
//       and MaskBitWidth(AndMask) == BitWidth(Node) - BitWidthDiff
//       and XorMask has more trailing ones than AndMask
//       -->
//
//     (ctlz_zero_poison (not (shl Src BitWidthDiff)))
template <class MatchContextClass>
static SDValue foldSubCtlzNot(SDNode *N, SelectionDAG &DAG) {
  const SDLoc DL(N);
  SDValue N0 = N->getOperand(0);
  EVT VT = N0.getValueType();
  unsigned BitWidth = VT.getScalarSizeInBits();
 
  MatchContextClass Matcher(DAG, DAG.getTargetLoweringInfo(), N);
 
  APInt AndMask;
  APInt XorMask;
  uint64_t BitWidthDiff;
 
  SDValue CtlzOp;
  SDValue Src;
 
  if (!sd_context_match(
          N, Matcher, m_Sub(m_Ctlz(m_Value(CtlzOp)), m_ConstInt(BitWidthDiff))))
    return SDValue();
 
  if (sd_context_match(CtlzOp, Matcher, m_ZExt(m_Not(m_Value(Src))))) {
    // DAG Legalisation Pattern:
    // (sub (ctlz (zero_extend (not Op)) BitWidthDiff))
    if ((BitWidth - Src.getValueType().getScalarSizeInBits()) != BitWidthDiff)
      return SDValue();
 
    Src = DAG.getNode(ISD::ANY_EXTEND, DL, VT, Src);
  } else if (sd_context_match(CtlzOp, Matcher,
                              m_And(m_Xor(m_Value(Src), m_ConstInt(XorMask)),
                                    m_ConstInt(AndMask)))) {
    // Type Legalisation Pattern:
    // (sub (ctlz (and (xor Op XorMask) AndMask)) BitWidthDiff)
    if (BitWidthDiff >= BitWidth)
      return SDValue();
    unsigned AndMaskWidth = BitWidth - BitWidthDiff;
    if (!(AndMask.isMask(AndMaskWidth) && XorMask.countr_one() >= AndMaskWidth))
      return SDValue();
  } else
    return SDValue();
 
  SDValue ShiftConst = DAG.getShiftAmountConstant(BitWidthDiff, VT, DL);
  SDValue LShift = Matcher.getNode(ISD::SHL, DL, VT, Src, ShiftConst);
  SDValue Not =
      Matcher.getNode(ISD::XOR, DL, VT, LShift, DAG.getAllOnesConstant(DL, VT));
 
  return Matcher.getNode(ISD::CTLZ_ZERO_POISON, DL, VT, Not);
}
 
// Fold sub(x, mul(divrem(x,y)[0], y)) to divrem(x, y)[1]
static SDValue foldRemainderIdiom(SDNode *N, SelectionDAG &DAG,
                                  const SDLoc &DL) {
  assert(N->getOpcode() == ISD::SUB && "Node must be a SUB");
  SDValue Sub0 = N->getOperand(0);
  SDValue Sub1 = N->getOperand(1);
 
  auto CheckAndFoldMulCase = [&](SDValue DivRem, SDValue MaybeY) -> SDValue {
    if ((DivRem.getOpcode() == ISD::SDIVREM ||
         DivRem.getOpcode() == ISD::UDIVREM) &&
        DivRem.getResNo() == 0 && DivRem.getOperand(0) == Sub0 &&
        DivRem.getOperand(1) == MaybeY) {
      return SDValue(DivRem.getNode(), 1);
    }
    return SDValue();
  };
 
  if (Sub1.getOpcode() == ISD::MUL) {
    // (sub x, (mul divrem(x,y)[0], y))
    SDValue Mul0 = Sub1.getOperand(0);
    SDValue Mul1 = Sub1.getOperand(1);
 
    if (SDValue Res = CheckAndFoldMulCase(Mul0, Mul1))
      return Res;
 
    if (SDValue Res = CheckAndFoldMulCase(Mul1, Mul0))
      return Res;
 
  } else if (Sub1.getOpcode() == ISD::SHL) {
    // Handle (sub x, (shl divrem(x,y)[0], C)) where y = 1 << C
    SDValue Shl0 = Sub1.getOperand(0);
    SDValue Shl1 = Sub1.getOperand(1);
    // Check if Shl0 is divrem(x, Y)[0]
    if ((Shl0.getOpcode() == ISD::SDIVREM ||
         Shl0.getOpcode() == ISD::UDIVREM) &&
        Shl0.getResNo() == 0 && Shl0.getOperand(0) == Sub0) {
 
      SDValue Divisor = Shl0.getOperand(1);
 
      ConstantSDNode *DivC = isConstOrConstSplat(Divisor);
      ConstantSDNode *ShC = isConstOrConstSplat(Shl1);
      if (!DivC || !ShC)
        return SDValue();
 
      if (DivC->getAPIntValue().isPowerOf2() &&
          DivC->getAPIntValue().logBase2() == ShC->getAPIntValue())
        return SDValue(Shl0.getNode(), 1);
    }
  }
  return SDValue();
}
 
// Since it may not be valid to emit a fold to zero for vector initializers
// check if we can before folding.
static SDValue tryFoldToZero(const SDLoc &DL, const TargetLowering &TLI, EVT VT,
                             SelectionDAG &DAG, bool LegalOperations) {
  if (!VT.isVector())
    return DAG.getConstant(0, DL, VT);
  if (!LegalOperations || TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
    return DAG.getConstant(0, DL, VT);
  return SDValue();
}
 
SDValue DAGCombiner::visitSUB(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  unsigned BitWidth = VT.getScalarSizeInBits();
  SDLoc DL(N);
 
  if (SDValue V = foldSubCtlzNot<EmptyMatchContext>(N, DAG))
    return V;
 
  // fold (sub x, x) -> 0
  if (N0 == N1)
    return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
 
  // fold (sub c1, c2) -> c3
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N1}))
    return C;
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (sub x, 0) -> x, vector edition
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      return N0;
  }
 
  // (sub x, ([v]select (ult x, y), 0, y)) -> (umin x, (sub x, y))
  // (sub x, ([v]select (uge x, y), y, 0)) -> (umin x, (sub x, y))
  if (N1.hasOneUse() && hasUMin(VT)) {
    SDValue Y;
    auto MS0 = m_Specific(N0);
    auto MVY = m_Value(Y);
    auto MZ = m_Zero();
    auto MCC1 = m_SpecificCondCode(ISD::SETULT);
    auto MCC2 = m_SpecificCondCode(ISD::SETUGE);
 
    if (sd_match(N1, m_SelectCCLike(MS0, MVY, MZ, m_Deferred(Y), MCC1)) ||
        sd_match(N1, m_SelectCCLike(MS0, MVY, m_Deferred(Y), MZ, MCC2)) ||
        sd_match(N1, m_VSelect(m_SetCC(MS0, MVY, MCC1), MZ, m_Deferred(Y))) ||
        sd_match(N1, m_VSelect(m_SetCC(MS0, MVY, MCC2), m_Deferred(Y), MZ)))
 
      return DAG.getNode(ISD::UMIN, DL, VT, N0,
                         DAG.getNode(ISD::SUB, DL, VT, N0, Y));
  }
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // fold (sub x, c) -> (add x, -c)
  if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N1))
    return DAG.getNode(ISD::ADD, DL, VT, N0,
                       DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
 
  if (isNullOrNullSplat(N0)) {
    // Right-shifting everything out but the sign bit followed by negation is
    // the same as flipping arithmetic/logical shift type without the negation:
    // -(X >>u 31) -> (X >>s 31)
    // -(X >>s 31) -> (X >>u 31)
    if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) {
      ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1));
      if (ShiftAmt && ShiftAmt->getAPIntValue() == (BitWidth - 1)) {
        auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA;
        if (!LegalOperations || TLI.isOperationLegal(NewSh, VT))
          return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1));
      }
    }
 
    // 0 - X --> 0 if the sub is NUW.
    if (N->getFlags().hasNoUnsignedWrap())
      return N0;
 
    if (DAG.MaskedValueIsZero(N1, ~APInt::getSignMask(BitWidth))) {
      // N1 is either 0 or the minimum signed value. If the sub is NSW, then
      // N1 must be 0 because negating the minimum signed value is undefined.
      if (N->getFlags().hasNoSignedWrap())
        return N0;
 
      // 0 - X --> X if X is 0 or the minimum signed value.
      return N1;
    }
 
    // Convert 0 - abs(x).
    if (ISD::isAbsOpcode(N1.getOpcode()) && N1.hasOneUse() &&
        !TLI.isOperationLegalOrCustom(N1.getOpcode(), VT))
      if (SDValue Result = TLI.expandABS(N1.getNode(), DAG, true))
        return Result;
 
    // Similar to the previous rule, but this time targeting an expanded abs.
    // (sub 0, (max X, (sub 0, X))) --> (min X, (sub 0, X))
    // as well as
    // (sub 0, (min X, (sub 0, X))) --> (max X, (sub 0, X))
    // Note that these two are applicable to both signed and unsigned min/max.
    SDValue X;
    SDValue S0;
    auto NegPat = m_AllOf(m_Neg(m_Deferred(X)), m_Value(S0));
    if (sd_match(N1, m_OneUse(m_AnyOf(m_SMax(m_Value(X), NegPat),
                                      m_UMax(m_Value(X), NegPat),
                                      m_SMin(m_Value(X), NegPat),
                                      m_UMin(m_Value(X), NegPat))))) {
      unsigned NewOpc = ISD::getInverseMinMaxOpcode(N1->getOpcode());
      if (hasOperation(NewOpc, VT))
        return DAG.getNode(NewOpc, DL, VT, X, S0);
    }
 
    // Fold neg(splat(neg(x)) -> splat(x)
    if (VT.isVector()) {
      SDValue N1S = DAG.getSplatValue(N1, true);
      if (N1S && N1S.getOpcode() == ISD::SUB &&
          isNullConstant(N1S.getOperand(0)))
        return DAG.getSplat(VT, DL, N1S.getOperand(1));
    }
 
    // sub 0, (and x, 1)  -->  SIGN_EXTEND_INREG x, i1
    if (N1.getOpcode() == ISD::AND && N1.hasOneUse() &&
        isOneOrOneSplat(N1->getOperand(1))) {
      EVT ExtVT = VT.changeElementType(*DAG.getContext(), MVT::i1);
      if (TLI.getOperationAction(ISD::SIGN_EXTEND_INREG, ExtVT) ==
          TargetLowering::Legal) {
        return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N1->getOperand(0),
                           DAG.getValueType(ExtVT));
      }
    }
  }
 
  // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1)
  if (isAllOnesOrAllOnesSplat(N0))
    return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
 
  // fold (A - (0-B)) -> A+B
  if (N1.getOpcode() == ISD::SUB && isNullOrNullSplat(N1.getOperand(0)))
    return DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(1));
 
  // fold A-(A-B) -> B
  if (N1.getOpcode() == ISD::SUB && N0 == N1.getOperand(0))
    return N1.getOperand(1);
 
  // fold (A+B)-A -> B
  if (N0.getOpcode() == ISD::ADD && N0.getOperand(0) == N1)
    return N0.getOperand(1);
 
  // fold (A+B)-B -> A
  if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1)
    return N0.getOperand(0);
 
  // fold (A+C1)-C2 -> A+(C1-C2)
  if (N0.getOpcode() == ISD::ADD) {
    SDValue N01 = N0.getOperand(1);
    if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N01, N1}))
      return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC);
  }
 
  // fold C2-(A+C1) -> (C2-C1)-A
  if (N1.getOpcode() == ISD::ADD) {
    SDValue N11 = N1.getOperand(1);
    if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N0, N11}))
      return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0));
  }
 
  // fold (A-C1)-C2 -> A-(C1+C2)
  if (N0.getOpcode() == ISD::SUB) {
    SDValue N01 = N0.getOperand(1);
    if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, {N01, N1}))
      return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC);
  }
 
  // fold (c1-A)-c2 -> (c1-c2)-A
  if (N0.getOpcode() == ISD::SUB) {
    SDValue N00 = N0.getOperand(0);
    if (SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, {N00, N1}))
      return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1));
  }
 
  SDValue A, B, C;
 
  // fold ((A+(B+C))-B) -> A+C
  if (sd_match(N0, m_Add(m_Value(A), m_Add(m_Specific(N1), m_Value(C)))))
    return DAG.getNode(ISD::ADD, DL, VT, A, C);
 
  // fold ((A+(B-C))-B) -> A-C
  if (sd_match(N0, m_Add(m_Value(A), m_Sub(m_Specific(N1), m_Value(C)))))
    return DAG.getNode(ISD::SUB, DL, VT, A, C);
 
  // fold ((A-(B-C))-C) -> A-B
  if (sd_match(N0, m_Sub(m_Value(A), m_Sub(m_Value(B), m_Specific(N1)))))
    return DAG.getNode(ISD::SUB, DL, VT, A, B);
 
  // fold (A-(B-C)) -> A+(C-B)
  if (sd_match(N1, m_OneUse(m_Sub(m_Value(B), m_Value(C)))))
    return DAG.getNode(ISD::ADD, DL, VT, N0,
                       DAG.getNode(ISD::SUB, DL, VT, C, B));
 
  // A - (A & B)  ->  A & (~B)
  if (sd_match(N1, m_And(m_Specific(N0), m_Value(B))) &&
      (N1.hasOneUse() || isConstantOrConstantVector(B, /*NoOpaques=*/true)))
    return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getNOT(DL, B, VT));
 
  // fold (A - (-B * C)) -> (A + (B * C))
  if (sd_match(N1, m_OneUse(m_Mul(m_Neg(m_Value(B)), m_Value(C)))))
    return DAG.getNode(ISD::ADD, DL, VT, N0,
                       DAG.getNode(ISD::MUL, DL, VT, B, C));
 
  // If either operand of a sub is undef, the result is undef
  if (N0.isUndef())
    return N0;
  if (N1.isUndef())
    return N1;
 
  if (SDValue V = foldAddSubBoolOfMaskedVal(N, DL, DAG))
    return V;
 
  if (SDValue V = foldAddSubOfSignBit(N, DL, DAG))
    return V;
 
  // Try to match AVGCEIL fixedwidth pattern
  if (SDValue V = foldSubToAvg(N, DL))
    return V;
 
  if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, DL))
    return V;
 
  if (SDValue V = foldSubToUSubSat(VT, N, DL))
    return V;
 
  if (SDValue V = foldRemainderIdiom(N, DAG, DL))
    return V;
 
  // (A - B) - 1  ->  add (xor B, -1), A
  if (sd_match(N, m_Sub(m_OneUse(m_Sub(m_Value(A), m_Value(B))),
                        m_One(/*AllowUndefs=*/true))))
    return DAG.getNode(ISD::ADD, DL, VT, A, DAG.getNOT(DL, B, VT));
 
  // Look for:
  //   sub y, (xor x, -1)
  // And if the target does not like this form then turn into:
  //   add (add x, y), 1
  if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) {
    SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0));
    return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT));
  }
 
  // Hoist one-use addition by non-opaque constant:
  //   (x + C) - y  ->  (x - y) + C
  if (!reassociationCanBreakAddressingModePattern(ISD::SUB, DL, N, N0, N1) &&
      N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
      isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
    SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
    return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1));
  }
  // y - (x + C)  ->  (y - x) - C
  if (N1.getOpcode() == ISD::ADD && N1.hasOneUse() &&
      isConstantOrConstantVector(N1.getOperand(1), /*NoOpaques=*/true)) {
    SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0));
    return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1));
  }
  // (x - C) - y  ->  (x - y) - C
  // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors.
  if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
      isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) {
    SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1);
    return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1));
  }
  // (C - x) - y  ->  C - (x + y)
  if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
      isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) {
    SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1);
    return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add);
  }
 
  // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1'
  // rather than 'sub 0/1' (the sext should get folded).
  // sub X, (zext i1 Y) --> add X, (sext i1 Y)
  if (N1.getOpcode() == ISD::ZERO_EXTEND &&
      N1.getOperand(0).getScalarValueSizeInBits() == 1 &&
      TLI.getBooleanContents(VT) ==
          TargetLowering::ZeroOrNegativeOneBooleanContent) {
    SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0));
    return DAG.getNode(ISD::ADD, DL, VT, N0, SExt);
  }
 
  // fold B = sra (A, size(A)-1); sub (xor (A, B), B) -> (abs A)
  if ((!LegalOperations || hasOperation(ISD::ABS, VT)) &&
      sd_match(N1, m_Sra(m_Value(A), m_SpecificInt(BitWidth - 1))) &&
      sd_match(N0, m_Xor(m_Specific(A), m_Specific(N1))))
    return DAG.getNode(ISD::ABS, DL, VT, A);
 
  // If the relocation model supports it, consider symbol offsets.
  if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N0))
    if (!LegalOperations && TLI.isOffsetFoldingLegal(GA)) {
      // fold (sub Sym+c1, Sym+c2) -> c1-c2
      if (GlobalAddressSDNode *GB = dyn_cast<GlobalAddressSDNode>(N1))
        if (GA->getGlobal() == GB->getGlobal())
          return DAG.getConstant((uint64_t)GA->getOffset() - GB->getOffset(),
                                 DL, VT);
    }
 
  // sub X, (sextinreg Y i1) -> add X, (and Y 1)
  if (N1.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    VTSDNode *TN = cast<VTSDNode>(N1.getOperand(1));
    if (TN->getVT() == MVT::i1) {
      SDValue ZExt = DAG.getNode(ISD::AND, DL, VT, N1.getOperand(0),
                                 DAG.getConstant(1, DL, VT));
      return DAG.getNode(ISD::ADD, DL, VT, N0, ZExt);
    }
  }
 
  // canonicalize (sub X, (vscale * C)) to (add X, (vscale * -C))
  // avoid if ISD::MUL handling is poor and ISD::SHL isn't an option.
  if (N1.getOpcode() == ISD::VSCALE && N1.hasOneUse()) {
    const APInt &IntVal = N1.getConstantOperandAPInt(0);
    if (!IntVal.isPowerOf2() ||
        hasOperation(ISD::MUL, N1.getOperand(0).getValueType()))
      return DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getVScale(DL, VT, -IntVal));
  }
 
  // canonicalize (sub X, step_vector(C)) to (add X, step_vector(-C))
  if (N1.getOpcode() == ISD::STEP_VECTOR && N1.hasOneUse()) {
    APInt NewStep = -N1.getConstantOperandAPInt(0);
    return DAG.getNode(ISD::ADD, DL, VT, N0,
                       DAG.getStepVector(DL, VT, NewStep));
  }
 
  // Prefer an add for more folding potential and possibly better codegen:
  // sub N0, (lshr N10, width-1) --> add N0, (ashr N10, width-1)
  if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) {
    SDValue ShAmt = N1.getOperand(1);
    ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt);
    if (ShAmtC && ShAmtC->getAPIntValue() == (BitWidth - 1)) {
      SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt);
      return DAG.getNode(ISD::ADD, DL, VT, N0, SRA);
    }
  }
 
  // As with the previous fold, prefer add for more folding potential.
  // Subtracting SMIN/0 is the same as adding SMIN/0:
  // N0 - (X << BW-1) --> N0 + (X << BW-1)
  if (N1.getOpcode() == ISD::SHL) {
    ConstantSDNode *ShlC = isConstOrConstSplat(N1.getOperand(1));
    if (ShlC && ShlC->getAPIntValue() == (BitWidth - 1))
      return DAG.getNode(ISD::ADD, DL, VT, N1, N0);
  }
 
  // (sub (usubo_carry X, 0, Carry), Y) -> (usubo_carry X, Y, Carry)
  if (N0.getOpcode() == ISD::USUBO_CARRY && isNullConstant(N0.getOperand(1)) &&
      N0.getResNo() == 0 && N0.hasOneUse())
    return DAG.getNode(ISD::USUBO_CARRY, DL, N0->getVTList(),
                       N0.getOperand(0), N1, N0.getOperand(2));
 
  if (TLI.isOperationLegalOrCustom(ISD::UADDO_CARRY, VT)) {
    // (sub Carry, X)  ->  (uaddo_carry (sub 0, X), 0, Carry)
    if (SDValue Carry = getAsCarry(TLI, N0)) {
      SDValue X = N1;
      SDValue Zero = DAG.getConstant(0, DL, VT);
      SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
      return DAG.getNode(ISD::UADDO_CARRY, DL,
                         DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
                         Carry);
    }
  }
 
  if (ConstantSDNode *C0 = isConstOrConstSplat(N0)) {
    const APInt &C0Val = C0->getAPIntValue();
 
    // sub nuw C, x --> xor x, C when C is a mask (2^k - 1)
    if (N->getFlags().hasNoUnsignedWrap() && C0Val.isMask())
      return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
 
    // If there's no chance of borrowing from adjacent bits, then sub is xor:
    // sub C0, X --> xor X, C0
    if (!C0->isOpaque()) {
      const APInt &MaybeOnes = ~DAG.computeKnownBits(N1).Zero;
      if ((C0Val - MaybeOnes) == (C0Val ^ MaybeOnes))
        return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
    }
  }
 
  // smax(a,b) - smin(a,b) --> abds(a,b)
  if ((!LegalOperations || hasOperation(ISD::ABDS, VT)) &&
      sd_match(N0, &DAG, m_SMaxLike(m_Value(A), m_Value(B))) &&
      sd_match(N1, &DAG, m_SMinLike(m_Specific(A), m_Specific(B))))
    return DAG.getNode(ISD::ABDS, DL, VT, A, B);
 
  // smin(a,b) - smax(a,b) --> neg(abds(a,b))
  if (hasOperation(ISD::ABDS, VT) &&
      sd_match(N0, &DAG, m_SMinLike(m_Value(A), m_Value(B))) &&
      sd_match(N1, &DAG, m_SMaxLike(m_Specific(A), m_Specific(B))))
    return DAG.getNegative(DAG.getNode(ISD::ABDS, DL, VT, A, B), DL, VT);
 
  // umax(a,b) - umin(a,b) --> abdu(a,b)
  if ((!LegalOperations || hasOperation(ISD::ABDU, VT)) &&
      sd_match(N0, &DAG, m_UMaxLike(m_Value(A), m_Value(B))) &&
      sd_match(N1, &DAG, m_UMinLike(m_Specific(A), m_Specific(B))))
    return DAG.getNode(ISD::ABDU, DL, VT, A, B);
 
  // umin(a,b) - umax(a,b) --> neg(abdu(a,b))
  if (hasOperation(ISD::ABDU, VT) &&
      sd_match(N0, &DAG, m_UMinLike(m_Value(A), m_Value(B))) &&
      sd_match(N1, &DAG, m_UMaxLike(m_Specific(A), m_Specific(B))))
    return DAG.getNegative(DAG.getNode(ISD::ABDU, DL, VT, A, B), DL, VT);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSUBSAT(SDNode *N) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  bool IsSigned = Opcode == ISD::SSUBSAT;
  SDLoc DL(N);
 
  // fold (sub_sat x, undef) -> 0
  if (N0.isUndef() || N1.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  // fold (sub_sat x, x) -> 0
  if (N0 == N1)
    return DAG.getConstant(0, DL, VT);
 
  // fold (sub_sat c1, c2) -> c3
  if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
    return C;
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (sub_sat x, 0) -> x, vector edition
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      return N0;
  }
 
  // fold (sub_sat x, 0) -> x
  if (isNullConstant(N1))
    return N0;
 
  // If it cannot overflow, transform into an sub.
  if (DAG.willNotOverflowSub(IsSigned, N0, N1))
    return DAG.getNode(ISD::SUB, DL, VT, N0, N1);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSUBC(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  SDLoc DL(N);
 
  // If the flag result is dead, turn this into an SUB.
  if (!N->hasAnyUseOfValue(1))
    return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
                     DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
 
  // fold (subc x, x) -> 0 + no borrow
  if (N0 == N1)
    return CombineTo(N, DAG.getConstant(0, DL, VT),
                     DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
 
  // fold (subc x, 0) -> x + no borrow
  if (isNullConstant(N1))
    return CombineTo(N, N0, DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
 
  // Canonicalize (sub -1, x) -> ~x, i.e. (xor x, -1) + no borrow
  if (isAllOnesConstant(N0))
    return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
                     DAG.getNode(ISD::CARRY_FALSE, DL, MVT::Glue));
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSUBO(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  bool IsSigned = (ISD::SSUBO == N->getOpcode());
 
  EVT CarryVT = N->getValueType(1);
  SDLoc DL(N);
 
  // If the flag result is dead, turn this into an SUB.
  if (!N->hasAnyUseOfValue(1))
    return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
                     DAG.getUNDEF(CarryVT));
 
  // fold (subo x, x) -> 0 + no borrow
  if (N0 == N1)
    return CombineTo(N, DAG.getConstant(0, DL, VT),
                     DAG.getConstant(0, DL, CarryVT));
 
  // fold (subox, c) -> (addo x, -c)
  if (ConstantSDNode *N1C = getAsNonOpaqueConstant(N1))
    if (IsSigned && !N1C->isMinSignedValue())
      return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0,
                         DAG.getConstant(-N1C->getAPIntValue(), DL, VT));
 
  // fold (subo x, 0) -> x + no borrow
  if (isNullOrNullSplat(N1))
    return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT));
 
  // If it cannot overflow, transform into an sub.
  if (DAG.willNotOverflowSub(IsSigned, N0, N1))
    return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1),
                     DAG.getConstant(0, DL, CarryVT));
 
  // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow
  if (!IsSigned && isAllOnesOrAllOnesSplat(N0))
    return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0),
                     DAG.getConstant(0, DL, CarryVT));
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSUBE(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue CarryIn = N->getOperand(2);
 
  // fold (sube x, y, false) -> (subc x, y)
  if (CarryIn.getOpcode() == ISD::CARRY_FALSE)
    return DAG.getNode(ISD::SUBC, SDLoc(N), N->getVTList(), N0, N1);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitUSUBO_CARRY(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue CarryIn = N->getOperand(2);
 
  // fold (usubo_carry x, y, false) -> (usubo x, y)
  if (isNullConstant(CarryIn)) {
    if (!LegalOperations ||
        TLI.isOperationLegalOrCustom(ISD::USUBO, N->getValueType(0)))
      return DAG.getNode(ISD::USUBO, SDLoc(N), N->getVTList(), N0, N1);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSSUBO_CARRY(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue CarryIn = N->getOperand(2);
 
  // fold (ssubo_carry x, y, false) -> (ssubo x, y)
  if (isNullConstant(CarryIn)) {
    if (!LegalOperations ||
        TLI.isOperationLegalOrCustom(ISD::SSUBO, N->getValueType(0)))
      return DAG.getNode(ISD::SSUBO, SDLoc(N), N->getVTList(), N0, N1);
  }
 
  return SDValue();
}
 
// Notice that "mulfix" can be any of SMULFIX, SMULFIXSAT, UMULFIX and
// UMULFIXSAT here.
SDValue DAGCombiner::visitMULFIX(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue Scale = N->getOperand(2);
  EVT VT = N0.getValueType();
 
  // fold (mulfix x, undef, scale) -> 0
  if (N0.isUndef() || N1.isUndef())
    return DAG.getConstant(0, SDLoc(N), VT);
 
  // Canonicalize constant to RHS (vector doesn't have to splat)
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
     !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N1, N0, Scale);
 
  // fold (mulfix x, 0, scale) -> 0
  if (isNullConstant(N1))
    return DAG.getConstant(0, SDLoc(N), VT);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMUL(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  unsigned BitWidth = VT.getScalarSizeInBits();
  SDLoc DL(N);
 
  // fold (mul x, undef) -> 0
  if (N0.isUndef() || N1.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  // fold (mul c1, c2) -> c1*c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MUL, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS (vector doesn't have to splat)
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::MUL, DL, VT, N1, N0);
 
  bool N1IsConst = false;
  bool N1IsOpaqueConst = false;
  APInt ConstValue1;
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    N1IsConst = ISD::isConstantSplatVector(N1.getNode(), ConstValue1);
    assert((!N1IsConst || ConstValue1.getBitWidth() == BitWidth) &&
           "Splat APInt should be element width");
  } else {
    N1IsConst = isa<ConstantSDNode>(N1);
    if (N1IsConst) {
      ConstValue1 = N1->getAsAPIntVal();
      N1IsOpaqueConst = cast<ConstantSDNode>(N1)->isOpaque();
    }
  }
 
  // fold (mul x, 0) -> 0
  if (N1IsConst && ConstValue1.isZero())
    return N1;
 
  // fold (mul x, 1) -> x
  if (N1IsConst && ConstValue1.isOne())
    return N0;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // fold (mul x, -1) -> 0-x
  if (N1IsConst && ConstValue1.isAllOnes())
    return DAG.getNegative(N0, DL, VT);
 
  // fold (mul x, (1 << c)) -> x << c
  if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) &&
      (!VT.isVector() || Level <= AfterLegalizeVectorOps)) {
    if (SDValue LogBase2 = BuildLogBase2(N1, DL)) {
      EVT ShiftVT = getShiftAmountTy(N0.getValueType());
      SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
      SDNodeFlags Flags;
      Flags.setNoUnsignedWrap(N->getFlags().hasNoUnsignedWrap());
      // Preserve nsw when the shift amount is strictly less than BitWidth - 1,
      // i.e. the multiplier is not the signed minimum value.
      if (N->getFlags().hasNoSignedWrap() && N1IsConst &&
          ConstValue1.logBase2() < BitWidth - 1)
        Flags.setNoSignedWrap(true);
      return DAG.getNode(ISD::SHL, DL, VT, N0, Trunc, Flags);
    }
  }
 
  // fold (mul x, -(1 << c)) -> -(x << c) or (-x) << c
  if (N1IsConst && !N1IsOpaqueConst && ConstValue1.isNegatedPowerOf2()) {
    unsigned Log2Val = (-ConstValue1).logBase2();
 
    // FIXME: If the input is something that is easily negated (e.g. a
    // single-use add), we should put the negate there.
    return DAG.getNode(
        ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
        DAG.getNode(ISD::SHL, DL, VT, N0,
                    DAG.getShiftAmountConstant(Log2Val, VT, DL)));
  }
 
  // Attempt to reuse an existing umul_lohi/smul_lohi node, but only if the
  // hi result is in use in case we hit this mid-legalization.
  for (unsigned LoHiOpc : {ISD::UMUL_LOHI, ISD::SMUL_LOHI}) {
    if (!LegalOperations || TLI.isOperationLegalOrCustom(LoHiOpc, VT)) {
      SDVTList LoHiVT = DAG.getVTList(VT, VT);
      // TODO: Can we match commutable operands with getNodeIfExists?
      if (SDNode *LoHi = DAG.getNodeIfExists(LoHiOpc, LoHiVT, {N0, N1}))
        if (LoHi->hasAnyUseOfValue(1))
          return SDValue(LoHi, 0);
      if (SDNode *LoHi = DAG.getNodeIfExists(LoHiOpc, LoHiVT, {N1, N0}))
        if (LoHi->hasAnyUseOfValue(1))
          return SDValue(LoHi, 0);
    }
  }
 
  // Try to transform:
  // (1) multiply-by-(power-of-2 +/- 1) into shift and add/sub.
  // mul x, (2^N + 1) --> add (shl x, N), x
  // mul x, (2^N - 1) --> sub (shl x, N), x
  // Examples: x * 33 --> (x << 5) + x
  //           x * 15 --> (x << 4) - x
  //           x * -33 --> -((x << 5) + x)
  //           x * -15 --> -((x << 4) - x) ; this reduces --> x - (x << 4)
  // (2) multiply-by-(power-of-2 +/- power-of-2) into shifts and add/sub.
  // mul x, (2^N + 2^M) --> (add (shl x, N), (shl x, M))
  // mul x, (2^N - 2^M) --> (sub (shl x, N), (shl x, M))
  // Examples: x * 0x8800 --> (x << 15) + (x << 11)
  //           x * 0xf800 --> (x << 16) - (x << 11)
  //           x * -0x8800 --> -((x << 15) + (x << 11))
  //           x * -0xf800 --> -((x << 16) - (x << 11)) ; (x << 11) - (x << 16)
  if (N1IsConst && TLI.decomposeMulByConstant(*DAG.getContext(), VT, N1)) {
    // TODO: We could handle more general decomposition of any constant by
    //       having the target set a limit on number of ops and making a
    //       callback to determine that sequence (similar to sqrt expansion).
    unsigned MathOp = ISD::DELETED_NODE;
    APInt MulC = ConstValue1.abs();
    // The constant `2` should be treated as (2^0 + 1).
    unsigned TZeros = MulC == 2 ? 0 : MulC.countr_zero();
    MulC.lshrInPlace(TZeros);
    if ((MulC - 1).isPowerOf2())
      MathOp = ISD::ADD;
    else if ((MulC + 1).isPowerOf2())
      MathOp = ISD::SUB;
 
    if (MathOp != ISD::DELETED_NODE) {
      unsigned ShAmt =
          MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2();
      ShAmt += TZeros;
      assert(ShAmt < BitWidth &&
             "multiply-by-constant generated out of bounds shift");
      SDValue Shl =
          DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT));
      SDValue R =
          TZeros ? DAG.getNode(MathOp, DL, VT, Shl,
                               DAG.getNode(ISD::SHL, DL, VT, N0,
                                           DAG.getConstant(TZeros, DL, VT)))
                 : DAG.getNode(MathOp, DL, VT, Shl, N0);
      if (ConstValue1.isNegative())
        R = DAG.getNegative(R, DL, VT);
      return R;
    }
  }
 
  // (mul (shl X, c1), c2) -> (mul X, c2 << c1)
  {
    SDValue X, C1;
    if (sd_match(N0, m_Shl(m_Value(X), m_Value(C1))))
      if (SDValue C3 = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N1, C1}))
        return DAG.getNode(ISD::MUL, DL, VT, X, C3);
  }
 
  // Change (mul (shl X, C), Y) -> (shl (mul X, Y), C) when the shift has one
  // use.
  {
    SDValue X, C, Y;
    if (sd_match(N,
                 m_Mul(m_OneUse(m_Shl(m_Value(X), m_Value(C))), m_Value(Y))) &&
        isConstantOrConstantVector(C)) {
      SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, X, Y);
      return DAG.getNode(ISD::SHL, DL, VT, Mul, C);
    }
  }
 
  // fold (mul (add x, c1), c2) -> (add (mul x, c2), c1*c2)
  if (sd_match(N0, m_SpecificOpc(ISD::ADD)) && isConstantOrConstantVector(N1) &&
      isConstantOrConstantVector(N0.getOperand(1)) &&
      isMulAddWithConstProfitable(N, N0, N1))
    return DAG.getNode(
        ISD::ADD, DL, VT,
        DAG.getNode(ISD::MUL, SDLoc(N0), VT, N0.getOperand(0), N1),
        DAG.getNode(ISD::MUL, SDLoc(N1), VT, N0.getOperand(1), N1));
 
  // Fold (mul (vscale * C0), C1) to (vscale * (C0 * C1)).
  // avoid if ISD::MUL handling is poor and ISD::SHL isn't an option.
  ConstantSDNode *NC1 = isConstOrConstSplat(N1);
  if (N0.getOpcode() == ISD::VSCALE && NC1) {
    const APInt &C0 = N0.getConstantOperandAPInt(0);
    const APInt &C1 = NC1->getAPIntValue();
    if (!C0.isPowerOf2() || C1.isPowerOf2() ||
        hasOperation(ISD::MUL, NC1->getValueType(0)))
      return DAG.getVScale(DL, VT, C0 * C1);
  }
 
  // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
  APInt MulVal;
  if (N0.getOpcode() == ISD::STEP_VECTOR &&
      ISD::isConstantSplatVector(N1.getNode(), MulVal)) {
    const APInt &C0 = N0.getConstantOperandAPInt(0);
    APInt NewStep = C0 * MulVal;
    return DAG.getStepVector(DL, VT, NewStep);
  }
 
  // Fold Y = sra (X, size(X)-1); mul (or (Y, 1), X) -> (abs X)
  SDValue X;
  if ((!LegalOperations || hasOperation(ISD::ABS, VT)) &&
      sd_match(N, m_Mul(m_Or(m_Sra(m_Value(X), m_SpecificInt(BitWidth - 1)),
                             m_One()),
                        m_Deferred(X)))) {
    return DAG.getNode(ISD::ABS, DL, VT, X);
  }
 
  // Fold ((mul x, 0/undef) -> 0,
  //       (mul x, 1) -> x) -> x)
  // -> and(x, mask)
  // We can replace vectors with '0' and '1' factors with a clearing mask.
  if (VT.isFixedLengthVector()) {
    unsigned NumElts = VT.getVectorNumElements();
    SmallBitVector ClearMask;
    ClearMask.reserve(NumElts);
    auto IsClearMask = [&ClearMask](ConstantSDNode *V) {
      if (!V || V->isZero()) {
        ClearMask.push_back(true);
        return true;
      }
      ClearMask.push_back(false);
      return V->isOne();
    };
    if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::AND, VT)) &&
        ISD::matchUnaryPredicate(N1, IsClearMask, /*AllowUndefs*/ true)) {
      assert(N1.getOpcode() == ISD::BUILD_VECTOR && "Unknown constant vector");
      EVT LegalSVT = N1.getOperand(0).getValueType();
      SDValue Zero = DAG.getConstant(0, DL, LegalSVT);
      SDValue AllOnes = DAG.getAllOnesConstant(DL, LegalSVT);
      SmallVector<SDValue, 16> Mask(NumElts, AllOnes);
      for (unsigned I = 0; I != NumElts; ++I)
        if (ClearMask[I])
          Mask[I] = Zero;
      return DAG.getNode(ISD::AND, DL, VT, N0, DAG.getBuildVector(VT, DL, Mask));
    }
  }
 
  // reassociate mul
  if (SDValue RMUL = reassociateOps(ISD::MUL, DL, N0, N1, N->getFlags()))
    return RMUL;
 
  // Fold mul(vecreduce(x), vecreduce(y)) -> vecreduce(mul(x, y))
  if (SDValue SD =
          reassociateReduction(ISD::VECREDUCE_MUL, ISD::MUL, DL, VT, N0, N1))
    return SD;
 
  // Simplify the operands using demanded-bits information.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  return SDValue();
}
 
/// Return true if divmod libcall is available.
static bool isDivRemLibcallAvailable(SDNode *Node, bool isSigned,
                                     const SelectionDAG &DAG) {
  RTLIB::Libcall LC;
  EVT NodeType = Node->getValueType(0);
  if (!NodeType.isSimple())
    return false;
  switch (NodeType.getSimpleVT().SimpleTy) {
  default: return false; // No libcall for vector types.
  case MVT::i8:   LC= isSigned ? RTLIB::SDIVREM_I8  : RTLIB::UDIVREM_I8;  break;
  case MVT::i16:  LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
  case MVT::i32:  LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
  case MVT::i64:  LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
  }
 
  return DAG.getLibcalls().getLibcallImpl(LC) != RTLIB::Unsupported;
}
 
/// Issue divrem if both quotient and remainder are needed.
SDValue DAGCombiner::useDivRem(SDNode *Node) {
  if (Node->use_empty())
    return SDValue(); // This is a dead node, leave it alone.
 
  unsigned Opcode = Node->getOpcode();
  bool isSigned = (Opcode == ISD::SDIV) || (Opcode == ISD::SREM);
  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
 
  // DivMod lib calls can still work on non-legal types if using lib-calls.
  EVT VT = Node->getValueType(0);
  if (VT.isVector() || !VT.isInteger())
    return SDValue();
 
  if (!TLI.isTypeLegal(VT) && !TLI.isOperationCustom(DivRemOpc, VT))
    return SDValue();
 
  // If DIVREM is going to get expanded into a libcall,
  // but there is no libcall available, then don't combine.
  if (!TLI.isOperationLegalOrCustom(DivRemOpc, VT) &&
      !isDivRemLibcallAvailable(Node, isSigned, DAG))
    return SDValue();
 
  // If div is legal, it's better to do the normal expansion
  unsigned OtherOpcode = 0;
  if ((Opcode == ISD::SDIV) || (Opcode == ISD::UDIV)) {
    OtherOpcode = isSigned ? ISD::SREM : ISD::UREM;
    if (TLI.isOperationLegalOrCustom(Opcode, VT))
      return SDValue();
  } else {
    OtherOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
    if (TLI.isOperationLegalOrCustom(OtherOpcode, VT))
      return SDValue();
  }
 
  SDValue Op0 = Node->getOperand(0);
  SDValue Op1 = Node->getOperand(1);
  SDValue combined;
  for (SDNode *User : Op0->users()) {
    if (User == Node || User->getOpcode() == ISD::DELETED_NODE ||
        User->use_empty())
      continue;
    // Convert the other matching node(s), too;
    // otherwise, the DIVREM may get target-legalized into something
    // target-specific that we won't be able to recognize.
    unsigned UserOpc = User->getOpcode();
    if ((UserOpc == Opcode || UserOpc == OtherOpcode || UserOpc == DivRemOpc) &&
        User->getOperand(0) == Op0 &&
        User->getOperand(1) == Op1) {
      if (!combined) {
        if (UserOpc == OtherOpcode) {
          SDVTList VTs = DAG.getVTList(VT, VT);
          combined = DAG.getNode(DivRemOpc, SDLoc(Node), VTs, Op0, Op1);
        } else if (UserOpc == DivRemOpc) {
          combined = SDValue(User, 0);
        } else {
          assert(UserOpc == Opcode);
          continue;
        }
      }
      if (UserOpc == ISD::SDIV || UserOpc == ISD::UDIV)
        CombineTo(User, combined);
      else if (UserOpc == ISD::SREM || UserOpc == ISD::UREM)
        CombineTo(User, combined.getValue(1));
    }
  }
  return combined;
}
 
static SDValue simplifyDivRem(SDNode *N, SelectionDAG &DAG) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  unsigned Opc = N->getOpcode();
  bool IsDiv = (ISD::SDIV == Opc) || (ISD::UDIV == Opc);
 
  // X / undef -> undef
  // X % undef -> undef
  // X / 0 -> undef
  // X % 0 -> undef
  // NOTE: This includes vectors where any divisor element is zero/undef.
  if (DAG.isUndef(Opc, {N0, N1}))
    return DAG.getUNDEF(VT);
 
  // undef / X -> 0
  // undef % X -> 0
  if (N0.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  // 0 / X -> 0
  // 0 % X -> 0
  ConstantSDNode *N0C = isConstOrConstSplat(N0);
  if (N0C && N0C->isZero())
    return N0;
 
  // X / X -> 1
  // X % X -> 0
  if (N0 == N1)
    return DAG.getConstant(IsDiv ? 1 : 0, DL, VT);
 
  // X / 1 -> X
  // X % 1 -> 0
  // If this is a boolean op (single-bit element type), we can't have
  // division-by-zero or remainder-by-zero, so assume the divisor is 1.
  // TODO: Similarly, if we're zero-extending a boolean divisor, then assume
  // it's a 1.
  if (isOneOrOneSplat(N1) || (VT.getScalarType() == MVT::i1))
    return IsDiv ? N0 : DAG.getConstant(0, DL, VT);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSDIV(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  EVT CCVT = getSetCCResultType(VT);
  SDLoc DL(N);
 
  // fold (sdiv c1, c2) -> c1/c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SDIV, DL, VT, {N0, N1}))
    return C;
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  // fold (sdiv X, -1) -> 0-X
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
  if (N1C && N1C->isAllOnes())
    return DAG.getNegative(N0, DL, VT);
 
  // fold (sdiv X, MIN_SIGNED) -> select(X == MIN_SIGNED, 1, 0)
  if (N1C && N1C->isMinSignedValue())
    return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
                         DAG.getConstant(1, DL, VT),
                         DAG.getConstant(0, DL, VT));
 
  if (SDValue V = simplifyDivRem(N, DAG))
    return V;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // If we know the sign bits of both operands are zero, strength reduce to a
  // udiv instead.  Handles (X&15) /s 4 -> X&15 >> 2
  if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
    return DAG.getNode(ISD::UDIV, DL, N1.getValueType(), N0, N1);
 
  if (SDValue V = visitSDIVLike(N0, N1, N)) {
    // If the corresponding remainder node exists, update its users with
    // (Dividend - (Quotient * Divisor).
    if (SDNode *RemNode = DAG.getNodeIfExists(ISD::SREM, N->getVTList(),
                                              { N0, N1 })) {
      // If the sdiv has the exact flag we shouldn't propagate it to the
      // remainder node.
      if (!N->getFlags().hasExact()) {
        SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
        SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
        AddToWorklist(Mul.getNode());
        AddToWorklist(Sub.getNode());
        CombineTo(RemNode, Sub);
      }
    }
    return V;
  }
 
  // sdiv, srem -> sdivrem
  // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
  // true.  Otherwise, we break the simplification logic in visitREM().
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
  if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
    if (SDValue DivRem = useDivRem(N))
        return DivRem;
 
  return SDValue();
}
 
static bool isDivisorPowerOfTwo(SDValue Divisor) {
  // Helper for determining whether a value is a power-2 constant scalar or a
  // vector of such elements.
  auto IsPowerOfTwo = [](ConstantSDNode *C) {
    if (C->isZero() || C->isOpaque())
      return false;
    if (C->getAPIntValue().isPowerOf2())
      return true;
    if (C->getAPIntValue().isNegatedPowerOf2())
      return true;
    return false;
  };
 
  return ISD::matchUnaryPredicate(Divisor, IsPowerOfTwo, /*AllowUndefs=*/false,
                                  /*AllowTruncation=*/true);
}
 
SDValue DAGCombiner::visitSDIVLike(SDValue N0, SDValue N1, SDNode *N) {
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
  EVT CCVT = getSetCCResultType(VT);
  unsigned BitWidth = VT.getScalarSizeInBits();
  unsigned MaxLegalDivRemBitWidth = TLI.getMaxDivRemBitWidthSupported();
 
  // fold (sdiv X, pow2) -> simple ops after legalize
  // FIXME: We check for the exact bit here because the generic lowering gives
  // better results in that case. The target-specific lowering should learn how
  // to handle exact sdivs efficiently. An exception is made for large bitwidths
  // exceeding what the target can natively support, as division expansion was
  // skipped in favor of this optimization.
  if ((!N->getFlags().hasExact() || BitWidth > MaxLegalDivRemBitWidth) &&
      isDivisorPowerOfTwo(N1)) {
    // Target-specific implementation of sdiv x, pow2.
    if (SDValue Res = BuildSDIVPow2(N))
      return Res;
 
    // Create constants that are functions of the shift amount value.
    EVT ShiftAmtTy = getShiftAmountTy(N0.getValueType());
    SDValue Bits = DAG.getConstant(BitWidth, DL, ShiftAmtTy);
    SDValue C1 = DAG.getNode(ISD::CTTZ, DL, VT, N1);
    C1 = DAG.getZExtOrTrunc(C1, DL, ShiftAmtTy);
    SDValue Inexact = DAG.getNode(ISD::SUB, DL, ShiftAmtTy, Bits, C1);
    if (!isConstantOrConstantVector(Inexact))
      return SDValue();
 
    // Splat the sign bit into the register
    SDValue Sign = DAG.getNode(ISD::SRA, DL, VT, N0,
                               DAG.getConstant(BitWidth - 1, DL, ShiftAmtTy));
    AddToWorklist(Sign.getNode());
 
    // Add (N0 < 0) ? abs2 - 1 : 0;
    SDValue Srl = DAG.getNode(ISD::SRL, DL, VT, Sign, Inexact);
    AddToWorklist(Srl.getNode());
    SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Srl);
    AddToWorklist(Add.getNode());
    SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Add, C1);
    AddToWorklist(Sra.getNode());
 
    // Special case: (sdiv X, 1) -> X
    // Special Case: (sdiv X, -1) -> 0-X
    SDValue One = DAG.getConstant(1, DL, VT);
    SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
    SDValue IsOne = DAG.getSetCC(DL, CCVT, N1, One, ISD::SETEQ);
    SDValue IsAllOnes = DAG.getSetCC(DL, CCVT, N1, AllOnes, ISD::SETEQ);
    SDValue IsOneOrAllOnes = DAG.getNode(ISD::OR, DL, CCVT, IsOne, IsAllOnes);
    Sra = DAG.getSelect(DL, VT, IsOneOrAllOnes, N0, Sra);
 
    // If dividing by a positive value, we're done. Otherwise, the result must
    // be negated.
    SDValue Zero = DAG.getConstant(0, DL, VT);
    SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, Zero, Sra);
 
    // FIXME: Use SELECT_CC once we improve SELECT_CC constant-folding.
    SDValue IsNeg = DAG.getSetCC(DL, CCVT, N1, Zero, ISD::SETLT);
    SDValue Res = DAG.getSelect(DL, VT, IsNeg, Sub, Sra);
    return Res;
  }
 
  // If integer divide is expensive and we satisfy the requirements, emit an
  // alternate sequence.  Targets may check function attributes for size/speed
  // trade-offs.
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
  if (isConstantOrConstantVector(N1, /*NoOpaques=*/false,
                                 /*AllowTruncation=*/true) &&
      !TLI.isIntDivCheap(N->getValueType(0), Attr))
    if (SDValue Op = BuildSDIV(N))
      return Op;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitUDIV(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  EVT CCVT = getSetCCResultType(VT);
  SDLoc DL(N);
 
  // fold (udiv c1, c2) -> c1/c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::UDIV, DL, VT, {N0, N1}))
    return C;
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  // fold (udiv X, -1) -> select(X == -1, 1, 0)
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
  if (N1C && N1C->isAllOnes() && CCVT.isVector() == VT.isVector()) {
    return DAG.getSelect(DL, VT, DAG.getSetCC(DL, CCVT, N0, N1, ISD::SETEQ),
                         DAG.getConstant(1, DL, VT),
                         DAG.getConstant(0, DL, VT));
  }
 
  if (SDValue V = simplifyDivRem(N, DAG))
    return V;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  if (SDValue V = visitUDIVLike(N0, N1, N)) {
    // If the corresponding remainder node exists, update its users with
    // (Dividend - (Quotient * Divisor).
    if (SDNode *RemNode = DAG.getNodeIfExists(ISD::UREM, N->getVTList(),
                                              { N0, N1 })) {
      // If the udiv has the exact flag we shouldn't propagate it to the
      // remainder node.
      if (!N->getFlags().hasExact()) {
        SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, V, N1);
        SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
        AddToWorklist(Mul.getNode());
        AddToWorklist(Sub.getNode());
        CombineTo(RemNode, Sub);
      }
    }
    return V;
  }
 
  // sdiv, srem -> sdivrem
  // If the divisor is constant, then return DIVREM only if isIntDivCheap() is
  // true.  Otherwise, we break the simplification logic in visitREM().
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
  if (!N1C || TLI.isIntDivCheap(N->getValueType(0), Attr))
    if (SDValue DivRem = useDivRem(N))
        return DivRem;
 
  // Simplify the operands using demanded-bits information.
  // We don't have demanded bits support for UDIV so this just enables constant
  // folding based on known bits.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitUDIVLike(SDValue N0, SDValue N1, SDNode *N) {
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
 
  // fold (udiv x, (1 << c)) -> x >>u c
  if (isConstantOrConstantVector(N1, /*NoOpaques=*/true,
                                 /*AllowTruncation=*/true)) {
    if (SDValue LogBase2 = BuildLogBase2(N1, DL)) {
      AddToWorklist(LogBase2.getNode());
 
      EVT ShiftVT = getShiftAmountTy(N0.getValueType());
      SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ShiftVT);
      AddToWorklist(Trunc.getNode());
      return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
    }
  }
 
  // fold (udiv x, (shl c, y)) -> x >>u (log2(c)+y) iff c is power of 2
  if (N1.getOpcode() == ISD::SHL) {
    SDValue N10 = N1.getOperand(0);
    if (isConstantOrConstantVector(N10, /*NoOpaques=*/true,
                                   /*AllowTruncation=*/true)) {
      if (SDValue LogBase2 = BuildLogBase2(N10, DL)) {
        AddToWorklist(LogBase2.getNode());
 
        EVT ADDVT = N1.getOperand(1).getValueType();
        SDValue Trunc = DAG.getZExtOrTrunc(LogBase2, DL, ADDVT);
        AddToWorklist(Trunc.getNode());
        SDValue Add = DAG.getNode(ISD::ADD, DL, ADDVT, N1.getOperand(1), Trunc);
        AddToWorklist(Add.getNode());
        return DAG.getNode(ISD::SRL, DL, VT, N0, Add);
      }
    }
  }
 
  // fold (udiv x, c) -> alternate
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
  if (isConstantOrConstantVector(N1, /*NoOpaques=*/false,
                                 /*AllowTruncation=*/true) &&
      !TLI.isIntDivCheap(N->getValueType(0), Attr))
    if (SDValue Op = BuildUDIV(N))
      return Op;
 
  return SDValue();
}
 
SDValue DAGCombiner::buildOptimizedSREM(SDValue N0, SDValue N1, SDNode *N) {
  if (!N->getFlags().hasExact() && isDivisorPowerOfTwo(N1) &&
      !DAG.doesNodeExist(ISD::SDIV, N->getVTList(), {N0, N1})) {
    // Target-specific implementation of srem x, pow2.
    if (SDValue Res = BuildSREMPow2(N))
      return Res;
  }
  return SDValue();
}
 
// handles ISD::SREM and ISD::UREM
SDValue DAGCombiner::visitREM(SDNode *N) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  EVT CCVT = getSetCCResultType(VT);
 
  bool isSigned = (Opcode == ISD::SREM);
  SDLoc DL(N);
 
  // fold (rem c1, c2) -> c1%c2
  if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
    return C;
 
  // fold (urem X, -1) -> select(FX == -1, 0, FX)
  // Freeze the numerator to avoid a miscompile with an undefined value.
  if (!isSigned && llvm::isAllOnesOrAllOnesSplat(N1, /*AllowUndefs*/ false) &&
      CCVT.isVector() == VT.isVector()) {
    SDValue F0 = DAG.getFreeze(N0);
    SDValue EqualsNeg1 = DAG.getSetCC(DL, CCVT, F0, N1, ISD::SETEQ);
    return DAG.getSelect(DL, VT, EqualsNeg1, DAG.getConstant(0, DL, VT), F0);
  }
 
  if (SDValue V = simplifyDivRem(N, DAG))
    return V;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  if (isSigned) {
    // If we know the sign bits of both operands are zero, strength reduce to a
    // urem instead.  Handles (X & 0x0FFFFFFF) %s 16 -> X&15
    if (DAG.SignBitIsZero(N1) && DAG.SignBitIsZero(N0))
      return DAG.getNode(ISD::UREM, DL, VT, N0, N1);
  } else {
    if (DAG.isKnownToBeAPowerOfTwo(N1, /*OrZero=*/true)) {
      // fold (urem x, pow2) -> (and x, pow2-1)
      SDValue NegOne = DAG.getAllOnesConstant(DL, VT);
      SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N1, NegOne);
      AddToWorklist(Add.getNode());
      return DAG.getNode(ISD::AND, DL, VT, N0, Add);
    }
  }
 
  AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
 
  // If X/C can be simplified by the division-by-constant logic, lower
  // X%C to the equivalent of X-X/C*C.
  // Reuse the SDIVLike/UDIVLike combines - to avoid mangling nodes, the
  // speculative DIV must not cause a DIVREM conversion.  We guard against this
  // by skipping the simplification if isIntDivCheap().  When div is not cheap,
  // combine will not return a DIVREM.  Regardless, checking cheapness here
  // makes sense since the simplification results in fatter code.
  if (DAG.isKnownNeverZero(N1) && !TLI.isIntDivCheap(VT, Attr)) {
    if (isSigned) {
      // check if we can build faster implementation for srem
      if (SDValue OptimizedRem = buildOptimizedSREM(N0, N1, N))
        return OptimizedRem;
    }
 
    SDValue OptimizedDiv =
        isSigned ? visitSDIVLike(N0, N1, N) : visitUDIVLike(N0, N1, N);
    if (OptimizedDiv.getNode() && OptimizedDiv.getNode() != N) {
      // If the equivalent Div node also exists, update its users.
      unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
      if (SDNode *DivNode = DAG.getNodeIfExists(DivOpcode, N->getVTList(),
                                                { N0, N1 }))
        CombineTo(DivNode, OptimizedDiv);
      SDValue Mul = DAG.getNode(ISD::MUL, DL, VT, OptimizedDiv, N1);
      SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, Mul);
      AddToWorklist(OptimizedDiv.getNode());
      AddToWorklist(Mul.getNode());
      return Sub;
    }
  }
 
  // sdiv, srem -> sdivrem
  if (SDValue DivRem = useDivRem(N))
    return DivRem.getValue(1);
 
  // fold urem(urem(A, BCst), Op1Cst) -> urem(A, Op1Cst)
  // iff urem(BCst, Op1Cst) == 0
  SDValue A;
  APInt Op1Cst, BCst;
  if (sd_match(N, m_URem(m_URem(m_Value(A), m_ConstInt(BCst)),
                         m_ConstInt(Op1Cst))) &&
      BCst.urem(Op1Cst).isZero()) {
    return DAG.getNode(ISD::UREM, DL, VT, A, DAG.getConstant(Op1Cst, DL, VT));
  }
 
  // fold srem(srem(A, BCst), Op1Cst) -> srem(A, Op1Cst)
  // iff srem(BCst, Op1Cst) == 0 && Op1Cst != 1
  if (sd_match(N, m_SRem(m_SRem(m_Value(A), m_ConstInt(BCst)),
                         m_ConstInt(Op1Cst))) &&
      BCst.srem(Op1Cst).isZero() && !Op1Cst.isAllOnes()) {
    return DAG.getNode(ISD::SREM, DL, VT, A, DAG.getConstant(Op1Cst, DL, VT));
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMULHS(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (mulhs c1, c2)
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHS, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS.
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::MULHS, DL, N->getVTList(), N1, N0);
 
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (mulhs x, 0) -> 0
    // do not return N1, because undef node may exist.
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      return DAG.getConstant(0, DL, VT);
  }
 
  // fold (mulhs x, 0) -> 0
  if (isNullConstant(N1))
    return N1;
 
  // fold (mulhs x, 1) -> (sra x, size(x)-1)
  if (isOneConstant(N1))
    return DAG.getNode(
        ISD::SRA, DL, VT, N0,
        DAG.getShiftAmountConstant(N0.getScalarValueSizeInBits() - 1, VT, DL));
 
  // fold (mulhs x, undef) -> 0
  if (N0.isUndef() || N1.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  // If the type twice as wide is legal, transform the mulhs to a wider multiply
  // plus a shift.
  if (!TLI.isOperationLegalOrCustom(ISD::MULHS, VT) && VT.isSimple() &&
      !VT.isVector()) {
    MVT Simple = VT.getSimpleVT();
    unsigned SimpleSize = Simple.getSizeInBits();
    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
      N0 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
      N1 = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
      N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
      N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
                       DAG.getShiftAmountConstant(SimpleSize, NewVT, DL));
      return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMULHU(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (mulhu c1, c2)
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::MULHU, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS.
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::MULHU, DL, N->getVTList(), N1, N0);
 
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (mulhu x, 0) -> 0
    // do not return N1, because undef node may exist.
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      return DAG.getConstant(0, DL, VT);
  }
 
  // fold (mulhu x, 0) -> 0
  if (isNullConstant(N1))
    return N1;
 
  // fold (mulhu x, 1) -> 0
  if (isOneConstant(N1))
    return DAG.getConstant(0, DL, VT);
 
  // fold (mulhu x, undef) -> 0
  if (N0.isUndef() || N1.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c)
  if (isConstantOrConstantVector(N1, /*NoOpaques=*/true,
                                 /*AllowTruncation=*/true) &&
      (!LegalOperations || hasOperation(ISD::SRL, VT))) {
    if (SDValue LogBase2 = BuildLogBase2(N1, DL)) {
      unsigned NumEltBits = VT.getScalarSizeInBits();
      SDValue SRLAmt = DAG.getNode(
          ISD::SUB, DL, VT, DAG.getConstant(NumEltBits, DL, VT), LogBase2);
      EVT ShiftVT = getShiftAmountTy(N0.getValueType());
      SDValue Trunc = DAG.getZExtOrTrunc(SRLAmt, DL, ShiftVT);
      return DAG.getNode(ISD::SRL, DL, VT, N0, Trunc);
    }
  }
 
  // If the type twice as wide is legal, transform the mulhu to a wider multiply
  // plus a shift.
  if (!TLI.isOperationLegalOrCustom(ISD::MULHU, VT) && VT.isSimple() &&
      !VT.isVector()) {
    MVT Simple = VT.getSimpleVT();
    unsigned SimpleSize = Simple.getSizeInBits();
    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
      N0 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
      N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
      N1 = DAG.getNode(ISD::MUL, DL, NewVT, N0, N1);
      N1 = DAG.getNode(ISD::SRL, DL, NewVT, N1,
                       DAG.getShiftAmountConstant(SimpleSize, NewVT, DL));
      return DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
    }
  }
 
  // Simplify the operands using demanded-bits information.
  // We don't have demanded bits support for MULHU so this just enables constant
  // folding based on known bits.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitAVG(SDNode *N) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  bool IsSigned = Opcode == ISD::AVGCEILS || Opcode == ISD::AVGFLOORS;
 
  // fold (avg c1, c2)
  if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS.
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0);
 
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  // fold (avg x, undef) -> x
  if (N0.isUndef())
    return N1;
  if (N1.isUndef())
    return N0;
 
  // fold (avg x, x) --> x
  if (N0 == N1 && Level >= AfterLegalizeTypes)
    return N0;
 
  // fold (avgfloor x, 0) -> x >> 1
  SDValue X, Y;
  if (sd_match(N, m_c_BinOp(ISD::AVGFLOORS, m_Value(X), m_Zero())))
    return DAG.getNode(ISD::SRA, DL, VT, X,
                       DAG.getShiftAmountConstant(1, VT, DL));
  if (sd_match(N, m_c_BinOp(ISD::AVGFLOORU, m_Value(X), m_Zero())))
    return DAG.getNode(ISD::SRL, DL, VT, X,
                       DAG.getShiftAmountConstant(1, VT, DL));
 
  // fold avgu(zext(x), zext(y)) -> zext(avgu(x, y))
  // fold avgs(sext(x), sext(y)) -> sext(avgs(x, y))
  if (!IsSigned &&
      sd_match(N, m_BinOp(Opcode, m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) &&
      X.getValueType() == Y.getValueType() &&
      hasOperation(Opcode, X.getValueType())) {
    SDValue AvgU = DAG.getNode(Opcode, DL, X.getValueType(), X, Y);
    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, AvgU);
  }
  if (IsSigned &&
      sd_match(N, m_BinOp(Opcode, m_SExt(m_Value(X)), m_SExt(m_Value(Y)))) &&
      X.getValueType() == Y.getValueType() &&
      hasOperation(Opcode, X.getValueType())) {
    SDValue AvgS = DAG.getNode(Opcode, DL, X.getValueType(), X, Y);
    return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, AvgS);
  }
 
  // Fold avgflooru(x,y) -> avgceilu(x,y-1) iff y != 0
  // Fold avgflooru(x,y) -> avgceilu(x-1,y) iff x != 0
  // Check if avgflooru isn't legal/custom but avgceilu is.
  if (Opcode == ISD::AVGFLOORU && !hasOperation(ISD::AVGFLOORU, VT) &&
      (!LegalOperations || hasOperation(ISD::AVGCEILU, VT))) {
    if (DAG.isKnownNeverZero(N1))
      return DAG.getNode(
          ISD::AVGCEILU, DL, VT, N0,
          DAG.getNode(ISD::ADD, DL, VT, N1, DAG.getAllOnesConstant(DL, VT)));
    if (DAG.isKnownNeverZero(N0))
      return DAG.getNode(
          ISD::AVGCEILU, DL, VT, N1,
          DAG.getNode(ISD::ADD, DL, VT, N0, DAG.getAllOnesConstant(DL, VT)));
  }
 
  // Fold avgfloor((add nw x,y), 1) -> avgceil(x,y)
  // Fold avgfloor((add nw x,1), y) -> avgceil(x,y)
  if ((Opcode == ISD::AVGFLOORU && hasOperation(ISD::AVGCEILU, VT)) ||
      (Opcode == ISD::AVGFLOORS && hasOperation(ISD::AVGCEILS, VT))) {
    SDValue Add;
    if (sd_match(N,
                 m_c_BinOp(Opcode,
                           m_AllOf(m_Value(Add), m_Add(m_Value(X), m_Value(Y))),
                           m_One())) ||
        sd_match(N, m_c_BinOp(Opcode,
                              m_AllOf(m_Value(Add), m_Add(m_Value(X), m_One())),
                              m_Value(Y)))) {
 
      if (IsSigned && Add->getFlags().hasNoSignedWrap())
        return DAG.getNode(ISD::AVGCEILS, DL, VT, X, Y);
 
      if (!IsSigned && Add->getFlags().hasNoUnsignedWrap())
        return DAG.getNode(ISD::AVGCEILU, DL, VT, X, Y);
    }
  }
 
  // Fold avgfloors(x,y) -> avgflooru(x,y) if both x and y are non-negative
  if (Opcode == ISD::AVGFLOORS && hasOperation(ISD::AVGFLOORU, VT)) {
    if (DAG.SignBitIsZero(N0) && DAG.SignBitIsZero(N1))
      return DAG.getNode(ISD::AVGFLOORU, DL, VT, N0, N1);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitABD(SDNode *N) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (abd c1, c2)
  if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS.
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(Opcode, DL, N->getVTList(), N1, N0);
 
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  // fold (abd x, undef) -> 0
  if (N0.isUndef() || N1.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  // fold (abd x, x) -> 0
  if (N0 == N1)
    return DAG.getConstant(0, DL, VT);
 
  SDValue X, Y;
 
  // fold (abds x, 0) -> abs x
  if (sd_match(N, m_c_BinOp(ISD::ABDS, m_Value(X), m_Zero())) &&
      (!LegalOperations || hasOperation(ISD::ABS, VT)))
    return DAG.getNode(ISD::ABS, DL, VT, X);
 
  // fold (abdu x, 0) -> x
  if (sd_match(N, m_c_BinOp(ISD::ABDU, m_Value(X), m_Zero())))
    return X;
 
  // fold (abds x, y) -> (abdu x, y) iff both args are known positive
  if (Opcode == ISD::ABDS && hasOperation(ISD::ABDU, VT) &&
      DAG.SignBitIsZero(N0) && DAG.SignBitIsZero(N1))
    return DAG.getNode(ISD::ABDU, DL, VT, N1, N0);
 
  // fold (abd? (?ext x), (?ext y)) -> (zext (abd? x, y))
  if (sd_match(N, m_BinOp(ISD::ABDU, m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) ||
      sd_match(N, m_BinOp(ISD::ABDS, m_SExt(m_Value(X)), m_SExt(m_Value(Y))))) {
    EVT SmallVT = X.getScalarValueSizeInBits() > Y.getScalarValueSizeInBits()
                      ? X.getValueType()
                      : Y.getValueType();
    if (!LegalOperations || hasOperation(Opcode, SmallVT)) {
      SDValue ExtedX = DAG.getExtOrTrunc(X, SDLoc(X), SmallVT, N0->getOpcode());
      SDValue ExtedY = DAG.getExtOrTrunc(Y, SDLoc(Y), SmallVT, N0->getOpcode());
      SDValue SmallABD = DAG.getNode(Opcode, DL, SmallVT, {ExtedX, ExtedY});
      SDValue ZExted = DAG.getZExtOrTrunc(SmallABD, DL, VT);
      return ZExted;
    }
  }
 
  // fold (abd? (?ext ty:x), small_const:c) -> (zext (abd? x, c))
  if (sd_match(N, m_c_BinOp(ISD::ABDU, m_ZExt(m_Value(X)), m_Value(Y))) ||
      sd_match(N, m_c_BinOp(ISD::ABDS, m_SExt(m_Value(X)), m_Value(Y)))) {
    EVT SmallVT = X.getValueType();
    if (!LegalOperations || hasOperation(Opcode, SmallVT)) {
      uint64_t Bits = SmallVT.getScalarSizeInBits();
      unsigned RelevantBits =
          (Opcode == ISD::ABDS) ? DAG.ComputeMaxSignificantBits(Y)
                                : DAG.computeKnownBits(Y).countMaxActiveBits();
      bool TruncatingYIsCheap = TLI.isTruncateFree(Y, SmallVT) ||
                                ISD::matchUnaryPredicate(
                                    Y,
                                    [&](auto *C) {
                                      const APInt &YConst = C->getAsAPIntVal();
                                      return (Opcode == ISD::ABDS)
                                                 ? YConst.isSignedIntN(Bits)
                                                 : YConst.isIntN(Bits);
                                    },
                                    /*AllowUndefs=*/true);
 
      if (RelevantBits <= Bits && TruncatingYIsCheap) {
        SDValue NewY = DAG.getNode(ISD::TRUNCATE, SDLoc(Y), SmallVT, Y);
        SDValue SmallABD = DAG.getNode(Opcode, DL, SmallVT, {X, NewY});
        return DAG.getZExtOrTrunc(SmallABD, DL, VT);
      }
    }
  }
 
  return SDValue();
}
 
/// Perform optimizations common to nodes that compute two values. LoOp and HiOp
/// give the opcodes for the two computations that are being performed. Return
/// true if a simplification was made.
SDValue DAGCombiner::SimplifyNodeWithTwoResults(SDNode *N, unsigned LoOp,
                                                unsigned HiOp) {
  // If the high half is not needed, just compute the low half.
  bool HiExists = N->hasAnyUseOfValue(1);
  if (!HiExists && (!LegalOperations ||
                    TLI.isOperationLegalOrCustom(LoOp, N->getValueType(0)))) {
    SDValue Res = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
    return CombineTo(N, Res, Res);
  }
 
  // If the low half is not needed, just compute the high half.
  bool LoExists = N->hasAnyUseOfValue(0);
  if (!LoExists && (!LegalOperations ||
                    TLI.isOperationLegalOrCustom(HiOp, N->getValueType(1)))) {
    SDValue Res = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
    return CombineTo(N, Res, Res);
  }
 
  // If both halves are used, return as it is.
  if (LoExists && HiExists)
    return SDValue();
 
  // If the two computed results can be simplified separately, separate them.
  if (LoExists) {
    SDValue Lo = DAG.getNode(LoOp, SDLoc(N), N->getValueType(0), N->ops());
    AddToWorklist(Lo.getNode());
    SDValue LoOpt = combine(Lo.getNode());
    if (LoOpt.getNode() && LoOpt.getNode() != Lo.getNode() &&
        (!LegalOperations ||
         TLI.isOperationLegalOrCustom(LoOpt.getOpcode(), LoOpt.getValueType())))
      return CombineTo(N, LoOpt, LoOpt);
  }
 
  if (HiExists) {
    SDValue Hi = DAG.getNode(HiOp, SDLoc(N), N->getValueType(1), N->ops());
    AddToWorklist(Hi.getNode());
    SDValue HiOpt = combine(Hi.getNode());
    if (HiOpt.getNode() && HiOpt != Hi &&
        (!LegalOperations ||
         TLI.isOperationLegalOrCustom(HiOpt.getOpcode(), HiOpt.getValueType())))
      return CombineTo(N, HiOpt, HiOpt);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSMUL_LOHI(SDNode *N) {
  if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHS))
    return Res;
 
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // Constant fold.
  if (isa<ConstantSDNode>(N0) && isa<ConstantSDNode>(N1))
    return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N0, N1);
 
  // canonicalize constant to RHS (vector doesn't have to splat)
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::SMUL_LOHI, DL, N->getVTList(), N1, N0);
 
  // If the type is twice as wide is legal, transform the mulhu to a wider
  // multiply plus a shift.
  if (VT.isSimple() && !VT.isVector()) {
    MVT Simple = VT.getSimpleVT();
    unsigned SimpleSize = Simple.getSizeInBits();
    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
      SDValue Lo = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N0);
      SDValue Hi = DAG.getNode(ISD::SIGN_EXTEND, DL, NewVT, N1);
      Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
      // Compute the high part as N1.
      Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
                       DAG.getShiftAmountConstant(SimpleSize, NewVT, DL));
      Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
      // Compute the low part as N0.
      Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
      return CombineTo(N, Lo, Hi);
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) {
  if (SDValue Res = SimplifyNodeWithTwoResults(N, ISD::MUL, ISD::MULHU))
    return Res;
 
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // Constant fold.
  if (isa<ConstantSDNode>(N0) && isa<ConstantSDNode>(N1))
    return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N0, N1);
 
  // canonicalize constant to RHS (vector doesn't have to splat)
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::UMUL_LOHI, DL, N->getVTList(), N1, N0);
 
  // (umul_lohi N0, 0) -> (0, 0)
  if (isNullConstant(N1)) {
    SDValue Zero = DAG.getConstant(0, DL, VT);
    return CombineTo(N, Zero, Zero);
  }
 
  // (umul_lohi N0, 1) -> (N0, 0)
  if (isOneConstant(N1)) {
    SDValue Zero = DAG.getConstant(0, DL, VT);
    return CombineTo(N, N0, Zero);
  }
 
  // If the type is twice as wide is legal, transform the mulhu to a wider
  // multiply plus a shift.
  if (VT.isSimple() && !VT.isVector()) {
    MVT Simple = VT.getSimpleVT();
    unsigned SimpleSize = Simple.getSizeInBits();
    EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), SimpleSize*2);
    if (TLI.isOperationLegal(ISD::MUL, NewVT)) {
      SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N0);
      SDValue Hi = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, N1);
      Lo = DAG.getNode(ISD::MUL, DL, NewVT, Lo, Hi);
      // Compute the high part as N1.
      Hi = DAG.getNode(ISD::SRL, DL, NewVT, Lo,
                       DAG.getShiftAmountConstant(SimpleSize, NewVT, DL));
      Hi = DAG.getNode(ISD::TRUNCATE, DL, VT, Hi);
      // Compute the low part as N0.
      Lo = DAG.getNode(ISD::TRUNCATE, DL, VT, Lo);
      return CombineTo(N, Lo, Hi);
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMULO(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  bool IsSigned = (ISD::SMULO == N->getOpcode());
 
  EVT CarryVT = N->getValueType(1);
  SDLoc DL(N);
 
  ConstantSDNode *N0C = isConstOrConstSplat(N0);
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
 
  // fold operation with constant operands.
  // TODO: Move this to FoldConstantArithmetic when it supports nodes with
  // multiple results.
  if (N0C && N1C) {
    bool Overflow;
    APInt Result =
        IsSigned ? N0C->getAPIntValue().smul_ov(N1C->getAPIntValue(), Overflow)
                 : N0C->getAPIntValue().umul_ov(N1C->getAPIntValue(), Overflow);
    return CombineTo(N, DAG.getConstant(Result, DL, VT),
                     DAG.getBoolConstant(Overflow, DL, CarryVT, CarryVT));
  }
 
  // canonicalize constant to RHS.
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0);
 
  // fold (mulo x, 0) -> 0 + no carry out
  if (isNullOrNullSplat(N1))
    return CombineTo(N, DAG.getConstant(0, DL, VT),
                     DAG.getConstant(0, DL, CarryVT));
 
  // (mulo x, 2) -> (addo x, x)
  // FIXME: This needs a freeze.
  if (N1C && N1C->getAPIntValue() == 2 &&
      (!IsSigned || VT.getScalarSizeInBits() > 2))
    return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, DL,
                       N->getVTList(), N0, N0);
 
  // A 1 bit SMULO overflows if both inputs are 1.
  if (IsSigned && VT.getScalarSizeInBits() == 1) {
    SDValue And = DAG.getNode(ISD::AND, DL, VT, N0, N1);
    SDValue Cmp = DAG.getSetCC(DL, CarryVT, And,
                               DAG.getConstant(0, DL, VT), ISD::SETNE);
    return CombineTo(N, And, Cmp);
  }
 
  // If it cannot overflow, transform into a mul.
  if (DAG.willNotOverflowMul(IsSigned, N0, N1))
    return CombineTo(N, DAG.getNode(ISD::MUL, DL, VT, N0, N1),
                     DAG.getConstant(0, DL, CarryVT));
  return SDValue();
}
 
// Function to calculate whether the Min/Max pair of SDNodes (potentially
// swapped around) make a signed saturate pattern, clamping to between a signed
// saturate of -2^(BW-1) and 2^(BW-1)-1, or an unsigned saturate of 0 and 2^BW.
// Returns the node being clamped and the bitwidth of the clamp in BW. Should
// work with both SMIN/SMAX nodes and setcc/select combo. The operands are the
// same as SimplifySelectCC. N0<N1 ? N2 : N3.
static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
                                  SDValue N3, ISD::CondCode CC, unsigned &BW,
                                  bool &Unsigned, SelectionDAG &DAG) {
  auto isSignedMinMax = [&](SDValue N0, SDValue N1, SDValue N2, SDValue N3,
                            ISD::CondCode CC) {
    // The compare and select operand should be the same or the select operands
    // should be truncated versions of the comparison.
    if (N0 != N2 && (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0)))
      return 0;
    // The constants need to be the same or a truncated version of each other.
    ConstantSDNode *N1C = isConstOrConstSplat(peekThroughTruncates(N1));
    ConstantSDNode *N3C = isConstOrConstSplat(peekThroughTruncates(N3));
    if (!N1C || !N3C)
      return 0;
    const APInt &C1 = N1C->getAPIntValue().trunc(N1.getScalarValueSizeInBits());
    const APInt &C2 = N3C->getAPIntValue().trunc(N3.getScalarValueSizeInBits());
    if (C1.getBitWidth() < C2.getBitWidth() || C1 != C2.sext(C1.getBitWidth()))
      return 0;
    return CC == ISD::SETLT ? ISD::SMIN : (CC == ISD::SETGT ? ISD::SMAX : 0);
  };
 
  // Check the initial value is a SMIN/SMAX equivalent.
  unsigned Opcode0 = isSignedMinMax(N0, N1, N2, N3, CC);
  if (!Opcode0)
    return SDValue();
 
  // We could only need one range check, if the fptosi could never produce
  // the upper value.
  if (N0.getOpcode() == ISD::FP_TO_SINT && Opcode0 == ISD::SMAX) {
    if (isNullOrNullSplat(N3)) {
      EVT IntVT = N0.getValueType().getScalarType();
      EVT FPVT = N0.getOperand(0).getValueType().getScalarType();
      if (FPVT.isSimple()) {
        Type *InputTy = FPVT.getTypeForEVT(*DAG.getContext());
        const fltSemantics &Semantics = InputTy->getFltSemantics();
        uint32_t MinBitWidth =
          APFloatBase::semanticsIntSizeInBits(Semantics, /*isSigned*/ true);
        if (IntVT.getSizeInBits() >= MinBitWidth) {
          Unsigned = true;
          BW = PowerOf2Ceil(MinBitWidth);
          return N0;
        }
      }
    }
  }
 
  SDValue N00, N01, N02, N03;
  ISD::CondCode N0CC;
  switch (N0.getOpcode()) {
  case ISD::SMIN:
  case ISD::SMAX:
    N00 = N02 = N0.getOperand(0);
    N01 = N03 = N0.getOperand(1);
    N0CC = N0.getOpcode() == ISD::SMIN ? ISD::SETLT : ISD::SETGT;
    break;
  case ISD::SELECT_CC:
    N00 = N0.getOperand(0);
    N01 = N0.getOperand(1);
    N02 = N0.getOperand(2);
    N03 = N0.getOperand(3);
    N0CC = cast<CondCodeSDNode>(N0.getOperand(4))->get();
    break;
  case ISD::SELECT:
  case ISD::VSELECT:
    if (N0.getOperand(0).getOpcode() != ISD::SETCC)
      return SDValue();
    N00 = N0.getOperand(0).getOperand(0);
    N01 = N0.getOperand(0).getOperand(1);
    N02 = N0.getOperand(1);
    N03 = N0.getOperand(2);
    N0CC = cast<CondCodeSDNode>(N0.getOperand(0).getOperand(2))->get();
    break;
  default:
    return SDValue();
  }
 
  unsigned Opcode1 = isSignedMinMax(N00, N01, N02, N03, N0CC);
  if (!Opcode1 || Opcode0 == Opcode1)
    return SDValue();
 
  ConstantSDNode *MinCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N1 : N01);
  ConstantSDNode *MaxCOp = isConstOrConstSplat(Opcode0 == ISD::SMIN ? N01 : N1);
  if (!MinCOp || !MaxCOp || MinCOp->getValueType(0) != MaxCOp->getValueType(0))
    return SDValue();
 
  const APInt &MinC = MinCOp->getAPIntValue();
  const APInt &MaxC = MaxCOp->getAPIntValue();
  APInt MinCPlus1 = MinC + 1;
  if (-MaxC == MinCPlus1 && MinCPlus1.isPowerOf2()) {
    BW = MinCPlus1.exactLogBase2() + 1;
    Unsigned = false;
    return N02;
  }
 
  if (MaxC == 0 && MinC != 0 && MinCPlus1.isPowerOf2()) {
    BW = MinCPlus1.exactLogBase2();
    Unsigned = true;
    return N02;
  }
 
  return SDValue();
}
 
static SDValue PerformMinMaxFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
                                           SDValue N3, ISD::CondCode CC,
                                           SelectionDAG &DAG) {
  unsigned BW;
  bool Unsigned;
  SDValue Fp = isSaturatingMinMax(N0, N1, N2, N3, CC, BW, Unsigned, DAG);
  if (!Fp || Fp.getOpcode() != ISD::FP_TO_SINT)
    return SDValue();
  EVT FPVT = Fp.getOperand(0).getValueType();
  EVT NewVT = FPVT.changeElementType(*DAG.getContext(),
                                     EVT::getIntegerVT(*DAG.getContext(), BW));
  unsigned NewOpc = Unsigned ? ISD::FP_TO_UINT_SAT : ISD::FP_TO_SINT_SAT;
  if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(NewOpc, FPVT, NewVT))
    return SDValue();
  SDLoc DL(Fp);
  SDValue Sat = DAG.getNode(NewOpc, DL, NewVT, Fp.getOperand(0),
                            DAG.getValueType(NewVT.getScalarType()));
  return DAG.getExtOrTrunc(!Unsigned, Sat, DL, N2->getValueType(0));
}
 
static SDValue PerformUMinFpToSatCombine(SDValue N0, SDValue N1, SDValue N2,
                                         SDValue N3, ISD::CondCode CC,
                                         SelectionDAG &DAG) {
  // We are looking for UMIN(FPTOUI(X), (2^n)-1), which may have come via a
  // select/vselect/select_cc. The two operands pairs for the select (N2/N3) may
  // be truncated versions of the setcc (N0/N1).
  if ((N0 != N2 &&
       (N2.getOpcode() != ISD::TRUNCATE || N0 != N2.getOperand(0))) ||
      N0.getOpcode() != ISD::FP_TO_UINT || CC != ISD::SETULT)
    return SDValue();
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
  ConstantSDNode *N3C = isConstOrConstSplat(N3);
  if (!N1C || !N3C)
    return SDValue();
  const APInt &C1 = N1C->getAPIntValue();
  const APInt &C3 = N3C->getAPIntValue();
  if (!(C1 + 1).isPowerOf2() || C1.getBitWidth() < C3.getBitWidth() ||
      C1 != C3.zext(C1.getBitWidth()))
    return SDValue();
 
  unsigned BW = (C1 + 1).exactLogBase2();
  EVT FPVT = N0.getOperand(0).getValueType();
  EVT NewVT = FPVT.changeElementType(*DAG.getContext(),
                                     EVT::getIntegerVT(*DAG.getContext(), BW));
  if (!DAG.getTargetLoweringInfo().shouldConvertFpToSat(ISD::FP_TO_UINT_SAT,
                                                        FPVT, NewVT))
    return SDValue();
 
  SDValue Sat =
      DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), NewVT, N0.getOperand(0),
                  DAG.getValueType(NewVT.getScalarType()));
  return DAG.getZExtOrTrunc(Sat, SDLoc(N0), N3.getValueType());
}
 
SDValue DAGCombiner::visitIMINMAX(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  unsigned Opcode = N->getOpcode();
  SDLoc DL(N);
 
  // fold operation with constant operands.
  if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
    return C;
 
  // If the operands are the same, this is a no-op.
  if (N0 == N1)
    return N0;
 
  // canonicalize constant to RHS
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(Opcode, DL, VT, N1, N0);
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  // reassociate minmax
  if (SDValue RMINMAX = reassociateOps(Opcode, DL, N0, N1, N->getFlags()))
    return RMINMAX;
 
  // If both operands are known to have the same sign (both non-negative or both
  // negative), flip between UMIN/UMAX and SMIN/SMAX.
  // Only do this if:
  // 1. The current op isn't legal and the flipped is.
  // 2. The saturation pattern is broken by canonicalization in InstCombine.
  bool IsOpIllegal = !TLI.isOperationLegal(Opcode, VT);
  bool IsSatBroken = Opcode == ISD::UMIN && N0.getOpcode() == ISD::SMAX;
 
  if (IsSatBroken || IsOpIllegal) {
    auto HasKnownSameSign = [&](SDValue A, SDValue B) {
      if (A.isUndef() || B.isUndef())
        return true;
 
      KnownBits KA = DAG.computeKnownBits(A);
      if (!KA.isNonNegative() && !KA.isNegative())
        return false;
 
      KnownBits KB = DAG.computeKnownBits(B);
      if (KA.isNonNegative())
        return KB.isNonNegative();
      return KB.isNegative();
    };
 
    if (HasKnownSameSign(N0, N1)) {
      unsigned AltOpcode = ISD::getOppositeSignednessMinMaxOpcode(Opcode);
      if ((IsSatBroken && IsOpIllegal) || TLI.isOperationLegal(AltOpcode, VT))
        return DAG.getNode(AltOpcode, DL, VT, N0, N1);
    }
  }
 
  if (Opcode == ISD::SMIN || Opcode == ISD::SMAX)
    if (SDValue S = PerformMinMaxFpToSatCombine(
            N0, N1, N0, N1, Opcode == ISD::SMIN ? ISD::SETLT : ISD::SETGT, DAG))
      return S;
  if (Opcode == ISD::UMIN)
    if (SDValue S = PerformUMinFpToSatCombine(N0, N1, N0, N1, ISD::SETULT, DAG))
      return S;
 
  // Fold min/max(vecreduce(x), vecreduce(y)) -> vecreduce(min/max(x, y))
  auto ReductionOpcode = [](unsigned Opcode) {
    switch (Opcode) {
    case ISD::SMIN:
      return ISD::VECREDUCE_SMIN;
    case ISD::SMAX:
      return ISD::VECREDUCE_SMAX;
    case ISD::UMIN:
      return ISD::VECREDUCE_UMIN;
    case ISD::UMAX:
      return ISD::VECREDUCE_UMAX;
    default:
      llvm_unreachable("Unexpected opcode");
    }
  };
  if (SDValue SD = reassociateReduction(ReductionOpcode(Opcode), Opcode,
                                        SDLoc(N), VT, N0, N1))
    return SD;
 
  // Fold operation with vscale operands.
  if (N0.getOpcode() == ISD::VSCALE && N1.getOpcode() == ISD::VSCALE) {
    uint64_t C0 = N0->getConstantOperandVal(0);
    uint64_t C1 = N1->getConstantOperandVal(0);
    if (Opcode == ISD::UMAX)
      return C0 > C1 ? N0 : N1;
    else if (Opcode == ISD::UMIN)
      return C0 > C1 ? N1 : N0;
  }
 
  // If we know the range of vscale, see if we can fold it given a constant.
  if (N0.getOpcode() == ISD::VSCALE) {
    if (auto *C1 = dyn_cast<ConstantSDNode>(N1)) {
      bool ForSigned = (Opcode == ISD::SMAX || Opcode == ISD::SMIN);
      ConstantRange Range = DAG.computeConstantRange(N0, ForSigned);
 
      const APInt &C1V = C1->getAPIntValue();
      if ((Opcode == ISD::UMAX && Range.getUnsignedMax().ule(C1V)) ||
          (Opcode == ISD::UMIN && Range.getUnsignedMin().uge(C1V)) ||
          (Opcode == ISD::SMAX && Range.getSignedMax().sle(C1V)) ||
          (Opcode == ISD::SMIN && Range.getSignedMin().sge(C1V))) {
        return N1;
      }
    }
  }
 
  // Simplify the operands using demanded-bits information.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  return SDValue();
}
 
/// If this is a bitwise logic instruction and both operands have the same
/// opcode, try to sink the other opcode after the logic instruction.
SDValue DAGCombiner::hoistLogicOpWithSameOpcodeHands(SDNode *N) {
  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  unsigned LogicOpcode = N->getOpcode();
  unsigned HandOpcode = N0.getOpcode();
  assert(ISD::isBitwiseLogicOp(LogicOpcode) && "Expected logic opcode");
  assert(HandOpcode == N1.getOpcode() && "Bad input!");
 
  // Bail early if none of these transforms apply.
  if (N0.getNumOperands() == 0)
    return SDValue();
 
  // FIXME: We should check number of uses of the operands to not increase
  //        the instruction count for all transforms.
 
  // Handle size-changing casts (or sign_extend_inreg).
  SDValue X = N0.getOperand(0);
  SDValue Y = N1.getOperand(0);
  EVT XVT = X.getValueType();
  SDLoc DL(N);
  if (ISD::isExtOpcode(HandOpcode) || ISD::isExtVecInRegOpcode(HandOpcode) ||
      (HandOpcode == ISD::SIGN_EXTEND_INREG &&
       N0.getOperand(1) == N1.getOperand(1))) {
    // If both operands have other uses, this transform would create extra
    // instructions without eliminating anything.
    if (!N0.hasOneUse() && !N1.hasOneUse())
      return SDValue();
    // We need matching integer source types.
    if (XVT != Y.getValueType())
      return SDValue();
    // Don't create an illegal op during or after legalization. Don't ever
    // create an unsupported vector op.
    if ((VT.isVector() || LegalOperations) &&
        !TLI.isOperationLegalOrCustom(LogicOpcode, XVT))
      return SDValue();
    // Avoid infinite looping with PromoteIntBinOp.
    // TODO: Should we apply desirable/legal constraints to all opcodes?
    if ((HandOpcode == ISD::ANY_EXTEND ||
         HandOpcode == ISD::ANY_EXTEND_VECTOR_INREG) &&
        LegalTypes && !TLI.isTypeDesirableForOp(LogicOpcode, XVT))
      return SDValue();
    // logic_op (hand_op X), (hand_op Y) --> hand_op (logic_op X, Y)
    SDNodeFlags LogicFlags;
    LogicFlags.setDisjoint(N->getFlags().hasDisjoint() &&
                           ISD::isExtOpcode(HandOpcode));
    SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y, LogicFlags);
    if (HandOpcode == ISD::SIGN_EXTEND_INREG)
      return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
    return DAG.getNode(HandOpcode, DL, VT, Logic);
  }
 
  // logic_op (truncate x), (truncate y) --> truncate (logic_op x, y)
  if (HandOpcode == ISD::TRUNCATE) {
    // If both operands have other uses, this transform would create extra
    // instructions without eliminating anything.
    if (!N0.hasOneUse() && !N1.hasOneUse())
      return SDValue();
    // We need matching source types.
    if (XVT != Y.getValueType())
      return SDValue();
    // Don't create an illegal op during or after legalization.
    if (LegalOperations && !TLI.isOperationLegal(LogicOpcode, XVT))
      return SDValue();
    // Be extra careful sinking truncate. If it's free, there's no benefit in
    // widening a binop. Also, don't create a logic op on an illegal type.
    if (TLI.isZExtFree(VT, XVT) && TLI.isTruncateFree(XVT, VT))
      return SDValue();
    if (!TLI.isTypeLegal(XVT))
      return SDValue();
    SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
    return DAG.getNode(HandOpcode, DL, VT, Logic);
  }
 
  // For binops SHL/SRL/SRA/AND:
  //   logic_op (OP x, z), (OP y, z) --> OP (logic_op x, y), z
  if ((HandOpcode == ISD::SHL || HandOpcode == ISD::SRL ||
       HandOpcode == ISD::SRA || HandOpcode == ISD::AND) &&
      N0.getOperand(1) == N1.getOperand(1)) {
    // If either operand has other uses, this transform is not an improvement.
    if (!N0.hasOneUse() || !N1.hasOneUse())
      return SDValue();
    SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
    return DAG.getNode(HandOpcode, DL, VT, Logic, N0.getOperand(1));
  }
 
  // Unary ops: logic_op (bswap x), (bswap y) --> bswap (logic_op x, y)
  if (HandOpcode == ISD::BSWAP) {
    // If either operand has other uses, this transform is not an improvement.
    if (!N0.hasOneUse() || !N1.hasOneUse())
      return SDValue();
    SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
    return DAG.getNode(HandOpcode, DL, VT, Logic);
  }
 
  // For funnel shifts FSHL/FSHR:
  // logic_op (OP x, x1, s), (OP y, y1, s) -->
  // --> OP (logic_op x, y), (logic_op, x1, y1), s
  if ((HandOpcode == ISD::FSHL || HandOpcode == ISD::FSHR) &&
      N0.getOperand(2) == N1.getOperand(2)) {
    if (!N0.hasOneUse() || !N1.hasOneUse())
      return SDValue();
    SDValue X1 = N0.getOperand(1);
    SDValue Y1 = N1.getOperand(1);
    SDValue S = N0.getOperand(2);
    SDValue Logic0 = DAG.getNode(LogicOpcode, DL, VT, X, Y);
    SDValue Logic1 = DAG.getNode(LogicOpcode, DL, VT, X1, Y1);
    return DAG.getNode(HandOpcode, DL, VT, Logic0, Logic1, S);
  }
 
  // Simplify xor/and/or (bitcast(A), bitcast(B)) -> bitcast(op (A,B))
  // Only perform this optimization up until type legalization, before
  // LegalizeVectorOprs. LegalizeVectorOprs promotes vector operations by
  // adding bitcasts. For example (xor v4i32) is promoted to (v2i64), and
  // we don't want to undo this promotion.
  // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
  // on scalars.
  if ((HandOpcode == ISD::BITCAST || HandOpcode == ISD::SCALAR_TO_VECTOR) &&
       Level <= AfterLegalizeTypes) {
    // Input types must be integer and the same.
    if (XVT.isInteger() && XVT == Y.getValueType() &&
        !(VT.isVector() && TLI.isTypeLegal(VT) &&
          !XVT.isVector() && !TLI.isTypeLegal(XVT))) {
      SDValue Logic = DAG.getNode(LogicOpcode, DL, XVT, X, Y);
      return DAG.getNode(HandOpcode, DL, VT, Logic);
    }
  }
 
  // Xor/and/or are indifferent to the swizzle operation (shuffle of one value).
  // Simplify xor/and/or (shuff(A), shuff(B)) -> shuff(op (A,B))
  // If both shuffles use the same mask, and both shuffle within a single
  // vector, then it is worthwhile to move the swizzle after the operation.
  // The type-legalizer generates this pattern when loading illegal
  // vector types from memory. In many cases this allows additional shuffle
  // optimizations.
  // There are other cases where moving the shuffle after the xor/and/or
  // is profitable even if shuffles don't perform a swizzle.
  // If both shuffles use the same mask, and both shuffles have the same first
  // or second operand, then it might still be profitable to move the shuffle
  // after the xor/and/or operation.
  if (HandOpcode == ISD::VECTOR_SHUFFLE && Level < AfterLegalizeDAG) {
    auto *SVN0 = cast<ShuffleVectorSDNode>(N0);
    auto *SVN1 = cast<ShuffleVectorSDNode>(N1);
    assert(X.getValueType() == Y.getValueType() &&
           "Inputs to shuffles are not the same type");
 
    // Check that both shuffles use the same mask. The masks are known to be of
    // the same length because the result vector type is the same.
    // Check also that shuffles have only one use to avoid introducing extra
    // instructions.
    if (!SVN0->hasOneUse() || !SVN1->hasOneUse() ||
        !SVN0->getMask().equals(SVN1->getMask()))
      return SDValue();
 
    // Don't try to fold this node if it requires introducing a
    // build vector of all zeros that might be illegal at this stage.
    SDValue ShOp = N0.getOperand(1);
    if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
      ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
 
    // (logic_op (shuf (A, C), shuf (B, C))) --> shuf (logic_op (A, B), C)
    if (N0.getOperand(1) == N1.getOperand(1) && ShOp.getNode()) {
      SDValue Logic = DAG.getNode(LogicOpcode, DL, VT,
                                  N0.getOperand(0), N1.getOperand(0));
      return DAG.getVectorShuffle(VT, DL, Logic, ShOp, SVN0->getMask());
    }
 
    // Don't try to fold this node if it requires introducing a
    // build vector of all zeros that might be illegal at this stage.
    ShOp = N0.getOperand(0);
    if (LogicOpcode == ISD::XOR && !ShOp.isUndef())
      ShOp = tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
 
    // (logic_op (shuf (C, A), shuf (C, B))) --> shuf (C, logic_op (A, B))
    if (N0.getOperand(0) == N1.getOperand(0) && ShOp.getNode()) {
      SDValue Logic = DAG.getNode(LogicOpcode, DL, VT, N0.getOperand(1),
                                  N1.getOperand(1));
      return DAG.getVectorShuffle(VT, DL, ShOp, Logic, SVN0->getMask());
    }
  }
 
  return SDValue();
}
 
/// Try to make (and/or setcc (LL, LR), setcc (RL, RR)) more efficient.
SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1,
                                       const SDLoc &DL) {
  SDValue LL, LR, RL, RR, N0CC, N1CC;
  if (!isSetCCEquivalent(N0, LL, LR, N0CC) ||
      !isSetCCEquivalent(N1, RL, RR, N1CC))
    return SDValue();
 
  assert(N0.getValueType() == N1.getValueType() &&
         "Unexpected operand types for bitwise logic op");
  assert(LL.getValueType() == LR.getValueType() &&
         RL.getValueType() == RR.getValueType() &&
         "Unexpected operand types for setcc");
 
  // If we're here post-legalization or the logic op type is not i1, the logic
  // op type must match a setcc result type. Also, all folds require new
  // operations on the left and right operands, so those types must match.
  EVT VT = N0.getValueType();
  EVT OpVT = LL.getValueType();
  if (LegalOperations || VT.getScalarType() != MVT::i1)
    if (VT != getSetCCResultType(OpVT))
      return SDValue();
  if (OpVT != RL.getValueType())
    return SDValue();
 
  ISD::CondCode CC0 = cast<CondCodeSDNode>(N0CC)->get();
  ISD::CondCode CC1 = cast<CondCodeSDNode>(N1CC)->get();
  bool IsInteger = OpVT.isInteger();
  if (LR == RR && CC0 == CC1 && IsInteger) {
    bool IsZero = isNullOrNullSplat(LR);
    bool IsNeg1 = isAllOnesOrAllOnesSplat(LR);
 
    // All bits clear?
    bool AndEqZero = IsAnd && CC1 == ISD::SETEQ && IsZero;
    // All sign bits clear?
    bool AndGtNeg1 = IsAnd && CC1 == ISD::SETGT && IsNeg1;
    // Any bits set?
    bool OrNeZero = !IsAnd && CC1 == ISD::SETNE && IsZero;
    // Any sign bits set?
    bool OrLtZero = !IsAnd && CC1 == ISD::SETLT && IsZero;
 
    // (and (seteq X,  0), (seteq Y,  0)) --> (seteq (or X, Y),  0)
    // (and (setgt X, -1), (setgt Y, -1)) --> (setgt (or X, Y), -1)
    // (or  (setne X,  0), (setne Y,  0)) --> (setne (or X, Y),  0)
    // (or  (setlt X,  0), (setlt Y,  0)) --> (setlt (or X, Y),  0)
    if (AndEqZero || AndGtNeg1 || OrNeZero || OrLtZero) {
      SDValue Or = DAG.getNode(ISD::OR, SDLoc(N0), OpVT, LL, RL);
      AddToWorklist(Or.getNode());
      return DAG.getSetCC(DL, VT, Or, LR, CC1);
    }
 
    // All bits set?
    bool AndEqNeg1 = IsAnd && CC1 == ISD::SETEQ && IsNeg1;
    // All sign bits set?
    bool AndLtZero = IsAnd && CC1 == ISD::SETLT && IsZero;
    // Any bits clear?
    bool OrNeNeg1 = !IsAnd && CC1 == ISD::SETNE && IsNeg1;
    // Any sign bits clear?
    bool OrGtNeg1 = !IsAnd && CC1 == ISD::SETGT && IsNeg1;
 
    // (and (seteq X, -1), (seteq Y, -1)) --> (seteq (and X, Y), -1)
    // (and (setlt X,  0), (setlt Y,  0)) --> (setlt (and X, Y),  0)
    // (or  (setne X, -1), (setne Y, -1)) --> (setne (and X, Y), -1)
    // (or  (setgt X, -1), (setgt Y  -1)) --> (setgt (and X, Y), -1)
    if (AndEqNeg1 || AndLtZero || OrNeNeg1 || OrGtNeg1) {
      SDValue And = DAG.getNode(ISD::AND, SDLoc(N0), OpVT, LL, RL);
      AddToWorklist(And.getNode());
      return DAG.getSetCC(DL, VT, And, LR, CC1);
    }
  }
 
  // (and (setne X, 0), (setne X, -1)) --> (setuge (add X, 1), 2)
  // (or  (seteq X, 0), (seteq X, -1)) --> (setult (add X, 1), 2)
  if (LL == RL && CC0 == CC1 && OpVT.getScalarSizeInBits() > 1 && IsInteger &&
      ((IsAnd && CC0 == ISD::SETNE) || (!IsAnd && CC0 == ISD::SETEQ)) &&
      ((isNullConstant(LR) && isAllOnesConstant(RR)) ||
       (isAllOnesConstant(LR) && isNullConstant(RR)))) {
    SDValue One = DAG.getConstant(1, DL, OpVT);
    SDValue Two = DAG.getConstant(2, DL, OpVT);
    SDValue Add = DAG.getNode(ISD::ADD, SDLoc(N0), OpVT, LL, One);
    AddToWorklist(Add.getNode());
    return DAG.getSetCC(DL, VT, Add, Two, IsAnd ? ISD::SETUGE : ISD::SETULT);
  }
 
  // Try more general transforms if the predicates match and the only user of
  // the compares is the 'and' or 'or'.
  if (IsInteger && TLI.convertSetCCLogicToBitwiseLogic(OpVT) && CC0 == CC1 &&
      N0.hasOneUse() && N1.hasOneUse()) {
    // and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
    // or  (setne A, B), (setne C, D) --> setne (or (xor A, B), (xor C, D)), 0
    if ((IsAnd && CC1 == ISD::SETEQ) || (!IsAnd && CC1 == ISD::SETNE)) {
      SDValue XorL = DAG.getNode(ISD::XOR, SDLoc(N0), OpVT, LL, LR);
      SDValue XorR = DAG.getNode(ISD::XOR, SDLoc(N1), OpVT, RL, RR);
      SDValue Or = DAG.getNode(ISD::OR, DL, OpVT, XorL, XorR);
      SDValue Zero = DAG.getConstant(0, DL, OpVT);
      return DAG.getSetCC(DL, VT, Or, Zero, CC1);
    }
 
    // Turn compare of constants whose difference is 1 bit into add+and+setcc.
    if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) {
      // Match a shared variable operand and 2 non-opaque constant operands.
      auto MatchDiffPow2 = [&](ConstantSDNode *C0, ConstantSDNode *C1) {
        // The difference of the constants must be a single bit.
        const APInt &CMax =
            APIntOps::umax(C0->getAPIntValue(), C1->getAPIntValue());
        const APInt &CMin =
            APIntOps::umin(C0->getAPIntValue(), C1->getAPIntValue());
        return !C0->isOpaque() && !C1->isOpaque() && (CMax - CMin).isPowerOf2();
      };
      if (LL == RL && ISD::matchBinaryPredicate(LR, RR, MatchDiffPow2)) {
        // and/or (setcc X, CMax, ne), (setcc X, CMin, ne/eq) -->
        // setcc ((sub X, CMin), ~(CMax - CMin)), 0, ne/eq
        SDValue Max = DAG.getNode(ISD::UMAX, DL, OpVT, LR, RR);
        SDValue Min = DAG.getNode(ISD::UMIN, DL, OpVT, LR, RR);
        SDValue Offset = DAG.getNode(ISD::SUB, DL, OpVT, LL, Min);
        SDValue Diff = DAG.getNode(ISD::SUB, DL, OpVT, Max, Min);
        SDValue Mask = DAG.getNOT(DL, Diff, OpVT);
        SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Offset, Mask);
        SDValue Zero = DAG.getConstant(0, DL, OpVT);
        return DAG.getSetCC(DL, VT, And, Zero, CC0);
      }
    }
  }
 
  // Canonicalize equivalent operands to LL == RL.
  if (LL == RR && LR == RL) {
    CC1 = ISD::getSetCCSwappedOperands(CC1);
    std::swap(RL, RR);
  }
 
  // (and (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
  // (or  (setcc X, Y, CC0), (setcc X, Y, CC1)) --> (setcc X, Y, NewCC)
  if (LL == RL && LR == RR) {
    ISD::CondCode NewCC = IsAnd ? ISD::getSetCCAndOperation(CC0, CC1, OpVT)
                                : ISD::getSetCCOrOperation(CC0, CC1, OpVT);
    if (NewCC != ISD::SETCC_INVALID &&
        (!LegalOperations ||
         (TLI.isCondCodeLegal(NewCC, LL.getSimpleValueType()) &&
          TLI.isOperationLegal(ISD::SETCC, OpVT))))
      return DAG.getSetCC(DL, VT, LL, LR, NewCC);
  }
 
  return SDValue();
}
 
static bool arebothOperandsNotSNan(SDValue Operand1, SDValue Operand2,
                                   SelectionDAG &DAG) {
  return DAG.isKnownNeverSNaN(Operand2) && DAG.isKnownNeverSNaN(Operand1);
}
 
static bool arebothOperandsNotNan(SDValue Operand1, SDValue Operand2,
                                  SelectionDAG &DAG) {
  return DAG.isKnownNeverNaN(Operand2) && DAG.isKnownNeverNaN(Operand1);
}
 
/// Returns an appropriate FP min/max opcode for clamping operations.
static unsigned getMinMaxOpcodeForClamp(bool IsMin, SDValue Operand1,
                                        SDValue Operand2, SelectionDAG &DAG,
                                        const TargetLowering &TLI) {
  EVT VT = Operand1.getValueType();
  unsigned IEEEOp = IsMin ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
  if (TLI.isOperationLegalOrCustom(IEEEOp, VT) &&
      arebothOperandsNotNan(Operand1, Operand2, DAG))
    return IEEEOp;
  unsigned PreferredOp = IsMin ? ISD::FMINNUM : ISD::FMAXNUM;
  if (TLI.isOperationLegalOrCustom(PreferredOp, VT))
    return PreferredOp;
  return ISD::DELETED_NODE;
}
 
// FIXME: use FMINIMUMNUM if possible, such as for RISC-V.
static unsigned getMinMaxOpcodeForCompareFold(
    SDValue Operand1, SDValue Operand2, bool SetCCNoNaNs, ISD::CondCode CC,
    unsigned OrAndOpcode, SelectionDAG &DAG, bool isFMAXNUMFMINNUM_IEEE,
    bool isFMAXNUMFMINNUM) {
  // The optimization cannot be applied for all the predicates because
  // of the way FMINNUM/FMAXNUM and FMINNUM_IEEE/FMAXNUM_IEEE handle
  // NaNs. For FMINNUM_IEEE/FMAXNUM_IEEE, the optimization cannot be
  // applied at all if one of the operands is a signaling NaN.
 
  // It is safe to use FMINNUM_IEEE/FMAXNUM_IEEE if all the operands
  // are non NaN values.
  if (((CC == ISD::SETLT || CC == ISD::SETLE) && (OrAndOpcode == ISD::OR)) ||
      ((CC == ISD::SETGT || CC == ISD::SETGE) && (OrAndOpcode == ISD::AND))) {
    return (SetCCNoNaNs || arebothOperandsNotNan(Operand1, Operand2, DAG)) &&
                   isFMAXNUMFMINNUM_IEEE
               ? ISD::FMINNUM_IEEE
               : ISD::DELETED_NODE;
  }
 
  if (((CC == ISD::SETGT || CC == ISD::SETGE) && (OrAndOpcode == ISD::OR)) ||
      ((CC == ISD::SETLT || CC == ISD::SETLE) && (OrAndOpcode == ISD::AND))) {
    return (SetCCNoNaNs || arebothOperandsNotNan(Operand1, Operand2, DAG)) &&
                   isFMAXNUMFMINNUM_IEEE
               ? ISD::FMAXNUM_IEEE
               : ISD::DELETED_NODE;
  }
 
  // Both FMINNUM/FMAXNUM and FMINNUM_IEEE/FMAXNUM_IEEE handle quiet
  // NaNs in the same way. But, FMINNUM/FMAXNUM and FMINNUM_IEEE/
  // FMAXNUM_IEEE handle signaling NaNs differently. If we cannot prove
  // that there are not any sNaNs, then the optimization is not valid
  // for FMINNUM_IEEE/FMAXNUM_IEEE. In the presence of sNaNs, we apply
  // the optimization using FMINNUM/FMAXNUM for the following cases. If
  // we can prove that we do not have any sNaNs, then we can do the
  // optimization using FMINNUM_IEEE/FMAXNUM_IEEE for the following
  // cases.
  if (((CC == ISD::SETOLT || CC == ISD::SETOLE) && (OrAndOpcode == ISD::OR)) ||
      ((CC == ISD::SETUGT || CC == ISD::SETUGE) && (OrAndOpcode == ISD::AND))) {
    return isFMAXNUMFMINNUM ? ISD::FMINNUM
           : arebothOperandsNotSNan(Operand1, Operand2, DAG) &&
                   isFMAXNUMFMINNUM_IEEE
               ? ISD::FMINNUM_IEEE
               : ISD::DELETED_NODE;
  }
 
  if (((CC == ISD::SETOGT || CC == ISD::SETOGE) && (OrAndOpcode == ISD::OR)) ||
      ((CC == ISD::SETULT || CC == ISD::SETULE) && (OrAndOpcode == ISD::AND))) {
    return isFMAXNUMFMINNUM ? ISD::FMAXNUM
           : arebothOperandsNotSNan(Operand1, Operand2, DAG) &&
                   isFMAXNUMFMINNUM_IEEE
               ? ISD::FMAXNUM_IEEE
               : ISD::DELETED_NODE;
  }
 
  return ISD::DELETED_NODE;
}
 
static SDValue foldAndOrOfSETCC(SDNode *LogicOp, SelectionDAG &DAG) {
  using AndOrSETCCFoldKind = TargetLowering::AndOrSETCCFoldKind;
  assert(
      (LogicOp->getOpcode() == ISD::AND || LogicOp->getOpcode() == ISD::OR) &&
      "Invalid Op to combine SETCC with");
 
  // TODO: Search past casts/truncates.
  SDValue LHS = LogicOp->getOperand(0);
  SDValue RHS = LogicOp->getOperand(1);
  if (LHS->getOpcode() != ISD::SETCC || RHS->getOpcode() != ISD::SETCC ||
      !LHS->hasOneUse() || !RHS->hasOneUse())
    return SDValue();
 
  SDNodeFlags LHSSetCCFlags = LHS->getFlags();
  SDNodeFlags RHSSetCCFlags = RHS->getFlags();
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  AndOrSETCCFoldKind TargetPreference = TLI.isDesirableToCombineLogicOpOfSETCC(
      LogicOp, LHS.getNode(), RHS.getNode());
 
  SDValue LHS0 = LHS->getOperand(0);
  SDValue RHS0 = RHS->getOperand(0);
  SDValue LHS1 = LHS->getOperand(1);
  SDValue RHS1 = RHS->getOperand(1);
  // TODO: We don't actually need a splat here, for vectors we just need the
  // invariants to hold for each element.
  auto *LHS1C = isConstOrConstSplat(LHS1);
  auto *RHS1C = isConstOrConstSplat(RHS1);
  ISD::CondCode CCL = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
  ISD::CondCode CCR = cast<CondCodeSDNode>(RHS.getOperand(2))->get();
  EVT VT = LogicOp->getValueType(0);
  EVT OpVT = LHS0.getValueType();
  SDLoc DL(LogicOp);
 
  // Check if the operands of an and/or operation are comparisons and if they
  // compare against the same value. Replace the and/or-cmp-cmp sequence with
  // min/max cmp sequence. If LHS1 is equal to RHS1, then the or-cmp-cmp
  // sequence will be replaced with min-cmp sequence:
  // (LHS0 < LHS1) | (RHS0 < RHS1) -> min(LHS0, RHS0) < LHS1
  // and and-cmp-cmp will be replaced with max-cmp sequence:
  // (LHS0 < LHS1) & (RHS0 < RHS1) -> max(LHS0, RHS0) < LHS1
  // The optimization does not work for `==` or `!=` .
  // The two comparisons should have either the same predicate or the
  // predicate of one of the comparisons is the opposite of the other one.
  bool isFMAXNUMFMINNUM_IEEE = TLI.isOperationLegal(ISD::FMAXNUM_IEEE, OpVT) &&
                               TLI.isOperationLegal(ISD::FMINNUM_IEEE, OpVT);
  bool isFMAXNUMFMINNUM = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, OpVT) &&
                          TLI.isOperationLegalOrCustom(ISD::FMINNUM, OpVT);
  if (((OpVT.isInteger() && TLI.isOperationLegal(ISD::UMAX, OpVT) &&
        TLI.isOperationLegal(ISD::SMAX, OpVT) &&
        TLI.isOperationLegal(ISD::UMIN, OpVT) &&
        TLI.isOperationLegal(ISD::SMIN, OpVT)) ||
       (OpVT.isFloatingPoint() &&
        (isFMAXNUMFMINNUM_IEEE || isFMAXNUMFMINNUM))) &&
      !ISD::isIntEqualitySetCC(CCL) && !ISD::isFPEqualitySetCC(CCL) &&
      CCL != ISD::SETFALSE && CCL != ISD::SETO && CCL != ISD::SETUO &&
      CCL != ISD::SETTRUE &&
      (CCL == CCR || CCL == ISD::getSetCCSwappedOperands(CCR))) {
 
    SDValue CommonValue, Operand1, Operand2;
    ISD::CondCode CC = ISD::SETCC_INVALID;
    if (CCL == CCR) {
      if (LHS0 == RHS0) {
        CommonValue = LHS0;
        Operand1 = LHS1;
        Operand2 = RHS1;
        CC = ISD::getSetCCSwappedOperands(CCL);
      } else if (LHS1 == RHS1) {
        CommonValue = LHS1;
        Operand1 = LHS0;
        Operand2 = RHS0;
        CC = CCL;
      }
    } else {
      assert(CCL == ISD::getSetCCSwappedOperands(CCR) && "Unexpected CC");
      if (LHS0 == RHS1) {
        CommonValue = LHS0;
        Operand1 = LHS1;
        Operand2 = RHS0;
        CC = CCR;
      } else if (RHS0 == LHS1) {
        CommonValue = LHS1;
        Operand1 = LHS0;
        Operand2 = RHS1;
        CC = CCL;
      }
    }
 
    // Don't do this transform for sign bit tests. Let foldLogicOfSetCCs
    // handle it using OR/AND.
    if (CC == ISD::SETLT && isNullOrNullSplat(CommonValue))
      CC = ISD::SETCC_INVALID;
    else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CommonValue))
      CC = ISD::SETCC_INVALID;
 
    if (CC != ISD::SETCC_INVALID) {
      unsigned NewOpcode = ISD::DELETED_NODE;
      bool IsSigned = isSignedIntSetCC(CC);
      if (OpVT.isInteger()) {
        bool IsLess = (CC == ISD::SETLE || CC == ISD::SETULE ||
                       CC == ISD::SETLT || CC == ISD::SETULT);
        bool IsOr = (LogicOp->getOpcode() == ISD::OR);
        if (IsLess == IsOr)
          NewOpcode = IsSigned ? ISD::SMIN : ISD::UMIN;
        else
          NewOpcode = IsSigned ? ISD::SMAX : ISD::UMAX;
      } else if (OpVT.isFloatingPoint())
        NewOpcode = getMinMaxOpcodeForCompareFold(
            Operand1, Operand2,
            LHSSetCCFlags.hasNoNaNs() && RHSSetCCFlags.hasNoNaNs(), CC,
            LogicOp->getOpcode(), DAG, isFMAXNUMFMINNUM_IEEE, isFMAXNUMFMINNUM);
 
      if (NewOpcode != ISD::DELETED_NODE) {
        // Propagate fast-math flags from setcc.
        SDNodeFlags Flags = LHS->getFlags() & RHS->getFlags();
        SDValue MinMaxValue =
            DAG.getNode(NewOpcode, DL, OpVT, Operand1, Operand2, Flags);
        return DAG.getSetCC(DL, VT, MinMaxValue, CommonValue, CC, /*Chain=*/{},
                            /*IsSignaling=*/false, Flags);
      }
    }
  }
 
  if (LHS0 == LHS1 && RHS0 == RHS1 && CCL == CCR &&
      LHS0.getValueType() == RHS0.getValueType() &&
      ((LogicOp->getOpcode() == ISD::AND && CCL == ISD::SETO) ||
       (LogicOp->getOpcode() == ISD::OR && CCL == ISD::SETUO)))
    return DAG.getSetCC(DL, VT, LHS0, RHS0, CCL);
 
  if (TargetPreference == AndOrSETCCFoldKind::None)
    return SDValue();
 
  if (CCL == CCR &&
      CCL == (LogicOp->getOpcode() == ISD::AND ? ISD::SETNE : ISD::SETEQ) &&
      LHS0 == RHS0 && LHS1C && RHS1C && OpVT.isInteger()) {
    const APInt &APLhs = LHS1C->getAPIntValue();
    const APInt &APRhs = RHS1C->getAPIntValue();
 
    // Preference is to use ISD::ABS or we already have an ISD::ABS (in which
    // case this is just a compare).
    if (APLhs == (-APRhs) &&
        ((TargetPreference & AndOrSETCCFoldKind::ABS) ||
         DAG.doesNodeExist(ISD::ABS, DAG.getVTList(OpVT), {LHS0}))) {
      const APInt &C = APLhs.isNegative() ? APRhs : APLhs;
      // (icmp eq A, C) | (icmp eq A, -C)
      //    -> (icmp eq Abs(A), C)
      // (icmp ne A, C) & (icmp ne A, -C)
      //    -> (icmp ne Abs(A), C)
      SDValue AbsOp = DAG.getNode(ISD::ABS, DL, OpVT, LHS0);
      return DAG.getNode(ISD::SETCC, DL, VT, AbsOp,
                         DAG.getConstant(C, DL, OpVT), LHS.getOperand(2));
    } else if (TargetPreference &
               (AndOrSETCCFoldKind::AddAnd | AndOrSETCCFoldKind::NotAnd)) {
 
      // AndOrSETCCFoldKind::AddAnd:
      // A == C0 | A == C1
      //  IF IsPow2(smax(C0, C1)-smin(C0, C1))
      //    -> ((A - smin(C0, C1)) & ~(smax(C0, C1)-smin(C0, C1))) == 0
      // A != C0 & A != C1
      //  IF IsPow2(smax(C0, C1)-smin(C0, C1))
      //    -> ((A - smin(C0, C1)) & ~(smax(C0, C1)-smin(C0, C1))) != 0
 
      // AndOrSETCCFoldKind::NotAnd:
      // A == C0 | A == C1
      //  IF smax(C0, C1) == -1 AND IsPow2(smax(C0, C1) - smin(C0, C1))
      //    -> ~A & smin(C0, C1) == 0
      // A != C0 & A != C1
      //  IF smax(C0, C1) == -1 AND IsPow2(smax(C0, C1) - smin(C0, C1))
      //    -> ~A & smin(C0, C1) != 0
 
      const APInt &MaxC = APIntOps::smax(APRhs, APLhs);
      const APInt &MinC = APIntOps::smin(APRhs, APLhs);
      APInt Dif = MaxC - MinC;
      if (!Dif.isZero() && Dif.isPowerOf2()) {
        if (MaxC.isAllOnes() &&
            (TargetPreference & AndOrSETCCFoldKind::NotAnd)) {
          SDValue NotOp = DAG.getNOT(DL, LHS0, OpVT);
          SDValue AndOp = DAG.getNode(ISD::AND, DL, OpVT, NotOp,
                                      DAG.getConstant(MinC, DL, OpVT));
          return DAG.getNode(ISD::SETCC, DL, VT, AndOp,
                             DAG.getConstant(0, DL, OpVT), LHS.getOperand(2));
        } else if (TargetPreference & AndOrSETCCFoldKind::AddAnd) {
 
          SDValue AddOp = DAG.getNode(ISD::ADD, DL, OpVT, LHS0,
                                      DAG.getConstant(-MinC, DL, OpVT));
          SDValue AndOp = DAG.getNode(ISD::AND, DL, OpVT, AddOp,
                                      DAG.getConstant(~Dif, DL, OpVT));
          return DAG.getNode(ISD::SETCC, DL, VT, AndOp,
                             DAG.getConstant(0, DL, OpVT), LHS.getOperand(2));
        }
      }
    }
  }
 
  return SDValue();
}
 
// Combine `(select c, (X & 1), 0)` -> `(and (zext c), X)`.
// We canonicalize to the `select` form in the middle end, but the `and` form
// gets better codegen and all tested targets (arm, x86, riscv)
static SDValue combineSelectAsExtAnd(SDValue Cond, SDValue T, SDValue F,
                                     const SDLoc &DL, SelectionDAG &DAG) {
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  if (!isNullConstant(F))
    return SDValue();
 
  EVT CondVT = Cond.getValueType();
  if (TLI.getBooleanContents(CondVT) !=
      TargetLoweringBase::ZeroOrOneBooleanContent)
    return SDValue();
 
  if (T.getOpcode() != ISD::AND)
    return SDValue();
 
  if (!isOneConstant(T.getOperand(1)))
    return SDValue();
 
  EVT OpVT = T.getValueType();
 
  SDValue CondMask =
      OpVT == CondVT ? Cond : DAG.getBoolExtOrTrunc(Cond, DL, OpVT, CondVT);
  return DAG.getNode(ISD::AND, DL, OpVT, CondMask, T.getOperand(0));
}
 
/// This contains all DAGCombine rules which reduce two values combined by
/// an And operation to a single value. This makes them reusable in the context
/// of visitSELECT(). Rules involving constants are not included as
/// visitSELECT() already handles those cases.
SDValue DAGCombiner::visitANDLike(SDValue N0, SDValue N1, SDNode *N) {
  EVT VT = N1.getValueType();
  SDLoc DL(N);
 
  // fold (and x, undef) -> 0
  if (N0.isUndef() || N1.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  if (SDValue V = foldLogicOfSetCCs(true, N0, N1, DL))
    return V;
 
  // Canonicalize:
  //   and(x, add) -> and(add, x)
  if (N1.getOpcode() == ISD::ADD)
    std::swap(N0, N1);
 
  // TODO: Rewrite this to return a new 'AND' instead of using CombineTo.
  if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::SRL &&
      VT.isScalarInteger() && VT.getSizeInBits() <= 64 && N0->hasOneUse()) {
    if (ConstantSDNode *ADDI = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
      if (ConstantSDNode *SRLI = dyn_cast<ConstantSDNode>(N1.getOperand(1))) {
        // Look for (and (add x, c1), (lshr y, c2)). If C1 wasn't a legal
        // immediate for an add, but it is legal if its top c2 bits are set,
        // transform the ADD so the immediate doesn't need to be materialized
        // in a register.
        APInt ADDC = ADDI->getAPIntValue();
        APInt SRLC = SRLI->getAPIntValue();
        if (ADDC.getSignificantBits() <= 64 && SRLC.ult(VT.getSizeInBits()) &&
            !TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
          APInt Mask = APInt::getHighBitsSet(VT.getSizeInBits(),
                                             SRLC.getZExtValue());
          if (DAG.MaskedValueIsZero(N0.getOperand(1), Mask)) {
            ADDC |= Mask;
            if (TLI.isLegalAddImmediate(ADDC.getSExtValue())) {
              SDLoc DL0(N0);
              SDValue NewAdd =
                DAG.getNode(ISD::ADD, DL0, VT,
                            N0.getOperand(0), DAG.getConstant(ADDC, DL, VT));
              CombineTo(N0.getNode(), NewAdd);
              // Return N so it doesn't get rechecked!
              return SDValue(N, 0);
            }
          }
        }
      }
    }
  }
 
  return SDValue();
}
 
bool DAGCombiner::isAndLoadExtLoad(ConstantSDNode *AndC, LoadSDNode *LoadN,
                                   EVT LoadResultTy, EVT &ExtVT) {
  if (!AndC->getAPIntValue().isMask())
    return false;
 
  unsigned ActiveBits = AndC->getAPIntValue().countr_one();
 
  ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
  EVT LoadedVT = LoadN->getMemoryVT();
 
  if (ExtVT == LoadedVT &&
      (!LegalOperations ||
       TLI.isLoadLegal(LoadResultTy, ExtVT, LoadN->getAlign(),
                       LoadN->getAddressSpace(), ISD::ZEXTLOAD, false))) {
    // ZEXTLOAD will match without needing to change the size of the value being
    // loaded.
    return true;
  }
 
  // Do not change the width of a volatile or atomic loads.
  if (!LoadN->isSimple())
    return false;
 
  // Do not generate loads of non-round integer types since these can
  // be expensive (and would be wrong if the type is not byte sized).
  if (!LoadedVT.bitsGT(ExtVT) || !ExtVT.isRound())
    return false;
 
  if (LegalOperations &&
      !TLI.isLoadLegal(LoadResultTy, ExtVT, LoadN->getAlign(),
                       LoadN->getAddressSpace(), ISD::ZEXTLOAD, false))
    return false;
 
  if (!TLI.shouldReduceLoadWidth(LoadN, ISD::ZEXTLOAD, ExtVT, /*ByteOffset=*/0))
    return false;
 
  return true;
}
 
bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
                                    ISD::LoadExtType ExtType, EVT &MemVT,
                                    unsigned ShAmt) {
  if (!LDST)
    return false;
 
  // Only allow byte offsets.
  if (ShAmt % 8)
    return false;
  const unsigned ByteShAmt = ShAmt / 8;
 
  // Do not generate loads of non-round integer types since these can
  // be expensive (and would be wrong if the type is not byte sized).
  if (!MemVT.isRound())
    return false;
 
  // Don't change the width of a volatile or atomic loads.
  if (!LDST->isSimple())
    return false;
 
  EVT LdStMemVT = LDST->getMemoryVT();
 
  // Bail out when changing the scalable property, since we can't be sure that
  // we're actually narrowing here.
  if (LdStMemVT.isScalableVector() != MemVT.isScalableVector())
    return false;
 
  // Verify that we are actually reducing a load width here.
  if (LdStMemVT.bitsLT(MemVT))
    return false;
 
  // Ensure that this isn't going to produce an unsupported memory access.
  if (ShAmt) {
    const Align LDSTAlign = LDST->getAlign();
    const Align NarrowAlign = commonAlignment(LDSTAlign, ByteShAmt);
    if (!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
                                LDST->getAddressSpace(), NarrowAlign,
                                LDST->getMemOperand()->getFlags()))
      return false;
  }
 
  // It's not possible to generate a constant of extended or untyped type.
  EVT PtrType = LDST->getBasePtr().getValueType();
  if (PtrType == MVT::Untyped || PtrType.isExtended())
    return false;
 
  if (isa<LoadSDNode>(LDST)) {
    LoadSDNode *Load = cast<LoadSDNode>(LDST);
    // Don't transform one with multiple uses, this would require adding a new
    // load.
    if (!SDValue(Load, 0).hasOneUse())
      return false;
 
    if (LegalOperations &&
        !TLI.isLoadLegal(Load->getValueType(0), MemVT, Load->getAlign(),
                         Load->getAddressSpace(), ExtType, false))
      return false;
 
    // For the transform to be legal, the load must produce only two values
    // (the value loaded and the chain).  Don't transform a pre-increment
    // load, for example, which produces an extra value.  Otherwise the
    // transformation is not equivalent, and the downstream logic to replace
    // uses gets things wrong.
    if (Load->getNumValues() > 2)
      return false;
 
    // If the load that we're shrinking is an extload and we're not just
    // discarding the extension we can't simply shrink the load. Bail.
    // TODO: It would be possible to merge the extensions in some cases.
    if (Load->getExtensionType() != ISD::NON_EXTLOAD &&
        Load->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
      return false;
 
    if (!TLI.shouldReduceLoadWidth(Load, ExtType, MemVT, ByteShAmt))
      return false;
  } else {
    assert(isa<StoreSDNode>(LDST) && "It is not a Load nor a Store SDNode");
    StoreSDNode *Store = cast<StoreSDNode>(LDST);
    // Can't write outside the original store
    if (Store->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits() + ShAmt)
      return false;
 
    if (LegalOperations &&
        !TLI.isTruncStoreLegal(Store->getValue().getValueType(), MemVT,
                               Store->getAlign(), Store->getAddressSpace()))
      return false;
  }
  return true;
}
 
bool DAGCombiner::SearchForAndLoads(SDNode *N,
                                    SmallVectorImpl<LoadSDNode*> &Loads,
                                    SmallPtrSetImpl<SDNode*> &NodesWithConsts,
                                    ConstantSDNode *Mask,
                                    SDNode *&NodeToMask) {
  // Recursively search for the operands, looking for loads which can be
  // narrowed.
  for (SDValue Op : N->op_values()) {
    if (Op.getValueType().isVector())
      return false;
 
    // Some constants may need fixing up later if they are too large.
    if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
      assert(ISD::isBitwiseLogicOp(N->getOpcode()) &&
             "Expected bitwise logic operation");
      if (!C->getAPIntValue().isSubsetOf(Mask->getAPIntValue()))
        NodesWithConsts.insert(N);
      continue;
    }
 
    if (!Op.hasOneUse())
      return false;
 
    switch(Op.getOpcode()) {
    case ISD::LOAD: {
      auto *Load = cast<LoadSDNode>(Op);
      EVT ExtVT;
      if (isAndLoadExtLoad(Mask, Load, Load->getValueType(0), ExtVT) &&
          isLegalNarrowLdSt(Load, ISD::ZEXTLOAD, ExtVT)) {
 
        // ZEXTLOAD is already small enough.
        if (Load->getExtensionType() == ISD::ZEXTLOAD &&
            ExtVT.bitsGE(Load->getMemoryVT()))
          continue;
 
        // Use LE to convert equal sized loads to zext.
        if (ExtVT.bitsLE(Load->getMemoryVT()))
          Loads.push_back(Load);
 
        continue;
      }
      return false;
    }
    case ISD::ZERO_EXTEND:
    case ISD::AssertZext: {
      unsigned ActiveBits = Mask->getAPIntValue().countr_one();
      EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
      EVT VT = Op.getOpcode() == ISD::AssertZext ?
        cast<VTSDNode>(Op.getOperand(1))->getVT() :
        Op.getOperand(0).getValueType();
 
      // We can accept extending nodes if the mask is wider or an equal
      // width to the original type.
      if (ExtVT.bitsGE(VT))
        continue;
      break;
    }
    case ISD::OR:
    case ISD::XOR:
    case ISD::AND:
      if (!SearchForAndLoads(Op.getNode(), Loads, NodesWithConsts, Mask,
                             NodeToMask))
        return false;
      continue;
    }
 
    // Allow one node which will masked along with any loads found.
    if (NodeToMask)
      return false;
 
    // Also ensure that the node to be masked only produces one data result.
    NodeToMask = Op.getNode();
    if (NodeToMask->getNumValues() > 1) {
      bool HasValue = false;
      for (unsigned i = 0, e = NodeToMask->getNumValues(); i < e; ++i) {
        MVT VT = SDValue(NodeToMask, i).getSimpleValueType();
        if (VT != MVT::Glue && VT != MVT::Other) {
          if (HasValue) {
            NodeToMask = nullptr;
            return false;
          }
          HasValue = true;
        }
      }
      assert(HasValue && "Node to be masked has no data result?");
    }
  }
  return true;
}
 
bool DAGCombiner::BackwardsPropagateMask(SDNode *N) {
  auto *Mask = dyn_cast<ConstantSDNode>(N->getOperand(1));
  if (!Mask)
    return false;
 
  if (!Mask->getAPIntValue().isMask())
    return false;
 
  // No need to do anything if the and directly uses a load.
  if (isa<LoadSDNode>(N->getOperand(0)))
    return false;
 
  SmallVector<LoadSDNode*, 8> Loads;
  SmallPtrSet<SDNode*, 2> NodesWithConsts;
  SDNode *FixupNode = nullptr;
  if (SearchForAndLoads(N, Loads, NodesWithConsts, Mask, FixupNode)) {
    if (Loads.empty())
      return false;
 
    LLVM_DEBUG(dbgs() << "Backwards propagate AND: "; N->dump());
    SDValue MaskOp = N->getOperand(1);
 
    // If it exists, fixup the single node we allow in the tree that needs
    // masking.
    if (FixupNode) {
      LLVM_DEBUG(dbgs() << "First, need to fix up: "; FixupNode->dump());
      SDValue And = DAG.getNode(ISD::AND, SDLoc(FixupNode),
                                FixupNode->getValueType(0),
                                SDValue(FixupNode, 0), MaskOp);
      DAG.ReplaceAllUsesOfValueWith(SDValue(FixupNode, 0), And);
      if (And.getOpcode() == ISD ::AND)
        DAG.UpdateNodeOperands(And.getNode(), SDValue(FixupNode, 0), MaskOp);
    }
 
    // Narrow any constants that need it.
    for (auto *LogicN : NodesWithConsts) {
      SDValue Op0 = LogicN->getOperand(0);
      SDValue Op1 = LogicN->getOperand(1);
 
      // We only need to fix AND if both inputs are constants. And we only need
      // to fix one of the constants.
      if (LogicN->getOpcode() == ISD::AND &&
          (!isa<ConstantSDNode>(Op0) || !isa<ConstantSDNode>(Op1)))
        continue;
 
      if (isa<ConstantSDNode>(Op0) && LogicN->getOpcode() != ISD::AND)
        Op0 =
            DAG.getNode(ISD::AND, SDLoc(Op0), Op0.getValueType(), Op0, MaskOp);
 
      if (isa<ConstantSDNode>(Op1))
        Op1 =
            DAG.getNode(ISD::AND, SDLoc(Op1), Op1.getValueType(), Op1, MaskOp);
 
      if (isa<ConstantSDNode>(Op0) && !isa<ConstantSDNode>(Op1))
        std::swap(Op0, Op1);
 
      DAG.UpdateNodeOperands(LogicN, Op0, Op1);
    }
 
    // Create narrow loads.
    for (auto *Load : Loads) {
      LLVM_DEBUG(dbgs() << "Propagate AND back to: "; Load->dump());
      SDValue And = DAG.getNode(ISD::AND, SDLoc(Load), Load->getValueType(0),
                                SDValue(Load, 0), MaskOp);
      DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 0), And);
      if (And.getOpcode() == ISD ::AND)
        And = SDValue(
            DAG.UpdateNodeOperands(And.getNode(), SDValue(Load, 0), MaskOp), 0);
      SDValue NewLoad = reduceLoadWidth(And.getNode());
      assert(NewLoad &&
             "Shouldn't be masking the load if it can't be narrowed");
      CombineTo(Load, NewLoad, NewLoad.getValue(1));
    }
    DAG.ReplaceAllUsesWith(N, N->getOperand(0).getNode());
    return true;
  }
  return false;
}
 
// Unfold
//    x &  (-1 'logical shift' y)
// To
//    (x 'opposite logical shift' y) 'logical shift' y
// if it is better for performance.
SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) {
  assert(N->getOpcode() == ISD::AND);
 
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
 
  // Do we actually prefer shifts over mask?
  if (!TLI.shouldFoldMaskToVariableShiftPair(N0))
    return SDValue();
 
  // Try to match  (-1 '[outer] logical shift' y)
  unsigned OuterShift;
  unsigned InnerShift; // The opposite direction to the OuterShift.
  SDValue Y;           // Shift amount.
  auto matchMask = [&OuterShift, &InnerShift, &Y](SDValue M) -> bool {
    if (!M.hasOneUse())
      return false;
    OuterShift = M->getOpcode();
    if (OuterShift == ISD::SHL)
      InnerShift = ISD::SRL;
    else if (OuterShift == ISD::SRL)
      InnerShift = ISD::SHL;
    else
      return false;
    if (!isAllOnesConstant(M->getOperand(0)))
      return false;
    Y = M->getOperand(1);
    return true;
  };
 
  SDValue X;
  if (matchMask(N1))
    X = N0;
  else if (matchMask(N0))
    X = N1;
  else
    return SDValue();
 
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
 
  //     tmp = x   'opposite logical shift' y
  SDValue T0 = DAG.getNode(InnerShift, DL, VT, X, Y);
  //     ret = tmp 'logical shift' y
  SDValue T1 = DAG.getNode(OuterShift, DL, VT, T0, Y);
 
  return T1;
}
 
/// Try to replace shift/logic that tests if a bit is clear with mask + setcc.
/// For a target with a bit test, this is expected to become test + set and save
/// at least 1 instruction.
static SDValue combineShiftAnd1ToBitTest(SDNode *And, SelectionDAG &DAG) {
  assert(And->getOpcode() == ISD::AND && "Expected an 'and' op");
 
  // Look through an optional extension.
  SDValue And0 = And->getOperand(0), And1 = And->getOperand(1);
  if (And0.getOpcode() == ISD::ANY_EXTEND && And0.hasOneUse())
    And0 = And0.getOperand(0);
  if (!isOneConstant(And1) || !And0.hasOneUse())
    return SDValue();
 
  SDValue Src = And0;
 
  // Attempt to find a 'not' op.
  // TODO: Should we favor test+set even without the 'not' op?
  bool FoundNot = false;
  if (isBitwiseNot(Src)) {
    FoundNot = true;
    Src = Src.getOperand(0);
 
    // Look though an optional truncation. The source operand may not be the
    // same type as the original 'and', but that is ok because we are masking
    // off everything but the low bit.
    if (Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse())
      Src = Src.getOperand(0);
  }
 
  // Match a shift-right by constant.
  if (Src.getOpcode() != ISD::SRL || !Src.hasOneUse())
    return SDValue();
 
  // This is probably not worthwhile without a supported type.
  EVT SrcVT = Src.getValueType();
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  if (!TLI.isTypeLegal(SrcVT))
    return SDValue();
 
  // We might have looked through casts that make this transform invalid.
  unsigned BitWidth = SrcVT.getScalarSizeInBits();
  SDValue ShiftAmt = Src.getOperand(1);
  auto *ShiftAmtC = dyn_cast<ConstantSDNode>(ShiftAmt);
  if (!ShiftAmtC || !ShiftAmtC->getAPIntValue().ult(BitWidth))
    return SDValue();
 
  // Set source to shift source.
  Src = Src.getOperand(0);
 
  // Try again to find a 'not' op.
  // TODO: Should we favor test+set even with two 'not' ops?
  if (!FoundNot) {
    if (!isBitwiseNot(Src))
      return SDValue();
    Src = Src.getOperand(0);
  }
 
  if (!TLI.hasBitTest(Src, ShiftAmt))
    return SDValue();
 
  // Turn this into a bit-test pattern using mask op + setcc:
  // and (not (srl X, C)), 1 --> (and X, 1<<C) == 0
  // and (srl (not X), C)), 1 --> (and X, 1<<C) == 0
  SDLoc DL(And);
  SDValue X = DAG.getZExtOrTrunc(Src, DL, SrcVT);
  EVT CCVT =
      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), SrcVT);
  SDValue Mask = DAG.getConstant(
      APInt::getOneBitSet(BitWidth, ShiftAmtC->getZExtValue()), DL, SrcVT);
  SDValue NewAnd = DAG.getNode(ISD::AND, DL, SrcVT, X, Mask);
  SDValue Zero = DAG.getConstant(0, DL, SrcVT);
  SDValue Setcc = DAG.getSetCC(DL, CCVT, NewAnd, Zero, ISD::SETEQ);
  return DAG.getZExtOrTrunc(Setcc, DL, And->getValueType(0));
}
 
/// For targets that support usubsat, match a bit-hack form of that operation
/// that ends in 'and' and convert it.
static SDValue foldAndToUsubsat(SDNode *N, SelectionDAG &DAG, const SDLoc &DL) {
  EVT VT = N->getValueType(0);
  unsigned BitWidth = VT.getScalarSizeInBits();
  APInt SignMask = APInt::getSignMask(BitWidth);
 
  // (i8 X ^ 128) & (i8 X s>> 7) --> usubsat X, 128
  // (i8 X + 128) & (i8 X s>> 7) --> usubsat X, 128
  // xor/add with SMIN (signmask) are logically equivalent.
  SDValue X;
  if (!sd_match(N, m_And(m_OneUse(m_Xor(m_Value(X), m_SpecificInt(SignMask))),
                         m_OneUse(m_Sra(m_Deferred(X),
                                        m_SpecificInt(BitWidth - 1))))) &&
      !sd_match(N, m_And(m_OneUse(m_Add(m_Value(X), m_SpecificInt(SignMask))),
                         m_OneUse(m_Sra(m_Deferred(X),
                                        m_SpecificInt(BitWidth - 1))))))
    return SDValue();
 
  return DAG.getNode(ISD::USUBSAT, DL, VT, X,
                     DAG.getConstant(SignMask, DL, VT));
}
 
/// Given a bitwise logic operation N with a matching bitwise logic operand,
/// fold a pattern where 2 of the source operands are identically shifted
/// values. For example:
/// ((X0 << Y) | Z) | (X1 << Y) --> ((X0 | X1) << Y) | Z
static SDValue foldLogicOfShifts(SDNode *N, SDValue LogicOp, SDValue ShiftOp,
                                 SelectionDAG &DAG) {
  unsigned LogicOpcode = N->getOpcode();
  assert(ISD::isBitwiseLogicOp(LogicOpcode) &&
         "Expected bitwise logic operation");
 
  if (!LogicOp.hasOneUse() || !ShiftOp.hasOneUse())
    return SDValue();
 
  // Match another bitwise logic op and a shift.
  unsigned ShiftOpcode = ShiftOp.getOpcode();
  if (LogicOp.getOpcode() != LogicOpcode ||
      !(ShiftOpcode == ISD::SHL || ShiftOpcode == ISD::SRL ||
        ShiftOpcode == ISD::SRA))
    return SDValue();
 
  // Match another shift op inside the first logic operand. Handle both commuted
  // possibilities.
  // LOGIC (LOGIC (SH X0, Y), Z), (SH X1, Y) --> LOGIC (SH (LOGIC X0, X1), Y), Z
  // LOGIC (LOGIC Z, (SH X0, Y)), (SH X1, Y) --> LOGIC (SH (LOGIC X0, X1), Y), Z
  SDValue X1 = ShiftOp.getOperand(0);
  SDValue Y = ShiftOp.getOperand(1);
  SDValue X0, Z;
  if (LogicOp.getOperand(0).getOpcode() == ShiftOpcode &&
      LogicOp.getOperand(0).getOperand(1) == Y) {
    X0 = LogicOp.getOperand(0).getOperand(0);
    Z = LogicOp.getOperand(1);
  } else if (LogicOp.getOperand(1).getOpcode() == ShiftOpcode &&
             LogicOp.getOperand(1).getOperand(1) == Y) {
    X0 = LogicOp.getOperand(1).getOperand(0);
    Z = LogicOp.getOperand(0);
  } else {
    return SDValue();
  }
 
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  SDValue LogicX = DAG.getNode(LogicOpcode, DL, VT, X0, X1);
  SDValue NewShift = DAG.getNode(ShiftOpcode, DL, VT, LogicX, Y);
  return DAG.getNode(LogicOpcode, DL, VT, NewShift, Z);
}
 
/// Given a tree of logic operations with shape like
/// (LOGIC (LOGIC (X, Y), LOGIC (Z, Y)))
/// try to match and fold shift operations with the same shift amount.
/// For example:
/// LOGIC (LOGIC (SH X0, Y), Z), (LOGIC (SH X1, Y), W) -->
/// --> LOGIC (SH (LOGIC X0, X1), Y), (LOGIC Z, W)
static SDValue foldLogicTreeOfShifts(SDNode *N, SDValue LeftHand,
                                     SDValue RightHand, SelectionDAG &DAG) {
  unsigned LogicOpcode = N->getOpcode();
  assert(ISD::isBitwiseLogicOp(LogicOpcode) &&
         "Expected bitwise logic operation");
  if (LeftHand.getOpcode() != LogicOpcode ||
      RightHand.getOpcode() != LogicOpcode)
    return SDValue();
  if (!LeftHand.hasOneUse() || !RightHand.hasOneUse())
    return SDValue();
 
  // Try to match one of following patterns:
  // LOGIC (LOGIC (SH X0, Y), Z), (LOGIC (SH X1, Y), W)
  // LOGIC (LOGIC (SH X0, Y), Z), (LOGIC W, (SH X1, Y))
  // Note that foldLogicOfShifts will handle commuted versions of the left hand
  // itself.
  SDValue CombinedShifts, W;
  SDValue R0 = RightHand.getOperand(0);
  SDValue R1 = RightHand.getOperand(1);
  if ((CombinedShifts = foldLogicOfShifts(N, LeftHand, R0, DAG)))
    W = R1;
  else if ((CombinedShifts = foldLogicOfShifts(N, LeftHand, R1, DAG)))
    W = R0;
  else
    return SDValue();
 
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  return DAG.getNode(LogicOpcode, DL, VT, CombinedShifts, W);
}
 
/// Fold "masked merge" expressions like `(m & x) | (~m & y)` and its DeMorgan
/// variant `(~m | x) & (m | y)` into the equivalent `((x ^ y) & m) ^ y)`
/// pattern. This is typically a better representation for targets without a
/// fused "and-not" operation.
static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG,
                               const TargetLowering &TLI, const SDLoc &DL) {
  // Note that masked-merge variants using XOR or ADD expressions are
  // normalized to OR by InstCombine so we only check for OR or AND.
  assert((Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::AND) &&
         "Must be called with ISD::OR or ISD::AND node");
 
  // If the target supports and-not, don't fold this.
  if (TLI.hasAndNot(SDValue(Node, 0)))
    return SDValue();
 
  SDValue M, X, Y;
 
  if (sd_match(Node,
               m_Or(m_OneUse(m_And(m_OneUse(m_Not(m_Value(M))), m_Value(Y))),
                    m_OneUse(m_And(m_Deferred(M), m_Value(X))))) ||
      sd_match(Node,
               m_And(m_OneUse(m_Or(m_OneUse(m_Not(m_Value(M))), m_Value(X))),
                     m_OneUse(m_Or(m_Deferred(M), m_Value(Y)))))) {
    EVT VT = M.getValueType();
    SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, Y);
    SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor, M);
    return DAG.getNode(ISD::XOR, DL, VT, And, Y);
  }
  return SDValue();
}
 
SDValue DAGCombiner::visitAND(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N1.getValueType();
  SDLoc DL(N);
 
  // x & x --> x
  if (N0 == N1)
    return N0;
 
  // fold (and c1, c2) -> c1&c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::AND, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::AND, DL, VT, N1, N0);
 
  if (areBitwiseNotOfEachother(N0, N1))
    return DAG.getConstant(APInt::getZero(VT.getScalarSizeInBits()), DL, VT);
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (and x, 0) -> 0, vector edition
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      // do not return N1, because undef node may exist in N1
      return DAG.getConstant(APInt::getZero(N1.getScalarValueSizeInBits()), DL,
                             N1.getValueType());
 
    // fold (and x, -1) -> x, vector edition
    if (ISD::isConstantSplatVectorAllOnes(N1.getNode()))
      return N0;
 
    // fold (and buildvector(x,0,-1,w), buildvector(0,y,z,w))
    // --> buildvector(0,0,z,w)
    auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
    auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
    if (BV0 && BV1 && !BV0->getSplatValue() && !BV1->getSplatValue() &&
        N0.hasOneUse() && N1.hasOneUse() &&
        BV0->getOperand(0).getValueType() ==
            BV1->getOperand(0).getValueType()) {
      SmallVector<SDValue> MergedOps;
      unsigned NumElts = VT.getVectorNumElements();
      EVT EltVT = BV0->getOperand(0).getValueType();
      for (unsigned I = 0; I != NumElts; ++I) {
        auto *C0 = dyn_cast<ConstantSDNode>(BV0->getOperand(I));
        auto *C1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
        if (C0 && C1)
          MergedOps.push_back(DAG.getConstant(
              C0->getAPIntValue() & C1->getAPIntValue(), DL, EltVT));
        else if (C0 && C0->isZero())
          MergedOps.push_back(BV0->getOperand(I));
        else if (C1 && C1->isZero())
          MergedOps.push_back(BV1->getOperand(I));
        else if (C0 && C0->isAllOnes())
          MergedOps.push_back(BV1->getOperand(I));
        else if (C1 && C1->isAllOnes())
          MergedOps.push_back(BV0->getOperand(I));
        else if (BV0->getOperand(I) == BV1->getOperand(I))
          MergedOps.push_back(BV0->getOperand(I));
        else
          break;
      }
      if (MergedOps.size() == NumElts)
        return DAG.getBuildVector(VT, DL, MergedOps);
    }
 
    // fold (and (masked_load) (splat_vec (x, ...))) to zext_masked_load
    bool Frozen = N0.getOpcode() == ISD::FREEZE;
    auto *MLoad = dyn_cast<MaskedLoadSDNode>(Frozen ? N0.getOperand(0) : N0);
    ConstantSDNode *Splat = isConstOrConstSplat(N1, true, true);
    if (MLoad && MLoad->getExtensionType() == ISD::EXTLOAD && Splat) {
      EVT MemVT = MLoad->getMemoryVT();
      if (TLI.isLoadLegal(VT, MemVT, MLoad->getAlign(),
                          MLoad->getAddressSpace(), ISD::ZEXTLOAD, false)) {
        // For this AND to be a zero extension of the masked load the elements
        // of the BuildVec must mask the bottom bits of the extended element
        // type
        if (Splat->getAPIntValue().isMask(MemVT.getScalarSizeInBits())) {
          SDValue NewLoad = DAG.getMaskedLoad(
              VT, DL, MLoad->getChain(), MLoad->getBasePtr(),
              MLoad->getOffset(), MLoad->getMask(), MLoad->getPassThru(), MemVT,
              MLoad->getMemOperand(), MLoad->getAddressingMode(), ISD::ZEXTLOAD,
              MLoad->isExpandingLoad());
          CombineTo(N, Frozen ? N0 : NewLoad);
          CombineTo(MLoad, NewLoad, NewLoad.getValue(1));
          return SDValue(N, 0);
        }
      }
    }
  }
 
  // fold (and x, -1) -> x
  if (isAllOnesConstant(N1))
    return N0;
 
  // if (and x, c) is known to be zero, return 0
  unsigned BitWidth = VT.getScalarSizeInBits();
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
  if (N1C && DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(BitWidth)))
    return DAG.getConstant(0, DL, VT);
 
  if (SDValue R = foldAndOrOfSETCC(N, DAG))
    return R;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // reassociate and
  if (SDValue RAND = reassociateOps(ISD::AND, DL, N0, N1, N->getFlags()))
    return RAND;
 
  // Fold and(vecreduce(x), vecreduce(y)) -> vecreduce(and(x, y))
  if (SDValue SD =
          reassociateReduction(ISD::VECREDUCE_AND, ISD::AND, DL, VT, N0, N1))
    return SD;
 
  // fold (and (or x, C), D) -> D if (C & D) == D
  auto MatchSubset = [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
    return RHS->getAPIntValue().isSubsetOf(LHS->getAPIntValue());
  };
  if (N0.getOpcode() == ISD::OR &&
      ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchSubset))
    return N1;
 
  if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
    SDValue N0Op0 = N0.getOperand(0);
    EVT SrcVT = N0Op0.getValueType();
    unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
    APInt Mask = ~N1C->getAPIntValue();
    Mask = Mask.trunc(SrcBitWidth);
 
    // fold (and (any_ext V), c) -> (zero_ext V) if 'and' only clears top bits.
    if (DAG.MaskedValueIsZero(N0Op0, Mask))
      return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0Op0);
 
    // fold (and (any_ext V), c) -> (zero_ext (and (trunc V), c)) if profitable.
    if (N1C->getAPIntValue().countLeadingZeros() >= (BitWidth - SrcBitWidth) &&
        TLI.isTruncateFree(VT, SrcVT) && TLI.isZExtFree(SrcVT, VT) &&
        TLI.isTypeDesirableForOp(ISD::AND, SrcVT) &&
        TLI.isNarrowingProfitable(N, VT, SrcVT))
      return DAG.getNode(ISD::ZERO_EXTEND, DL, VT,
                         DAG.getNode(ISD::AND, DL, SrcVT, N0Op0,
                                     DAG.getZExtOrTrunc(N1, DL, SrcVT)));
  }
 
  // fold (and (ext (and V, c1)), c2) -> (and (ext V), (and c1, (ext c2)))
  if (ISD::isExtOpcode(N0.getOpcode())) {
    unsigned ExtOpc = N0.getOpcode();
    SDValue N0Op0 = N0.getOperand(0);
    if (N0Op0.getOpcode() == ISD::AND &&
        (ExtOpc != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0Op0, VT)) &&
        N0->hasOneUse() && N0Op0->hasOneUse()) {
      if (SDValue NewExt = DAG.FoldConstantArithmetic(ExtOpc, DL, VT,
                                                      {N0Op0.getOperand(1)})) {
        if (SDValue NewMask =
                DAG.FoldConstantArithmetic(ISD::AND, DL, VT, {N1, NewExt})) {
          return DAG.getNode(ISD::AND, DL, VT,
                             DAG.getNode(ExtOpc, DL, VT, N0Op0.getOperand(0)),
                             NewMask);
        }
      }
    }
  }
 
  // similarly fold (and (X (load ([non_ext|any_ext|zero_ext] V))), c) ->
  // (X (load ([non_ext|zero_ext] V))) if 'and' only clears top bits which must
  // already be zero by virtue of the width of the base type of the load.
  //
  // the 'X' node here can either be nothing or an extract_vector_elt to catch
  // more cases.
  if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
       N0.getValueSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits() &&
       N0.getOperand(0).getOpcode() == ISD::LOAD &&
       N0.getOperand(0).getResNo() == 0) ||
      (N0.getOpcode() == ISD::LOAD && N0.getResNo() == 0)) {
    auto *Load =
        cast<LoadSDNode>((N0.getOpcode() == ISD::LOAD) ? N0 : N0.getOperand(0));
 
    // Get the constant (if applicable) the zero'th operand is being ANDed with.
    // This can be a pure constant or a vector splat, in which case we treat the
    // vector as a scalar and use the splat value.
    APInt Constant = APInt::getZero(1);
    if (const ConstantSDNode *C = isConstOrConstSplat(
            N1, /*AllowUndefs=*/false, /*AllowTruncation=*/true)) {
      Constant = C->getAPIntValue();
    } else if (BuildVectorSDNode *Vector = dyn_cast<BuildVectorSDNode>(N1)) {
      unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits();
      APInt SplatValue, SplatUndef;
      unsigned SplatBitSize;
      bool HasAnyUndefs;
      // Endianness should not matter here. Code below makes sure that we only
      // use the result if the SplatBitSize is a multiple of the vector element
      // size. And after that we AND all element sized parts of the splat
      // together. So the end result should be the same regardless of in which
      // order we do those operations.
      const bool IsBigEndian = false;
      bool IsSplat =
          Vector->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
                                  HasAnyUndefs, EltBitWidth, IsBigEndian);
 
      // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a
      // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value.
      if (IsSplat && (SplatBitSize % EltBitWidth) == 0) {
        // Undef bits can contribute to a possible optimisation if set, so
        // set them.
        SplatValue |= SplatUndef;
 
        // The splat value may be something like "0x00FFFFFF", which means 0 for
        // the first vector value and FF for the rest, repeating. We need a mask
        // that will apply equally to all members of the vector, so AND all the
        // lanes of the constant together.
        Constant = APInt::getAllOnes(EltBitWidth);
        for (unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i)
          Constant &= SplatValue.extractBits(EltBitWidth, i * EltBitWidth);
      }
    }
 
    // If we want to change an EXTLOAD to a ZEXTLOAD, ensure a ZEXTLOAD is
    // actually legal and isn't going to get expanded, else this is a false
    // optimisation.
    bool CanZextLoadProfitably = TLI.isLoadLegal(
        Load->getValueType(0), Load->getMemoryVT(), Load->getAlign(),
        Load->getAddressSpace(), ISD::ZEXTLOAD, false);
 
    // Resize the constant to the same size as the original memory access before
    // extension. If it is still the AllOnesValue then this AND is completely
    // unneeded.
    Constant = Constant.zextOrTrunc(Load->getMemoryVT().getScalarSizeInBits());
 
    bool B;
    switch (Load->getExtensionType()) {
    default: B = false; break;
    case ISD::EXTLOAD: B = CanZextLoadProfitably; break;
    case ISD::ZEXTLOAD:
    case ISD::NON_EXTLOAD: B = true; break;
    }
 
    if (B && Constant.isAllOnes()) {
      // If the load type was an EXTLOAD, convert to ZEXTLOAD in order to
      // preserve semantics once we get rid of the AND.
      SDValue NewLoad(Load, 0);
 
      // Fold the AND away. NewLoad may get replaced immediately.
      CombineTo(N, (N0.getNode() == Load) ? NewLoad : N0);
 
      if (Load->getExtensionType() == ISD::EXTLOAD) {
        NewLoad = DAG.getLoad(Load->getAddressingMode(), ISD::ZEXTLOAD,
                              Load->getValueType(0), SDLoc(Load),
                              Load->getChain(), Load->getBasePtr(),
                              Load->getOffset(), Load->getMemoryVT(),
                              Load->getMemOperand());
        // Replace uses of the EXTLOAD with the new ZEXTLOAD.
        if (Load->getNumValues() == 3) {
          // PRE/POST_INC loads have 3 values.
          SDValue To[] = { NewLoad.getValue(0), NewLoad.getValue(1),
                           NewLoad.getValue(2) };
          CombineTo(Load, To, 3, true);
        } else {
          CombineTo(Load, NewLoad.getValue(0), NewLoad.getValue(1));
        }
      }
 
      return SDValue(N, 0); // Return N so it doesn't get rechecked!
    }
  }
 
  // Try to convert a constant mask AND into a shuffle clear mask.
  if (VT.isVector())
    if (SDValue Shuffle = XformToShuffleWithZero(N))
      return Shuffle;
 
  if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
    return Combined;
 
  if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() && N1C &&
      ISD::isExtOpcode(N0.getOperand(0).getOpcode())) {
    SDValue Ext = N0.getOperand(0);
    EVT ExtVT = Ext->getValueType(0);
    SDValue Extendee = Ext->getOperand(0);
 
    unsigned ScalarWidth = Extendee.getValueType().getScalarSizeInBits();
    if (N1C->getAPIntValue().isMask(ScalarWidth) &&
        (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, ExtVT))) {
      //    (and (extract_subvector (zext|anyext|sext v) _) iN_mask)
      // => (extract_subvector (iN_zeroext v))
      SDValue ZeroExtExtendee =
          DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVT, Extendee);
 
      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ZeroExtExtendee,
                         N0.getOperand(1));
    }
  }
 
  // fold (and (masked_gather x)) -> (zext_masked_gather x)
  if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
    EVT MemVT = GN0->getMemoryVT();
    EVT ScalarVT = MemVT.getScalarType();
 
    if (SDValue(GN0, 0).hasOneUse() &&
        isConstantSplatVectorMaskForType(N1.getNode(), ScalarVT) &&
        TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
      SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
                       GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
 
      SDValue ZExtLoad = DAG.getMaskedGather(
          DAG.getVTList(VT, MVT::Other), MemVT, DL, Ops, GN0->getMemOperand(),
          GN0->getIndexType(), ISD::ZEXTLOAD);
 
      CombineTo(N, ZExtLoad);
      AddToWorklist(ZExtLoad.getNode());
      // Avoid recheck of N.
      return SDValue(N, 0);
    }
  }
 
  // fold (and (load x), 255) -> (zextload x, i8)
  // fold (and (extload x, i16), 255) -> (zextload x, i8)
  // fold (and (freeze (load x)), 255) -> (freeze (zextload x, i8))
  // fold (and (freeze (extload x, i16)), 255) -> (freeze (zextload x, i8))
  if (N1C && !VT.isVector()) {
    SDValue Inner = peekThroughFreeze(N0);
    if (Inner.getOpcode() == ISD::LOAD)
      if (SDValue Res = reduceLoadWidth(N))
        return Res;
  }
 
  if (LegalTypes) {
    // Attempt to propagate the AND back up to the leaves which, if they're
    // loads, can be combined to narrow loads and the AND node can be removed.
    // Perform after legalization so that extend nodes will already be
    // combined into the loads.
    if (BackwardsPropagateMask(N))
      return SDValue(N, 0);
  }
 
  if (SDValue Combined = visitANDLike(N0, N1, N))
    return Combined;
 
  // Simplify: (and (op x...), (op y...))  -> (op (and x, y))
  if (N0.getOpcode() == N1.getOpcode())
    if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
      return V;
 
  if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG))
    return R;
  if (SDValue R = foldLogicOfShifts(N, N1, N0, DAG))
    return R;
 
  // Fold (and X, (bswap (not Y))) -> (and X, (not (bswap Y)))
  // Fold (and X, (bitreverse (not Y))) -> (and X, (not (bitreverse Y)))
  SDValue X, Y, Z, NotY;
  for (unsigned Opc : {ISD::BSWAP, ISD::BITREVERSE})
    if (sd_match(N,
                 m_And(m_Value(X), m_OneUse(m_UnaryOp(Opc, m_Value(NotY))))) &&
        sd_match(NotY, m_Not(m_Value(Y))) &&
        (TLI.hasAndNot(SDValue(N, 0)) || NotY->hasOneUse()))
      return DAG.getNode(ISD::AND, DL, VT, X,
                         DAG.getNOT(DL, DAG.getNode(Opc, DL, VT, Y), VT));
 
  // Fold (and X, (rot (not Y), Z)) -> (and X, (not (rot Y, Z)))
  for (unsigned Opc : {ISD::ROTL, ISD::ROTR})
    if (sd_match(N, m_And(m_Value(X),
                          m_OneUse(m_BinOp(Opc, m_Value(NotY), m_Value(Z))))) &&
        sd_match(NotY, m_Not(m_Value(Y))) &&
        (TLI.hasAndNot(SDValue(N, 0)) || NotY->hasOneUse()))
      return DAG.getNode(ISD::AND, DL, VT, X,
                         DAG.getNOT(DL, DAG.getNode(Opc, DL, VT, Y, Z), VT));
 
  // Fold (and X, (add (not Y), Z)) -> (and X, (not (sub Y, Z)))
  // Fold (and X, (sub (not Y), Z)) -> (and X, (not (add Y, Z)))
  if (TLI.hasAndNot(SDValue(N, 0)))
    if (SDValue Folded = foldBitwiseOpWithNeg(N, DL, VT))
      return Folded;
 
  // Fold (and (srl X, C), 1) -> (srl X, BW-1) for signbit extraction
  // If we are shifting down an extended sign bit, see if we can simplify
  // this to shifting the MSB directly to expose further simplifications.
  // This pattern often appears after sext_inreg legalization.
  APInt Amt;
  if (sd_match(N, m_And(m_Srl(m_Value(X), m_ConstInt(Amt)), m_One())) &&
      Amt.ult(BitWidth - 1) && Amt.uge(BitWidth - DAG.ComputeNumSignBits(X)))
    return DAG.getNode(ISD::SRL, DL, VT, X,
                       DAG.getShiftAmountConstant(BitWidth - 1, VT, DL));
 
  // Masking the negated extension of a boolean is just the zero-extended
  // boolean:
  // and (sub 0, zext(bool X)), 1 --> zext(bool X)
  // and (sub 0, sext(bool X)), 1 --> zext(bool X)
  //
  // Note: the SimplifyDemandedBits fold below can make an information-losing
  // transform, and then we have no way to find this better fold.
  if (sd_match(N, m_And(m_Sub(m_Zero(), m_Value(X)), m_One()))) {
    if (X.getOpcode() == ISD::ZERO_EXTEND &&
        X.getOperand(0).getScalarValueSizeInBits() == 1)
      return X;
    if (X.getOpcode() == ISD::SIGN_EXTEND &&
        X.getOperand(0).getScalarValueSizeInBits() == 1)
      return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, X.getOperand(0));
  }
 
  // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1)
  // fold (and (sra)) -> (and (srl)) when possible.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  // fold (zext_inreg (extload x)) -> (zextload x)
  // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use
  if (ISD::isUNINDEXEDLoad(N0.getNode()) &&
      (ISD::isEXTLoad(N0.getNode()) ||
       (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) {
    auto *LN0 = cast<LoadSDNode>(N0);
    EVT MemVT = LN0->getMemoryVT();
    // If we zero all the possible extended bits, then we can turn this into
    // a zextload if we are running before legalize or the operation is legal.
    unsigned ExtBitSize = N1.getScalarValueSizeInBits();
    unsigned MemBitSize = MemVT.getScalarSizeInBits();
    APInt ExtBits = APInt::getHighBitsSet(ExtBitSize, ExtBitSize - MemBitSize);
    if (DAG.MaskedValueIsZero(N1, ExtBits) &&
        ((!LegalOperations && LN0->isSimple()) ||
         TLI.isLoadLegal(VT, MemVT, LN0->getAlign(), LN0->getAddressSpace(),
                         ISD::ZEXTLOAD, false))) {
      SDValue ExtLoad =
          DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(),
                         LN0->getBasePtr(), MemVT, LN0->getMemOperand());
      AddToWorklist(N);
      CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
      return SDValue(N, 0); // Return N so it doesn't get rechecked!
    }
  }
 
  // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const)
  if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) {
    if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
                                           N0.getOperand(1), false))
      return BSwap;
  }
 
  if (SDValue Shifts = unfoldExtremeBitClearingToShifts(N))
    return Shifts;
 
  if (SDValue V = combineShiftAnd1ToBitTest(N, DAG))
    return V;
 
  // Recognize the following pattern:
  //
  // AndVT = (and (sign_extend NarrowVT to AndVT) #bitmask)
  //
  // where bitmask is a mask that clears the upper bits of AndVT. The
  // number of bits in bitmask must be a power of two.
  auto IsAndZeroExtMask = [](SDValue LHS, SDValue RHS) {
    if (LHS->getOpcode() != ISD::SIGN_EXTEND)
      return false;
 
    auto *C = isConstOrConstSplat(RHS, false, true);
    if (!C)
      return false;
 
    if (!C->getAPIntValue().isMask(
            LHS.getOperand(0).getValueType().getScalarSizeInBits()))
      return false;
 
    return true;
  };
 
  // Replace (and (sign_extend ...) #bitmask) with (zero_extend ...).
  if (IsAndZeroExtMask(N0, N1) &&
      (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)))
    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
 
  if (hasOperation(ISD::USUBSAT, VT))
    if (SDValue V = foldAndToUsubsat(N, DAG, DL))
      return V;
 
  // Postpone until legalization completed to avoid interference with bswap
  // folding
  if (LegalOperations || VT.isVector())
    if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG))
      return R;
 
  if (VT.isScalarInteger() && VT != MVT::i1)
    if (SDValue R = foldMaskedMerge(N, DAG, TLI, DL))
      return R;
 
  return SDValue();
}
 
/// Match (a >> 8) | (a << 8) as (bswap a) >> 16.
SDValue DAGCombiner::MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
                                        bool DemandHighBits) {
  if (!LegalOperations)
    return SDValue();
 
  EVT VT = N->getValueType(0);
  if (VT != MVT::i64 && VT != MVT::i32 && VT != MVT::i16)
    return SDValue();
  if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
    return SDValue();
 
  // Recognize (and (shl a, 8), 0xff00), (and (srl a, 8), 0xff)
  bool LookPassAnd0 = false;
  bool LookPassAnd1 = false;
  if (N0.getOpcode() == ISD::AND && N0.getOperand(0).getOpcode() == ISD::SRL)
    std::swap(N0, N1);
  if (N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL)
    std::swap(N0, N1);
  if (N0.getOpcode() == ISD::AND) {
    if (!N0->hasOneUse())
      return SDValue();
    ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    // Also handle 0xffff since the LHS is guaranteed to have zeros there.
    // This is needed for X86.
    if (!N01C || (N01C->getZExtValue() != 0xFF00 &&
                  N01C->getZExtValue() != 0xFFFF))
      return SDValue();
    N0 = N0.getOperand(0);
    LookPassAnd0 = true;
  }
 
  if (N1.getOpcode() == ISD::AND) {
    if (!N1->hasOneUse())
      return SDValue();
    ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
    if (!N11C || N11C->getZExtValue() != 0xFF)
      return SDValue();
    N1 = N1.getOperand(0);
    LookPassAnd1 = true;
  }
 
  if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
    std::swap(N0, N1);
  if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
    return SDValue();
  if (!N0->hasOneUse() || !N1->hasOneUse())
    return SDValue();
 
  ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
  ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
  if (!N01C || !N11C)
    return SDValue();
  if (N01C->getZExtValue() != 8 || N11C->getZExtValue() != 8)
    return SDValue();
 
  // Look for (shl (and a, 0xff), 8), (srl (and a, 0xff00), 8)
  SDValue N00 = N0->getOperand(0);
  if (!LookPassAnd0 && N00.getOpcode() == ISD::AND) {
    if (!N00->hasOneUse())
      return SDValue();
    ConstantSDNode *N001C = dyn_cast<ConstantSDNode>(N00.getOperand(1));
    if (!N001C || N001C->getZExtValue() != 0xFF)
      return SDValue();
    N00 = N00.getOperand(0);
    LookPassAnd0 = true;
  }
 
  SDValue N10 = N1->getOperand(0);
  if (!LookPassAnd1 && N10.getOpcode() == ISD::AND) {
    if (!N10->hasOneUse())
      return SDValue();
    ConstantSDNode *N101C = dyn_cast<ConstantSDNode>(N10.getOperand(1));
    // Also allow 0xFFFF since the bits will be shifted out. This is needed
    // for X86.
    if (!N101C || (N101C->getZExtValue() != 0xFF00 &&
                   N101C->getZExtValue() != 0xFFFF))
      return SDValue();
    N10 = N10.getOperand(0);
    LookPassAnd1 = true;
  }
 
  if (N00 != N10)
    return SDValue();
 
  // Make sure everything beyond the low halfword gets set to zero since the SRL
  // 16 will clear the top bits.
  unsigned OpSizeInBits = VT.getSizeInBits();
  if (OpSizeInBits > 16) {
    // If the left-shift isn't masked out then the only way this is a bswap is
    // if all bits beyond the low 8 are 0. In that case the entire pattern
    // reduces to a left shift anyway: leave it for other parts of the combiner.
    if (DemandHighBits && !LookPassAnd0)
      return SDValue();
 
    // However, if the right shift isn't masked out then it might be because
    // it's not needed. See if we can spot that too. If the high bits aren't
    // demanded, we only need bits 23:16 to be zero. Otherwise, we need all
    // upper bits to be zero.
    if (!LookPassAnd1) {
      unsigned HighBit = DemandHighBits ? OpSizeInBits : 24;
      if (!DAG.MaskedValueIsZero(N10,
                                 APInt::getBitsSet(OpSizeInBits, 16, HighBit)))
        return SDValue();
    }
  }
 
  SDValue Res = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, N00);
  if (OpSizeInBits > 16) {
    SDLoc DL(N);
    Res = DAG.getNode(ISD::SRL, DL, VT, Res,
                      DAG.getShiftAmountConstant(OpSizeInBits - 16, VT, DL));
  }
  return Res;
}
 
/// Return true if the specified node is an element that makes up a 32-bit
/// packed halfword byteswap.
/// ((x & 0x000000ff) << 8) |
/// ((x & 0x0000ff00) >> 8) |
/// ((x & 0x00ff0000) << 8) |
/// ((x & 0xff000000) >> 8)
static bool isBSwapHWordElement(SDValue N, MutableArrayRef<SDNode *> Parts) {
  if (!N->hasOneUse())
    return false;
 
  unsigned Opc = N.getOpcode();
  if (Opc != ISD::AND && Opc != ISD::SHL && Opc != ISD::SRL)
    return false;
 
  SDValue N0 = N.getOperand(0);
  unsigned Opc0 = N0.getOpcode();
  if (Opc0 != ISD::AND && Opc0 != ISD::SHL && Opc0 != ISD::SRL)
    return false;
 
  ConstantSDNode *N1C = nullptr;
  // SHL or SRL: look upstream for AND mask operand
  if (Opc == ISD::AND)
    N1C = dyn_cast<ConstantSDNode>(N.getOperand(1));
  else if (Opc0 == ISD::AND)
    N1C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
  if (!N1C)
    return false;
 
  unsigned MaskByteOffset;
  switch (N1C->getZExtValue()) {
  default:
    return false;
  case 0xFF:       MaskByteOffset = 0; break;
  case 0xFF00:     MaskByteOffset = 1; break;
  case 0xFFFF:
    // In case demanded bits didn't clear the bits that will be shifted out.
    // This is needed for X86.
    if (Opc == ISD::SRL || (Opc == ISD::AND && Opc0 == ISD::SHL)) {
      MaskByteOffset = 1;
      break;
    }
    return false;
  case 0xFF0000:   MaskByteOffset = 2; break;
  case 0xFF000000: MaskByteOffset = 3; break;
  }
 
  // Look for (x & 0xff) << 8 as well as ((x << 8) & 0xff00).
  if (Opc == ISD::AND) {
    if (MaskByteOffset == 0 || MaskByteOffset == 2) {
      // (x >> 8) & 0xff
      // (x >> 8) & 0xff0000
      if (Opc0 != ISD::SRL)
        return false;
      ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
      if (!C || C->getZExtValue() != 8)
        return false;
    } else {
      // (x << 8) & 0xff00
      // (x << 8) & 0xff000000
      if (Opc0 != ISD::SHL)
        return false;
      ConstantSDNode *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
      if (!C || C->getZExtValue() != 8)
        return false;
    }
  } else if (Opc == ISD::SHL) {
    // (x & 0xff) << 8
    // (x & 0xff0000) << 8
    if (MaskByteOffset != 0 && MaskByteOffset != 2)
      return false;
    ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    if (!C || C->getZExtValue() != 8)
      return false;
  } else { // Opc == ISD::SRL
    // (x & 0xff00) >> 8
    // (x & 0xff000000) >> 8
    if (MaskByteOffset != 1 && MaskByteOffset != 3)
      return false;
    ConstantSDNode *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
    if (!C || C->getZExtValue() != 8)
      return false;
  }
 
  if (Parts[MaskByteOffset])
    return false;
 
  Parts[MaskByteOffset] = N0.getOperand(0).getNode();
  return true;
}
 
// Match 2 elements of a packed halfword bswap.
static bool isBSwapHWordPair(SDValue N, MutableArrayRef<SDNode *> Parts) {
  if (N.getOpcode() == ISD::OR)
    return isBSwapHWordElement(N.getOperand(0), Parts) &&
           isBSwapHWordElement(N.getOperand(1), Parts);
 
  if (N.getOpcode() == ISD::SRL && N.getOperand(0).getOpcode() == ISD::BSWAP) {
    ConstantSDNode *C = isConstOrConstSplat(N.getOperand(1));
    if (!C || C->getAPIntValue() != 16)
      return false;
    Parts[0] = Parts[1] = N.getOperand(0).getOperand(0).getNode();
    return true;
  }
 
  return false;
}
 
// Match this pattern:
//   (or (and (shl (A, 8)), 0xff00ff00), (and (srl (A, 8)), 0x00ff00ff))
// And rewrite this to:
//   (rotr (bswap A), 16)
static SDValue matchBSwapHWordOrAndAnd(const TargetLowering &TLI,
                                       SelectionDAG &DAG, SDNode *N, SDValue N0,
                                       SDValue N1, EVT VT) {
  assert(N->getOpcode() == ISD::OR && VT == MVT::i32 &&
         "MatchBSwapHWordOrAndAnd: expecting i32");
  if (!TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
    return SDValue();
  if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
    return SDValue();
  // TODO: this is too restrictive; lifting this restriction requires more tests
  if (!N0->hasOneUse() || !N1->hasOneUse())
    return SDValue();
  ConstantSDNode *Mask0 = isConstOrConstSplat(N0.getOperand(1));
  ConstantSDNode *Mask1 = isConstOrConstSplat(N1.getOperand(1));
  if (!Mask0 || !Mask1)
    return SDValue();
  if (Mask0->getAPIntValue() != 0xff00ff00 ||
      Mask1->getAPIntValue() != 0x00ff00ff)
    return SDValue();
  SDValue Shift0 = N0.getOperand(0);
  SDValue Shift1 = N1.getOperand(0);
  if (Shift0.getOpcode() != ISD::SHL || Shift1.getOpcode() != ISD::SRL)
    return SDValue();
  ConstantSDNode *ShiftAmt0 = isConstOrConstSplat(Shift0.getOperand(1));
  ConstantSDNode *ShiftAmt1 = isConstOrConstSplat(Shift1.getOperand(1));
  if (!ShiftAmt0 || !ShiftAmt1)
    return SDValue();
  if (ShiftAmt0->getAPIntValue() != 8 || ShiftAmt1->getAPIntValue() != 8)
    return SDValue();
  if (Shift0.getOperand(0) != Shift1.getOperand(0))
    return SDValue();
 
  SDLoc DL(N);
  SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Shift0.getOperand(0));
  SDValue ShAmt = DAG.getShiftAmountConstant(16, VT, DL);
  return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
}
 
/// Match a 32-bit packed halfword bswap. That is
/// ((x & 0x000000ff) << 8) |
/// ((x & 0x0000ff00) >> 8) |
/// ((x & 0x00ff0000) << 8) |
/// ((x & 0xff000000) >> 8)
/// => (rotl (bswap x), 16)
SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
  if (!LegalOperations)
    return SDValue();
 
  EVT VT = N->getValueType(0);
  if (VT != MVT::i32)
    return SDValue();
  if (!TLI.isOperationLegalOrCustom(ISD::BSWAP, VT))
    return SDValue();
 
  if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N0, N1, VT))
    return BSwap;
 
  // Try again with commuted operands.
  if (SDValue BSwap = matchBSwapHWordOrAndAnd(TLI, DAG, N, N1, N0, VT))
    return BSwap;
 
 
  // Look for either
  // (or (bswaphpair), (bswaphpair))
  // (or (or (bswaphpair), (and)), (and))
  // (or (or (and), (bswaphpair)), (and))
  SDNode *Parts[4] = {};
 
  if (isBSwapHWordPair(N0, Parts)) {
    // (or (or (and), (and)), (or (and), (and)))
    if (!isBSwapHWordPair(N1, Parts))
      return SDValue();
  } else if (N0.getOpcode() == ISD::OR) {
    // (or (or (or (and), (and)), (and)), (and))
    if (!isBSwapHWordElement(N1, Parts))
      return SDValue();
    SDValue N00 = N0.getOperand(0);
    SDValue N01 = N0.getOperand(1);
    if (!(isBSwapHWordElement(N01, Parts) && isBSwapHWordPair(N00, Parts)) &&
        !(isBSwapHWordElement(N00, Parts) && isBSwapHWordPair(N01, Parts)))
      return SDValue();
  } else {
    return SDValue();
  }
 
  // Make sure the parts are all coming from the same node.
  if (Parts[0] != Parts[1] || Parts[0] != Parts[2] || Parts[0] != Parts[3])
    return SDValue();
 
  SDLoc DL(N);
  SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT,
                              SDValue(Parts[0], 0));
 
  // Result of the bswap should be rotated by 16. If it's not legal, then
  // do  (x << 16) | (x >> 16).
  SDValue ShAmt = DAG.getShiftAmountConstant(16, VT, DL);
  if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
    return DAG.getNode(ISD::ROTL, DL, VT, BSwap, ShAmt);
  if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
    return DAG.getNode(ISD::ROTR, DL, VT, BSwap, ShAmt);
  return DAG.getNode(ISD::OR, DL, VT,
                     DAG.getNode(ISD::SHL, DL, VT, BSwap, ShAmt),
                     DAG.getNode(ISD::SRL, DL, VT, BSwap, ShAmt));
}
 
/// This contains all DAGCombine rules which reduce two values combined by
/// an Or operation to a single value \see visitANDLike().
SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, const SDLoc &DL) {
  EVT VT = N1.getValueType();
 
  // fold (or x, undef) -> -1
  if (!LegalOperations && (N0.isUndef() || N1.isUndef()))
    return DAG.getAllOnesConstant(DL, VT);
 
  if (SDValue V = foldLogicOfSetCCs(false, N0, N1, DL))
    return V;
 
  // (or (and X, C1), (and Y, C2))  -> (and (or X, Y), C3) if possible.
  if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
      // Don't increase # computations.
      (N0->hasOneUse() || N1->hasOneUse())) {
    // We can only do this xform if we know that bits from X that are set in C2
    // but not in C1 are already zero.  Likewise for Y.
    if (const ConstantSDNode *N0O1C =
        getAsNonOpaqueConstant(N0.getOperand(1))) {
      if (const ConstantSDNode *N1O1C =
          getAsNonOpaqueConstant(N1.getOperand(1))) {
        // We can only do this xform if we know that bits from X that are set in
        // C2 but not in C1 are already zero.  Likewise for Y.
        const APInt &LHSMask = N0O1C->getAPIntValue();
        const APInt &RHSMask = N1O1C->getAPIntValue();
 
        if (DAG.MaskedValueIsZero(N0.getOperand(0), RHSMask&~LHSMask) &&
            DAG.MaskedValueIsZero(N1.getOperand(0), LHSMask&~RHSMask)) {
          SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
                                  N0.getOperand(0), N1.getOperand(0));
          return DAG.getNode(ISD::AND, DL, VT, X,
                             DAG.getConstant(LHSMask | RHSMask, DL, VT));
        }
      }
    }
  }
 
  // (or (and X, M), (and X, N)) -> (and X, (or M, N))
  if (N0.getOpcode() == ISD::AND &&
      N1.getOpcode() == ISD::AND &&
      N0.getOperand(0) == N1.getOperand(0) &&
      // Don't increase # computations.
      (N0->hasOneUse() || N1->hasOneUse())) {
    SDValue X = DAG.getNode(ISD::OR, SDLoc(N0), VT,
                            N0.getOperand(1), N1.getOperand(1));
    return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), X);
  }
 
  return SDValue();
}
 
/// OR combines for which the commuted variant will be tried as well.
static SDValue visitORCommutative(SelectionDAG &DAG, SDValue N0, SDValue N1,
                                  SDNode *N) {
  EVT VT = N0.getValueType();
  unsigned BW = VT.getScalarSizeInBits();
  SDLoc DL(N);
 
  auto peekThroughResize = [](SDValue V) {
    if (V->getOpcode() == ISD::ZERO_EXTEND || V->getOpcode() == ISD::TRUNCATE)
      return V->getOperand(0);
    return V;
  };
 
  SDValue N0Resized = peekThroughResize(N0);
  if (N0Resized.getOpcode() == ISD::AND) {
    SDValue N1Resized = peekThroughResize(N1);
    SDValue N00 = N0Resized.getOperand(0);
    SDValue N01 = N0Resized.getOperand(1);
 
    // fold or (and x, y), x --> x
    if (N00 == N1Resized || N01 == N1Resized)
      return N1;
 
    // fold (or (and X, (xor Y, -1)), Y) -> (or X, Y)
    // TODO: Set AllowUndefs = true.
    if (SDValue NotOperand = getBitwiseNotOperand(N01, N00,
                                                  /* AllowUndefs */ false)) {
      if (peekThroughResize(NotOperand) == N1Resized)
        return DAG.getNode(ISD::OR, DL, VT, DAG.getZExtOrTrunc(N00, DL, VT),
                           N1);
    }
 
    // fold (or (and (xor Y, -1), X), Y) -> (or X, Y)
    if (SDValue NotOperand = getBitwiseNotOperand(N00, N01,
                                                  /* AllowUndefs */ false)) {
      if (peekThroughResize(NotOperand) == N1Resized)
        return DAG.getNode(ISD::OR, DL, VT, DAG.getZExtOrTrunc(N01, DL, VT),
                           N1);
    }
  }
 
  SDValue X, Y;
 
  // fold or (xor X, N1), N1 --> or X, N1
  if (sd_match(N0, m_Xor(m_Value(X), m_Specific(N1))))
    return DAG.getNode(ISD::OR, DL, VT, X, N1);
 
  // fold or (xor x, y), (x and/or y) --> or x, y
  if (sd_match(N0, m_Xor(m_Value(X), m_Value(Y))) &&
      (sd_match(N1, m_And(m_Specific(X), m_Specific(Y))) ||
       sd_match(N1, m_Or(m_Specific(X), m_Specific(Y)))))
    return DAG.getNode(ISD::OR, DL, VT, X, Y);
 
  if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG))
    return R;
 
  auto peekThroughZext = [](SDValue V) {
    if (V->getOpcode() == ISD::ZERO_EXTEND)
      return V->getOperand(0);
    return V;
  };
 
  if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::SHL &&
      peekThroughZext(N0.getOperand(2)) == peekThroughZext(N1.getOperand(1))) {
    // (fshl X, ?, Y) | (shl X, Y) --> fshl X, ?, Y
    if (N0.getOperand(0) == N1.getOperand(0))
      return N0;
    // (fshl A, X, Y) | (shl X, Y) --> fshl (A|X), X, Y
    if (N0.getOperand(1) == N1.getOperand(0) && N0.hasOneUse() &&
        N1.hasOneUse()) {
      SDValue A = N0.getOperand(0);
      SDValue X = N1.getOperand(0);
      SDValue NewLHS = DAG.getNode(ISD::OR, DL, VT, A, X);
      return DAG.getNode(ISD::FSHL, DL, VT, NewLHS, X, N0.getOperand(2));
    }
  }
 
  if (N0.getOpcode() == ISD::FSHR && N1.getOpcode() == ISD::SRL &&
      peekThroughZext(N0.getOperand(2)) == peekThroughZext(N1.getOperand(1))) {
    // (fshr ?, X, Y) | (srl X, Y) --> fshr ?, X, Y
    if (N0.getOperand(1) == N1.getOperand(0))
      return N0;
    // (fshr X, B, Y) | (srl X, Y) --> fshr X, (X|B), Y
    if (N0.getOperand(0) == N1.getOperand(0) && N0.hasOneUse() &&
        N1.hasOneUse()) {
      SDValue X = N1.getOperand(0);
      SDValue B = N0.getOperand(1);
      SDValue NewRHS = DAG.getNode(ISD::OR, DL, VT, X, B);
      return DAG.getNode(ISD::FSHR, DL, VT, X, NewRHS, N0.getOperand(2));
    }
  }
 
  // (fshl A, B, S0) | (fshr C, D, S1) --> fshl (A|C), (B|D), S0
  // iff S0 + S1 == bitwidth(S1)
  if (N0.getOpcode() == ISD::FSHL && N1.getOpcode() == ISD::FSHR &&
      N0.hasOneUse() && N1.hasOneUse()) {
    auto *S0 = dyn_cast<ConstantSDNode>(N0.getOperand(2));
    auto *S1 = dyn_cast<ConstantSDNode>(N1.getOperand(2));
    if (S0 && S1 && S0->getZExtValue() < BW && S1->getZExtValue() < BW &&
        S0->getZExtValue() == (BW - S1->getZExtValue())) {
      SDValue A = N0.getOperand(0);
      SDValue B = N0.getOperand(1);
      SDValue C = N1.getOperand(0);
      SDValue D = N1.getOperand(1);
      SDValue NewLHS = DAG.getNode(ISD::OR, DL, VT, A, C);
      SDValue NewRHS = DAG.getNode(ISD::OR, DL, VT, B, D);
      return DAG.getNode(ISD::FSHL, DL, VT, NewLHS, NewRHS, N0.getOperand(2));
    }
  }
 
  // Attempt to match a legalized build_pair-esque pattern:
  // or(shl(aext(Hi),BW/2),zext(Lo))
  SDValue Lo, Hi;
  if (sd_match(N0,
               m_OneUse(m_Shl(m_AnyExt(m_Value(Hi)), m_SpecificInt(BW / 2)))) &&
      sd_match(N1, m_ZExt(m_Value(Lo))) &&
      Lo.getScalarValueSizeInBits() == (BW / 2) &&
      Lo.getValueType() == Hi.getValueType()) {
    // Fold build_pair(not(Lo),not(Hi)) -> not(build_pair(Lo,Hi)).
    SDValue NotLo, NotHi;
    if (sd_match(Lo, m_OneUse(m_Not(m_Value(NotLo)))) &&
        sd_match(Hi, m_OneUse(m_Not(m_Value(NotHi))))) {
      Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, NotLo);
      Hi = DAG.getNode(ISD::ANY_EXTEND, DL, VT, NotHi);
      Hi = DAG.getNode(ISD::SHL, DL, VT, Hi,
                       DAG.getShiftAmountConstant(BW / 2, VT, DL));
      return DAG.getNOT(DL, DAG.getNode(ISD::OR, DL, VT, Lo, Hi), VT);
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitOR(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N1.getValueType();
  SDLoc DL(N);
 
  // x | x --> x
  if (N0 == N1)
    return N0;
 
  // fold (or c1, c2) -> c1|c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::OR, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::OR, DL, VT, N1, N0);
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (or x, 0) -> x, vector edition
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      return N0;
 
    // fold (or x, -1) -> -1, vector edition
    if (ISD::isConstantSplatVectorAllOnes(N1.getNode()))
      // do not return N1, because undef node may exist in N1
      return DAG.getAllOnesConstant(DL, N1.getValueType());
 
    // fold (or buildvector(x,0,-1,w), buildvector(0,y,z,w))
    // --> buildvector(x,y,-1,w)
    auto *BV0 = dyn_cast<BuildVectorSDNode>(N0);
    auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
    if (BV0 && BV1 && !BV0->getSplatValue() && !BV1->getSplatValue() &&
        N0.hasOneUse() && N1.hasOneUse() &&
        BV0->getOperand(0).getValueType() ==
            BV1->getOperand(0).getValueType()) {
      SmallVector<SDValue> MergedOps;
      unsigned NumElts = VT.getVectorNumElements();
      EVT EltVT = BV0->getOperand(0).getValueType();
      for (unsigned I = 0; I != NumElts; ++I) {
        auto *C0 = dyn_cast<ConstantSDNode>(BV0->getOperand(I));
        auto *C1 = dyn_cast<ConstantSDNode>(BV1->getOperand(I));
        if (C0 && C1)
          MergedOps.push_back(DAG.getConstant(
              C0->getAPIntValue() | C1->getAPIntValue(), DL, EltVT));
        else if (C0 && C0->isZero())
          MergedOps.push_back(BV1->getOperand(I));
        else if (C1 && C1->isZero())
          MergedOps.push_back(BV0->getOperand(I));
        else if (C0 && C0->isAllOnes())
          MergedOps.push_back(BV0->getOperand(I));
        else if (C1 && C1->isAllOnes())
          MergedOps.push_back(BV1->getOperand(I));
        else if (BV0->getOperand(I) == BV1->getOperand(I))
          MergedOps.push_back(BV0->getOperand(I));
        else
          break;
      }
      if (MergedOps.size() == NumElts)
        return DAG.getBuildVector(VT, DL, MergedOps);
    }
 
    // fold (or (shuf A, V_0, MA), (shuf B, V_0, MB)) -> (shuf A, B, Mask)
    // Do this only if the resulting type / shuffle is legal.
    auto *SV0 = dyn_cast<ShuffleVectorSDNode>(N0);
    auto *SV1 = dyn_cast<ShuffleVectorSDNode>(N1);
    if (SV0 && SV1 && TLI.isTypeLegal(VT)) {
      bool ZeroN00 = ISD::isBuildVectorAllZeros(N0.getOperand(0).getNode());
      bool ZeroN01 = ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode());
      bool ZeroN10 = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
      bool ZeroN11 = ISD::isBuildVectorAllZeros(N1.getOperand(1).getNode());
      // Ensure both shuffles have a zero input.
      if ((ZeroN00 != ZeroN01) && (ZeroN10 != ZeroN11)) {
        assert((!ZeroN00 || !ZeroN01) && "Both inputs zero!");
        assert((!ZeroN10 || !ZeroN11) && "Both inputs zero!");
        bool CanFold = true;
        int NumElts = VT.getVectorNumElements();
        SmallVector<int, 4> Mask(NumElts, -1);
 
        for (int i = 0; i != NumElts; ++i) {
          int M0 = SV0->getMaskElt(i);
          int M1 = SV1->getMaskElt(i);
 
          // Determine if either index is pointing to a zero vector.
          bool M0Zero = M0 < 0 || (ZeroN00 == (M0 < NumElts));
          bool M1Zero = M1 < 0 || (ZeroN10 == (M1 < NumElts));
 
          // If one element is zero and the otherside is undef, keep undef.
          // This also handles the case that both are undef.
          if ((M0Zero && M1 < 0) || (M1Zero && M0 < 0))
            continue;
 
          // Make sure only one of the elements is zero.
          if (M0Zero == M1Zero) {
            CanFold = false;
            break;
          }
 
          assert((M0 >= 0 || M1 >= 0) && "Undef index!");
 
          // We have a zero and non-zero element. If the non-zero came from
          // SV0 make the index a LHS index. If it came from SV1, make it
          // a RHS index. We need to mod by NumElts because we don't care
          // which operand it came from in the original shuffles.
          Mask[i] = M1Zero ? M0 % NumElts : (M1 % NumElts) + NumElts;
        }
 
        if (CanFold) {
          SDValue NewLHS = ZeroN00 ? N0.getOperand(1) : N0.getOperand(0);
          SDValue NewRHS = ZeroN10 ? N1.getOperand(1) : N1.getOperand(0);
          SDValue LegalShuffle =
              TLI.buildLegalVectorShuffle(VT, DL, NewLHS, NewRHS, Mask, DAG);
          if (LegalShuffle)
            return LegalShuffle;
        }
      }
    }
  }
 
  // fold (or x, 0) -> x
  if (isNullConstant(N1))
    return N0;
 
  // fold (or x, -1) -> -1
  if (isAllOnesConstant(N1))
    return N1;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // fold (or x, c) -> c iff (x & ~c) == 0
  ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
  if (N1C && DAG.MaskedValueIsZero(N0, ~N1C->getAPIntValue()))
    return N1;
 
  if (SDValue R = foldAndOrOfSETCC(N, DAG))
    return R;
 
  if (SDValue Combined = visitORLike(N0, N1, DL))
    return Combined;
 
  if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
    return Combined;
 
  if (SDValue Combined = combineOrOfSetCCToUSUBOCarry(N, DAG, TLI))
    return Combined;
 
  // Recognize halfword bswaps as (bswap + rotl 16) or (bswap + shl 16)
  if (SDValue BSwap = MatchBSwapHWord(N, N0, N1))
    return BSwap;
  if (SDValue BSwap = MatchBSwapHWordLow(N, N0, N1))
    return BSwap;
 
  // reassociate or
  if (SDValue ROR = reassociateOps(ISD::OR, DL, N0, N1, N->getFlags()))
    return ROR;
 
  // Fold or(vecreduce(x), vecreduce(y)) -> vecreduce(or(x, y))
  if (SDValue SD =
          reassociateReduction(ISD::VECREDUCE_OR, ISD::OR, DL, VT, N0, N1))
    return SD;
 
  // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2)
  // iff (c1 & c2) != 0 or c1/c2 are undef.
  auto MatchIntersect = [](ConstantSDNode *C1, ConstantSDNode *C2) {
    return !C1 || !C2 || C1->getAPIntValue().intersects(C2->getAPIntValue());
  };
  if (N0.getOpcode() == ISD::AND && N0->hasOneUse() &&
      ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchIntersect, true)) {
    if (SDValue COR = DAG.FoldConstantArithmetic(ISD::OR, SDLoc(N1), VT,
                                                 {N1, N0.getOperand(1)})) {
      SDValue IOR = DAG.getNode(ISD::OR, SDLoc(N0), VT, N0.getOperand(0), N1);
      AddToWorklist(IOR.getNode());
      return DAG.getNode(ISD::AND, DL, VT, COR, IOR);
    }
  }
 
  if (SDValue Combined = visitORCommutative(DAG, N0, N1, N))
    return Combined;
  if (SDValue Combined = visitORCommutative(DAG, N1, N0, N))
    return Combined;
 
  // Simplify: (or (op x...), (op y...))  -> (op (or x, y))
  if (N0.getOpcode() == N1.getOpcode())
    if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
      return V;
 
  // See if this is some rotate idiom.
  if (SDValue Rot = MatchRotate(N0, N1, DL, /*FromAdd=*/false))
    return Rot;
 
  if (SDValue Load = MatchLoadCombine(N))
    return Load;
 
  // Simplify the operands using demanded-bits information.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  // If OR can be rewritten into ADD, try combines based on ADD.
  if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) &&
      DAG.isADDLike(SDValue(N, 0)))
    if (SDValue Combined = visitADDLike(N))
      return Combined;
 
  // Postpone until legalization completed to avoid interference with bswap
  // folding
  if (LegalOperations || VT.isVector())
    if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG))
      return R;
 
  if (VT.isScalarInteger() && VT != MVT::i1)
    if (SDValue R = foldMaskedMerge(N, DAG, TLI, DL))
      return R;
 
  return SDValue();
}
 
static SDValue stripConstantMask(const SelectionDAG &DAG, SDValue Op,
                                 SDValue &Mask) {
  if (Op.getOpcode() == ISD::AND &&
      DAG.isConstantIntBuildVectorOrConstantInt(Op.getOperand(1))) {
    Mask = Op.getOperand(1);
    return Op.getOperand(0);
  }
  return Op;
}
 
/// Match "(X shl/srl V1) & V2" where V2 may not be present.
static bool matchRotateHalf(const SelectionDAG &DAG, SDValue Op, SDValue &Shift,
                            SDValue &Mask) {
  Op = stripConstantMask(DAG, Op, Mask);
  if (Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) {
    Shift = Op;
    return true;
  }
  return false;
}
 
/// Helper function for visitOR to extract the needed side of a rotate idiom
/// from a shl/srl/mul/udiv.  This is meant to handle cases where
/// InstCombine merged some outside op with one of the shifts from
/// the rotate pattern.
/// \returns An empty \c SDValue if the needed shift couldn't be extracted.
/// Otherwise, returns an expansion of \p ExtractFrom based on the following
/// patterns:
///
///   (or (add v v) (shrl v bitwidth-1)):
///     expands (add v v) -> (shl v 1)
///
///   (or (mul v c0) (shrl (mul v c1) c2)):
///     expands (mul v c0) -> (shl (mul v c1) c3)
///
///   (or (udiv v c0) (shl (udiv v c1) c2)):
///     expands (udiv v c0) -> (shrl (udiv v c1) c3)
///
///   (or (shl v c0) (shrl (shl v c1) c2)):
///     expands (shl v c0) -> (shl (shl v c1) c3)
///
///   (or (shrl v c0) (shl (shrl v c1) c2)):
///     expands (shrl v c0) -> (shrl (shrl v c1) c3)
///
/// Such that in all cases, c3+c2==bitwidth(op v c1).
static SDValue extractShiftForRotate(SelectionDAG &DAG, SDValue OppShift,
                                     SDValue ExtractFrom, SDValue &Mask,
                                     const SDLoc &DL) {
  assert(OppShift && ExtractFrom && "Empty SDValue");
  if (OppShift.getOpcode() != ISD::SHL && OppShift.getOpcode() != ISD::SRL)
    return SDValue();
 
  ExtractFrom = stripConstantMask(DAG, ExtractFrom, Mask);
 
  // Value and Type of the shift.
  SDValue OppShiftLHS = OppShift.getOperand(0);
  EVT ShiftedVT = OppShiftLHS.getValueType();
 
  // Amount of the existing shift.
  ConstantSDNode *OppShiftCst = isConstOrConstSplat(OppShift.getOperand(1));
 
  // (add v v) -> (shl v 1)
  // TODO: Should this be a general DAG canonicalization?
  if (OppShift.getOpcode() == ISD::SRL && OppShiftCst &&
      ExtractFrom.getOpcode() == ISD::ADD &&
      ExtractFrom.getOperand(0) == ExtractFrom.getOperand(1) &&
      ExtractFrom.getOperand(0) == OppShiftLHS &&
      OppShiftCst->getAPIntValue() == ShiftedVT.getScalarSizeInBits() - 1)
    return DAG.getNode(ISD::SHL, DL, ShiftedVT, OppShiftLHS,
                       DAG.getShiftAmountConstant(1, ShiftedVT, DL));
 
  // Preconditions:
  //    (or (op0 v c0) (shiftl/r (op0 v c1) c2))
  //
  // Find opcode of the needed shift to be extracted from (op0 v c0).
  unsigned Opcode = ISD::DELETED_NODE;
  bool IsMulOrDiv = false;
  // Set Opcode and IsMulOrDiv if the extract opcode matches the needed shift
  // opcode or its arithmetic (mul or udiv) variant.
  auto SelectOpcode = [&](unsigned NeededShift, unsigned MulOrDivVariant) {
    IsMulOrDiv = ExtractFrom.getOpcode() == MulOrDivVariant;
    if (!IsMulOrDiv && ExtractFrom.getOpcode() != NeededShift)
      return false;
    Opcode = NeededShift;
    return true;
  };
  // op0 must be either the needed shift opcode or the mul/udiv equivalent
  // that the needed shift can be extracted from.
  if ((OppShift.getOpcode() != ISD::SRL || !SelectOpcode(ISD::SHL, ISD::MUL)) &&
      (OppShift.getOpcode() != ISD::SHL || !SelectOpcode(ISD::SRL, ISD::UDIV)))
    return SDValue();
 
  // op0 must be the same opcode on both sides, have the same LHS argument,
  // and produce the same value type.
  if (OppShiftLHS.getOpcode() != ExtractFrom.getOpcode() ||
      OppShiftLHS.getOperand(0) != ExtractFrom.getOperand(0) ||
      ShiftedVT != ExtractFrom.getValueType())
    return SDValue();
 
  // Constant mul/udiv/shift amount from the RHS of the shift's LHS op.
  ConstantSDNode *OppLHSCst = isConstOrConstSplat(OppShiftLHS.getOperand(1));
  // Constant mul/udiv/shift amount from the RHS of the ExtractFrom op.
  ConstantSDNode *ExtractFromCst =
      isConstOrConstSplat(ExtractFrom.getOperand(1));
  // TODO: We should be able to handle non-uniform constant vectors for these values
  // Check that we have constant values.
  if (!OppShiftCst || !OppShiftCst->getAPIntValue() ||
      !OppLHSCst || !OppLHSCst->getAPIntValue() ||
      !ExtractFromCst || !ExtractFromCst->getAPIntValue())
    return SDValue();
 
  // Compute the shift amount we need to extract to complete the rotate.
  const unsigned VTWidth = ShiftedVT.getScalarSizeInBits();
  if (OppShiftCst->getAPIntValue().ugt(VTWidth))
    return SDValue();
  APInt NeededShiftAmt = VTWidth - OppShiftCst->getAPIntValue();
  // Normalize the bitwidth of the two mul/udiv/shift constant operands.
  APInt ExtractFromAmt = ExtractFromCst->getAPIntValue();
  APInt OppLHSAmt = OppLHSCst->getAPIntValue();
  zeroExtendToMatch(ExtractFromAmt, OppLHSAmt);
 
  // Now try extract the needed shift from the ExtractFrom op and see if the
  // result matches up with the existing shift's LHS op.
  if (IsMulOrDiv) {
    // Op to extract from is a mul or udiv by a constant.
    // Check:
    //     c2 / (1 << (bitwidth(op0 v c0) - c1)) == c0
    //     c2 % (1 << (bitwidth(op0 v c0) - c1)) == 0
    const APInt ExtractDiv = APInt::getOneBitSet(ExtractFromAmt.getBitWidth(),
                                                 NeededShiftAmt.getZExtValue());
    APInt ResultAmt;
    APInt Rem;
    APInt::udivrem(ExtractFromAmt, ExtractDiv, ResultAmt, Rem);
    if (Rem != 0 || ResultAmt != OppLHSAmt)
      return SDValue();
  } else {
    // Op to extract from is a shift by a constant.
    // Check:
    //      c2 - (bitwidth(op0 v c0) - c1) == c0
    if (OppLHSAmt != ExtractFromAmt - NeededShiftAmt.zextOrTrunc(
                                          ExtractFromAmt.getBitWidth()))
      return SDValue();
  }
 
  // Return the expanded shift op that should allow a rotate to be formed.
  EVT ShiftVT = OppShift.getOperand(1).getValueType();
  EVT ResVT = ExtractFrom.getValueType();
  SDValue NewShiftNode = DAG.getConstant(NeededShiftAmt, DL, ShiftVT);
  return DAG.getNode(Opcode, DL, ResVT, OppShiftLHS, NewShiftNode);
}
 
// Return true if we can prove that, whenever Neg and Pos are both in the
// range [0, EltSize), Neg == (Pos == 0 ? 0 : EltSize - Pos).  This means that
// for two opposing shifts shift1 and shift2 and a value X with OpBits bits:
//
//     (or (shift1 X, Neg), (shift2 X, Pos))
//
// reduces to a rotate in direction shift2 by Pos or (equivalently) a rotate
// in direction shift1 by Neg.  The range [0, EltSize) means that we only need
// to consider shift amounts with defined behavior.
//
// The IsRotate flag should be set when the LHS of both shifts is the same.
// Otherwise if matching a general funnel shift, it should be clear.
static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
                           SelectionDAG &DAG, bool IsRotate, bool FromAdd) {
  const auto &TLI = DAG.getTargetLoweringInfo();
  // If EltSize is a power of 2 then:
  //
  //  (a) (Pos == 0 ? 0 : EltSize - Pos) == (EltSize - Pos) & (EltSize - 1)
  //  (b) Neg == Neg & (EltSize - 1) whenever Neg is in [0, EltSize).
  //
  // So if EltSize is a power of 2 and Neg is (and Neg', EltSize-1), we check
  // for the stronger condition:
  //
  //     Neg & (EltSize - 1) == (EltSize - Pos) & (EltSize - 1)    [A]
  //
  // for all Neg and Pos.  Since Neg & (EltSize - 1) == Neg' & (EltSize - 1)
  // we can just replace Neg with Neg' for the rest of the function.
  //
  // In other cases we check for the even stronger condition:
  //
  //     Neg == EltSize - Pos                                    [B]
  //
  // for all Neg and Pos.  Note that the (or ...) then invokes undefined
  // behavior if Pos == 0 (and consequently Neg == EltSize).
  //
  // We could actually use [A] whenever EltSize is a power of 2, but the
  // only extra cases that it would match are those uninteresting ones
  // where Neg and Pos are never in range at the same time.  E.g. for
  // EltSize == 32, using [A] would allow a Neg of the form (sub 64, Pos)
  // as well as (sub 32, Pos), but:
  //
  //     (or (shift1 X, (sub 64, Pos)), (shift2 X, Pos))
  //
  // always invokes undefined behavior for 32-bit X.
  //
  // Below, Mask == EltSize - 1 when using [A] and is all-ones otherwise.
  // This allows us to peek through any operations that only affect Mask's
  // un-demanded bits.
  //
  // NOTE: We can only do this when matching operations which won't modify the
  // least Log2(EltSize) significant bits and not a general funnel shift.
  unsigned MaskLoBits = 0;
  if (IsRotate && !FromAdd && isPowerOf2_64(EltSize)) {
    unsigned Bits = Log2_64(EltSize);
    unsigned NegBits = Neg.getScalarValueSizeInBits();
    if (NegBits >= Bits) {
      APInt DemandedBits = APInt::getLowBitsSet(NegBits, Bits);
      if (SDValue Inner =
              TLI.SimplifyMultipleUseDemandedBits(Neg, DemandedBits, DAG)) {
        Neg = Inner;
        MaskLoBits = Bits;
      }
    }
  }
 
  // Check whether Neg has the form (sub NegC, NegOp1) for some NegC and NegOp1.
  if (Neg.getOpcode() != ISD::SUB)
    return false;
  ConstantSDNode *NegC = isConstOrConstSplat(Neg.getOperand(0));
  if (!NegC)
    return false;
  SDValue NegOp1 = Neg.getOperand(1);
 
  // On the RHS of [A], if Pos is the result of operation on Pos' that won't
  // affect Mask's demanded bits, just replace Pos with Pos'. These operations
  // are redundant for the purpose of the equality.
  if (MaskLoBits) {
    unsigned PosBits = Pos.getScalarValueSizeInBits();
    if (PosBits >= MaskLoBits) {
      APInt DemandedBits = APInt::getLowBitsSet(PosBits, MaskLoBits);
      if (SDValue Inner =
              TLI.SimplifyMultipleUseDemandedBits(Pos, DemandedBits, DAG)) {
        Pos = Inner;
      }
    }
  }
 
  // The condition we need is now:
  //
  //     (NegC - NegOp1) & Mask == (EltSize - Pos) & Mask
  //
  // If NegOp1 == Pos then we need:
  //
  //              EltSize & Mask == NegC & Mask
  //
  // (because "x & Mask" is a truncation and distributes through subtraction).
  //
  // We also need to account for a potential truncation of NegOp1 if the amount
  // has already been legalized to a shift amount type.
  APInt Width;
  if ((Pos == NegOp1) ||
      (NegOp1.getOpcode() == ISD::TRUNCATE && Pos == NegOp1.getOperand(0)))
    Width = NegC->getAPIntValue();
 
  // Check for cases where Pos has the form (add NegOp1, PosC) for some PosC.
  // Then the condition we want to prove becomes:
  //
  //     (NegC - NegOp1) & Mask == (EltSize - (NegOp1 + PosC)) & Mask
  //
  // which, again because "x & Mask" is a truncation, becomes:
  //
  //                NegC & Mask == (EltSize - PosC) & Mask
  //             EltSize & Mask == (NegC + PosC) & Mask
  else if (Pos.getOpcode() == ISD::ADD && Pos.getOperand(0) == NegOp1) {
    if (ConstantSDNode *PosC = isConstOrConstSplat(Pos.getOperand(1)))
      Width = PosC->getAPIntValue() + NegC->getAPIntValue();
    else
      return false;
  } else
    return false;
 
  // Now we just need to check that EltSize & Mask == Width & Mask.
  if (MaskLoBits)
    // EltSize & Mask is 0 since Mask is EltSize - 1.
    return Width.getLoBits(MaskLoBits) == 0;
  return Width == EltSize;
}
 
// A subroutine of MatchRotate used once we have found an OR of two opposite
// shifts of Shifted.  If Neg == <operand size> - Pos then the OR reduces
// to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
// former being preferred if supported.  InnerPos and InnerNeg are Pos and
// Neg with outer conversions stripped away.
SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
                                       SDValue Neg, SDValue InnerPos,
                                       SDValue InnerNeg, bool FromAdd,
                                       bool HasPos, unsigned PosOpcode,
                                       unsigned NegOpcode, const SDLoc &DL) {
  // fold (or/add (shl x, (*ext y)),
  //              (srl x, (*ext (sub 32, y)))) ->
  //   (rotl x, y) or (rotr x, (sub 32, y))
  //
  // fold (or/add (shl x, (*ext (sub 32, y))),
  //              (srl x, (*ext y))) ->
  //   (rotr x, y) or (rotl x, (sub 32, y))
  EVT VT = Shifted.getValueType();
  if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG,
                     /*IsRotate*/ true, FromAdd))
    return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
                       HasPos ? Pos : Neg);
 
  return SDValue();
}
 
// A subroutine of MatchRotate used once we have found an OR of two opposite
// shifts of N0 + N1.  If Neg == <operand size> - Pos then the OR reduces
// to both (PosOpcode N0, N1, Pos) and (NegOpcode N0, N1, Neg), with the
// former being preferred if supported.  InnerPos and InnerNeg are Pos and
// Neg with outer conversions stripped away.
// TODO: Merge with MatchRotatePosNeg.
SDValue DAGCombiner::MatchFunnelPosNeg(SDValue N0, SDValue N1, SDValue Pos,
                                       SDValue Neg, SDValue InnerPos,
                                       SDValue InnerNeg, bool FromAdd,
                                       bool HasPos, unsigned PosOpcode,
                                       unsigned NegOpcode, const SDLoc &DL) {
  EVT VT = N0.getValueType();
  unsigned EltBits = VT.getScalarSizeInBits();
 
  // fold (or/add (shl x0, (*ext y)),
  //              (srl x1, (*ext (sub 32, y)))) ->
  //   (fshl x0, x1, y) or (fshr x0, x1, (sub 32, y))
  //
  // fold (or/add (shl x0, (*ext (sub 32, y))),
  //              (srl x1, (*ext y))) ->
  //   (fshr x0, x1, y) or (fshl x0, x1, (sub 32, y))
  if (matchRotateSub(InnerPos, InnerNeg, EltBits, DAG, /*IsRotate*/ N0 == N1,
                     FromAdd))
    return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, N0, N1,
                       HasPos ? Pos : Neg);
 
  // Matching the shift+xor cases, we can't easily use the xor'd shift amount
  // so for now just use the PosOpcode case if its legal.
  // TODO: When can we use the NegOpcode case?
  if (PosOpcode == ISD::FSHL && isPowerOf2_32(EltBits)) {
    SDValue X;
    // fold (or/add (shl x0, y), (srl (srl x1, 1), (xor y, 31)))
    //   -> (fshl x0, x1, y)
    if (sd_match(N1, m_Srl(m_Value(X), m_One())) &&
        sd_match(InnerNeg,
                 m_Xor(m_Specific(InnerPos), m_SpecificInt(EltBits - 1))) &&
        TLI.isOperationLegalOrCustom(ISD::FSHL, VT)) {
      return DAG.getNode(ISD::FSHL, DL, VT, N0, X, Pos);
    }
 
    // fold (or/add (shl (shl x0, 1), (xor y, 31)), (srl x1, y))
    //   -> (fshr x0, x1, y)
    if (sd_match(N0, m_Shl(m_Value(X), m_One())) &&
        sd_match(InnerPos,
                 m_Xor(m_Specific(InnerNeg), m_SpecificInt(EltBits - 1))) &&
        TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) {
      return DAG.getNode(ISD::FSHR, DL, VT, X, N1, Neg);
    }
 
    // fold (or/add (shl (add x0, x0), (xor y, 31)), (srl x1, y))
    //   -> (fshr x0, x1, y)
    // TODO: Should add(x,x) -> shl(x,1) be a general DAG canonicalization?
    if (sd_match(N0, m_Add(m_Value(X), m_Deferred(X))) &&
        sd_match(InnerPos,
                 m_Xor(m_Specific(InnerNeg), m_SpecificInt(EltBits - 1))) &&
        TLI.isOperationLegalOrCustom(ISD::FSHR, VT)) {
      return DAG.getNode(ISD::FSHR, DL, VT, X, N1, Neg);
    }
  }
 
  return SDValue();
}
 
// MatchRotate - Handle an 'or' or 'add' of two operands.  If this is one of the
// many idioms for rotate, and if the target supports rotation instructions,
// generate a rot[lr]. This also matches funnel shift patterns, similar to
// rotation but with different shifted sources.
SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL,
                                 bool FromAdd) {
  EVT VT = LHS.getValueType();
 
  // The target must have at least one rotate/funnel flavor.
  // We still try to match rotate by constant pre-legalization.
  // TODO: Support pre-legalization funnel-shift by constant.
  bool HasROTL = hasOperation(ISD::ROTL, VT);
  bool HasROTR = hasOperation(ISD::ROTR, VT);
  bool HasFSHL = hasOperation(ISD::FSHL, VT);
  bool HasFSHR = hasOperation(ISD::FSHR, VT);
 
  // If the type is going to be promoted and the target has enabled custom
  // lowering for rotate, allow matching rotate by non-constants. Only allow
  // this for scalar types.
  if (VT.isScalarInteger() && TLI.getTypeAction(*DAG.getContext(), VT) ==
                                  TargetLowering::TypePromoteInteger) {
    HasROTL |= TLI.getOperationAction(ISD::ROTL, VT) == TargetLowering::Custom;
    HasROTR |= TLI.getOperationAction(ISD::ROTR, VT) == TargetLowering::Custom;
  }
 
  if (LegalOperations && !HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
    return SDValue();
 
  // Check for truncated rotate.
  if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
      LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
    assert(LHS.getValueType() == RHS.getValueType());
    if (SDValue Rot =
            MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL, FromAdd))
      return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot);
  }
 
  // Match "(X shl/srl V1) & V2" where V2 may not be present.
  SDValue LHSShift;   // The shift.
  SDValue LHSMask;    // AND value if any.
  matchRotateHalf(DAG, LHS, LHSShift, LHSMask);
 
  SDValue RHSShift;   // The shift.
  SDValue RHSMask;    // AND value if any.
  matchRotateHalf(DAG, RHS, RHSShift, RHSMask);
 
  // If neither side matched a rotate half, bail
  if (!LHSShift && !RHSShift)
    return SDValue();
 
  // InstCombine may have combined a constant shl, srl, mul, or udiv with one
  // side of the rotate, so try to handle that here. In all cases we need to
  // pass the matched shift from the opposite side to compute the opcode and
  // needed shift amount to extract.  We still want to do this if both sides
  // matched a rotate half because one half may be a potential overshift that
  // can be broken down (ie if InstCombine merged two shl or srl ops into a
  // single one).
 
  // Have LHS side of the rotate, try to extract the needed shift from the RHS.
  if (LHSShift)
    if (SDValue NewRHSShift =
            extractShiftForRotate(DAG, LHSShift, RHS, RHSMask, DL))
      RHSShift = NewRHSShift;
  // Have RHS side of the rotate, try to extract the needed shift from the LHS.
  if (RHSShift)
    if (SDValue NewLHSShift =
            extractShiftForRotate(DAG, RHSShift, LHS, LHSMask, DL))
      LHSShift = NewLHSShift;
 
  // If a side is still missing, nothing else we can do.
  if (!RHSShift || !LHSShift)
    return SDValue();
 
  // At this point we've matched or extracted a shift op on each side.
 
  if (LHSShift.getOpcode() == RHSShift.getOpcode())
    return SDValue(); // Shifts must disagree.
 
  // Canonicalize shl to left side in a shl/srl pair.
  if (RHSShift.getOpcode() == ISD::SHL) {
    std::swap(LHS, RHS);
    std::swap(LHSShift, RHSShift);
    std::swap(LHSMask, RHSMask);
  }
 
  // Something has gone wrong - we've lost the shl/srl pair - bail.
  if (LHSShift.getOpcode() != ISD::SHL || RHSShift.getOpcode() != ISD::SRL)
    return SDValue();
 
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
  SDValue LHSShiftArg = LHSShift.getOperand(0);
  SDValue LHSShiftAmt = LHSShift.getOperand(1);
  SDValue RHSShiftArg = RHSShift.getOperand(0);
  SDValue RHSShiftAmt = RHSShift.getOperand(1);
 
  auto MatchRotateSum = [EltSizeInBits](ConstantSDNode *LHS,
                                        ConstantSDNode *RHS) {
    return (LHS->getAPIntValue() + RHS->getAPIntValue()) == EltSizeInBits;
  };
 
  auto ApplyMasks = [&](SDValue Res) {
    // If there is an AND of either shifted operand, apply it to the result.
    if (LHSMask.getNode() || RHSMask.getNode()) {
      SDValue AllOnes = DAG.getAllOnesConstant(DL, VT);
      SDValue Mask = AllOnes;
 
      if (LHSMask.getNode()) {
        SDValue RHSBits = DAG.getNode(ISD::SRL, DL, VT, AllOnes, RHSShiftAmt);
        Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
                           DAG.getNode(ISD::OR, DL, VT, LHSMask, RHSBits));
      }
      if (RHSMask.getNode()) {
        SDValue LHSBits = DAG.getNode(ISD::SHL, DL, VT, AllOnes, LHSShiftAmt);
        Mask = DAG.getNode(ISD::AND, DL, VT, Mask,
                           DAG.getNode(ISD::OR, DL, VT, RHSMask, LHSBits));
      }
 
      Res = DAG.getNode(ISD::AND, DL, VT, Res, Mask);
    }
 
    return Res;
  };
 
  // TODO: Support pre-legalization funnel-shift by constant.
  bool IsRotate = LHSShiftArg == RHSShiftArg;
  if (!IsRotate && !(HasFSHL || HasFSHR)) {
    if (TLI.isTypeLegal(VT) && LHS.hasOneUse() && RHS.hasOneUse() &&
        ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
      // Look for a disguised rotate by constant.
      // The common shifted operand X may be hidden inside another 'or'.
      SDValue X, Y;
      auto matchOr = [&X, &Y](SDValue Or, SDValue CommonOp) {
        if (!Or.hasOneUse() || Or.getOpcode() != ISD::OR)
          return false;
        if (CommonOp == Or.getOperand(0)) {
          X = CommonOp;
          Y = Or.getOperand(1);
          return true;
        }
        if (CommonOp == Or.getOperand(1)) {
          X = CommonOp;
          Y = Or.getOperand(0);
          return true;
        }
        return false;
      };
 
      SDValue Res;
      if (matchOr(LHSShiftArg, RHSShiftArg)) {
        // (shl (X | Y), C1) | (srl X, C2) --> (rotl X, C1) | (shl Y, C1)
        SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt);
        SDValue ShlY = DAG.getNode(ISD::SHL, DL, VT, Y, LHSShiftAmt);
        Res = DAG.getNode(ISD::OR, DL, VT, RotX, ShlY);
      } else if (matchOr(RHSShiftArg, LHSShiftArg)) {
        // (shl X, C1) | (srl (X | Y), C2) --> (rotl X, C1) | (srl Y, C2)
        SDValue RotX = DAG.getNode(ISD::ROTL, DL, VT, X, LHSShiftAmt);
        SDValue SrlY = DAG.getNode(ISD::SRL, DL, VT, Y, RHSShiftAmt);
        Res = DAG.getNode(ISD::OR, DL, VT, RotX, SrlY);
      } else {
        return SDValue();
      }
 
      return ApplyMasks(Res);
    }
 
    return SDValue(); // Requires funnel shift support.
  }
 
  // fold (or/add (shl x, C1), (srl x, C2)) -> (rotl x, C1)
  // fold (or/add (shl x, C1), (srl x, C2)) -> (rotr x, C2)
  // fold (or/add (shl x, C1), (srl y, C2)) -> (fshl x, y, C1)
  // fold (or/add (shl x, C1), (srl y, C2)) -> (fshr x, y, C2)
  // iff C1+C2 == EltSizeInBits
  if (ISD::matchBinaryPredicate(LHSShiftAmt, RHSShiftAmt, MatchRotateSum)) {
    SDValue Res;
    if (IsRotate && (HasROTL || HasROTR || !(HasFSHL || HasFSHR))) {
      bool UseROTL = !LegalOperations || HasROTL;
      Res = DAG.getNode(UseROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg,
                        UseROTL ? LHSShiftAmt : RHSShiftAmt);
    } else {
      bool UseFSHL = !LegalOperations || HasFSHL;
      Res = DAG.getNode(UseFSHL ? ISD::FSHL : ISD::FSHR, DL, VT, LHSShiftArg,
                        RHSShiftArg, UseFSHL ? LHSShiftAmt : RHSShiftAmt);
    }
 
    return ApplyMasks(Res);
  }
 
  // Even pre-legalization, we can't easily rotate/funnel-shift by a variable
  // shift.
  if (!HasROTL && !HasROTR && !HasFSHL && !HasFSHR)
    return SDValue();
 
  // If there is a mask here, and we have a variable shift, we can't be sure
  // that we're masking out the right stuff.
  if (LHSMask.getNode() || RHSMask.getNode())
    return SDValue();
 
  // If the shift amount is sign/zext/any-extended just peel it off.
  SDValue LExtOp0 = LHSShiftAmt;
  SDValue RExtOp0 = RHSShiftAmt;
  if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
       LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
       LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
       LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
      (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
       RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
       RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
       RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
    LExtOp0 = LHSShiftAmt.getOperand(0);
    RExtOp0 = RHSShiftAmt.getOperand(0);
  }
 
  if (IsRotate && (HasROTL || HasROTR)) {
    if (SDValue TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
                                         LExtOp0, RExtOp0, FromAdd, HasROTL,
                                         ISD::ROTL, ISD::ROTR, DL))
      return TryL;
 
    if (SDValue TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
                                         RExtOp0, LExtOp0, FromAdd, HasROTR,
                                         ISD::ROTR, ISD::ROTL, DL))
      return TryR;
  }
 
  if (SDValue TryL = MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, LHSShiftAmt,
                                       RHSShiftAmt, LExtOp0, RExtOp0, FromAdd,
                                       HasFSHL, ISD::FSHL, ISD::FSHR, DL))
    return TryL;
 
  if (SDValue TryR = MatchFunnelPosNeg(LHSShiftArg, RHSShiftArg, RHSShiftAmt,
                                       LHSShiftAmt, RExtOp0, LExtOp0, FromAdd,
                                       HasFSHR, ISD::FSHR, ISD::FSHL, DL))
    return TryR;
 
  return SDValue();
}
 
/// Recursively traverses the expression calculating the origin of the requested
/// byte of the given value. Returns std::nullopt if the provider can't be
/// calculated.
///
/// For all the values except the root of the expression, we verify that the
/// value has exactly one use and if not then return std::nullopt. This way if
/// the origin of the byte is returned it's guaranteed that the values which
/// contribute to the byte are not used outside of this expression.
 
/// However, there is a special case when dealing with vector loads -- we allow
/// more than one use if the load is a vector type.  Since the values that
/// contribute to the byte ultimately come from the ExtractVectorElements of the
/// Load, we don't care if the Load has uses other than ExtractVectorElements,
/// because those operations are independent from the pattern to be combined.
/// For vector loads, we simply care that the ByteProviders are adjacent
/// positions of the same vector, and their index matches the byte that is being
/// provided. This is captured by the \p VectorIndex algorithm. \p VectorIndex
/// is the index used in an ExtractVectorElement, and \p StartingIndex is the
/// byte position we are trying to provide for the LoadCombine. If these do
/// not match, then we can not combine the vector loads. \p Index uses the
/// byte position we are trying to provide for and is matched against the
/// shl and load size. The \p Index algorithm ensures the requested byte is
/// provided for by the pattern, and the pattern does not over provide bytes.
///
///
/// The supported LoadCombine pattern for vector loads is as follows
///                              or
///                          /        \
///                         or        shl
///                       /     \      |
///                     or      shl   zext
///                   /    \     |     |
///                 shl   zext  zext  EVE*
///                  |     |     |     |
///                 zext  EVE*  EVE*  LOAD
///                  |     |     |
///                 EVE*  LOAD  LOAD
///                  |
///                 LOAD
///
/// *ExtractVectorElement
using SDByteProvider = ByteProvider<SDNode *>;
 
static std::optional<SDByteProvider>
calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
                      std::optional<uint64_t> VectorIndex,
                      unsigned StartingIndex = 0) {
 
  // Typical i64 by i8 pattern requires recursion up to 8 calls depth
  if (Depth == 10)
    return std::nullopt;
 
  // Only allow multiple uses if the instruction is a vector load (in which
  // case we will use the load for every ExtractVectorElement)
  if (Depth && !Op.hasOneUse() &&
      (Op.getOpcode() != ISD::LOAD || !Op.getValueType().isVector()))
    return std::nullopt;
 
  // Fail to combine if we have encountered anything but a LOAD after handling
  // an ExtractVectorElement.
  if (Op.getOpcode() != ISD::LOAD && VectorIndex.has_value())
    return std::nullopt;
 
  unsigned BitWidth = Op.getScalarValueSizeInBits();
  if (BitWidth % 8 != 0)
    return std::nullopt;
  unsigned ByteWidth = BitWidth / 8;
  assert(Index < ByteWidth && "invalid index requested");
  (void) ByteWidth;
 
  switch (Op.getOpcode()) {
  case ISD::OR: {
    auto LHS =
        calculateByteProvider(Op->getOperand(0), Index, Depth + 1, VectorIndex);
    if (!LHS)
      return std::nullopt;
    auto RHS =
        calculateByteProvider(Op->getOperand(1), Index, Depth + 1, VectorIndex);
    if (!RHS)
      return std::nullopt;
 
    if (LHS->isConstantZero())
      return RHS;
    if (RHS->isConstantZero())
      return LHS;
    return std::nullopt;
  }
  case ISD::SHL: {
    auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
    if (!ShiftOp)
      return std::nullopt;
 
    uint64_t BitShift = ShiftOp->getZExtValue();
 
    if (BitShift % 8 != 0)
      return std::nullopt;
    uint64_t ByteShift = BitShift / 8;
 
    // If we are shifting by an amount greater than the index we are trying to
    // provide, then do not provide anything. Otherwise, subtract the index by
    // the amount we shifted by.
    return Index < ByteShift
               ? SDByteProvider::getConstantZero()
               : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
                                       Depth + 1, VectorIndex, Index);
  }
  case ISD::ANY_EXTEND:
  case ISD::SIGN_EXTEND:
  case ISD::ZERO_EXTEND: {
    SDValue NarrowOp = Op->getOperand(0);
    unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
    if (NarrowBitWidth % 8 != 0)
      return std::nullopt;
    uint64_t NarrowByteWidth = NarrowBitWidth / 8;
 
    if (Index >= NarrowByteWidth)
      return Op.getOpcode() == ISD::ZERO_EXTEND
                 ? std::optional<SDByteProvider>(
                       SDByteProvider::getConstantZero())
                 : std::nullopt;
    return calculateByteProvider(NarrowOp, Index, Depth + 1, VectorIndex,
                                 StartingIndex);
  }
  case ISD::BSWAP:
    return calculateByteProvider(Op->getOperand(0), ByteWidth - Index - 1,
                                 Depth + 1, VectorIndex, StartingIndex);
  case ISD::EXTRACT_VECTOR_ELT: {
    auto OffsetOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
    if (!OffsetOp)
      return std::nullopt;
 
    VectorIndex = OffsetOp->getZExtValue();
 
    SDValue NarrowOp = Op->getOperand(0);
    unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
    if (NarrowBitWidth % 8 != 0)
      return std::nullopt;
    uint64_t NarrowByteWidth = NarrowBitWidth / 8;
    // EXTRACT_VECTOR_ELT can extend the element type to the width of the return
    // type, leaving the high bits undefined.
    if (Index >= NarrowByteWidth)
      return std::nullopt;
 
    // Check to see if the position of the element in the vector corresponds
    // with the byte we are trying to provide for. In the case of a vector of
    // i8, this simply means the VectorIndex == StartingIndex. For non i8 cases,
    // the element will provide a range of bytes. For example, if we have a
    // vector of i16s, each element provides two bytes (V[1] provides byte 2 and
    // 3).
    if (*VectorIndex * NarrowByteWidth > StartingIndex)
      return std::nullopt;
    if ((*VectorIndex + 1) * NarrowByteWidth <= StartingIndex)
      return std::nullopt;
 
    return calculateByteProvider(Op->getOperand(0), Index, Depth + 1,
                                 VectorIndex, StartingIndex);
  }
  case ISD::LOAD: {
    auto L = cast<LoadSDNode>(Op.getNode());
    if (!L->isSimple() || L->isIndexed())
      return std::nullopt;
 
    unsigned NarrowBitWidth = L->getMemoryVT().getScalarSizeInBits();
    if (NarrowBitWidth % 8 != 0)
      return std::nullopt;
    uint64_t NarrowByteWidth = NarrowBitWidth / 8;
 
    // If the width of the load does not reach byte we are trying to provide for
    // and it is not a ZEXTLOAD, then the load does not provide for the byte in
    // question
    if (Index >= NarrowByteWidth)
      return L->getExtensionType() == ISD::ZEXTLOAD
                 ? std::optional<SDByteProvider>(
                       SDByteProvider::getConstantZero())
                 : std::nullopt;
 
    unsigned BPVectorIndex = VectorIndex.value_or(0U);
    return SDByteProvider::getSrc(L, Index, BPVectorIndex);
  }
  }
 
  return std::nullopt;
}
 
static unsigned littleEndianByteAt(unsigned BW, unsigned i) {
  return i;
}
 
static unsigned bigEndianByteAt(unsigned BW, unsigned i) {
  return BW - i - 1;
}
 
// Check if the bytes offsets we are looking at match with either big or
// little endian value loaded. Return true for big endian, false for little
// endian, and std::nullopt if match failed.
static std::optional<bool> isBigEndian(ArrayRef<int64_t> ByteOffsets,
                                       int64_t FirstOffset) {
  // The endian can be decided only when it is 2 bytes at least.
  unsigned Width = ByteOffsets.size();
  if (Width < 2)
    return std::nullopt;
 
  bool BigEndian = true, LittleEndian = true;
  for (unsigned i = 0; i < Width; i++) {
    int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset;
    LittleEndian &= CurrentByteOffset == littleEndianByteAt(Width, i);
    BigEndian &= CurrentByteOffset == bigEndianByteAt(Width, i);
    if (!BigEndian && !LittleEndian)
      return std::nullopt;
  }
 
  assert((BigEndian != LittleEndian) && "It should be either big endian or"
                                        "little endian");
  return BigEndian;
}
 
// Look through one layer of truncate or extend.
static SDValue stripTruncAndExt(SDValue Value) {
  switch (Value.getOpcode()) {
  case ISD::TRUNCATE:
  case ISD::ZERO_EXTEND:
  case ISD::SIGN_EXTEND:
  case ISD::ANY_EXTEND:
    return Value.getOperand(0);
  }
  return SDValue();
}
 
/// Match a pattern where a wide type scalar value is stored by several narrow
/// stores. Fold it into a single store or a BSWAP and a store if the targets
/// supports it.
///
/// Assuming little endian target:
///  i8 *p = ...
///  i32 val = ...
///  p[0] = (val >> 0) & 0xFF;
///  p[1] = (val >> 8) & 0xFF;
///  p[2] = (val >> 16) & 0xFF;
///  p[3] = (val >> 24) & 0xFF;
/// =>
///  *((i32)p) = val;
///
///  i8 *p = ...
///  i32 val = ...
///  p[0] = (val >> 24) & 0xFF;
///  p[1] = (val >> 16) & 0xFF;
///  p[2] = (val >> 8) & 0xFF;
///  p[3] = (val >> 0) & 0xFF;
/// =>
///  *((i32)p) = BSWAP(val);
SDValue DAGCombiner::mergeTruncStores(StoreSDNode *N) {
  // The matching looks for "store (trunc x)" patterns that appear early but are
  // likely to be replaced by truncating store nodes during combining.
  // TODO: If there is evidence that running this later would help, this
  //       limitation could be removed. Legality checks may need to be added
  //       for the created store and optional bswap/rotate.
  if (LegalOperations || OptLevel == CodeGenOptLevel::None)
    return SDValue();
 
  // We only handle merging simple stores of 1-4 bytes.
  // TODO: Allow unordered atomics when wider type is legal (see D66309)
  EVT MemVT = N->getMemoryVT();
  if (!(MemVT == MVT::i8 || MemVT == MVT::i16 || MemVT == MVT::i32) ||
      !N->isSimple() || N->isIndexed())
    return SDValue();
 
  // Collect all of the stores in the chain, upto the maximum store width (i64).
  SDValue Chain = N->getChain();
  SmallVector<StoreSDNode *, 8> Stores = {N};
  unsigned NarrowNumBits = MemVT.getScalarSizeInBits();
  unsigned MaxWideNumBits = 64;
  unsigned MaxStores = MaxWideNumBits / NarrowNumBits;
  while (auto *Store = dyn_cast<StoreSDNode>(Chain)) {
    // All stores must be the same size to ensure that we are writing all of the
    // bytes in the wide value.
    // This store should have exactly one use as a chain operand for another
    // store in the merging set. If there are other chain uses, then the
    // transform may not be safe because order of loads/stores outside of this
    // set may not be preserved.
    // TODO: We could allow multiple sizes by tracking each stored byte.
    if (Store->getMemoryVT() != MemVT || !Store->isSimple() ||
        Store->isIndexed() || !Store->hasOneUse())
      return SDValue();
    Stores.push_back(Store);
    Chain = Store->getChain();
    if (MaxStores < Stores.size())
      return SDValue();
  }
  // There is no reason to continue if we do not have at least a pair of stores.
  if (Stores.size() < 2)
    return SDValue();
 
  // Handle simple types only.
  LLVMContext &Context = *DAG.getContext();
  unsigned NumStores = Stores.size();
  unsigned WideNumBits = NumStores * NarrowNumBits;
  if (WideNumBits != 16 && WideNumBits != 32 && WideNumBits != 64)
    return SDValue();
 
  // Check if all bytes of the source value that we are looking at are stored
  // to the same base address. Collect offsets from Base address into OffsetMap.
  SDValue SourceValue;
  SmallVector<int64_t, 8> OffsetMap(NumStores, INT64_MAX);
  int64_t FirstOffset = INT64_MAX;
  StoreSDNode *FirstStore = nullptr;
  std::optional<BaseIndexOffset> Base;
  for (auto *Store : Stores) {
    // All the stores store different parts of the CombinedValue. A truncate is
    // required to get the partial value.
    SDValue Trunc = Store->getValue();
    if (Trunc.getOpcode() != ISD::TRUNCATE)
      return SDValue();
    // Other than the first/last part, a shift operation is required to get the
    // offset.
    int64_t Offset = 0;
    SDValue WideVal = Trunc.getOperand(0);
    if ((WideVal.getOpcode() == ISD::SRL || WideVal.getOpcode() == ISD::SRA) &&
        isa<ConstantSDNode>(WideVal.getOperand(1))) {
      // The shift amount must be a constant multiple of the narrow type.
      // It is translated to the offset address in the wide source value "y".
      //
      // x = srl y, ShiftAmtC
      // i8 z = trunc x
      // store z, ...
      uint64_t ShiftAmtC = WideVal.getConstantOperandVal(1);
      if (ShiftAmtC % NarrowNumBits != 0)
        return SDValue();
 
      // Make sure we aren't reading bits that are shifted in.
      if (ShiftAmtC > WideVal.getScalarValueSizeInBits() - NarrowNumBits)
        return SDValue();
 
      Offset = ShiftAmtC / NarrowNumBits;
      WideVal = WideVal.getOperand(0);
    }
 
    // Stores must share the same source value with different offsets.
    if (!SourceValue)
      SourceValue = WideVal;
    else if (SourceValue != WideVal) {
      // Truncate and extends can be stripped to see if the values are related.
      if (stripTruncAndExt(SourceValue) != WideVal &&
          stripTruncAndExt(WideVal) != SourceValue)
        return SDValue();
 
      if (WideVal.getScalarValueSizeInBits() >
          SourceValue.getScalarValueSizeInBits())
        SourceValue = WideVal;
 
      // Give up if the source value type is smaller than the store size.
      if (SourceValue.getScalarValueSizeInBits() < WideNumBits)
        return SDValue();
    }
 
    // Stores must share the same base address.
    BaseIndexOffset Ptr = BaseIndexOffset::match(Store, DAG);
    int64_t ByteOffsetFromBase = 0;
    if (!Base)
      Base = Ptr;
    else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
      return SDValue();
 
    // Remember the first store.
    if (ByteOffsetFromBase < FirstOffset) {
      FirstStore = Store;
      FirstOffset = ByteOffsetFromBase;
    }
    // Map the offset in the store and the offset in the combined value, and
    // early return if it has been set before.
    if (Offset < 0 || Offset >= NumStores || OffsetMap[Offset] != INT64_MAX)
      return SDValue();
    OffsetMap[Offset] = ByteOffsetFromBase;
  }
 
  EVT WideVT = EVT::getIntegerVT(Context, WideNumBits);
 
  assert(FirstOffset != INT64_MAX && "First byte offset must be set");
  assert(FirstStore && "First store must be set");
 
  // Check that a store of the wide type is both allowed and fast on the target
  const DataLayout &Layout = DAG.getDataLayout();
  unsigned Fast = 0;
  bool Allowed = TLI.allowsMemoryAccess(Context, Layout, WideVT,
                                        *FirstStore->getMemOperand(), &Fast);
  if (!Allowed || !Fast)
    return SDValue();
 
  // Check if the pieces of the value are going to the expected places in memory
  // to merge the stores.
  auto checkOffsets = [&](bool MatchLittleEndian) {
    if (MatchLittleEndian) {
      for (unsigned i = 0; i != NumStores; ++i)
        if (OffsetMap[i] != i * (NarrowNumBits / 8) + FirstOffset)
          return false;
    } else { // MatchBigEndian by reversing loop counter.
      for (unsigned i = 0, j = NumStores - 1; i != NumStores; ++i, --j)
        if (OffsetMap[j] != i * (NarrowNumBits / 8) + FirstOffset)
          return false;
    }
    return true;
  };
 
  // Check if the offsets line up for the native data layout of this target.
  bool NeedBswap = false;
  bool NeedRotate = false;
  if (!checkOffsets(Layout.isLittleEndian())) {
    // Special-case: check if byte offsets line up for the opposite endian.
    if (NarrowNumBits == 8 && checkOffsets(Layout.isBigEndian()))
      NeedBswap = true;
    else if (NumStores == 2 && checkOffsets(Layout.isBigEndian()))
      NeedRotate = true;
    else
      return SDValue();
  }
 
  SDLoc DL(N);
  if (WideVT != SourceValue.getValueType()) {
    assert(SourceValue.getValueType().getScalarSizeInBits() > WideNumBits &&
           "Unexpected store value to merge");
    SourceValue = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SourceValue);
  }
 
  // Before legalize we can introduce illegal bswaps/rotates which will be later
  // converted to an explicit bswap sequence. This way we end up with a single
  // store and byte shuffling instead of several stores and byte shuffling.
  if (NeedBswap) {
    SourceValue = DAG.getNode(ISD::BSWAP, DL, WideVT, SourceValue);
  } else if (NeedRotate) {
    assert(WideNumBits % 2 == 0 && "Unexpected type for rotate");
    SDValue RotAmt = DAG.getConstant(WideNumBits / 2, DL, WideVT);
    SourceValue = DAG.getNode(ISD::ROTR, DL, WideVT, SourceValue, RotAmt);
  }
 
  SDValue NewStore =
      DAG.getStore(Chain, DL, SourceValue, FirstStore->getBasePtr(),
                   FirstStore->getPointerInfo(), FirstStore->getAlign());
 
  // Rely on other DAG combine rules to remove the other individual stores.
  DAG.ReplaceAllUsesWith(N, NewStore.getNode());
  return NewStore;
}
 
/// Match a pattern where a wide type scalar value is loaded by several narrow
/// loads and combined by shifts and ors. Fold it into a single load or a load
/// and a BSWAP if the targets supports it.
///
/// Assuming little endian target:
///  i8 *a = ...
///  i32 val = a[0] | (a[1] << 8) | (a[2] << 16) | (a[3] << 24)
/// =>
///  i32 val = *((i32)a)
///
///  i8 *a = ...
///  i32 val = (a[0] << 24) | (a[1] << 16) | (a[2] << 8) | a[3]
/// =>
///  i32 val = BSWAP(*((i32)a))
///
/// TODO: This rule matches complex patterns with OR node roots and doesn't
/// interact well with the worklist mechanism. When a part of the pattern is
/// updated (e.g. one of the loads) its direct users are put into the worklist,
/// but the root node of the pattern which triggers the load combine is not
/// necessarily a direct user of the changed node. For example, once the address
/// of t28 load is reassociated load combine won't be triggered:
///             t25: i32 = add t4, Constant:i32<2>
///           t26: i64 = sign_extend t25
///        t27: i64 = add t2, t26
///       t28: i8,ch = load<LD1[%tmp9]> t0, t27, undef:i64
///     t29: i32 = zero_extend t28
///   t32: i32 = shl t29, Constant:i8<8>
/// t33: i32 = or t23, t32
/// As a possible fix visitLoad can check if the load can be a part of a load
/// combine pattern and add corresponding OR roots to the worklist.
SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
  assert(N->getOpcode() == ISD::OR &&
         "Can only match load combining against OR nodes");
 
  // Handles simple types only
  EVT VT = N->getValueType(0);
  if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64)
    return SDValue();
  unsigned ByteWidth = VT.getSizeInBits() / 8;
 
  bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
  auto MemoryByteOffset = [&](SDByteProvider P) {
    assert(P.hasSrc() && "Must be a memory byte provider");
    auto *Load = cast<LoadSDNode>(P.Src.value());
 
    unsigned LoadBitWidth = Load->getMemoryVT().getScalarSizeInBits();
 
    assert(LoadBitWidth % 8 == 0 &&
           "can only analyze providers for individual bytes not bit");
    unsigned LoadByteWidth = LoadBitWidth / 8;
    return IsBigEndianTarget ? bigEndianByteAt(LoadByteWidth, P.DestOffset)
                             : littleEndianByteAt(LoadByteWidth, P.DestOffset);
  };
 
  std::optional<BaseIndexOffset> Base;
  SDValue Chain;
 
  SmallPtrSet<LoadSDNode *, 8> Loads;
  std::optional<SDByteProvider> FirstByteProvider;
  int64_t FirstOffset = INT64_MAX;
 
  // Check if all the bytes of the OR we are looking at are loaded from the same
  // base address. Collect bytes offsets from Base address in ByteOffsets.
  SmallVector<int64_t, 8> ByteOffsets(ByteWidth);
  unsigned ZeroExtendedBytes = 0;
  for (int i = ByteWidth - 1; i >= 0; --i) {
    auto P =
        calculateByteProvider(SDValue(N, 0), i, 0, /*VectorIndex*/ std::nullopt,
                              /*StartingIndex*/ i);
    if (!P)
      return SDValue();
 
    if (P->isConstantZero()) {
      // It's OK for the N most significant bytes to be 0, we can just
      // zero-extend the load.
      if (++ZeroExtendedBytes != (ByteWidth - static_cast<unsigned>(i)))
        return SDValue();
      continue;
    }
    assert(P->hasSrc() && "provenance should either be memory or zero");
    auto *L = cast<LoadSDNode>(P->Src.value());
 
    // All loads must share the same chain
    SDValue LChain = L->getChain();
    if (!Chain)
      Chain = LChain;
    else if (Chain != LChain)
      return SDValue();
 
    // Loads must share the same base address
    BaseIndexOffset Ptr = BaseIndexOffset::match(L, DAG);
    int64_t ByteOffsetFromBase = 0;
 
    // For vector loads, the expected load combine pattern will have an
    // ExtractElement for each index in the vector. While each of these
    // ExtractElements will be accessing the same base address as determined
    // by the load instruction, the actual bytes they interact with will differ
    // due to different ExtractElement indices. To accurately determine the
    // byte position of an ExtractElement, we offset the base load ptr with
    // the index multiplied by the byte size of each element in the vector.
    if (L->getMemoryVT().isVector()) {
      unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits();
      if (LoadWidthInBit % 8 != 0)
        return SDValue();
      unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
      Ptr.addToOffset(ByteOffsetFromVector);
    }
 
    if (!Base)
      Base = Ptr;
 
    else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase))
      return SDValue();
 
    // Calculate the offset of the current byte from the base address
    ByteOffsetFromBase += MemoryByteOffset(*P);
    ByteOffsets[i] = ByteOffsetFromBase;
 
    // Remember the first byte load
    if (ByteOffsetFromBase < FirstOffset) {
      FirstByteProvider = P;
      FirstOffset = ByteOffsetFromBase;
    }
 
    Loads.insert(L);
  }
 
  assert(!Loads.empty() && "All the bytes of the value must be loaded from "
         "memory, so there must be at least one load which produces the value");
  assert(Base && "Base address of the accessed memory location must be set");
  assert(FirstOffset != INT64_MAX && "First byte offset must be set");
 
  bool NeedsZext = ZeroExtendedBytes > 0;
 
  EVT MemVT =
      EVT::getIntegerVT(*DAG.getContext(), (ByteWidth - ZeroExtendedBytes) * 8);
 
  if (!MemVT.isSimple())
    return SDValue();
 
  // Check if the bytes of the OR we are looking at match with either big or
  // little endian value load
  std::optional<bool> IsBigEndian = isBigEndian(
      ArrayRef(ByteOffsets).drop_back(ZeroExtendedBytes), FirstOffset);
  if (!IsBigEndian)
    return SDValue();
 
  assert(FirstByteProvider && "must be set");
 
  // Ensure that the first byte is loaded from zero offset of the first load.
  // So the combined value can be loaded from the first load address.
  if (MemoryByteOffset(*FirstByteProvider) != 0)
    return SDValue();
  auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value());
 
  // Before legalization we allow introducing loads that are wider than legal,
  // which will later be split into legally sized loads. This enables us to
  // combine, for example, i8 loads forming an i64 into an i64 load, which get
  // then gets split up into couple of i32 loads on 32 bit targets.
  if (LegalOperations &&
      !TLI.isLoadLegal(VT, MemVT, FirstLoad->getAlign(),
                       FirstLoad->getAddressSpace(),
                       NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, false))
    return SDValue();
 
  // The node we are looking at matches with the pattern, check if we can
  // replace it with a single (possibly zero-extended) load and bswap + shift if
  // needed.
 
  // If the load needs byte swap check if the target supports it
  bool NeedsBswap = IsBigEndianTarget != *IsBigEndian;
 
  // Before legalize we can introduce illegal bswaps which will be later
  // converted to an explicit bswap sequence. This way we end up with a single
  // load and byte shuffling instead of several loads and byte shuffling.
  // We do not introduce illegal bswaps when zero-extending as this tends to
  // introduce too many arithmetic instructions.
  if (NeedsBswap && (LegalOperations || NeedsZext) &&
      !TLI.isOperationLegal(ISD::BSWAP, VT))
    return SDValue();
 
  // If we need to bswap and zero extend, we have to insert a shift. Check that
  // it is legal.
  if (NeedsBswap && NeedsZext && LegalOperations &&
      !TLI.isOperationLegal(ISD::SHL, VT))
    return SDValue();
 
  // Check that a load of the wide type is both allowed and fast on the target
  unsigned Fast = 0;
  bool Allowed =
      TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
                             *FirstLoad->getMemOperand(), &Fast);
  if (!Allowed || !Fast)
    return SDValue();
 
  SDValue NewLoad =
      DAG.getExtLoad(NeedsZext ? ISD::ZEXTLOAD : ISD::NON_EXTLOAD, SDLoc(N), VT,
                     Chain, FirstLoad->getBasePtr(),
                     FirstLoad->getPointerInfo(), MemVT, FirstLoad->getAlign());
 
  // Transfer chain users from old loads to the new load.
  for (LoadSDNode *L : Loads)
    DAG.makeEquivalentMemoryOrdering(L, NewLoad);
 
  if (!NeedsBswap)
    return NewLoad;
 
  SDValue ShiftedLoad =
      NeedsZext ? DAG.getNode(ISD::SHL, SDLoc(N), VT, NewLoad,
                              DAG.getShiftAmountConstant(ZeroExtendedBytes * 8,
                                                         VT, SDLoc(N)))
                : NewLoad;
  return DAG.getNode(ISD::BSWAP, SDLoc(N), VT, ShiftedLoad);
}
 
// If the target has andn, bsl, or a similar bit-select instruction,
// we want to unfold masked merge, with canonical pattern of:
//   |        A  |  |B|
//   ((x ^ y) & m) ^ y
//    |  D  |
// Into:
//   (x & m) | (y & ~m)
// If y is a constant, m is not a 'not', and the 'andn' does not work with
// immediates, we unfold into a different pattern:
//   ~(~x & m) & (m | y)
// If x is a constant, m is a 'not', and the 'andn' does not work with
// immediates, we unfold into a different pattern:
//   (x | ~m) & ~(~m & ~y)
// NOTE: we don't unfold the pattern if 'xor' is actually a 'not', because at
//       the very least that breaks andnpd / andnps patterns, and because those
//       patterns are simplified in IR and shouldn't be created in the DAG
SDValue DAGCombiner::unfoldMaskedMerge(SDNode *N) {
  assert(N->getOpcode() == ISD::XOR);
 
  // Don't touch 'not' (i.e. where y = -1).
  if (isAllOnesOrAllOnesSplat(N->getOperand(1)))
    return SDValue();
 
  EVT VT = N->getValueType(0);
 
  // There are 3 commutable operators in the pattern,
  // so we have to deal with 8 possible variants of the basic pattern.
  SDValue X, Y, M;
  auto matchAndXor = [&X, &Y, &M](SDValue And, unsigned XorIdx, SDValue Other) {
    if (And.getOpcode() != ISD::AND || !And.hasOneUse())
      return false;
    SDValue Xor = And.getOperand(XorIdx);
    if (Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
      return false;
    SDValue Xor0 = Xor.getOperand(0);
    SDValue Xor1 = Xor.getOperand(1);
    // Don't touch 'not' (i.e. where y = -1).
    if (isAllOnesOrAllOnesSplat(Xor1))
      return false;
    if (Other == Xor0)
      std::swap(Xor0, Xor1);
    if (Other != Xor1)
      return false;
    X = Xor0;
    Y = Xor1;
    M = And.getOperand(XorIdx ? 0 : 1);
    return true;
  };
 
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (!matchAndXor(N0, 0, N1) && !matchAndXor(N0, 1, N1) &&
      !matchAndXor(N1, 0, N0) && !matchAndXor(N1, 1, N0))
    return SDValue();
 
  // Don't do anything if the mask is constant. This should not be reachable.
  // InstCombine should have already unfolded this pattern, and DAGCombiner
  // probably shouldn't produce it, too.
  if (isa<ConstantSDNode>(M.getNode()))
    return SDValue();
 
  // We can transform if the target has AndNot
  if (!TLI.hasAndNot(M))
    return SDValue();
 
  SDLoc DL(N);
 
  // If Y is a constant, check that 'andn' works with immediates. Unless M is
  // a bitwise not that would already allow ANDN to be used.
  if (!TLI.hasAndNot(Y) && !isBitwiseNot(M)) {
    assert(TLI.hasAndNot(X) && "Only mask is a variable? Unreachable.");
    // If not, we need to do a bit more work to make sure andn is still used.
    SDValue NotX = DAG.getNOT(DL, X, VT);
    SDValue LHS = DAG.getNode(ISD::AND, DL, VT, NotX, M);
    SDValue NotLHS = DAG.getNOT(DL, LHS, VT);
    SDValue RHS = DAG.getNode(ISD::OR, DL, VT, M, Y);
    return DAG.getNode(ISD::AND, DL, VT, NotLHS, RHS);
  }
 
  // If X is a constant and M is a bitwise not, check that 'andn' works with
  // immediates.
  if (!TLI.hasAndNot(X) && isBitwiseNot(M)) {
    assert(TLI.hasAndNot(Y) && "Only mask is a variable? Unreachable.");
    // If not, we need to do a bit more work to make sure andn is still used.
    SDValue NotM = M.getOperand(0);
    SDValue LHS = DAG.getNode(ISD::OR, DL, VT, X, NotM);
    SDValue NotY = DAG.getNOT(DL, Y, VT);
    SDValue RHS = DAG.getNode(ISD::AND, DL, VT, NotM, NotY);
    SDValue NotRHS = DAG.getNOT(DL, RHS, VT);
    return DAG.getNode(ISD::AND, DL, VT, LHS, NotRHS);
  }
 
  SDValue LHS = DAG.getNode(ISD::AND, DL, VT, X, M);
  SDValue NotM = DAG.getNOT(DL, M, VT);
  SDValue RHS = DAG.getNode(ISD::AND, DL, VT, Y, NotM);
 
  return DAG.getNode(ISD::OR, DL, VT, LHS, RHS);
}
 
SDValue DAGCombiner::visitXOR(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N0.getValueType();
  SDLoc DL(N);
 
  // fold (xor undef, undef) -> 0. This is a common idiom (misuse).
  if (N0.isUndef() && N1.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  // fold (xor x, undef) -> undef
  if (N0.isUndef())
    return N0;
  if (N1.isUndef())
    return N1;
 
  // fold (xor c1, c2) -> c1^c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::XOR, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(ISD::XOR, DL, VT, N1, N0);
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    // fold (xor x, 0) -> x, vector edition
    if (ISD::isConstantSplatVectorAllZeros(N1.getNode()))
      return N0;
  }
 
  // fold (xor x, 0) -> x
  if (isNullConstant(N1))
    return N0;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // reassociate xor
  if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags()))
    return RXOR;
 
  // Fold xor(vecreduce(x), vecreduce(y)) -> vecreduce(xor(x, y))
  if (SDValue SD =
          reassociateReduction(ISD::VECREDUCE_XOR, ISD::XOR, DL, VT, N0, N1))
    return SD;
 
  // fold (a^b) -> (a|b) iff a and b share no bits.
  if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) &&
      DAG.haveNoCommonBitsSet(N0, N1))
    return DAG.getNode(ISD::OR, DL, VT, N0, N1, SDNodeFlags::Disjoint);
 
  // look for 'add-like' folds:
  // XOR(N0,MIN_SIGNED_VALUE) == ADD(N0,MIN_SIGNED_VALUE)
  if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) &&
      isMinSignedConstant(N1))
    if (SDValue Combined = visitADDLike(N))
      return Combined;
 
  // fold not (setcc x, y, cc) -> setcc x y !cc
  // Avoid breaking: and (not(setcc x, y, cc), z) -> andn for vec
  unsigned N0Opcode = N0.getOpcode();
  SDValue LHS, RHS, CC;
  if (TLI.isConstTrueVal(N1) &&
      isSetCCEquivalent(N0, LHS, RHS, CC, /*MatchStrict*/ true) &&
      !(VT.isVector() && TLI.hasAndNot(SDValue(N, 0)) && N->hasOneUse() &&
        N->use_begin()->getUser()->getOpcode() == ISD::AND)) {
    ISD::CondCode NotCC = ISD::getSetCCInverse(cast<CondCodeSDNode>(CC)->get(),
                                               LHS.getValueType());
    if (!LegalOperations ||
        TLI.isCondCodeLegal(NotCC, LHS.getSimpleValueType())) {
      // Propagate fast-math-flags.
      SDNodeFlags Flags = N0->getFlags();
      switch (N0Opcode) {
      default:
        llvm_unreachable("Unhandled SetCC Equivalent!");
      case ISD::SETCC:
        return DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, SDValue(),
                            /*IsSignaling=*/false, Flags);
      case ISD::SELECT_CC:
        return DAG.getSelectCC(SDLoc(N0), LHS, RHS, N0.getOperand(2),
                               N0.getOperand(3), NotCC, Flags);
      case ISD::STRICT_FSETCC:
      case ISD::STRICT_FSETCCS: {
        if (N0.hasOneUse()) {
          // FIXME Can we handle multiple uses? Could we token factor the chain
          // results from the new/old setcc?
          SDValue SetCC =
              DAG.getSetCC(SDLoc(N0), VT, LHS, RHS, NotCC, N0.getOperand(0),
                           N0Opcode == ISD::STRICT_FSETCCS, Flags);
          CombineTo(N, SetCC);
          DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), SetCC.getValue(1));
          recursivelyDeleteUnusedNodes(N0.getNode());
          return SDValue(N, 0); // Return N so it doesn't get rechecked!
        }
        break;
      }
      }
    }
  }
 
  // fold (not (zext (setcc x, y))) -> (zext (not (setcc x, y)))
  if (isOneConstant(N1) && N0Opcode == ISD::ZERO_EXTEND && N0.hasOneUse() &&
      isSetCCEquivalent(N0.getOperand(0), LHS, RHS, CC)){
    SDValue V = N0.getOperand(0);
    SDLoc DL0(N0);
    V = DAG.getNode(ISD::XOR, DL0, V.getValueType(), V,
                    DAG.getConstant(1, DL0, V.getValueType()));
    AddToWorklist(V.getNode());
    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, V);
  }
 
  // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are setcc
  // fold (not (and x, y)) -> (or (not x), (not y)) iff x or y are setcc
  if (isOneConstant(N1) && VT == MVT::i1 && N0.hasOneUse() &&
      (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) {
    SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1);
    if (isOneUseSetCC(N01) || isOneUseSetCC(N00)) {
      unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND;
      N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00
      N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01
      AddToWorklist(N00.getNode()); AddToWorklist(N01.getNode());
      return DAG.getNode(NewOpcode, DL, VT, N00, N01);
    }
  }
  // fold (not (or x, y)) -> (and (not x), (not y)) iff x or y are constants
  // fold (not (and x, y)) -> (or (not x), (not y)) iff x or y are constants
  if (isAllOnesConstant(N1) && N0.hasOneUse() &&
      (N0Opcode == ISD::OR || N0Opcode == ISD::AND)) {
    SDValue N00 = N0.getOperand(0), N01 = N0.getOperand(1);
    if (isa<ConstantSDNode>(N01) || isa<ConstantSDNode>(N00)) {
      unsigned NewOpcode = N0Opcode == ISD::AND ? ISD::OR : ISD::AND;
      N00 = DAG.getNode(ISD::XOR, SDLoc(N00), VT, N00, N1); // N00 = ~N00
      N01 = DAG.getNode(ISD::XOR, SDLoc(N01), VT, N01, N1); // N01 = ~N01
      AddToWorklist(N00.getNode()); AddToWorklist(N01.getNode());
      return DAG.getNode(NewOpcode, DL, VT, N00, N01);
    }
  }
 
  // fold (not (sub Y, X)) -> (add X, ~Y) if Y is a constant
  if (N0.getOpcode() == ISD::SUB && isAllOnesConstant(N1)) {
    SDValue Y = N0.getOperand(0);
    SDValue X = N0.getOperand(1);
 
    if (auto *YConst = dyn_cast<ConstantSDNode>(Y)) {
      APInt NotYValue = ~YConst->getAPIntValue();
      SDValue NotY = DAG.getConstant(NotYValue, DL, VT);
      return DAG.getNode(ISD::ADD, DL, VT, X, NotY, N->getFlags());
    }
  }
 
  // fold (not (add X, -1)) -> (neg X)
  if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && isAllOnesConstant(N1) &&
      isAllOnesOrAllOnesSplat(N0.getOperand(1))) {
    return DAG.getNegative(N0.getOperand(0), DL, VT);
  }
 
  // fold (xor (and x, y), y) -> (and (not x), y)
  if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(1) == N1) {
    SDValue X = N0.getOperand(0);
    SDValue NotX = DAG.getNOT(SDLoc(X), X, VT);
    AddToWorklist(NotX.getNode());
    return DAG.getNode(ISD::AND, DL, VT, NotX, N1);
  }
 
  // fold Y = sra (X, size(X)-1); xor (add (X, Y), Y) -> (abs X)
  if (!LegalOperations || hasOperation(ISD::ABS, VT)) {
    SDValue A = N0Opcode == ISD::ADD ? N0 : N1;
    SDValue S = N0Opcode == ISD::SRA ? N0 : N1;
    if (A.getOpcode() == ISD::ADD && S.getOpcode() == ISD::SRA) {
      SDValue A0 = A.getOperand(0), A1 = A.getOperand(1);
      SDValue S0 = S.getOperand(0);
      if ((A0 == S && A1 == S0) || (A1 == S && A0 == S0))
        if (ConstantSDNode *C = isConstOrConstSplat(S.getOperand(1)))
          if (C->getAPIntValue() == (VT.getScalarSizeInBits() - 1))
            return DAG.getNode(ISD::ABS, DL, VT, S0);
    }
  }
 
  // fold (xor x, x) -> 0
  if (N0 == N1)
    return tryFoldToZero(DL, TLI, VT, DAG, LegalOperations);
 
  // fold (xor (shl 1, x), -1) -> (rotl ~1, x)
  // Here is a concrete example of this equivalence:
  // i16   x ==  14
  // i16 shl ==   1 << 14  == 16384 == 0b0100000000000000
  // i16 xor == ~(1 << 14) == 49151 == 0b1011111111111111
  //
  // =>
  //
  // i16     ~1      == 0b1111111111111110
  // i16 rol(~1, 14) == 0b1011111111111111
  //
  // Some additional tips to help conceptualize this transform:
  // - Try to see the operation as placing a single zero in a value of all ones.
  // - There exists no value for x which would allow the result to contain zero.
  // - Values of x larger than the bitwidth are undefined and do not require a
  //   consistent result.
  // - Pushing the zero left requires shifting one bits in from the right.
  // A rotate left of ~1 is a nice way of achieving the desired result.
  if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT) && N0Opcode == ISD::SHL &&
      isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0))) {
    return DAG.getNode(ISD::ROTL, DL, VT, DAG.getSignedConstant(~1, DL, VT),
                       N0.getOperand(1));
  }
 
  // Simplify: xor (op x...), (op y...)  -> (op (xor x, y))
  if (N0Opcode == N1.getOpcode())
    if (SDValue V = hoistLogicOpWithSameOpcodeHands(N))
      return V;
 
  if (SDValue R = foldLogicOfShifts(N, N0, N1, DAG))
    return R;
  if (SDValue R = foldLogicOfShifts(N, N1, N0, DAG))
    return R;
  if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG))
    return R;
 
  // Unfold  ((x ^ y) & m) ^ y  into  (x & m) | (y & ~m)  if profitable
  if (SDValue MM = unfoldMaskedMerge(N))
    return MM;
 
  // Simplify the expression using non-local knowledge.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  if (SDValue Combined = combineCarryDiamond(DAG, TLI, N0, N1, N))
    return Combined;
 
  // fold (xor (smin(x, C), C)) -> select (x < C), xor(x, C), 0
  // fold (xor (smax(x, C), C)) -> select (x > C), xor(x, C), 0
  // fold (xor (umin(x, C), C)) -> select (x < C), xor(x, C), 0
  // fold (xor (umax(x, C), C)) -> select (x > C), xor(x, C), 0
  SDValue Op0;
  if (sd_match(N0, m_OneUse(m_AnyOf(m_SMin(m_Value(Op0), m_Specific(N1)),
                                    m_SMax(m_Value(Op0), m_Specific(N1)),
                                    m_UMin(m_Value(Op0), m_Specific(N1)),
                                    m_UMax(m_Value(Op0), m_Specific(N1)))))) {
 
    if (isa<ConstantSDNode>(N1) ||
        ISD::isBuildVectorOfConstantSDNodes(N1.getNode())) {
      // For vectors, only optimize when the constant is zero or all-ones to
      // avoid generating more instructions
      if (VT.isVector()) {
        ConstantSDNode *N1C = isConstOrConstSplat(N1);
        if (!N1C || (!N1C->isZero() && !N1C->isAllOnes()))
          return SDValue();
      }
 
      // Avoid the fold if the minmax operation is legal and select is expensive
      if (TLI.isOperationLegal(N0.getOpcode(), VT) &&
          TLI.isPredictableSelectExpensive())
        return SDValue();
 
      EVT CCVT = getSetCCResultType(VT);
      ISD::CondCode CC;
      switch (N0.getOpcode()) {
      case ISD::SMIN:
        CC = ISD::SETLT;
        break;
      case ISD::SMAX:
        CC = ISD::SETGT;
        break;
      case ISD::UMIN:
        CC = ISD::SETULT;
        break;
      case ISD::UMAX:
        CC = ISD::SETUGT;
        break;
      }
      SDValue FN1 = DAG.getFreeze(N1);
      SDValue Cmp = DAG.getSetCC(DL, CCVT, Op0, FN1, CC);
      SDValue XorXC = DAG.getNode(ISD::XOR, DL, VT, Op0, FN1);
      SDValue Zero = DAG.getConstant(0, DL, VT);
      return DAG.getSelect(DL, VT, Cmp, XorXC, Zero);
    }
  }
 
  return SDValue();
}
 
/// If we have a shift-by-constant of a bitwise logic op that itself has a
/// shift-by-constant operand with identical opcode, we may be able to convert
/// that into 2 independent shifts followed by the logic op. This is a
/// throughput improvement.
static SDValue combineShiftOfShiftedLogic(SDNode *Shift, SelectionDAG &DAG) {
  // Match a one-use bitwise logic op.
  SDValue LogicOp = Shift->getOperand(0);
  if (!LogicOp.hasOneUse())
    return SDValue();
 
  unsigned LogicOpcode = LogicOp.getOpcode();
  if (LogicOpcode != ISD::AND && LogicOpcode != ISD::OR &&
      LogicOpcode != ISD::XOR)
    return SDValue();
 
  // Find a matching one-use shift by constant.
  unsigned ShiftOpcode = Shift->getOpcode();
  SDValue C1 = Shift->getOperand(1);
  ConstantSDNode *C1Node = isConstOrConstSplat(C1);
  assert(C1Node && "Expected a shift with constant operand");
  const APInt &C1Val = C1Node->getAPIntValue();
  auto matchFirstShift = [&](SDValue V, SDValue &ShiftOp,
                             const APInt *&ShiftAmtVal) {
    if (V.getOpcode() != ShiftOpcode || !V.hasOneUse())
      return false;
 
    ConstantSDNode *ShiftCNode = isConstOrConstSplat(V.getOperand(1));
    if (!ShiftCNode)
      return false;
 
    // Capture the shifted operand and shift amount value.
    ShiftOp = V.getOperand(0);
    ShiftAmtVal = &ShiftCNode->getAPIntValue();
 
    // Shift amount types do not have to match their operand type, so check that
    // the constants are the same width.
    if (ShiftAmtVal->getBitWidth() != C1Val.getBitWidth())
      return false;
 
    // The fold is not valid if the sum of the shift values doesn't fit in the
    // given shift amount type.
    bool Overflow = false;
    APInt NewShiftAmt = C1Val.uadd_ov(*ShiftAmtVal, Overflow);
    if (Overflow)
      return false;
 
    // The fold is not valid if the sum of the shift values exceeds bitwidth.
    if (NewShiftAmt.uge(V.getScalarValueSizeInBits()))
      return false;
 
    return true;
  };
 
  // Logic ops are commutative, so check each operand for a match.
  SDValue X, Y;
  const APInt *C0Val;
  if (matchFirstShift(LogicOp.getOperand(0), X, C0Val))
    Y = LogicOp.getOperand(1);
  else if (matchFirstShift(LogicOp.getOperand(1), X, C0Val))
    Y = LogicOp.getOperand(0);
  else
    return SDValue();
 
  // shift (logic (shift X, C0), Y), C1 -> logic (shift X, C0+C1), (shift Y, C1)
  SDLoc DL(Shift);
  EVT VT = Shift->getValueType(0);
  EVT ShiftAmtVT = Shift->getOperand(1).getValueType();
  SDValue ShiftSumC = DAG.getConstant(*C0Val + C1Val, DL, ShiftAmtVT);
  SDValue NewShift1 = DAG.getNode(ShiftOpcode, DL, VT, X, ShiftSumC);
  SDValue NewShift2 = DAG.getNode(ShiftOpcode, DL, VT, Y, C1);
  return DAG.getNode(LogicOpcode, DL, VT, NewShift1, NewShift2,
                     LogicOp->getFlags());
}
 
/// Handle transforms common to the three shifts, when the shift amount is a
/// constant.
/// We are looking for: (shift being one of shl/sra/srl)
///   shift (binop X, C0), C1
/// And want to transform into:
///   binop (shift X, C1), (shift C0, C1)
SDValue DAGCombiner::visitShiftByConstant(SDNode *N) {
  assert(isConstOrConstSplat(N->getOperand(1)) && "Expected constant operand");
 
  // Do not turn a 'not' into a regular xor.
  if (isBitwiseNot(N->getOperand(0)))
    return SDValue();
 
  // The inner binop must be one-use, since we want to replace it.
  SDValue LHS = N->getOperand(0);
  if (!LHS.hasOneUse() || !TLI.isDesirableToCommuteWithShift(N, Level))
    return SDValue();
 
  // Fold shift(bitop(shift(x,c1),y), c2) -> bitop(shift(x,c1+c2),shift(y,c2)).
  if (SDValue R = combineShiftOfShiftedLogic(N, DAG))
    return R;
 
  // We want to pull some binops through shifts, so that we have (and (shift))
  // instead of (shift (and)), likewise for add, or, xor, etc.  This sort of
  // thing happens with address calculations, so it's important to canonicalize
  // it.
  switch (LHS.getOpcode()) {
  default:
    return SDValue();
  case ISD::OR:
  case ISD::XOR:
  case ISD::AND:
    break;
  case ISD::ADD:
    if (N->getOpcode() != ISD::SHL)
      return SDValue(); // only shl(add) not sr[al](add).
    break;
  }
 
  // FIXME: disable this unless the input to the binop is a shift by a constant
  // or is copy/select. Enable this in other cases when figure out it's exactly
  // profitable.
  SDValue BinOpLHSVal = LHS.getOperand(0);
  bool IsShiftByConstant = (BinOpLHSVal.getOpcode() == ISD::SHL ||
                            BinOpLHSVal.getOpcode() == ISD::SRA ||
                            BinOpLHSVal.getOpcode() == ISD::SRL) &&
                           isa<ConstantSDNode>(BinOpLHSVal.getOperand(1));
  bool IsCopyOrSelect = BinOpLHSVal.getOpcode() == ISD::CopyFromReg ||
                        BinOpLHSVal.getOpcode() == ISD::SELECT;
 
  if (!IsShiftByConstant && !IsCopyOrSelect)
    return SDValue();
 
  if (IsCopyOrSelect && N->hasOneUse())
    return SDValue();
 
  // Attempt to fold the constants, shifting the binop RHS by the shift amount.
  SDLoc DL(N);
  EVT VT = N->getValueType(0);
  if (SDValue NewRHS = DAG.FoldConstantArithmetic(
          N->getOpcode(), DL, VT, {LHS.getOperand(1), N->getOperand(1)})) {
    SDValue NewShift = DAG.getNode(N->getOpcode(), DL, VT, LHS.getOperand(0),
                                   N->getOperand(1));
    return DAG.getNode(LHS.getOpcode(), DL, VT, NewShift, NewRHS);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
  assert(N->getOpcode() == ISD::TRUNCATE);
  assert(N->getOperand(0).getOpcode() == ISD::AND);
 
  // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
  EVT TruncVT = N->getValueType(0);
  if (N->hasOneUse() && N->getOperand(0).hasOneUse() &&
      TLI.isTypeDesirableForOp(ISD::AND, TruncVT)) {
    SDValue N01 = N->getOperand(0).getOperand(1);
    if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
      SDLoc DL(N);
      SDValue N00 = N->getOperand(0).getOperand(0);
      SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
      SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
      AddToWorklist(Trunc00.getNode());
      AddToWorklist(Trunc01.getNode());
      return DAG.getNode(ISD::AND, DL, TruncVT, Trunc00, Trunc01);
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitRotate(SDNode *N) {
  SDLoc dl(N);
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  unsigned Bitsize = VT.getScalarSizeInBits();
 
  // fold (rot x, 0) -> x
  if (isNullOrNullSplat(N1))
    return N0;
 
  // fold (rot x, c) -> x iff (c % BitSize) == 0
  if (isPowerOf2_32(Bitsize) && Bitsize > 1) {
    APInt ModuloMask(N1.getScalarValueSizeInBits(), Bitsize - 1);
    if (DAG.MaskedValueIsZero(N1, ModuloMask))
      return N0;
  }
 
  // fold (rot x, c) -> (rot x, c % BitSize)
  bool OutOfRange = false;
  auto MatchOutOfRange = [Bitsize, &OutOfRange](ConstantSDNode *C) {
    OutOfRange |= C->getAPIntValue().uge(Bitsize);
    return true;
  };
  if (ISD::matchUnaryPredicate(N1, MatchOutOfRange) && OutOfRange) {
    EVT AmtVT = N1.getValueType();
    SDValue Bits = DAG.getConstant(Bitsize, dl, AmtVT);
    if (SDValue Amt =
            DAG.FoldConstantArithmetic(ISD::UREM, dl, AmtVT, {N1, Bits}))
      return DAG.getNode(N->getOpcode(), dl, VT, N0, Amt);
  }
 
  // rot i16 X, 8 --> bswap X
  auto *RotAmtC = isConstOrConstSplat(N1);
  if (RotAmtC && RotAmtC->getAPIntValue() == 8 &&
      VT.getScalarSizeInBits() == 16 && hasOperation(ISD::BSWAP, VT))
    return DAG.getNode(ISD::BSWAP, dl, VT, N0);
 
  // Simplify the operands using demanded-bits information.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  // fold (rot* x, (trunc (and y, c))) -> (rot* x, (and (trunc y), (trunc c))).
  if (N1.getOpcode() == ISD::TRUNCATE &&
      N1.getOperand(0).getOpcode() == ISD::AND) {
    if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
      return DAG.getNode(N->getOpcode(), dl, VT, N0, NewOp1);
  }
 
  unsigned NextOp = N0.getOpcode();
 
  // fold (rot* (rot* x, c2), c1)
  //   -> (rot* x, ((c1 % bitsize) +- (c2 % bitsize) + bitsize) % bitsize)
  if (NextOp == ISD::ROTL || NextOp == ISD::ROTR) {
    bool C1 = DAG.isConstantIntBuildVectorOrConstantInt(N1);
    bool C2 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1));
    if (C1 && C2 && N1.getValueType() == N0.getOperand(1).getValueType()) {
      EVT ShiftVT = N1.getValueType();
      bool SameSide = (N->getOpcode() == NextOp);
      unsigned CombineOp = SameSide ? ISD::ADD : ISD::SUB;
      SDValue BitsizeC = DAG.getConstant(Bitsize, dl, ShiftVT);
      SDValue Norm1 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT,
                                                 {N1, BitsizeC});
      SDValue Norm2 = DAG.FoldConstantArithmetic(ISD::UREM, dl, ShiftVT,
                                                 {N0.getOperand(1), BitsizeC});
      if (Norm1 && Norm2)
        if (SDValue CombinedShift = DAG.FoldConstantArithmetic(
                CombineOp, dl, ShiftVT, {Norm1, Norm2})) {
          CombinedShift = DAG.FoldConstantArithmetic(ISD::ADD, dl, ShiftVT,
                                                     {CombinedShift, BitsizeC});
          SDValue CombinedShiftNorm = DAG.FoldConstantArithmetic(
              ISD::UREM, dl, ShiftVT, {CombinedShift, BitsizeC});
          return DAG.getNode(N->getOpcode(), dl, VT, N0->getOperand(0),
                             CombinedShiftNorm);
        }
    }
  }
  return SDValue();
}
 
SDValue DAGCombiner::visitSHL(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (SDValue V = DAG.simplifyShift(N0, N1))
    return V;
 
  SDLoc DL(N);
  EVT VT = N0.getValueType();
  EVT ShiftVT = N1.getValueType();
  unsigned OpSizeInBits = VT.getScalarSizeInBits();
 
  // fold (shl c1, c2) -> c1<<c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N0, N1}))
    return C;
 
  // fold vector ops
  if (VT.isVector()) {
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
    BuildVectorSDNode *N1CV = dyn_cast<BuildVectorSDNode>(N1);
    // If setcc produces all-one true value then:
    // (shl (and (setcc) N01CV) N1CV) -> (and (setcc) N01CV<<N1CV)
    if (N1CV && N1CV->isConstant()) {
      if (N0.getOpcode() == ISD::AND) {
        SDValue N00 = N0->getOperand(0);
        SDValue N01 = N0->getOperand(1);
        BuildVectorSDNode *N01CV = dyn_cast<BuildVectorSDNode>(N01);
 
        if (N01CV && N01CV->isConstant() && N00.getOpcode() == ISD::SETCC &&
            TLI.getBooleanContents(N00.getOperand(0).getValueType()) ==
                TargetLowering::ZeroOrNegativeOneBooleanContent) {
          if (SDValue C =
                  DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {N01, N1}))
            return DAG.getNode(ISD::AND, DL, VT, N00, C);
        }
      }
    }
  }
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // if (shl x, c) is known to be zero, return 0
  if (DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(OpSizeInBits)))
    return DAG.getConstant(0, DL, VT);
 
  // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))).
  if (N1.getOpcode() == ISD::TRUNCATE &&
      N1.getOperand(0).getOpcode() == ISD::AND) {
    if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
      return DAG.getNode(ISD::SHL, DL, VT, N0, NewOp1);
  }
 
  // fold (shl (shl x, c1), c2) -> 0 or (shl x, (add c1, c2))
  if (N0.getOpcode() == ISD::SHL) {
    auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
                                          ConstantSDNode *RHS) {
      APInt c1 = LHS->getAPIntValue();
      APInt c2 = RHS->getAPIntValue();
      zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
      return (c1 + c2).uge(OpSizeInBits);
    };
    if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
      return DAG.getConstant(0, DL, VT);
 
    auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
                                       ConstantSDNode *RHS) {
      APInt c1 = LHS->getAPIntValue();
      APInt c2 = RHS->getAPIntValue();
      zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
      return (c1 + c2).ult(OpSizeInBits);
    };
    if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
      SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
      return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum);
    }
  }
 
  // fold (shl (ext (shl x, c1)), c2) -> (shl (ext x), (add c1, c2))
  // For this to be valid, the second form must not preserve any of the bits
  // that are shifted out by the inner shift in the first form.  This means
  // the outer shift size must be >= the number of bits added by the ext.
  // As a corollary, we don't care what kind of ext it is.
  if ((N0.getOpcode() == ISD::ZERO_EXTEND ||
       N0.getOpcode() == ISD::ANY_EXTEND ||
       N0.getOpcode() == ISD::SIGN_EXTEND) &&
      N0.getOperand(0).getOpcode() == ISD::SHL) {
    SDValue N0Op0 = N0.getOperand(0);
    SDValue InnerShiftAmt = N0Op0.getOperand(1);
    EVT InnerVT = N0Op0.getValueType();
    uint64_t InnerBitwidth = InnerVT.getScalarSizeInBits();
 
    auto MatchOutOfRange = [OpSizeInBits, InnerBitwidth](ConstantSDNode *LHS,
                                                         ConstantSDNode *RHS) {
      APInt c1 = LHS->getAPIntValue();
      APInt c2 = RHS->getAPIntValue();
      zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
      return c2.uge(OpSizeInBits - InnerBitwidth) &&
             (c1 + c2).uge(OpSizeInBits);
    };
    if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchOutOfRange,
                                  /*AllowUndefs*/ false,
                                  /*AllowTypeMismatch*/ true))
      return DAG.getConstant(0, DL, VT);
 
    auto MatchInRange = [OpSizeInBits, InnerBitwidth](ConstantSDNode *LHS,
                                                      ConstantSDNode *RHS) {
      APInt c1 = LHS->getAPIntValue();
      APInt c2 = RHS->getAPIntValue();
      zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
      return c2.uge(OpSizeInBits - InnerBitwidth) &&
             (c1 + c2).ult(OpSizeInBits);
    };
    if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchInRange,
                                  /*AllowUndefs*/ false,
                                  /*AllowTypeMismatch*/ true)) {
      SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0));
      SDValue Sum = DAG.getZExtOrTrunc(InnerShiftAmt, DL, ShiftVT);
      Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, Sum, N1);
      return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum);
    }
  }
 
  // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C))
  // Only fold this if the inner zext has no other uses to avoid increasing
  // the total number of instructions.
  if (N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() &&
      N0.getOperand(0).getOpcode() == ISD::SRL) {
    SDValue N0Op0 = N0.getOperand(0);
    SDValue InnerShiftAmt = N0Op0.getOperand(1);
 
    auto MatchEqual = [VT](ConstantSDNode *LHS, ConstantSDNode *RHS) {
      APInt c1 = LHS->getAPIntValue();
      APInt c2 = RHS->getAPIntValue();
      zeroExtendToMatch(c1, c2);
      return c1.ult(VT.getScalarSizeInBits()) && (c1 == c2);
    };
    if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchEqual,
                                  /*AllowUndefs*/ false,
                                  /*AllowTypeMismatch*/ true)) {
      EVT InnerShiftAmtVT = N0Op0.getOperand(1).getValueType();
      SDValue NewSHL = DAG.getZExtOrTrunc(N1, DL, InnerShiftAmtVT);
      NewSHL = DAG.getNode(ISD::SHL, DL, N0Op0.getValueType(), N0Op0, NewSHL);
      AddToWorklist(NewSHL.getNode());
      return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL);
    }
  }
 
  if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) {
    auto MatchShiftAmount = [OpSizeInBits](ConstantSDNode *LHS,
                                           ConstantSDNode *RHS) {
      const APInt &LHSC = LHS->getAPIntValue();
      const APInt &RHSC = RHS->getAPIntValue();
      return LHSC.ult(OpSizeInBits) && RHSC.ult(OpSizeInBits) &&
             LHSC.getZExtValue() <= RHSC.getZExtValue();
    };
 
    // fold (shl (sr[la] exact X,  C1), C2) -> (shl    X, (C2-C1)) if C1 <= C2
    // fold (shl (sr[la] exact X,  C1), C2) -> (sr[la] X, (C2-C1)) if C1 >= C2
    if (N0->getFlags().hasExact()) {
      if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount,
                                    /*AllowUndefs*/ false,
                                    /*AllowTypeMismatch*/ true)) {
        SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
        SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01);
        return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff);
      }
      if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount,
                                    /*AllowUndefs*/ false,
                                    /*AllowTypeMismatch*/ true)) {
        SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
        SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1);
        return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Diff);
      }
    }
 
    // fold (shl (srl x, c1), c2) -> (and (shl x, (sub c2, c1), MASK) or
    //                               (and (srl x, (sub c1, c2), MASK)
    // Only fold this if the inner shift has no other uses -- if it does,
    // folding this will increase the total number of instructions.
    if (N0.getOpcode() == ISD::SRL &&
        (N0.getOperand(1) == N1 || N0.hasOneUse()) &&
        TLI.shouldFoldConstantShiftPairToMask(N)) {
      if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount,
                                    /*AllowUndefs*/ false,
                                    /*AllowTypeMismatch*/ true)) {
        SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
        SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1);
        SDValue Mask = DAG.getAllOnesConstant(DL, VT);
        Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N01);
        Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, Diff);
        SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff);
        return DAG.getNode(ISD::AND, DL, VT, Shift, Mask);
      }
      if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount,
                                    /*AllowUndefs*/ false,
                                    /*AllowTypeMismatch*/ true)) {
        SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
        SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01);
        SDValue Mask = DAG.getAllOnesConstant(DL, VT);
        Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, N1);
        SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff);
        return DAG.getNode(ISD::AND, DL, VT, Shift, Mask);
      }
    }
  }
 
  // fold (shl (sra x, c1), c1) -> (and x, (shl -1, c1))
  if (N0.getOpcode() == ISD::SRA && N1 == N0.getOperand(1) &&
      isConstantOrConstantVector(N1, /* No Opaques */ true)) {
    SDValue AllBits = DAG.getAllOnesConstant(DL, VT);
    SDValue HiBitsMask = DAG.getNode(ISD::SHL, DL, VT, AllBits, N1);
    return DAG.getNode(ISD::AND, DL, VT, N0.getOperand(0), HiBitsMask);
  }
 
  // fold (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
  // fold (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
  // Variant of version done on multiply, except mul by a power of 2 is turned
  // into a shift.
  if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR) &&
      TLI.isDesirableToCommuteWithShift(N, Level)) {
    SDValue N01 = N0.getOperand(1);
    if (SDValue Shl1 =
            DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1})) {
      SDValue Shl0 = DAG.getNode(ISD::SHL, SDLoc(N0), VT, N0.getOperand(0), N1);
      AddToWorklist(Shl0.getNode());
      SDNodeFlags Flags;
      // Preserve the disjoint flag for Or.
      if (N0.getOpcode() == ISD::OR && N0->getFlags().hasDisjoint())
        Flags |= SDNodeFlags::Disjoint;
      return DAG.getNode(N0.getOpcode(), DL, VT, Shl0, Shl1, Flags);
    }
  }
 
  // fold (shl (sext (add_nsw x, c1)), c2) -> (add (shl (sext x), c2), c1 << c2)
  // TODO: Add zext/add_nuw variant with suitable test coverage
  // TODO: Should we limit this with isLegalAddImmediate?
  if (N0.getOpcode() == ISD::SIGN_EXTEND &&
      N0.getOperand(0).getOpcode() == ISD::ADD &&
      N0.getOperand(0)->getFlags().hasNoSignedWrap() &&
      TLI.isDesirableToCommuteWithShift(N, Level)) {
    SDValue Add = N0.getOperand(0);
    SDLoc DL(N0);
    if (SDValue ExtC = DAG.FoldConstantArithmetic(N0.getOpcode(), DL, VT,
                                                  {Add.getOperand(1)})) {
      if (SDValue ShlC =
              DAG.FoldConstantArithmetic(ISD::SHL, DL, VT, {ExtC, N1})) {
        SDValue ExtX = DAG.getNode(N0.getOpcode(), DL, VT, Add.getOperand(0));
        SDValue ShlX = DAG.getNode(ISD::SHL, DL, VT, ExtX, N1);
        return DAG.getNode(ISD::ADD, DL, VT, ShlX, ShlC);
      }
    }
  }
 
  // fold (shl (mul x, c1), c2) -> (mul x, c1 << c2)
  if (N0.getOpcode() == ISD::MUL && N0->hasOneUse()) {
    SDValue N01 = N0.getOperand(1);
    if (SDValue Shl =
            DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N1), VT, {N01, N1}))
      return DAG.getNode(ISD::MUL, DL, VT, N0.getOperand(0), Shl);
  }
 
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
  if (N1C && !N1C->isOpaque())
    if (SDValue NewSHL = visitShiftByConstant(N))
      return NewSHL;
 
  // fold (shl X, cttz(Y)) -> (mul (Y & -Y), X) if cttz is unsupported on the
  // target.
  if (((N1.getOpcode() == ISD::CTTZ &&
        VT.getScalarSizeInBits() <= ShiftVT.getScalarSizeInBits()) ||
       N1.getOpcode() == ISD::CTTZ_ZERO_POISON) &&
      N1.hasOneUse() && !TLI.isOperationLegalOrCustom(ISD::CTTZ, ShiftVT) &&
      TLI.isOperationLegalOrCustom(ISD::MUL, VT)) {
    SDValue Y = N1.getOperand(0);
    SDLoc DL(N);
    SDValue NegY = DAG.getNegative(Y, DL, ShiftVT);
    SDValue And =
        DAG.getZExtOrTrunc(DAG.getNode(ISD::AND, DL, ShiftVT, Y, NegY), DL, VT);
    return DAG.getNode(ISD::MUL, DL, VT, And, N0);
  }
 
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  // Fold (shl (vscale * C0), C1) to (vscale * (C0 << C1)).
  if (N0.getOpcode() == ISD::VSCALE && N1C) {
    const APInt &C0 = N0.getConstantOperandAPInt(0);
    const APInt &C1 = N1C->getAPIntValue();
    return DAG.getVScale(DL, VT, C0 << C1);
  }
 
  SDValue X;
  APInt VS0;
 
  // fold (shl (X * vscale(VS0)), C1) -> (X * vscale(VS0 << C1))
  if (N1C && sd_match(N0, m_Mul(m_Value(X), m_VScale(m_ConstInt(VS0))))) {
    SDNodeFlags Flags;
    Flags.setNoUnsignedWrap(N->getFlags().hasNoUnsignedWrap() &&
                            N0->getFlags().hasNoUnsignedWrap());
 
    SDValue VScale = DAG.getVScale(DL, VT, VS0 << N1C->getAPIntValue());
    return DAG.getNode(ISD::MUL, DL, VT, X, VScale, Flags);
  }
 
  // Fold (shl step_vector(C0), C1) to (step_vector(C0 << C1)).
  APInt ShlVal;
  if (N0.getOpcode() == ISD::STEP_VECTOR &&
      ISD::isConstantSplatVector(N1.getNode(), ShlVal)) {
    const APInt &C0 = N0.getConstantOperandAPInt(0);
    if (ShlVal.ult(C0.getBitWidth())) {
      APInt NewStep = C0 << ShlVal;
      return DAG.getStepVector(DL, VT, NewStep);
    }
  }
 
  return SDValue();
}
 
// Transform a right shift of a multiply into a multiply-high.
// Examples:
// (srl (mul (zext i32:$a to i64), (zext i32:$a to i64)), 32) -> (mulhu $a, $b)
// (sra (mul (sext i32:$a to i64), (sext i32:$a to i64)), 32) -> (mulhs $a, $b)
static SDValue combineShiftToMULH(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
                                  const TargetLowering &TLI) {
  assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA) &&
         "SRL or SRA node is required here!");
 
  // Check the shift amount. Proceed with the transformation if the shift
  // amount is constant.
  ConstantSDNode *ShiftAmtSrc = isConstOrConstSplat(N->getOperand(1));
  if (!ShiftAmtSrc)
    return SDValue();
 
  // The operation feeding into the shift must be a multiply.
  SDValue ShiftOperand = N->getOperand(0);
  if (ShiftOperand.getOpcode() != ISD::MUL)
    return SDValue();
 
  // Both operands must be equivalent extend nodes.
  SDValue LeftOp = ShiftOperand.getOperand(0);
  SDValue RightOp = ShiftOperand.getOperand(1);
 
  if (LeftOp.getOpcode() != ISD::SIGN_EXTEND &&
      LeftOp.getOpcode() != ISD::ZERO_EXTEND)
    std::swap(LeftOp, RightOp);
 
  bool IsSignExt = LeftOp.getOpcode() == ISD::SIGN_EXTEND;
  bool IsZeroExt = LeftOp.getOpcode() == ISD::ZERO_EXTEND;
 
  if (!IsSignExt && !IsZeroExt)
    return SDValue();
 
  EVT NarrowVT = LeftOp.getOperand(0).getValueType();
  unsigned NarrowVTSize = NarrowVT.getScalarSizeInBits();
 
  // return true if U may use the lower bits of its operands
  auto UserOfLowerBits = [NarrowVTSize](SDNode *U) {
    if (U->getOpcode() != ISD::SRL && U->getOpcode() != ISD::SRA) {
      return true;
    }
    ConstantSDNode *UShiftAmtSrc = isConstOrConstSplat(U->getOperand(1));
    if (!UShiftAmtSrc) {
      return true;
    }
    unsigned UShiftAmt = UShiftAmtSrc->getZExtValue();
    return UShiftAmt < NarrowVTSize;
  };
 
  // If the lower part of the MUL is also used and MUL_LOHI is supported
  // do not introduce the MULH in favor of MUL_LOHI
  unsigned MulLoHiOp = IsSignExt ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
  if (!ShiftOperand.hasOneUse() &&
      TLI.isOperationLegalOrCustom(MulLoHiOp, NarrowVT) &&
      llvm::any_of(ShiftOperand->users(), UserOfLowerBits)) {
    return SDValue();
  }
 
  SDValue MulhRightOp;
  if (LeftOp.getOpcode() != RightOp.getOpcode()) {
    if (IsZeroExt && ShiftOperand.hasOneUse() &&
        DAG.computeKnownBits(RightOp).countMaxActiveBits() <= NarrowVTSize) {
      MulhRightOp = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, RightOp);
    } else if (IsSignExt && ShiftOperand.hasOneUse() &&
               DAG.ComputeMaxSignificantBits(RightOp) <= NarrowVTSize) {
      MulhRightOp = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, RightOp);
    } else {
      return SDValue();
    }
  } else {
    // Check that the two extend nodes are the same type.
    if (NarrowVT != RightOp.getOperand(0).getValueType())
      return SDValue();
    MulhRightOp = RightOp.getOperand(0);
  }
 
  EVT WideVT = LeftOp.getValueType();
  // Proceed with the transformation if the wide types match.
  assert((WideVT == RightOp.getValueType()) &&
         "Cannot have a multiply node with two different operand types.");
 
  // Proceed with the transformation if the wide type is twice as large
  // as the narrow type.
  if (WideVT.getScalarSizeInBits() != 2 * NarrowVTSize)
    return SDValue();
 
  // Check the shift amount with the narrow type size.
  // Proceed with the transformation if the shift amount is the width
  // of the narrow type.
  unsigned ShiftAmt = ShiftAmtSrc->getZExtValue();
  if (ShiftAmt != NarrowVTSize)
    return SDValue();
 
  // If the operation feeding into the MUL is a sign extend (sext),
  // we use mulhs. Othewise, zero extends (zext) use mulhu.
  unsigned MulhOpcode = IsSignExt ? ISD::MULHS : ISD::MULHU;
 
  // Combine to mulh if mulh is legal/custom for the narrow type on the target
  // or if it is a vector type then we could transform to an acceptable type and
  // rely on legalization to split/combine the result.
  EVT TransformVT = NarrowVT;
  if (NarrowVT.isVector()) {
    TransformVT = TLI.getLegalTypeToTransformTo(*DAG.getContext(), NarrowVT);
    if (TransformVT.getScalarType() != NarrowVT.getScalarType())
      return SDValue();
  }
  if (!TLI.isOperationLegalOrCustom(MulhOpcode, TransformVT))
    return SDValue();
 
  SDValue Result =
      DAG.getNode(MulhOpcode, DL, NarrowVT, LeftOp.getOperand(0), MulhRightOp);
  bool IsSigned = N->getOpcode() == ISD::SRA;
  return DAG.getExtOrTrunc(IsSigned, Result, DL, WideVT);
}
 
// fold (bswap (logic_op(bswap(x),y))) -> logic_op(x,bswap(y))
// This helper function accept SDNode with opcode ISD::BSWAP and ISD::BITREVERSE
static SDValue foldBitOrderCrossLogicOp(SDNode *N, SelectionDAG &DAG) {
  unsigned Opcode = N->getOpcode();
  if (Opcode != ISD::BSWAP && Opcode != ISD::BITREVERSE)
    return SDValue();
 
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  SDValue X, Y;
 
  // If both operands are bswap/bitreverse, ignore the multiuse
  if (sd_match(N0, m_OneUse(m_BitwiseLogic(m_UnaryOp(Opcode, m_Value(X)),
                                           m_UnaryOp(Opcode, m_Value(Y))))))
    return DAG.getNode(N0.getOpcode(), DL, VT, X, Y);
 
  // Otherwise need to ensure logic_op and bswap/bitreverse(x) have one use.
  if (sd_match(N0, m_OneUse(m_BitwiseLogic(
                       m_OneUse(m_UnaryOp(Opcode, m_Value(X))), m_Value(Y))))) {
    SDValue NewBitReorder = DAG.getNode(Opcode, DL, VT, Y);
    return DAG.getNode(N0.getOpcode(), DL, VT, X, NewBitReorder);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSRA(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (SDValue V = DAG.simplifyShift(N0, N1))
    return V;
 
  SDLoc DL(N);
  EVT VT = N0.getValueType();
  unsigned OpSizeInBits = VT.getScalarSizeInBits();
 
  // fold (sra c1, c2) -> (sra c1, c2)
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRA, DL, VT, {N0, N1}))
    return C;
 
  // Arithmetic shifting an all-sign-bit value is a no-op.
  // fold (sra 0, x) -> 0
  // fold (sra -1, x) -> -1
  if (DAG.ComputeNumSignBits(N0) == OpSizeInBits)
    return N0;
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
 
  // fold (sra (sra x, c1), c2) -> (sra x, (add c1, c2))
  // clamp (add c1, c2) to max shift.
  if (N0.getOpcode() == ISD::SRA) {
    EVT ShiftVT = N1.getValueType();
    EVT ShiftSVT = ShiftVT.getScalarType();
    SmallVector<SDValue, 16> ShiftValues;
 
    auto SumOfShifts = [&](ConstantSDNode *LHS, ConstantSDNode *RHS) {
      APInt c1 = LHS->getAPIntValue();
      APInt c2 = RHS->getAPIntValue();
      zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
      APInt Sum = c1 + c2;
      unsigned ShiftSum =
          Sum.uge(OpSizeInBits) ? (OpSizeInBits - 1) : Sum.getZExtValue();
      ShiftValues.push_back(DAG.getConstant(ShiftSum, DL, ShiftSVT));
      return true;
    };
    if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), SumOfShifts)) {
      SDValue ShiftValue;
      if (N1.getOpcode() == ISD::BUILD_VECTOR)
        ShiftValue = DAG.getBuildVector(ShiftVT, DL, ShiftValues);
      else if (N1.getOpcode() == ISD::SPLAT_VECTOR) {
        assert(ShiftValues.size() == 1 &&
               "Expected matchBinaryPredicate to return one element for "
               "SPLAT_VECTORs");
        ShiftValue = DAG.getSplatVector(ShiftVT, DL, ShiftValues[0]);
      } else
        ShiftValue = ShiftValues[0];
      return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), ShiftValue);
    }
  }
 
  // fold (sra (xor (sra x, c1), -1), c2) -> (xor (sra x, c3), -1)
  // This allows merging two arithmetic shifts even when there's a NOT in
  // between.
  SDValue X;
  APInt C1;
  if (N1C && sd_match(N0, m_OneUse(m_Not(
                              m_OneUse(m_Sra(m_Value(X), m_ConstInt(C1))))))) {
    APInt C2 = N1C->getAPIntValue();
    zeroExtendToMatch(C1, C2, 1 /* Overflow Bit */);
    APInt Sum = C1 + C2;
    unsigned ShiftSum = Sum.getLimitedValue(OpSizeInBits - 1);
    SDValue NewShift = DAG.getNode(
        ISD::SRA, DL, VT, X, DAG.getShiftAmountConstant(ShiftSum, VT, DL));
    return DAG.getNOT(DL, NewShift, VT);
  }
 
  // fold (sra (shl X, m), (sub result_size, n))
  // -> (sign_extend (trunc (shl X, (sub (sub result_size, n), m)))) for
  // result_size - n != m.
  // If truncate is free for the target sext(shl) is likely to result in better
  // code.
  if (N0.getOpcode() == ISD::SHL && N1C) {
    // Get the two constants of the shifts, CN0 = m, CN = n.
    const ConstantSDNode *N01C = isConstOrConstSplat(N0.getOperand(1));
    if (N01C) {
      LLVMContext &Ctx = *DAG.getContext();
      // Determine what the truncate's result bitsize and type would be.
      EVT TruncVT = VT.changeElementType(
          Ctx, EVT::getIntegerVT(Ctx, OpSizeInBits - N1C->getZExtValue()));
 
      // Determine the residual right-shift amount.
      int ShiftAmt = N1C->getZExtValue() - N01C->getZExtValue();
 
      // If the shift is not a no-op (in which case this should be just a sign
      // extend already), the truncated to type is legal, sign_extend is legal
      // on that type, and the truncate to that type is both legal and free,
      // perform the transform.
      if ((ShiftAmt > 0) &&
          TLI.isOperationLegalOrCustom(ISD::SIGN_EXTEND, TruncVT) &&
          TLI.isOperationLegalOrCustom(ISD::TRUNCATE, VT) &&
          TLI.isTruncateFree(VT, TruncVT)) {
        SDValue Amt = DAG.getShiftAmountConstant(ShiftAmt, VT, DL);
        SDValue Shift = DAG.getNode(ISD::SRL, DL, VT,
                                    N0.getOperand(0), Amt);
        SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, TruncVT,
                                    Shift);
        return DAG.getNode(ISD::SIGN_EXTEND, DL,
                           N->getValueType(0), Trunc);
      }
    }
  }
 
  // We convert trunc/ext to opposing shifts in IR, but casts may be cheaper.
  //   sra (add (shl X, N1C), AddC), N1C -->
  //   sext (add (trunc X to (width - N1C)), AddC')
  //   sra (sub AddC, (shl X, N1C)), N1C -->
  //   sext (sub AddC1',(trunc X to (width - N1C)))
  if ((N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::SUB) && N1C &&
      N0.hasOneUse()) {
    bool IsAdd = N0.getOpcode() == ISD::ADD;
    SDValue Shl = N0.getOperand(IsAdd ? 0 : 1);
    if (Shl.getOpcode() == ISD::SHL && Shl.getOperand(1) == N1 &&
        Shl.hasOneUse()) {
      // TODO: AddC does not need to be a splat.
      if (ConstantSDNode *AddC =
              isConstOrConstSplat(N0.getOperand(IsAdd ? 1 : 0))) {
        // Determine what the truncate's type would be and ask the target if
        // that is a free operation.
        LLVMContext &Ctx = *DAG.getContext();
        unsigned ShiftAmt = N1C->getZExtValue();
        EVT TruncVT = VT.changeElementType(
            Ctx, EVT::getIntegerVT(Ctx, OpSizeInBits - ShiftAmt));
 
        // TODO: The simple type check probably belongs in the default hook
        //       implementation and/or target-specific overrides (because
        //       non-simple types likely require masking when legalized), but
        //       that restriction may conflict with other transforms.
        if (TruncVT.isSimple() && isTypeLegal(TruncVT) &&
            TLI.isTruncateFree(VT, TruncVT)) {
          SDValue Trunc = DAG.getZExtOrTrunc(Shl.getOperand(0), DL, TruncVT);
          SDValue ShiftC =
              DAG.getConstant(AddC->getAPIntValue().lshr(ShiftAmt).trunc(
                                  TruncVT.getScalarSizeInBits()),
                              DL, TruncVT);
          SDValue Add;
          if (IsAdd)
            Add = DAG.getNode(ISD::ADD, DL, TruncVT, Trunc, ShiftC);
          else
            Add = DAG.getNode(ISD::SUB, DL, TruncVT, ShiftC, Trunc);
          return DAG.getSExtOrTrunc(Add, DL, VT);
        }
      }
    }
  }
 
  // fold (sra x, (trunc (and y, c))) -> (sra x, (and (trunc y), (trunc c))).
  if (N1.getOpcode() == ISD::TRUNCATE &&
      N1.getOperand(0).getOpcode() == ISD::AND) {
    if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
      return DAG.getNode(ISD::SRA, DL, VT, N0, NewOp1);
  }
 
  // fold (sra (trunc (sra x, c1)), c2) -> (trunc (sra x, c1 + c2))
  // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2))
  //      if c1 is equal to the number of bits the trunc removes
  // TODO - support non-uniform vector shift amounts.
  if (N0.getOpcode() == ISD::TRUNCATE &&
      (N0.getOperand(0).getOpcode() == ISD::SRL ||
       N0.getOperand(0).getOpcode() == ISD::SRA) &&
      N0.getOperand(0).hasOneUse() &&
      N0.getOperand(0).getOperand(1).hasOneUse() && N1C) {
    SDValue N0Op0 = N0.getOperand(0);
    if (ConstantSDNode *LargeShift = isConstOrConstSplat(N0Op0.getOperand(1))) {
      EVT LargeVT = N0Op0.getValueType();
      unsigned TruncBits = LargeVT.getScalarSizeInBits() - OpSizeInBits;
      if (LargeShift->getAPIntValue() == TruncBits) {
        EVT LargeShiftVT = getShiftAmountTy(LargeVT);
        SDValue Amt = DAG.getZExtOrTrunc(N1, DL, LargeShiftVT);
        Amt = DAG.getNode(ISD::ADD, DL, LargeShiftVT, Amt,
                          DAG.getConstant(TruncBits, DL, LargeShiftVT));
        SDValue SRA =
            DAG.getNode(ISD::SRA, DL, LargeVT, N0Op0.getOperand(0), Amt);
        return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA);
      }
    }
  }
 
  // fold (sra (add nsw X, C), D) -> (add nsw (sra X, D), C s>> D)
  // when C has D trailing zeros (so C s>> D is exact).
  if (N1C && N0.hasOneUse() && N0.getOpcode() == ISD::ADD &&
      N0->getFlags().hasNoSignedWrap()) {
    if (ConstantSDNode *AddC = isConstOrConstSplat(N0.getOperand(1))) {
      const APInt &ShAmt = N1C->getAPIntValue();
      const APInt &AddVal = AddC->getAPIntValue();
      if (ShAmt.ult(AddVal.countr_zero())) {
        SDNodeFlags ShiftFlags = N->getFlags();
        SDValue NewSra =
            DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0), N1, ShiftFlags);
        SDValue NewC = DAG.getConstant(AddVal.ashr(ShAmt), DL, VT);
        SDNodeFlags AddFlags = N0->getFlags();
        return DAG.getNode(ISD::ADD, DL, VT, NewSra, NewC, AddFlags);
      }
    }
  }
 
  // Simplify, based on bits shifted out of the LHS.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  // If the sign bit is known to be zero, switch this to a SRL.
  if (DAG.SignBitIsZero(N0))
    return DAG.getNode(ISD::SRL, DL, VT, N0, N1);
 
  if (N1C && !N1C->isOpaque())
    if (SDValue NewSRA = visitShiftByConstant(N))
      return NewSRA;
 
  // Try to transform this shift into a multiply-high if
  // it matches the appropriate pattern detected in combineShiftToMULH.
  if (SDValue MULH = combineShiftToMULH(N, DL, DAG, TLI))
    return MULH;
 
  // Attempt to convert a sra of a load into a narrower sign-extending load.
  if (SDValue NarrowLoad = reduceLoadWidth(N))
    return NarrowLoad;
 
  if (SDValue AVG = foldShiftToAvg(N, DL))
    return AVG;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSRL(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (SDValue V = DAG.simplifyShift(N0, N1))
    return V;
 
  SDLoc DL(N);
  EVT VT = N0.getValueType();
  EVT ShiftVT = N1.getValueType();
  unsigned OpSizeInBits = VT.getScalarSizeInBits();
 
  // fold (srl c1, c2) -> c1 >>u c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::SRL, DL, VT, {N0, N1}))
    return C;
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // if (srl x, c) is known to be zero, return 0
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
  if (N1C &&
      DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnes(OpSizeInBits)))
    return DAG.getConstant(0, DL, VT);
 
  // fold (srl (srl x, c1), c2) -> 0 or (srl x, (add c1, c2))
  if (N0.getOpcode() == ISD::SRL) {
    auto MatchOutOfRange = [OpSizeInBits](ConstantSDNode *LHS,
                                          ConstantSDNode *RHS) {
      APInt c1 = LHS->getAPIntValue();
      APInt c2 = RHS->getAPIntValue();
      zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
      return (c1 + c2).uge(OpSizeInBits);
    };
    if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchOutOfRange))
      return DAG.getConstant(0, DL, VT);
 
    auto MatchInRange = [OpSizeInBits](ConstantSDNode *LHS,
                                       ConstantSDNode *RHS) {
      APInt c1 = LHS->getAPIntValue();
      APInt c2 = RHS->getAPIntValue();
      zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */);
      return (c1 + c2).ult(OpSizeInBits);
    };
    if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) {
      SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1));
      return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Sum);
    }
  }
 
  if (N1C && N0.getOpcode() == ISD::TRUNCATE &&
      N0.getOperand(0).getOpcode() == ISD::SRL) {
    SDValue InnerShift = N0.getOperand(0);
    // TODO - support non-uniform vector shift amounts.
    if (auto *N001C = isConstOrConstSplat(InnerShift.getOperand(1))) {
      uint64_t c1 = N001C->getZExtValue();
      uint64_t c2 = N1C->getZExtValue();
      EVT InnerShiftVT = InnerShift.getValueType();
      EVT ShiftAmtVT = InnerShift.getOperand(1).getValueType();
      uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits();
      // srl (trunc (srl x, c1)), c2 --> 0 or (trunc (srl x, (add c1, c2)))
      // This is only valid if the OpSizeInBits + c1 = size of inner shift.
      if (c1 + OpSizeInBits == InnerShiftSize) {
        if (c1 + c2 >= InnerShiftSize)
          return DAG.getConstant(0, DL, VT);
        SDValue NewShiftAmt = DAG.getConstant(c1 + c2, DL, ShiftAmtVT);
        SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT,
                                       InnerShift.getOperand(0), NewShiftAmt);
        return DAG.getNode(ISD::TRUNCATE, DL, VT, NewShift);
      }
      // In the more general case, we can clear the high bits after the shift:
      // srl (trunc (srl x, c1)), c2 --> trunc (and (srl x, (c1+c2)), Mask)
      if (N0.hasOneUse() && InnerShift.hasOneUse() &&
          c1 + c2 < InnerShiftSize) {
        SDValue NewShiftAmt = DAG.getConstant(c1 + c2, DL, ShiftAmtVT);
        SDValue NewShift = DAG.getNode(ISD::SRL, DL, InnerShiftVT,
                                       InnerShift.getOperand(0), NewShiftAmt);
        SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(InnerShiftSize,
                                                            OpSizeInBits - c2),
                                       DL, InnerShiftVT);
        SDValue And = DAG.getNode(ISD::AND, DL, InnerShiftVT, NewShift, Mask);
        return DAG.getNode(ISD::TRUNCATE, DL, VT, And);
      }
    }
  }
 
  if (N0.getOpcode() == ISD::SHL) {
    // fold (srl (shl nuw x, c), c) -> x
    if (N0.getOperand(1) == N1 && N0->getFlags().hasNoUnsignedWrap())
      return N0.getOperand(0);
 
    // fold (srl (shl x, c1), c2) -> (and (shl x, (sub c1, c2), MASK) or
    //                               (and (srl x, (sub c2, c1), MASK)
    if ((N0.getOperand(1) == N1 || N0->hasOneUse()) &&
        TLI.shouldFoldConstantShiftPairToMask(N)) {
      auto MatchShiftAmount = [OpSizeInBits](ConstantSDNode *LHS,
                                             ConstantSDNode *RHS) {
        const APInt &LHSC = LHS->getAPIntValue();
        const APInt &RHSC = RHS->getAPIntValue();
        return LHSC.ult(OpSizeInBits) && RHSC.ult(OpSizeInBits) &&
               LHSC.getZExtValue() <= RHSC.getZExtValue();
      };
      if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchShiftAmount,
                                    /*AllowUndefs*/ false,
                                    /*AllowTypeMismatch*/ true)) {
        SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
        SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N01, N1);
        SDValue Mask = DAG.getAllOnesConstant(DL, VT);
        Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N01);
        Mask = DAG.getNode(ISD::SHL, DL, VT, Mask, Diff);
        SDValue Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Diff);
        return DAG.getNode(ISD::AND, DL, VT, Shift, Mask);
      }
      if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchShiftAmount,
                                    /*AllowUndefs*/ false,
                                    /*AllowTypeMismatch*/ true)) {
        SDValue N01 = DAG.getZExtOrTrunc(N0.getOperand(1), DL, ShiftVT);
        SDValue Diff = DAG.getNode(ISD::SUB, DL, ShiftVT, N1, N01);
        SDValue Mask = DAG.getAllOnesConstant(DL, VT);
        Mask = DAG.getNode(ISD::SRL, DL, VT, Mask, N1);
        SDValue Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), Diff);
        return DAG.getNode(ISD::AND, DL, VT, Shift, Mask);
      }
    }
  }
 
  // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask)
  // TODO - support non-uniform vector shift amounts.
  if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) {
    // Shifting in all undef bits?
    EVT SmallVT = N0.getOperand(0).getValueType();
    unsigned BitSize = SmallVT.getScalarSizeInBits();
    if (N1C->getAPIntValue().uge(BitSize))
      return DAG.getUNDEF(VT);
 
    if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) {
      uint64_t ShiftAmt = N1C->getZExtValue();
      SDLoc DL0(N0);
      SDValue SmallShift =
          DAG.getNode(ISD::SRL, DL0, SmallVT, N0.getOperand(0),
                      DAG.getShiftAmountConstant(ShiftAmt, SmallVT, DL0));
      AddToWorklist(SmallShift.getNode());
      APInt Mask = APInt::getLowBitsSet(OpSizeInBits, OpSizeInBits - ShiftAmt);
      return DAG.getNode(ISD::AND, DL, VT,
                         DAG.getNode(ISD::ANY_EXTEND, DL, VT, SmallShift),
                         DAG.getConstant(Mask, DL, VT));
    }
  }
 
  // fold (srl (sra X, Y), 31) -> (srl X, 31).  This srl only looks at the sign
  // bit, which is unmodified by sra.
  if (N1C && N1C->getAPIntValue() == (OpSizeInBits - 1)) {
    if (N0.getOpcode() == ISD::SRA)
      return DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
  }
 
  // fold (srl (ctlz x), "5") -> x  iff x has one bit set (the low bit), and x has a power
  // of two bitwidth. The "5" represents (log2 (bitwidth x)).
  if (N1C && N0.getOpcode() == ISD::CTLZ &&
      isPowerOf2_32(OpSizeInBits) &&
      N1C->getAPIntValue() == Log2_32(OpSizeInBits)) {
    KnownBits Known = DAG.computeKnownBits(N0.getOperand(0));
 
    // If any of the input bits are KnownOne, then the input couldn't be all
    // zeros, thus the result of the srl will always be zero.
    if (Known.One.getBoolValue()) return DAG.getConstant(0, SDLoc(N0), VT);
 
    // If all of the bits input the to ctlz node are known to be zero, then
    // the result of the ctlz is "32" and the result of the shift is one.
    APInt UnknownBits = ~Known.Zero;
    if (UnknownBits == 0) return DAG.getConstant(1, SDLoc(N0), VT);
 
    // Otherwise, check to see if there is exactly one bit input to the ctlz.
    if (UnknownBits.isPowerOf2()) {
      // Okay, we know that only that the single bit specified by UnknownBits
      // could be set on input to the CTLZ node. If this bit is set, the SRL
      // will return 0, if it is clear, it returns 1. Change the CTLZ/SRL pair
      // to an SRL/XOR pair, which is likely to simplify more.
      unsigned ShAmt = UnknownBits.countr_zero();
      SDValue Op = N0.getOperand(0);
 
      if (ShAmt) {
        SDLoc DL(N0);
        Op = DAG.getNode(ISD::SRL, DL, VT, Op,
                         DAG.getShiftAmountConstant(ShAmt, VT, DL));
        AddToWorklist(Op.getNode());
      }
      return DAG.getNode(ISD::XOR, DL, VT, Op, DAG.getConstant(1, DL, VT));
    }
  }
 
  // fold (srl x, (trunc (and y, c))) -> (srl x, (and (trunc y), (trunc c))).
  if (N1.getOpcode() == ISD::TRUNCATE &&
      N1.getOperand(0).getOpcode() == ISD::AND) {
    if (SDValue NewOp1 = distributeTruncateThroughAnd(N1.getNode()))
      return DAG.getNode(ISD::SRL, DL, VT, N0, NewOp1);
  }
 
  // fold (srl (logic_op x, (shl (zext y), c1)), c1)
  //   -> (logic_op (srl x, c1), (zext y))
  // c1 <= leadingzeros(zext(y))
  // TODO: Replace c1 with valuetracking?
  SDValue X, ZExtY;
  if (sd_match(
          N0,
          m_OneUse(m_BitwiseLogic(
              m_Value(X),
              m_OneUse(m_Shl(m_Value(ZExtY, m_SpecificOpc(ISD::ZERO_EXTEND)),
                             m_Specific(N1))))))) {
    unsigned NumLeadingZeros = ZExtY.getScalarValueSizeInBits() -
                               ZExtY.getOperand(0).getScalarValueSizeInBits();
    if (N1C && N1C->getZExtValue() <= NumLeadingZeros)
      return DAG.getNode(N0.getOpcode(), SDLoc(N0), VT,
                         DAG.getNode(ISD::SRL, SDLoc(N0), VT, X, N1), ZExtY);
  }
 
  // fold (srl (bitcast (build_vector e1, ..., eN)), (N-1) * eltsize)
  //   -> (zext eN)
  if (N1C && VT.isScalarInteger() && DAG.getDataLayout().isLittleEndian()) {
    SDValue BV = peekThroughBitcasts(N0);
    if (BV.getOpcode() == ISD::BUILD_VECTOR) {
      EVT BVVT = BV.getValueType();
      unsigned EltSizeInBits = BVVT.getScalarSizeInBits();
      unsigned NumElts = BVVT.getVectorNumElements();
      if (N1C->getZExtValue() == (NumElts - 1) * EltSizeInBits) {
        SDValue LastElt = BV.getOperand(NumElts - 1);
        assert(LastElt.getScalarValueSizeInBits() >= EltSizeInBits &&
               "Expected BUILD_VECTOR operand as wide as element type");
        EVT IntEltVT = EVT::getIntegerVT(*DAG.getContext(), EltSizeInBits);
        LastElt = DAG.getBitcast(LastElt.getValueType().changeTypeToInteger(),
                                 LastElt);
        return DAG.getZExtOrTrunc(DAG.getZExtOrTrunc(LastElt, DL, IntEltVT), DL,
                                  VT);
      }
    }
  }
 
  // fold (srl (add nuw X, C), D) -> (add nuw (srl X, D), C u>> D)
  // when C has D trailing zeros (so C >> D is exact).
  if (N1C && N0.hasOneUse() && N0.getOpcode() == ISD::ADD &&
      N0->getFlags().hasNoUnsignedWrap()) {
    if (ConstantSDNode *AddC = isConstOrConstSplat(N0.getOperand(1))) {
      const APInt &ShAmt = N1C->getAPIntValue();
      const APInt &AddVal = AddC->getAPIntValue();
      if (ShAmt.ult(AddVal.countr_zero())) {
        SDNodeFlags ShiftFlags = N->getFlags();
        SDValue NewSrl =
            DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1, ShiftFlags);
        SDValue NewC = DAG.getConstant(AddVal.lshr(ShAmt), DL, VT);
        SDNodeFlags AddFlags = N0->getFlags();
        return DAG.getNode(ISD::ADD, DL, VT, NewSrl, NewC, AddFlags);
      }
    }
  }
 
  // fold operands of srl based on knowledge that the low bits are not
  // demanded.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  if (N1C && !N1C->isOpaque())
    if (SDValue NewSRL = visitShiftByConstant(N))
      return NewSRL;
 
  // Attempt to convert a srl of a load into a narrower zero-extending load.
  if (SDValue NarrowLoad = reduceLoadWidth(N))
    return NarrowLoad;
 
  // Here is a common situation. We want to optimize:
  //
  //   %a = ...
  //   %b = and i32 %a, 2
  //   %c = srl i32 %b, 1
  //   brcond i32 %c ...
  //
  // into
  //
  //   %a = ...
  //   %b = and %a, 2
  //   %c = setcc eq %b, 0
  //   brcond %c ...
  //
  // However when after the source operand of SRL is optimized into AND, the SRL
  // itself may not be optimized further. Look for it and add the BRCOND into
  // the worklist.
  //
  // The also tends to happen for binary operations when SimplifyDemandedBits
  // is involved.
  //
  // FIXME: This is unecessary if we process the DAG in topological order,
  // which we plan to do. This workaround can be removed once the DAG is
  // processed in topological order.
  if (N->hasOneUse()) {
    SDNode *User = *N->user_begin();
 
    // Look pass the truncate.
    if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse())
      User = *User->user_begin();
 
    if (User->getOpcode() == ISD::BRCOND || User->getOpcode() == ISD::AND ||
        User->getOpcode() == ISD::OR || User->getOpcode() == ISD::XOR)
      AddToWorklist(User);
  }
 
  // Try to transform this shift into a multiply-high if
  // it matches the appropriate pattern detected in combineShiftToMULH.
  if (SDValue MULH = combineShiftToMULH(N, DL, DAG, TLI))
    return MULH;
 
  if (SDValue AVG = foldShiftToAvg(N, DL))
    return AVG;
 
  SDValue Y;
  if (VT.getScalarSizeInBits() % 2 == 0 && N1C) {
    // Fold clmul(zext(x), zext(y)) >> (BW - 1 | BW) -> clmul(r|h)(x, y).
    unsigned HalfBW = VT.getScalarSizeInBits() / 2;
    if (sd_match(N0, m_Clmul(m_ZExt(m_Value(X)), m_ZExt(m_Value(Y)))) &&
        X.getScalarValueSizeInBits() == HalfBW &&
        Y.getScalarValueSizeInBits() == HalfBW) {
      if (N1C->getZExtValue() == HalfBW - 1 &&
          (!LegalOperations ||
           TLI.isOperationLegalOrCustom(ISD::CLMULR, X.getValueType())))
        return DAG.getNode(
            ISD::ZERO_EXTEND, DL, VT,
            DAG.getNode(ISD::CLMULR, DL, X.getValueType(), X, Y));
      if (N1C->getZExtValue() == HalfBW &&
          (!LegalOperations ||
           TLI.isOperationLegalOrCustom(ISD::CLMULH, X.getValueType())))
        return DAG.getNode(
            ISD::ZERO_EXTEND, DL, VT,
            DAG.getNode(ISD::CLMULH, DL, X.getValueType(), X, Y));
    }
  }
 
  // Fold bitreverse(clmul(bitreverse(x), bitreverse(y))) >> 1 ->
  // clmulh(x, y).
  if (N1C && N1C->getZExtValue() == 1 &&
      sd_match(N0, m_BitReverse(m_Clmul(m_BitReverse(m_Value(X)),
                                        m_BitReverse(m_Value(Y))))))
    return DAG.getNode(ISD::CLMULH, DL, VT, X, Y);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitFunnelShift(SDNode *N) {
  EVT VT = N->getValueType(0);
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  bool IsFSHL = N->getOpcode() == ISD::FSHL;
  unsigned BitWidth = VT.getScalarSizeInBits();
  SDLoc DL(N);
 
  // fold (fshl/fshr C0, C1, C2) -> C3
  if (SDValue C =
          DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1, N2}))
    return C;
 
  // fold (fshl N0, N1, 0) -> N0
  // fold (fshr N0, N1, 0) -> N1
  if (isPowerOf2_32(BitWidth))
    if (DAG.MaskedValueIsZero(
            N2, APInt(N2.getScalarValueSizeInBits(), BitWidth - 1)))
      return IsFSHL ? N0 : N1;
 
  auto IsUndefOrZero = [](SDValue V) {
    return V.isUndef() || isNullOrNullSplat(V, /*AllowUndefs*/ true);
  };
 
  // TODO - support non-uniform vector shift amounts.
  if (ConstantSDNode *Cst = isConstOrConstSplat(N2)) {
    EVT ShAmtTy = N2.getValueType();
 
    // fold (fsh* N0, N1, c) -> (fsh* N0, N1, c % BitWidth)
    if (Cst->getAPIntValue().uge(BitWidth)) {
      uint64_t RotAmt = Cst->getAPIntValue().urem(BitWidth);
      return DAG.getNode(N->getOpcode(), DL, VT, N0, N1,
                         DAG.getConstant(RotAmt, DL, ShAmtTy));
    }
 
    unsigned ShAmt = Cst->getZExtValue();
    if (ShAmt == 0)
      return IsFSHL ? N0 : N1;
 
    // fold fshl(undef_or_zero, N1, C) -> lshr(N1, BW-C)
    // fold fshr(undef_or_zero, N1, C) -> lshr(N1, C)
    // fold fshl(N0, undef_or_zero, C) -> shl(N0, C)
    // fold fshr(N0, undef_or_zero, C) -> shl(N0, BW-C)
    if (IsUndefOrZero(N0))
      return DAG.getNode(
          ISD::SRL, DL, VT, N1,
          DAG.getConstant(IsFSHL ? BitWidth - ShAmt : ShAmt, DL, ShAmtTy));
    if (IsUndefOrZero(N1))
      return DAG.getNode(
          ISD::SHL, DL, VT, N0,
          DAG.getConstant(IsFSHL ? ShAmt : BitWidth - ShAmt, DL, ShAmtTy));
 
    // fold fshl(N0, N1, c) -> x  and  fshr(N0, N1, c) -> x
    // where N0 is any node that contributes "x >> C0" to the result:
    //   lshr(x, C0)  |  fshr(_, x, C0)  |  fshl(_, x, C1)
    // and N1 is any node that contributes "x << C1" to the result:
    //   shl(x, C1)   |  fshl(x, _, C1)  |  fshr(x, _, C0)
    // with C0 = IsFSHL ? amnt : BW-amnt,  C1 = BW - C0
 
    // ShAmt == 0 was handled above; uge(BitWidth) was reduced via modulo above.
    assert(ShAmt >= 1 && ShAmt < BitWidth &&
           "ShAmt must be in [1, BW-1] for the identity fold to be valid");
    SDValue Val;
    unsigned C0Expected = IsFSHL ? ShAmt : BitWidth - ShAmt;
    unsigned C1Expected = IsFSHL ? BitWidth - ShAmt : ShAmt;
 
    if ((sd_match(N0, m_Srl(m_Value(Val), m_SpecificInt(C0Expected))) ||
         sd_match(N0, m_Node(ISD::FSHR, m_Value(), m_Value(Val),
                             m_SpecificInt(C0Expected))) ||
         sd_match(N0, m_Node(ISD::FSHL, m_Value(), m_Value(Val),
                             m_SpecificInt(C1Expected)))) &&
        (sd_match(N1, m_Shl(m_Specific(Val), m_SpecificInt(C1Expected))) ||
         sd_match(N1, m_Node(ISD::FSHL, m_Specific(Val), m_Value(),
                             m_SpecificInt(C1Expected))) ||
         sd_match(N1, m_Node(ISD::FSHR, m_Specific(Val), m_Value(),
                             m_SpecificInt(C0Expected)))))
      return Val;
 
    // fold (fshl ld1, ld0, c) -> (ld0[ofs]) iff ld0 and ld1 are consecutive.
    // fold (fshr ld1, ld0, c) -> (ld0[ofs]) iff ld0 and ld1 are consecutive.
    // TODO - bigendian support once we have test coverage.
    // TODO - can we merge this with CombineConseutiveLoads/MatchLoadCombine?
    // TODO - permit LHS EXTLOAD if extensions are shifted out.
    if ((BitWidth % 8) == 0 && (ShAmt % 8) == 0 && !VT.isVector() &&
        !DAG.getDataLayout().isBigEndian()) {
      auto *LHS = dyn_cast<LoadSDNode>(N0);
      auto *RHS = dyn_cast<LoadSDNode>(N1);
      if (LHS && RHS && LHS->isSimple() && RHS->isSimple() &&
          LHS->getAddressSpace() == RHS->getAddressSpace() &&
          (LHS->hasNUsesOfValue(1, 0) || RHS->hasNUsesOfValue(1, 0)) &&
          ISD::isNON_EXTLoad(RHS) && ISD::isNON_EXTLoad(LHS)) {
        if (DAG.areNonVolatileConsecutiveLoads(LHS, RHS, BitWidth / 8, 1)) {
          SDLoc DL(RHS);
          uint64_t PtrOff =
              IsFSHL ? (((BitWidth - ShAmt) % BitWidth) / 8) : (ShAmt / 8);
          Align NewAlign = commonAlignment(RHS->getAlign(), PtrOff);
          unsigned Fast = 0;
          if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
                                     RHS->getAddressSpace(), NewAlign,
                                     RHS->getMemOperand()->getFlags(), &Fast) &&
              Fast) {
            SDValue NewPtr = DAG.getMemBasePlusOffset(
                RHS->getBasePtr(), TypeSize::getFixed(PtrOff), DL);
            AddToWorklist(NewPtr.getNode());
            SDValue Load = DAG.getLoad(
                VT, DL, RHS->getChain(), NewPtr,
                RHS->getPointerInfo().getWithOffset(PtrOff), NewAlign,
                RHS->getMemOperand()->getFlags(), RHS->getAAInfo());
            DAG.makeEquivalentMemoryOrdering(LHS, Load.getValue(1));
            DAG.makeEquivalentMemoryOrdering(RHS, Load.getValue(1));
            return Load;
          }
        }
      }
    }
  }
 
  // fold fshr(undef_or_zero, N1, N2) -> lshr(N1, N2)
  // fold fshl(N0, undef_or_zero, N2) -> shl(N0, N2)
  // iff We know the shift amount is in range.
  // TODO: when is it worth doing SUB(BW, N2) as well?
  if (isPowerOf2_32(BitWidth)) {
    APInt ModuloBits(N2.getScalarValueSizeInBits(), BitWidth - 1);
    if (IsUndefOrZero(N0) && !IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
      return DAG.getNode(ISD::SRL, DL, VT, N1, N2);
    if (IsUndefOrZero(N1) && IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits))
      return DAG.getNode(ISD::SHL, DL, VT, N0, N2);
  }
 
  // fold (fshl N0, N0, N2) -> (rotl N0, N2)
  // fold (fshr N0, N0, N2) -> (rotr N0, N2)
  // TODO: Investigate flipping this rotate if only one is legal.
  // If funnel shift is legal as well we might be better off avoiding
  // non-constant (BW - N2).
  unsigned RotOpc = IsFSHL ? ISD::ROTL : ISD::ROTR;
  if (N0 == N1 && hasOperation(RotOpc, VT))
    return DAG.getNode(RotOpc, DL, VT, N0, N2);
 
  // Simplify, based on bits shifted out of N0/N1.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSHLSAT(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  if (SDValue V = DAG.simplifyShift(N0, N1))
    return V;
 
  SDLoc DL(N);
  EVT VT = N0.getValueType();
 
  // fold (*shlsat c1, c2) -> c1<<c2
  if (SDValue C = DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1}))
    return C;
 
  ConstantSDNode *N1C = isConstOrConstSplat(N1);
 
  if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) {
    // fold (sshlsat x, c) -> (shl x, c)
    if (N->getOpcode() == ISD::SSHLSAT && N1C &&
        N1C->getAPIntValue().ult(DAG.ComputeNumSignBits(N0)))
      return DAG.getNode(ISD::SHL, DL, VT, N0, N1);
 
    // fold (ushlsat x, c) -> (shl x, c)
    if (N->getOpcode() == ISD::USHLSAT && N1C &&
        N1C->getAPIntValue().ule(
            DAG.computeKnownBits(N0).countMinLeadingZeros()))
      return DAG.getNode(ISD::SHL, DL, VT, N0, N1);
  }
 
  return SDValue();
}
 
// Given a ABS node, detect the following patterns:
// (ABS (SUB (EXTEND a), (EXTEND b))).
// (TRUNC (ABS (SUB (EXTEND a), (EXTEND b)))).
// Generates UABD/SABD instruction.
SDValue DAGCombiner::foldABSToABD(SDNode *N, const SDLoc &DL) {
  EVT SrcVT = N->getValueType(0);
 
  if (N->getOpcode() == ISD::TRUNCATE)
    N = N->getOperand(0).getNode();
 
  EVT VT = N->getValueType(0);
  SDValue Op0, Op1;
 
  if (!sd_match(N, m_Abs(m_AnyOf(m_Sub(m_Value(Op0), m_Value(Op1)),
                                 m_Add(m_Value(Op0), m_Value(Op1))))))
    return SDValue();
 
  SDValue AbsOp0 = N->getOperand(0);
  bool IsAdd = AbsOp0.getOpcode() == ISD::ADD;
  // Make sure (abs B) is positive.
  if (IsAdd) {
    // Elements of Op1 must be constant and != VT.minSignedValue() (or undef)
    auto IsNotMinSignedInt = [VT](ConstantSDNode *C) {
      if (C == nullptr)
        return true;
      return !C->getAPIntValue()
                  .trunc(VT.getScalarSizeInBits())
                  .isMinSignedValue();
    };
 
    if (!ISD::matchUnaryPredicate(Op1, IsNotMinSignedInt, /*AllowUndefs=*/true,
                                  /*AllowTruncation=*/true))
      return SDValue();
  }
 
  unsigned Opc0 = Op0.getOpcode();
 
  // Check if the operands of the sub are (zero|sign)-extended, otherwise
  // fallback to ValueTracking.
  if (Opc0 != Op1.getOpcode() ||
      (Opc0 != ISD::ZERO_EXTEND && Opc0 != ISD::SIGN_EXTEND &&
       Opc0 != ISD::SIGN_EXTEND_INREG)) {
 
    auto CreateZextedAbd = [&](unsigned AbdOpc) {
      if (IsAdd)
        Op1 = DAG.getNegative(Op1, SDLoc(Op1), VT);
      SDValue ABD = DAG.getNode(AbdOpc, DL, VT, Op0, Op1);
      return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
    };
 
    // fold (abs (sub nsw x, y)) -> abds(x, y)
    // fold (abs (add nsw x, -y)) -> abds(x, y)
    bool AbsOpWillNSW =
        AbsOp0->getFlags().hasNoSignedWrap() ||
        (IsAdd ? DAG.willNotOverflowAdd(/*IsSigned=*/true, Op0, Op1)
               : DAG.willNotOverflowSub(/*IsSigned=*/true, Op0, Op1));
 
    // Don't fold this for unsupported types as we lose the NSW handling.
    if (hasOperation(ISD::ABDS, VT) && TLI.preferABDSToABSWithNSW(VT) &&
        AbsOpWillNSW)
      return CreateZextedAbd(ISD::ABDS);
 
    // fold (abs (sub x, y)) -> abdu(x, y)
    bool Op1SignBitIsOne = DAG.computeKnownBits(Op1).isNegative();
    bool AbsOpWillNUW = !IsAdd && DAG.SignBitIsZero(Op0) && Op1SignBitIsOne;
 
    if (hasOperation(ISD::ABDU, VT) && AbsOpWillNUW)
      return CreateZextedAbd(ISD::ABDU);
 
    return SDValue();
  }
 
  // The IsAdd case explicitly checks for const/bv-of-const. This implies either
  // (Opc0 != Op1.getOpcode() || Opc0 is not in {zext/sext/sign_ext_inreg}. This
  // implies it was alrady handled by the above if statement.
  assert(!IsAdd && "Unexpected abs(add(x,y)) pattern");
 
  EVT VT0, VT1;
  if (Opc0 == ISD::SIGN_EXTEND_INREG) {
    VT0 = cast<VTSDNode>(Op0.getOperand(1))->getVT();
    VT1 = cast<VTSDNode>(Op1.getOperand(1))->getVT();
  } else {
    VT0 = Op0.getOperand(0).getValueType();
    VT1 = Op1.getOperand(0).getValueType();
  }
  unsigned ABDOpcode = (Opc0 == ISD::ZERO_EXTEND) ? ISD::ABDU : ISD::ABDS;
 
  // fold abs(sext(x) - sext(y)) -> zext(abds(x, y))
  // fold abs(zext(x) - zext(y)) -> zext(abdu(x, y))
  EVT MaxVT = VT0.bitsGT(VT1) ? VT0 : VT1;
  if ((VT0 == MaxVT || Op0->hasOneUse()) &&
      (VT1 == MaxVT || Op1->hasOneUse()) &&
      (!LegalTypes || hasOperation(ABDOpcode, MaxVT))) {
    SDValue ABD = DAG.getNode(ABDOpcode, DL, MaxVT,
                              DAG.getNode(ISD::TRUNCATE, DL, MaxVT, Op0),
                              DAG.getNode(ISD::TRUNCATE, DL, MaxVT, Op1));
    ABD = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ABD);
    return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
  }
 
  // fold abs(sext(x) - sext(y)) -> abds(sext(x), sext(y))
  // fold abs(zext(x) - zext(y)) -> abdu(zext(x), zext(y))
  if (!LegalOperations || hasOperation(ABDOpcode, VT)) {
    SDValue ABD = DAG.getNode(ABDOpcode, DL, VT, Op0, Op1);
    return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitABS(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (abs c1) -> c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::ABS, DL, VT, {N0}))
    return C;
  // fold (abs (abs x)) -> (abs x)
  // fold (abs (abs_min_poison x)) -> (abs_min_poison x)
  if (ISD::isAbsOpcode(N0.getOpcode()))
    return N0;
  // fold (abs x) -> x iff not-negative
  if (DAG.SignBitIsZero(N0))
    return N0;
 
  if (SDValue ABD = foldABSToABD(N, DL))
    return ABD;
 
  // fold (abs (sign_extend_inreg x)) -> (zero_extend (abs (truncate x)))
  // iff zero_extend/truncate are free.
  if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    EVT ExtVT = cast<VTSDNode>(N0.getOperand(1))->getVT();
    if (TLI.isTruncateFree(VT, ExtVT) && TLI.isZExtFree(ExtVT, VT) &&
        TLI.isTypeDesirableForOp(ISD::ABS, ExtVT) &&
        hasOperation(ISD::ABS, ExtVT)) {
      return DAG.getNode(
          ISD::ZERO_EXTEND, DL, VT,
          DAG.getNode(ISD::ABS, DL, ExtVT,
                      DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N0.getOperand(0))));
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitABS_MIN_POISON(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (abs_min_poison c1) -> c2 (or poison if c1 == INT_MIN)
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::ABS_MIN_POISON, DL, VT, {N0}))
    return C;
  // fold (abs_min_poison (abs_min_poison x)) -> (abs_min_poison x)
  // fold (abs_min_poison (abs x)) -> (abs x)
  // fold (abs_min_poison (freeze (abs x))) -> (freeze (abs x))
  // fold (abs_min_poison (freeze (abs_min_poison x))) ->
  //   (freeze (abs_min_poison x))
  //
  // Freeze case is valid because: for x != INT_MIN both sides equal abs(x);
  // for x == INT_MIN both forms produce a non-deterministic but well-defined
  // value since freeze already consumed the poison.
  if (ISD::isAbsOpcode(peekThroughFreeze(N0).getOpcode()))
    return N0;
  // fold (abs_min_poison x) -> x iff not-negative
  if (DAG.SignBitIsZero(N0))
    return N0;
 
  if (SDValue ABD = foldABSToABD(N, DL))
    return ABD;
 
  // fold (abs_min_poison (sign_extend_inreg x)) ->
  //   (zero_extend (abs (truncate x)))
  // iff zero_extend/truncate are free. The sign_extend_inreg keeps the value
  // in the narrow type's range, so the wide abs_min_poison is never actually
  // poison.
  if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    EVT ExtVT = cast<VTSDNode>(N0.getOperand(1))->getVT();
    if (TLI.isTruncateFree(VT, ExtVT) && TLI.isZExtFree(ExtVT, VT) &&
        TLI.isTypeDesirableForOp(ISD::ABS, ExtVT) &&
        hasOperation(ISD::ABS, ExtVT)) {
      return DAG.getNode(
          ISD::ZERO_EXTEND, DL, VT,
          DAG.getNode(ISD::ABS, DL, ExtVT,
                      DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N0.getOperand(0))));
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitCLMUL(SDNode *N) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (clmul c1, c2)
  if (SDValue C = DAG.FoldConstantArithmetic(Opcode, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS
  if (DAG.isConstantIntBuildVectorOrConstantInt(N0) &&
      !DAG.isConstantIntBuildVectorOrConstantInt(N1))
    return DAG.getNode(Opcode, DL, VT, N1, N0);
 
  // fold (clmul x, 0) -> 0
  if (isNullConstant(N1) || ISD::isConstantSplatVectorAllZeros(N1.getNode()))
    return DAG.getConstant(0, DL, VT);
 
  // fold (clmul x, c_pow2) -> (shl x, log2(c_pow2))
  // This also handles (clmul x, 1) -> x since (shl x, 0) simplifies to x.
  if (Opcode == ISD::CLMUL) {
    if (ConstantSDNode *C = isConstOrConstSplat(N1)) {
      APInt CV = C->getAPIntValue().trunc(VT.getScalarSizeInBits());
      if (CV.isPowerOf2() &&
          (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)))
        return DAG.getNode(ISD::SHL, DL, VT, N0,
                           DAG.getShiftAmountConstant(CV.logBase2(), VT, DL));
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitBSWAP(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (bswap c1) -> c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::BSWAP, DL, VT, {N0}))
    return C;
  // fold (bswap (bswap x)) -> x
  if (N0.getOpcode() == ISD::BSWAP)
    return N0.getOperand(0);
 
  // Canonicalize bswap(bitreverse(x)) -> bitreverse(bswap(x)). If bitreverse
  // isn't supported, it will be expanded to bswap followed by a manual reversal
  // of bits in each byte. By placing bswaps before bitreverse, we can remove
  // the two bswaps if the bitreverse gets expanded.
  if (N0.getOpcode() == ISD::BITREVERSE && N0.hasOneUse()) {
    SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0));
    return DAG.getNode(ISD::BITREVERSE, DL, VT, BSwap);
  }
 
  unsigned BW = VT.getScalarSizeInBits();
  // fold (bswap shl(x,c)) -> (zext(bswap(trunc(shl(x,sub(c,bw/2))))))
  // iff x >= bw/2 (i.e. lower half is known zero)
  if (BW >= 32 && N0.getOpcode() == ISD::SHL && N0.hasOneUse()) {
    auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), BW / 2);
    if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
        ShAmt->getZExtValue() >= (BW / 2) && (ShAmt->getZExtValue() % 8) == 0 &&
        TLI.isTypeLegal(HalfVT) && TLI.isTruncateFree(VT, HalfVT) &&
        (!LegalOperations || hasOperation(ISD::BSWAP, HalfVT))) {
      SDValue Res = N0.getOperand(0);
      if (uint64_t NewShAmt = (ShAmt->getZExtValue() - (BW / 2)))
        Res = DAG.getNode(ISD::SHL, DL, VT, Res,
                          DAG.getShiftAmountConstant(NewShAmt, VT, DL));
      Res = DAG.getZExtOrTrunc(Res, DL, HalfVT);
      Res = DAG.getNode(ISD::BSWAP, DL, HalfVT, Res);
      return DAG.getZExtOrTrunc(Res, DL, VT);
    }
  }
 
  // Try to canonicalize bswap-of-logical-shift-by-8-bit-multiple as
  // inverse-shift-of-bswap:
  // bswap (X u<< C) --> (bswap X) u>> C
  // bswap (X u>> C) --> (bswap X) u<< C
  if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
      N0.hasOneUse()) {
    auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1));
    if (ShAmt && ShAmt->getAPIntValue().ult(BW) &&
        ShAmt->getZExtValue() % 8 == 0) {
      SDValue NewSwap = DAG.getNode(ISD::BSWAP, DL, VT, N0.getOperand(0));
      unsigned InverseShift = N0.getOpcode() == ISD::SHL ? ISD::SRL : ISD::SHL;
      return DAG.getNode(InverseShift, DL, VT, NewSwap, N0.getOperand(1));
    }
  }
 
  if (SDValue V = foldBitOrderCrossLogicOp(N, DAG))
    return V;
 
  // Folds that depend on computeKnownBits of the operand.
  KnownBits Known = DAG.computeKnownBits(N0);
  // bswap(0) = 0. Catch cases that computeKnownBits can prove are zero but
  // that structural combines haven't simplified to a constant yet
  // (e.g. and of disjoint byte masks).
  if (Known.isZero())
    return DAG.getConstant(0, DL, VT);
  // If only one byte of the operand may be nonzero, bswap becomes a shift
  // to the mirror byte.
  unsigned TZ = alignDown(Known.countMinTrailingZeros(), 8);
  unsigned LZ = alignDown(Known.countMinLeadingZeros(), 8);
  if (BW - (LZ + TZ) == 8) {
    unsigned Opc = LZ > TZ ? ISD::SHL : ISD::SRL;
    // Skip if the target would re-expand the produced shift post-legalize.
    // Targets that custom-lower byte-multiple shifts via bswap (e.g. MSP430
    // for shl i16) would loop with this combine.
    if (!LegalOperations || hasOperation(Opc, VT)) {
      unsigned Amt = AbsoluteDifference(LZ, TZ);
      SDNodeFlags Flags =
          Opc == ISD::SHL ? SDNodeFlags::NoUnsignedWrap : SDNodeFlags::Exact;
      return DAG.getNode(Opc, DL, VT, N0,
                         DAG.getShiftAmountConstant(Amt, VT, DL), Flags);
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitBITREVERSE(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (bitreverse c1) -> c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::BITREVERSE, DL, VT, {N0}))
    return C;
 
  // fold (bitreverse (bitreverse x)) -> x
  if (N0.getOpcode() == ISD::BITREVERSE)
    return N0.getOperand(0);
 
  SDValue X, Y;
 
  // fold (bitreverse (lshr (bitreverse x), y)) -> (shl x, y)
  if ((!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) &&
      sd_match(N0, m_Srl(m_BitReverse(m_Value(X)), m_Value(Y))))
    return DAG.getNode(ISD::SHL, DL, VT, X, Y);
 
  // fold (bitreverse (shl (bitreverse x), y)) -> (lshr x, y)
  if ((!LegalOperations || TLI.isOperationLegal(ISD::SRL, VT)) &&
      sd_match(N0, m_Shl(m_BitReverse(m_Value(X)), m_Value(Y))))
    return DAG.getNode(ISD::SRL, DL, VT, X, Y);
 
  // fold bitreverse(clmul(bitreverse(x), bitreverse(y))) -> clmulr(x, y)
  if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::CLMULR, VT)) &&
      sd_match(N0, m_Clmul(m_BitReverse(m_Value(X)), m_BitReverse(m_Value(Y)))))
    return DAG.getNode(ISD::CLMULR, DL, VT, X, Y);
 
  return SDValue();
}
 
// Fold (ctlz (xor x, (sra x, bitwidth-1))) -> (add (ctls x), 1).
// Fold (ctlz (or (shl (xor x, (sra x, bitwidth-1)), 1), 1) -> (ctls x)
SDValue DAGCombiner::foldCTLZToCTLS(SDValue Src, const SDLoc &DL) {
  EVT VT = Src.getValueType();
 
  auto LK = TLI.getTypeConversion(*DAG.getContext(), VT);
  if ((LK.first != TargetLoweringBase::TypeLegal &&
       LK.first != TargetLoweringBase::TypePromoteInteger) ||
      !TLI.isOperationLegalOrCustom(ISD::CTLS, LK.second))
    return SDValue();
 
  unsigned BitWidth = VT.getScalarSizeInBits();
 
  bool NeedAdd = true;
 
  SDValue X;
  if (sd_match(Src,
               m_OneUse(m_Or(m_OneUse(m_Shl(m_Value(X), m_One())), m_One())))) {
    NeedAdd = false;
    Src = X;
  }
 
  if (!sd_match(Src,
                m_OneUse(m_Xor(m_Value(X),
                               m_OneUse(m_Sra(m_Deferred(X),
                                              m_SpecificInt(BitWidth - 1)))))))
    return SDValue();
 
  SDValue Res = DAG.getNode(ISD::CTLS, DL, VT, X);
  if (!NeedAdd)
    return Res;
 
  return DAG.getNode(ISD::ADD, DL, VT, Res, DAG.getConstant(1, DL, VT));
}
 
SDValue DAGCombiner::visitCTLZ(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (ctlz c1) -> c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTLZ, DL, VT, {N0}))
    return C;
 
  // If the value is known never to be zero, switch to the poison version.
  if (!LegalOperations || TLI.isOperationLegal(ISD::CTLZ_ZERO_POISON, VT))
    if (DAG.isKnownNeverZero(N0))
      return DAG.getNode(ISD::CTLZ_ZERO_POISON, DL, VT, N0);
 
  if (SDValue V = foldCTLZToCTLS(N0, DL))
    return V;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitCTLZ_ZERO_POISON(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (ctlz_zero_poison c1) -> c2
  if (SDValue C =
          DAG.FoldConstantArithmetic(ISD::CTLZ_ZERO_POISON, DL, VT, {N0}))
    return C;
 
  if (SDValue V = foldCTLZToCTLS(N0, DL))
    return V;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitCTTZ(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (cttz c1) -> c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTTZ, DL, VT, {N0}))
    return C;
 
  // If the value is known never to be zero, switch to the poison version.
  if (!LegalOperations || TLI.isOperationLegal(ISD::CTTZ_ZERO_POISON, VT))
    if (DAG.isKnownNeverZero(N0))
      return DAG.getNode(ISD::CTTZ_ZERO_POISON, DL, VT, N0);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitCTTZ_ZERO_POISON(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (cttz_zero_poison c1) -> c2
  if (SDValue C =
          DAG.FoldConstantArithmetic(ISD::CTTZ_ZERO_POISON, DL, VT, {N0}))
    return C;
  return SDValue();
}
 
SDValue DAGCombiner::visitCTPOP(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  unsigned NumBits = VT.getScalarSizeInBits();
  SDLoc DL(N);
 
  // fold (ctpop c1) -> c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::CTPOP, DL, VT, {N0}))
    return C;
 
  // If the source is being shifted, but doesn't affect any active bits,
  // then we can call CTPOP on the shift source directly.
  if (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SHL) {
    if (ConstantSDNode *AmtC = isConstOrConstSplat(N0.getOperand(1))) {
      const APInt &Amt = AmtC->getAPIntValue();
      if (Amt.ult(NumBits)) {
        KnownBits KnownSrc = DAG.computeKnownBits(N0.getOperand(0));
        if ((N0.getOpcode() == ISD::SRL &&
             Amt.ule(KnownSrc.countMinTrailingZeros())) ||
            (N0.getOpcode() == ISD::SHL &&
             Amt.ule(KnownSrc.countMinLeadingZeros()))) {
          return DAG.getNode(ISD::CTPOP, DL, VT, N0.getOperand(0));
        }
      }
    }
  }
 
  // If the upper bits are known to be zero, then see if its profitable to
  // only count the lower bits.
  if (VT.isScalarInteger() && NumBits > 8 && (NumBits & 1) == 0) {
    EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), NumBits / 2);
    if (hasOperation(ISD::CTPOP, HalfVT) &&
        TLI.isTypeDesirableForOp(ISD::CTPOP, HalfVT) &&
        TLI.isTruncateFree(N0, HalfVT) && TLI.isZExtFree(HalfVT, VT)) {
      APInt UpperBits = APInt::getHighBitsSet(NumBits, NumBits / 2);
      if (DAG.MaskedValueIsZero(N0, UpperBits)) {
        SDValue PopCnt = DAG.getNode(ISD::CTPOP, DL, HalfVT,
                                     DAG.getZExtOrTrunc(N0, DL, HalfVT));
        return DAG.getZExtOrTrunc(PopCnt, DL, VT);
      }
    }
  }
 
  return SDValue();
}
 
static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS,
                                         SDValue RHS, const SDNodeFlags Flags,
                                         const TargetLowering &TLI) {
  EVT VT = LHS.getValueType();
  if (!VT.isFloatingPoint())
    return false;
 
  return Flags.hasNoSignedZeros() &&
         TLI.isProfitableToCombineMinNumMaxNum(VT) &&
         (Flags.hasNoNaNs() ||
          (DAG.isKnownNeverNaN(RHS) && DAG.isKnownNeverNaN(LHS)));
}
 
static SDValue combineMinNumMaxNumImpl(const SDLoc &DL, EVT VT, SDValue LHS,
                                       SDValue RHS, SDValue True, SDValue False,
                                       ISD::CondCode CC,
                                       const TargetLowering &TLI,
                                       SelectionDAG &DAG) {
  EVT TransformVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
  switch (CC) {
  case ISD::SETOLT:
  case ISD::SETOLE:
  case ISD::SETLT:
  case ISD::SETLE:
  case ISD::SETULT:
  case ISD::SETULE: {
    // Since it's known never nan to get here already, either fminnum or
    // fminnum_ieee are OK. Try the ieee version first, since it's fminnum is
    // expanded in terms of it.
    unsigned IEEEOpcode = (LHS == True) ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE;
    if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT))
      return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS);
 
    unsigned Opcode = (LHS == True) ? ISD::FMINNUM : ISD::FMAXNUM;
    if (TLI.isOperationLegalOrCustom(Opcode, TransformVT))
      return DAG.getNode(Opcode, DL, VT, LHS, RHS);
    return SDValue();
  }
  case ISD::SETOGT:
  case ISD::SETOGE:
  case ISD::SETGT:
  case ISD::SETGE:
  case ISD::SETUGT:
  case ISD::SETUGE: {
    unsigned IEEEOpcode = (LHS == True) ? ISD::FMAXNUM_IEEE : ISD::FMINNUM_IEEE;
    if (TLI.isOperationLegalOrCustom(IEEEOpcode, VT))
      return DAG.getNode(IEEEOpcode, DL, VT, LHS, RHS);
 
    unsigned Opcode = (LHS == True) ? ISD::FMAXNUM : ISD::FMINNUM;
    if (TLI.isOperationLegalOrCustom(Opcode, TransformVT))
      return DAG.getNode(Opcode, DL, VT, LHS, RHS);
    return SDValue();
  }
  default:
    return SDValue();
  }
}
 
// Convert (sr[al] (add n[su]w x, y)) -> (avgfloor[su] x, y)
SDValue DAGCombiner::foldShiftToAvg(SDNode *N, const SDLoc &DL) {
  const unsigned Opcode = N->getOpcode();
  if (Opcode != ISD::SRA && Opcode != ISD::SRL)
    return SDValue();
 
  EVT VT = N->getValueType(0);
  bool IsUnsigned = Opcode == ISD::SRL;
 
  // Captured values.
  SDValue A, B;
 
  // Match floor average as it is common to both floor/ceil avgs, ensure the add
  // doesn't wrap.
  SDNodeFlags Flags =
      IsUnsigned ? SDNodeFlags::NoUnsignedWrap : SDNodeFlags::NoSignedWrap;
  if (sd_match(N, m_BinOp(Opcode,
                          m_c_BinOp(ISD::ADD, m_Value(A), m_Value(B), Flags),
                          m_One()))) {
    // Decide whether signed or unsigned.
    unsigned FloorISD = IsUnsigned ? ISD::AVGFLOORU : ISD::AVGFLOORS;
    if (hasOperation(FloorISD, VT))
      return DAG.getNode(FloorISD, DL, VT, {A, B});
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::foldBitwiseOpWithNeg(SDNode *N, const SDLoc &DL, EVT VT) {
  unsigned Opc = N->getOpcode();
  SDValue X, Y, Z;
  if (sd_match(
          N, m_BitwiseLogic(m_Value(X), m_Add(m_Not(m_Value(Y)), m_Value(Z)))))
    return DAG.getNode(Opc, DL, VT, X,
                       DAG.getNOT(DL, DAG.getNode(ISD::SUB, DL, VT, Y, Z), VT));
 
  if (sd_match(N, m_BitwiseLogic(m_Value(X), m_Sub(m_OneUse(m_Not(m_Value(Y))),
                                                   m_Value(Z)))))
    return DAG.getNode(Opc, DL, VT, X,
                       DAG.getNOT(DL, DAG.getNode(ISD::ADD, DL, VT, Y, Z), VT));
 
  return SDValue();
}
 
/// Generate Min/Max node
SDValue DAGCombiner::combineMinNumMaxNum(const SDLoc &DL, EVT VT, SDValue LHS,
                                         SDValue RHS, SDValue True,
                                         SDValue False, ISD::CondCode CC) {
  if ((LHS == True && RHS == False) || (LHS == False && RHS == True))
    return combineMinNumMaxNumImpl(DL, VT, LHS, RHS, True, False, CC, TLI, DAG);
 
  // If we can't directly match this, try to see if we can pull an fneg out of
  // the select.
  SDValue NegTrue = TLI.getCheaperOrNeutralNegatedExpression(
      True, DAG, LegalOperations, ForCodeSize);
  if (!NegTrue)
    return SDValue();
 
  HandleSDNode NegTrueHandle(NegTrue);
 
  // Try to unfold an fneg from the select if we are comparing the negated
  // constant.
  //
  // select (setcc x, K) (fneg x), -K -> fneg(minnum(x, K))
  //
  // TODO: Handle fabs
  if (LHS == NegTrue) {
    // If we can't directly match this, try to see if we can pull an fneg out of
    // the select.
    SDValue NegRHS = TLI.getCheaperOrNeutralNegatedExpression(
        RHS, DAG, LegalOperations, ForCodeSize);
    if (NegRHS) {
      HandleSDNode NegRHSHandle(NegRHS);
      if (NegRHS == False) {
        SDValue Combined = combineMinNumMaxNumImpl(DL, VT, LHS, RHS, NegTrue,
                                                   False, CC, TLI, DAG);
        if (Combined)
          return DAG.getNode(ISD::FNEG, DL, VT, Combined);
      }
    }
  }
 
  return SDValue();
}
 
/// If a (v)select has a condition value that is a sign-bit test, try to smear
/// the condition operand sign-bit across the value width and use it as a mask.
static SDValue foldSelectOfConstantsUsingSra(SDNode *N, const SDLoc &DL,
                                             SelectionDAG &DAG) {
  SDValue Cond = N->getOperand(0);
  SDValue C1 = N->getOperand(1);
  SDValue C2 = N->getOperand(2);
  if (!isConstantOrConstantVector(C1) || !isConstantOrConstantVector(C2))
    return SDValue();
 
  EVT VT = N->getValueType(0);
  if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse() ||
      VT != Cond.getOperand(0).getValueType())
    return SDValue();
 
  // The inverted-condition + commuted-select variants of these patterns are
  // canonicalized to these forms in IR.
  SDValue X = Cond.getOperand(0);
  SDValue CondC = Cond.getOperand(1);
  ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
  if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(CondC) &&
      isAllOnesOrAllOnesSplat(C2)) {
    // i32 X > -1 ? C1 : -1 --> (X >>s 31) | C1
    SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
    SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
    return DAG.getNode(ISD::OR, DL, VT, Sra, C1);
  }
  if (CC == ISD::SETLT && isNullOrNullSplat(CondC) && isNullOrNullSplat(C2)) {
    // i8 X < 0 ? C1 : 0 --> (X >>s 7) & C1
    SDValue ShAmtC = DAG.getConstant(X.getScalarValueSizeInBits() - 1, DL, VT);
    SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, X, ShAmtC);
    return DAG.getNode(ISD::AND, DL, VT, Sra, C1);
  }
  return SDValue();
}
 
static bool shouldConvertSelectOfConstantsToMath(const SDValue &Cond, EVT VT,
                                                 const TargetLowering &TLI) {
  if (!TLI.convertSelectOfConstantsToMath(VT))
    return false;
 
  if (Cond.getOpcode() != ISD::SETCC || !Cond->hasOneUse())
    return true;
  if (!TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))
    return true;
 
  ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
  if (CC == ISD::SETLT && isNullOrNullSplat(Cond.getOperand(1)))
    return true;
  if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond.getOperand(1)))
    return true;
 
  return false;
}
 
SDValue DAGCombiner::foldSelectOfConstants(SDNode *N) {
  SDValue Cond = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  EVT VT = N->getValueType(0);
  EVT CondVT = Cond.getValueType();
  SDLoc DL(N);
 
  if (!VT.isInteger())
    return SDValue();
 
  auto *C1 = dyn_cast<ConstantSDNode>(N1);
  auto *C2 = dyn_cast<ConstantSDNode>(N2);
  if (!C1 || !C2)
    return SDValue();
 
  if (CondVT != MVT::i1 || LegalOperations) {
    // We can't do this reliably if integer based booleans have different contents
    // to floating point based booleans. This is because we can't tell whether we
    // have an integer-based boolean or a floating-point-based boolean unless we
    // can find the SETCC that produced it and inspect its operands. This is
    // fairly easy if C is the SETCC node, but it can potentially be
    // undiscoverable (or not reasonably discoverable). For example, it could be
    // in another basic block or it could require searching a complicated
    // expression.
    if (CondVT.isInteger() &&
        TLI.getBooleanContents(/*isVec*/false, /*isFloat*/true) ==
            TargetLowering::ZeroOrOneBooleanContent &&
        TLI.getBooleanContents(/*isVec*/false, /*isFloat*/false) ==
            TargetLowering::ZeroOrOneBooleanContent) {
      // fold (select Cond, 0, 1) -> (xor Cond, 1)
      if (C1->isZero() && C2->isOne()) {
        SDValue NotCond = DAG.getNode(ISD::XOR, DL, CondVT, Cond,
                                      DAG.getConstant(1, DL, CondVT));
        if (VT.bitsEq(CondVT))
          return NotCond;
        return DAG.getZExtOrTrunc(NotCond, DL, VT);
      }
 
      // fold (select Cond, 1, 0) -> Cond
      if (C1->isOne() && C2->isZero() && CondVT == VT)
        return Cond;
    }
 
    return SDValue();
  }
 
  // Only do this before legalization to avoid conflicting with target-specific
  // transforms in the other direction (create a select from a zext/sext). There
  // is also a target-independent combine here in DAGCombiner in the other
  // direction for (select Cond, -1, 0) when the condition is not i1.
  assert(CondVT == MVT::i1 && !LegalOperations);
 
  // select Cond, 1, 0 --> zext (Cond)
  if (C1->isOne() && C2->isZero())
    return DAG.getZExtOrTrunc(Cond, DL, VT);
 
  // select Cond, -1, 0 --> sext (Cond)
  if (C1->isAllOnes() && C2->isZero())
    return DAG.getSExtOrTrunc(Cond, DL, VT);
 
  // select Cond, 0, 1 --> zext (!Cond)
  if (C1->isZero() && C2->isOne()) {
    SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    NotCond = DAG.getZExtOrTrunc(NotCond, DL, VT);
    return NotCond;
  }
 
  // select Cond, 0, -1 --> sext (!Cond)
  if (C1->isZero() && C2->isAllOnes()) {
    SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    NotCond = DAG.getSExtOrTrunc(NotCond, DL, VT);
    return NotCond;
  }
 
  // Use a target hook because some targets may prefer to transform in the
  // other direction.
  if (!shouldConvertSelectOfConstantsToMath(Cond, VT, TLI))
    return SDValue();
 
  // For any constants that differ by 1, we can transform the select into
  // an extend and add.
  const APInt &C1Val = C1->getAPIntValue();
  const APInt &C2Val = C2->getAPIntValue();
 
  // select Cond, C1, C1-1 --> add (zext Cond), C1-1
  if (C1Val - 1 == C2Val) {
    Cond = DAG.getZExtOrTrunc(Cond, DL, VT);
    return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
  }
 
  // select Cond, C1, C1+1 --> add (sext Cond), C1+1
  if (C1Val + 1 == C2Val) {
    Cond = DAG.getSExtOrTrunc(Cond, DL, VT);
    return DAG.getNode(ISD::ADD, DL, VT, Cond, N2);
  }
 
  // select Cond, Pow2, 0 --> (zext Cond) << log2(Pow2)
  if (C1Val.isPowerOf2() && C2Val.isZero()) {
    Cond = DAG.getZExtOrTrunc(Cond, DL, VT);
    SDValue ShAmtC =
        DAG.getShiftAmountConstant(C1Val.exactLogBase2(), VT, DL);
    return DAG.getNode(ISD::SHL, DL, VT, Cond, ShAmtC);
  }
 
  // select Cond, -1, C --> or (sext Cond), C
  if (C1->isAllOnes()) {
    Cond = DAG.getSExtOrTrunc(Cond, DL, VT);
    return DAG.getNode(ISD::OR, DL, VT, Cond, N2);
  }
 
  // select Cond, C, -1 --> or (sext (not Cond)), C
  if (C2->isAllOnes()) {
    SDValue NotCond = DAG.getNOT(DL, Cond, MVT::i1);
    NotCond = DAG.getSExtOrTrunc(NotCond, DL, VT);
    return DAG.getNode(ISD::OR, DL, VT, NotCond, N1);
  }
 
  if (SDValue V = foldSelectOfConstantsUsingSra(N, DL, DAG))
    return V;
 
  return SDValue();
}
 
template <class MatchContextClass>
static SDValue foldBoolSelectToLogic(SDNode *N, const SDLoc &DL,
                                     SelectionDAG &DAG) {
  assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT ||
          N->getOpcode() == ISD::VP_SELECT) &&
         "Expected a (v)(vp.)select");
  SDValue Cond = N->getOperand(0);
  SDValue T = N->getOperand(1), F = N->getOperand(2);
  EVT VT = N->getValueType(0);
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  MatchContextClass matcher(DAG, TLI, N);
 
  if (VT != Cond.getValueType() || VT.getScalarSizeInBits() != 1)
    return SDValue();
 
  // select Cond, Cond, F --> or Cond, freeze(F)
  // select Cond, 1, F    --> or Cond, freeze(F)
  if (Cond == T || isOneOrOneSplat(T, /* AllowUndefs */ true))
    return matcher.getNode(ISD::OR, DL, VT, Cond, DAG.getFreeze(F));
 
  // select Cond, T, Cond --> and Cond, freeze(T)
  // select Cond, T, 0    --> and Cond, freeze(T)
  if (Cond == F || isNullOrNullSplat(F, /* AllowUndefs */ true))
    return matcher.getNode(ISD::AND, DL, VT, Cond, DAG.getFreeze(T));
 
  // select Cond, T, 1 --> or (not Cond), freeze(T)
  if (isOneOrOneSplat(F, /* AllowUndefs */ true)) {
    SDValue NotCond =
        matcher.getNode(ISD::XOR, DL, VT, Cond, DAG.getAllOnesConstant(DL, VT));
    return matcher.getNode(ISD::OR, DL, VT, NotCond, DAG.getFreeze(T));
  }
 
  // select Cond, 0, F --> and (not Cond), freeze(F)
  if (isNullOrNullSplat(T, /* AllowUndefs */ true)) {
    SDValue NotCond =
        matcher.getNode(ISD::XOR, DL, VT, Cond, DAG.getAllOnesConstant(DL, VT));
    return matcher.getNode(ISD::AND, DL, VT, NotCond, DAG.getFreeze(F));
  }
 
  return SDValue();
}
 
static SDValue foldVSelectToSignBitSplatMask(SDNode *N, SelectionDAG &DAG) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  EVT VT = N->getValueType(0);
  unsigned EltSizeInBits = VT.getScalarSizeInBits();
 
  SDValue Cond0, Cond1;
  ISD::CondCode CC;
  if (!sd_match(N0, m_OneUse(m_SetCC(m_Value(Cond0), m_Value(Cond1),
                                     m_CondCode(CC)))) ||
      VT != Cond0.getValueType())
    return SDValue();
 
  // Match a signbit check of Cond0 as "Cond0 s<0". Swap select operands if the
  // compare is inverted from that pattern ("Cond0 s> -1").
  if (CC == ISD::SETLT && isNullOrNullSplat(Cond1))
    ; // This is the pattern we are looking for.
  else if (CC == ISD::SETGT && isAllOnesOrAllOnesSplat(Cond1))
    std::swap(N1, N2);
  else
    return SDValue();
 
  // (Cond0 s< 0) ? N1 : 0 --> (Cond0 s>> BW-1) & freeze(N1)
  if (isNullOrNullSplat(N2)) {
    SDLoc DL(N);
    SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL);
    SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
    return DAG.getNode(ISD::AND, DL, VT, Sra, DAG.getFreeze(N1));
  }
 
  // (Cond0 s< 0) ? -1 : N2 --> (Cond0 s>> BW-1) | freeze(N2)
  if (isAllOnesOrAllOnesSplat(N1)) {
    SDLoc DL(N);
    SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL);
    SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
    return DAG.getNode(ISD::OR, DL, VT, Sra, DAG.getFreeze(N2));
  }
 
  // If we have to invert the sign bit mask, only do that transform if the
  // target has a bitwise 'and not' instruction (the invert is free).
  // (Cond0 s< -0) ? 0 : N2 --> ~(Cond0 s>> BW-1) & freeze(N2)
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  if (isNullOrNullSplat(N1) && TLI.hasAndNot(N1)) {
    SDLoc DL(N);
    SDValue ShiftAmt = DAG.getShiftAmountConstant(EltSizeInBits - 1, VT, DL);
    SDValue Sra = DAG.getNode(ISD::SRA, DL, VT, Cond0, ShiftAmt);
    SDValue Not = DAG.getNOT(DL, Sra, VT);
    return DAG.getNode(ISD::AND, DL, VT, Not, DAG.getFreeze(N2));
  }
 
  // TODO: There's another pattern in this family, but it may require
  //       implementing hasOrNot() to check for profitability:
  //       (Cond0 s> -1) ? -1 : N2 --> ~(Cond0 s>> BW-1) | freeze(N2)
 
  return SDValue();
}
 
// Match SELECTs with absolute difference patterns.
// (select (setcc a, b, set?gt), (sub a, b), (sub b, a)) --> (abd? a, b)
// (select (setcc a, b, set?ge), (sub a, b), (sub b, a)) --> (abd? a, b)
// (select (setcc a, b, set?lt), (sub b, a), (sub a, b)) --> (abd? a, b)
// (select (setcc a, b, set?le), (sub b, a), (sub a, b)) --> (abd? a, b)
SDValue DAGCombiner::foldSelectToABD(SDValue LHS, SDValue RHS, SDValue True,
                                     SDValue False, ISD::CondCode CC,
                                     const SDLoc &DL) {
  bool IsSigned = isSignedIntSetCC(CC);
  unsigned ABDOpc = IsSigned ? ISD::ABDS : ISD::ABDU;
  EVT VT = LHS.getValueType();
 
  if (LegalOperations && !hasOperation(ABDOpc, VT))
    return SDValue();
 
  // (setcc 0, b set???) --> (setcc b, 0, set???)
  if (isZeroOrZeroSplat(LHS)) {
    std::swap(LHS, RHS);
    CC = ISD::getSetCCSwappedOperands(CC);
  }
 
  // (setcc (add nsw A, Const), 0, sets??) --> (setcc A, -Const, sets??)
  SDValue A, B;
  if (ISD::isSignedIntSetCC(CC) && LHS->getFlags().hasNoSignedWrap() &&
      isZeroOrZeroSplat(RHS) && sd_match(LHS, m_Add(m_Value(A), m_Value(B))) &&
      DAG.isConstantIntBuildVectorOrConstantInt(B)) {
    RHS = DAG.getNegative(B, LHS, B.getValueType());
    LHS = A;
  }
 
  bool IsTypeLegalOrPromote =
      TLI.isTypeLegal(VT) || TLI.getTypeAction(*DAG.getContext(), VT) ==
                                 TargetLowering::TypePromoteInteger;
 
  switch (CC) {
  case ISD::SETGT:
  case ISD::SETGE:
  case ISD::SETUGT:
  case ISD::SETUGE:
    if (sd_match(True, m_AnyOf(m_Sub(m_Specific(LHS), m_Specific(RHS)),
                               m_Add(m_Specific(LHS), m_SpecificNeg(RHS)))) &&
        sd_match(False, m_AnyOf(m_Sub(m_Specific(RHS), m_Specific(LHS)),
                                m_Add(m_Specific(RHS), m_SpecificNeg(LHS)))))
      return DAG.getNode(ABDOpc, DL, VT, LHS, RHS);
    if (sd_match(True, m_AnyOf(m_Sub(m_Specific(RHS), m_Specific(LHS)),
                               m_Add(m_Specific(RHS), m_SpecificNeg(LHS)))) &&
        sd_match(False, m_AnyOf(m_Sub(m_Specific(LHS), m_Specific(RHS)),
                                m_Add(m_Specific(LHS), m_SpecificNeg(RHS)))) &&
        IsTypeLegalOrPromote)
      return DAG.getNegative(DAG.getNode(ABDOpc, DL, VT, LHS, RHS), DL, VT);
    break;
  case ISD::SETLT:
  case ISD::SETLE:
  case ISD::SETULT:
  case ISD::SETULE:
    if (sd_match(True, m_AnyOf(m_Sub(m_Specific(RHS), m_Specific(LHS)),
                               m_Add(m_Specific(RHS), m_SpecificNeg(LHS)))) &&
        sd_match(False, m_AnyOf(m_Sub(m_Specific(LHS), m_Specific(RHS)),
                                m_Add(m_Specific(LHS), m_SpecificNeg(RHS)))))
      return DAG.getNode(ABDOpc, DL, VT, LHS, RHS);
    if (sd_match(True, m_AnyOf(m_Sub(m_Specific(LHS), m_Specific(RHS)),
                               m_Add(m_Specific(LHS), m_SpecificNeg(RHS)))) &&
        sd_match(False, m_AnyOf(m_Sub(m_Specific(RHS), m_Specific(LHS)),
                                m_Add(m_Specific(RHS), m_SpecificNeg(LHS)))) &&
        IsTypeLegalOrPromote)
      return DAG.getNegative(DAG.getNode(ABDOpc, DL, VT, LHS, RHS), DL, VT);
    break;
  default:
    break;
  }
 
  return SDValue();
}
 
// ([v]select (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
// ([v]select (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
SDValue DAGCombiner::foldSelectToUMin(SDValue LHS, SDValue RHS, SDValue True,
                                      SDValue False, ISD::CondCode CC,
                                      const SDLoc &DL) {
  APInt C;
  EVT VT = True.getValueType();
  if (sd_match(RHS, m_ConstInt(C)) && hasUMin(VT)) {
    if (CC == ISD::SETUGT && LHS == False &&
        sd_match(True, m_Add(m_Specific(False), m_SpecificInt(~C)))) {
      SDValue AddC = DAG.getConstant(~C, DL, VT);
      SDValue Add = DAG.getNode(ISD::ADD, DL, VT, False, AddC);
      return DAG.getNode(ISD::UMIN, DL, VT, Add, False);
    }
    if (CC == ISD::SETULT && LHS == True &&
        sd_match(False, m_Add(m_Specific(True), m_SpecificInt(-C)))) {
      SDValue AddC = DAG.getConstant(-C, DL, VT);
      SDValue Add = DAG.getNode(ISD::ADD, DL, VT, True, AddC);
      return DAG.getNode(ISD::UMIN, DL, VT, True, Add);
    }
  }
  return SDValue();
}
 
SDValue DAGCombiner::visitSELECT(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  EVT VT = N->getValueType(0);
  EVT VT0 = N0.getValueType();
  SDLoc DL(N);
  SDNodeFlags Flags = N->getFlags();
 
  if (SDValue V = DAG.simplifySelect(N0, N1, N2))
    return V;
 
  if (SDValue V = foldBoolSelectToLogic<EmptyMatchContext>(N, DL, DAG))
    return V;
 
  // select (not Cond), N1, N2 -> select Cond, N2, N1
  if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
    return DAG.getSelect(DL, VT, F, N2, N1, Flags);
 
  if (SDValue V = foldSelectOfConstants(N))
    return V;
 
  // If we can fold this based on the true/false value, do so.
  if (SimplifySelectOps(N, N1, N2))
    return SDValue(N, 0); // Don't revisit N.
 
  if (VT0 == MVT::i1) {
    // The code in this block deals with the following 2 equivalences:
    //    select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
    //    select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
    // The target can specify its preferred form with the
    // shouldNormalizeToSelectSequence() callback. However we always transform
    // to the right anyway if we find the inner select exists in the DAG anyway
    // and we always transform to the left side if we know that we can further
    // optimize the combination of the conditions.
    bool normalizeToSequence =
        TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
    // select (and Cond0, Cond1), X, Y
    //   -> select Cond0, (select Cond1, X, Y), Y
    if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
      SDValue Cond0 = N0->getOperand(0);
      SDValue Cond1 = N0->getOperand(1);
      SDValue InnerSelect =
          DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2, Flags);
      if (normalizeToSequence || !InnerSelect.use_empty())
        return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0,
                           InnerSelect, N2, Flags);
      // Cleanup on failure.
      if (InnerSelect.use_empty())
        recursivelyDeleteUnusedNodes(InnerSelect.getNode());
    }
    // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y)
    if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
      SDValue Cond0 = N0->getOperand(0);
      SDValue Cond1 = N0->getOperand(1);
      SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(),
                                        Cond1, N1, N2, Flags);
      if (normalizeToSequence || !InnerSelect.use_empty())
        return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1,
                           InnerSelect, Flags);
      // Cleanup on failure.
      if (InnerSelect.use_empty())
        recursivelyDeleteUnusedNodes(InnerSelect.getNode());
    }
 
    // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y
    if (N1->getOpcode() == ISD::SELECT && N1->hasOneUse()) {
      SDValue N1_0 = N1->getOperand(0);
      SDValue N1_1 = N1->getOperand(1);
      SDValue N1_2 = N1->getOperand(2);
      if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
        // Create the actual and node if we can generate good code for it.
        if (!normalizeToSequence) {
          SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
          return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1,
                             N2, Flags);
        }
        // Otherwise see if we can optimize the "and" to a better pattern.
        if (SDValue Combined = visitANDLike(N0, N1_0, N)) {
          return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1,
                             N2, Flags);
        }
      }
    }
    // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y
    if (N2->getOpcode() == ISD::SELECT && N2->hasOneUse()) {
      SDValue N2_0 = N2->getOperand(0);
      SDValue N2_1 = N2->getOperand(1);
      SDValue N2_2 = N2->getOperand(2);
      if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
        // Create the actual or node if we can generate good code for it.
        if (!normalizeToSequence) {
          SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
          return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1,
                             N2_2, Flags);
        }
        // Otherwise see if we can optimize to a better pattern.
        if (SDValue Combined = visitORLike(N0, N2_0, DL))
          return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1,
                             N2_2, Flags);
      }
    }
 
    // select usubo(x, y).overflow, (sub y, x), (usubo x, y) -> abdu(x, y)
    if (N0.getOpcode() == ISD::USUBO && N0.getResNo() == 1 &&
        N2.getNode() == N0.getNode() && N2.getResNo() == 0 &&
        N1.getOpcode() == ISD::SUB && N2.getOperand(0) == N1.getOperand(1) &&
        N2.getOperand(1) == N1.getOperand(0) &&
        (!LegalOperations || TLI.isOperationLegal(ISD::ABDU, VT)))
      return DAG.getNode(ISD::ABDU, DL, VT, N0.getOperand(0), N0.getOperand(1));
 
    // select usubo(x, y).overflow, (usubo x, y), (sub y, x) -> neg (abdu x, y)
    if (N0.getOpcode() == ISD::USUBO && N0.getResNo() == 1 &&
        N1.getNode() == N0.getNode() && N1.getResNo() == 0 &&
        N2.getOpcode() == ISD::SUB && N2.getOperand(0) == N1.getOperand(1) &&
        N2.getOperand(1) == N1.getOperand(0) &&
        (!LegalOperations || TLI.isOperationLegal(ISD::ABDU, VT)))
      return DAG.getNegative(
          DAG.getNode(ISD::ABDU, DL, VT, N0.getOperand(0), N0.getOperand(1)),
          DL, VT);
  }
 
  // Fold selects based on a setcc into other things, such as min/max/abs.
  if (N0.getOpcode() == ISD::SETCC) {
    SDValue Cond0 = N0.getOperand(0), Cond1 = N0.getOperand(1);
    ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
 
    // select (fcmp lt x, y), x, y -> fminnum x, y
    // select (fcmp gt x, y), x, y -> fmaxnum x, y
    //
    // This is OK if we don't care what happens if either operand is a NaN.
    if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2, Flags, TLI))
      if (SDValue FMinMax =
              combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2, CC))
        return FMinMax;
 
    // Use 'unsigned add with overflow' to optimize an unsigned saturating add.
    // This is conservatively limited to pre-legal-operations to give targets
    // a chance to reverse the transform if they want to do that. Also, it is
    // unlikely that the pattern would be formed late, so it's probably not
    // worth going through the other checks.
    if (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::UADDO, VT) &&
        CC == ISD::SETUGT && N0.hasOneUse() && isAllOnesConstant(N1) &&
        N2.getOpcode() == ISD::ADD && Cond0 == N2.getOperand(0)) {
      auto *C = dyn_cast<ConstantSDNode>(N2.getOperand(1));
      auto *NotC = dyn_cast<ConstantSDNode>(Cond1);
      if (C && NotC && C->getAPIntValue() == ~NotC->getAPIntValue()) {
        // select (setcc Cond0, ~C, ugt), -1, (add Cond0, C) -->
        // uaddo Cond0, C; select uaddo.1, -1, uaddo.0
        //
        // The IR equivalent of this transform would have this form:
        //   %a = add %x, C
        //   %c = icmp ugt %x, ~C
        //   %r = select %c, -1, %a
        //   =>
        //   %u = call {iN,i1} llvm.uadd.with.overflow(%x, C)
        //   %u0 = extractvalue %u, 0
        //   %u1 = extractvalue %u, 1
        //   %r = select %u1, -1, %u0
        SDVTList VTs = DAG.getVTList(VT, VT0);
        SDValue UAO = DAG.getNode(ISD::UADDO, DL, VTs, Cond0, N2.getOperand(1));
        return DAG.getSelect(DL, VT, UAO.getValue(1), N1, UAO.getValue(0));
      }
    }
 
    if (TLI.isOperationLegal(ISD::SELECT_CC, VT) ||
        (!LegalOperations &&
         TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) {
      // Any flags available in a select/setcc fold will be on the setcc as they
      // migrated from fcmp
      return DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, N2,
                         N0.getOperand(2), N0->getFlags());
    }
 
    if (SDValue ABD = foldSelectToABD(Cond0, Cond1, N1, N2, CC, DL))
      return ABD;
 
    if (SDValue NewSel = SimplifySelect(DL, N0, N1, N2))
      return NewSel;
 
    // (select (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
    // (select (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
    if (SDValue UMin = foldSelectToUMin(Cond0, Cond1, N1, N2, CC, DL))
      return UMin;
  }
 
  if (!VT.isVector())
    if (SDValue BinOp = foldSelectOfBinops(N))
      return BinOp;
 
  if (SDValue R = combineSelectAsExtAnd(N0, N1, N2, DL, DAG))
    return R;
 
  return SDValue();
}
 
// This function assumes all the vselect's arguments are CONCAT_VECTOR
// nodes and that the condition is a BV of ConstantSDNodes (or undefs).
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
  SDLoc DL(N);
  SDValue Cond = N->getOperand(0);
  SDValue LHS = N->getOperand(1);
  SDValue RHS = N->getOperand(2);
  EVT VT = N->getValueType(0);
  int NumElems = VT.getVectorNumElements();
  assert(LHS.getOpcode() == ISD::CONCAT_VECTORS &&
         RHS.getOpcode() == ISD::CONCAT_VECTORS &&
         Cond.getOpcode() == ISD::BUILD_VECTOR);
 
  // CONCAT_VECTOR can take an arbitrary number of arguments. We only care about
  // binary ones here.
  if (LHS->getNumOperands() != 2 || RHS->getNumOperands() != 2)
    return SDValue();
 
  // We're sure we have an even number of elements due to the
  // concat_vectors we have as arguments to vselect.
  // Skip BV elements until we find one that's not an UNDEF
  // After we find an UNDEF element, keep looping until we get to half the
  // length of the BV and see if all the non-undef nodes are the same.
  ConstantSDNode *BottomHalf = nullptr;
  for (int i = 0; i < NumElems / 2; ++i) {
    if (Cond->getOperand(i)->isUndef())
      continue;
 
    if (BottomHalf == nullptr)
      BottomHalf = cast<ConstantSDNode>(Cond.getOperand(i));
    else if (Cond->getOperand(i).getNode() != BottomHalf)
      return SDValue();
  }
 
  // Do the same for the second half of the BuildVector
  ConstantSDNode *TopHalf = nullptr;
  for (int i = NumElems / 2; i < NumElems; ++i) {
    if (Cond->getOperand(i)->isUndef())
      continue;
 
    if (TopHalf == nullptr)
      TopHalf = cast<ConstantSDNode>(Cond.getOperand(i));
    else if (Cond->getOperand(i).getNode() != TopHalf)
      return SDValue();
  }
 
  assert(TopHalf && BottomHalf &&
         "One half of the selector was all UNDEFs and the other was all the "
         "same value. This should have been addressed before this function.");
  return DAG.getNode(
      ISD::CONCAT_VECTORS, DL, VT,
      BottomHalf->isZero() ? RHS->getOperand(0) : LHS->getOperand(0),
      TopHalf->isZero() ? RHS->getOperand(1) : LHS->getOperand(1));
}
 
bool refineUniformBase(SDValue &BasePtr, SDValue &Index, bool IndexIsScaled,
                       SelectionDAG &DAG, const SDLoc &DL) {
 
  // Only perform the transformation when existing operands can be reused.
  if (IndexIsScaled)
    return false;
 
  if (!isNullConstant(BasePtr) && !Index.hasOneUse())
    return false;
 
  EVT VT = BasePtr.getValueType();
 
  if (SDValue SplatVal = DAG.getSplatValue(Index);
      SplatVal && !isNullConstant(SplatVal) &&
      SplatVal.getValueType() == VT) {
    BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal);
    Index = DAG.getSplat(Index.getValueType(), DL, DAG.getConstant(0, DL, VT));
    return true;
  }
 
  if (Index.getOpcode() != ISD::ADD)
    return false;
 
  if (SDValue SplatVal = DAG.getSplatValue(Index.getOperand(0));
      SplatVal && SplatVal.getValueType() == VT) {
    BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal);
    Index = Index.getOperand(1);
    return true;
  }
  if (SDValue SplatVal = DAG.getSplatValue(Index.getOperand(1));
      SplatVal && SplatVal.getValueType() == VT) {
    BasePtr = DAG.getNode(ISD::ADD, DL, VT, BasePtr, SplatVal);
    Index = Index.getOperand(0);
    return true;
  }
  return false;
}
 
// Fold sext/zext of index into index type.
bool refineIndexType(SDValue &Index, ISD::MemIndexType &IndexType, EVT DataVT,
                     SelectionDAG &DAG) {
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
  // It's always safe to look through zero extends.
  if (Index.getOpcode() == ISD::ZERO_EXTEND) {
    if (TLI.shouldRemoveExtendFromGSIndex(Index, DataVT)) {
      IndexType = ISD::UNSIGNED_SCALED;
      Index = Index.getOperand(0);
      return true;
    }
    if (ISD::isIndexTypeSigned(IndexType)) {
      IndexType = ISD::UNSIGNED_SCALED;
      return true;
    }
  }
 
  // It's only safe to look through sign extends when Index is signed.
  if (Index.getOpcode() == ISD::SIGN_EXTEND &&
      ISD::isIndexTypeSigned(IndexType) &&
      TLI.shouldRemoveExtendFromGSIndex(Index, DataVT)) {
    Index = Index.getOperand(0);
    return true;
  }
 
  return false;
}
 
SDValue DAGCombiner::visitVPSCATTER(SDNode *N) {
  VPScatterSDNode *MSC = cast<VPScatterSDNode>(N);
  SDValue Mask = MSC->getMask();
  SDValue Chain = MSC->getChain();
  SDValue Index = MSC->getIndex();
  SDValue Scale = MSC->getScale();
  SDValue StoreVal = MSC->getValue();
  SDValue BasePtr = MSC->getBasePtr();
  SDValue VL = MSC->getVectorLength();
  ISD::MemIndexType IndexType = MSC->getIndexType();
  SDLoc DL(N);
 
  // Zap scatters with a zero mask.
  if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
    return Chain;
 
  if (refineUniformBase(BasePtr, Index, MSC->isIndexScaled(), DAG, DL)) {
    SDValue Ops[] = {Chain, StoreVal, BasePtr, Index, Scale, Mask, VL};
    return DAG.getScatterVP(DAG.getVTList(MVT::Other), MSC->getMemoryVT(),
                            DL, Ops, MSC->getMemOperand(), IndexType);
  }
 
  if (refineIndexType(Index, IndexType, StoreVal.getValueType(), DAG)) {
    SDValue Ops[] = {Chain, StoreVal, BasePtr, Index, Scale, Mask, VL};
    return DAG.getScatterVP(DAG.getVTList(MVT::Other), MSC->getMemoryVT(),
                            DL, Ops, MSC->getMemOperand(), IndexType);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMSCATTER(SDNode *N) {
  MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
  SDValue Mask = MSC->getMask();
  SDValue Chain = MSC->getChain();
  SDValue Index = MSC->getIndex();
  SDValue Scale = MSC->getScale();
  SDValue StoreVal = MSC->getValue();
  SDValue BasePtr = MSC->getBasePtr();
  ISD::MemIndexType IndexType = MSC->getIndexType();
  SDLoc DL(N);
 
  // Zap scatters with a zero mask.
  if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
    return Chain;
 
  if (refineUniformBase(BasePtr, Index, MSC->isIndexScaled(), DAG, DL)) {
    SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
    return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MSC->getMemoryVT(),
                                DL, Ops, MSC->getMemOperand(), IndexType,
                                MSC->isTruncatingStore());
  }
 
  if (refineIndexType(Index, IndexType, StoreVal.getValueType(), DAG)) {
    SDValue Ops[] = {Chain, StoreVal, Mask, BasePtr, Index, Scale};
    return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), MSC->getMemoryVT(),
                                DL, Ops, MSC->getMemOperand(), IndexType,
                                MSC->isTruncatingStore());
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMSTORE(SDNode *N) {
  MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
  SDValue Mask = MST->getMask();
  SDValue Chain = MST->getChain();
  SDValue Value = MST->getValue();
  SDValue Ptr = MST->getBasePtr();
 
  // Zap masked stores with a zero mask.
  if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
    return Chain;
 
  // Remove a masked store if base pointers and masks are equal.
  if (MaskedStoreSDNode *MST1 = dyn_cast<MaskedStoreSDNode>(Chain)) {
    if (MST->isUnindexed() && MST->isSimple() && MST1->isUnindexed() &&
        MST1->isSimple() && MST1->getBasePtr() == Ptr &&
        !MST->getBasePtr().isUndef() &&
        ((Mask == MST1->getMask() && MST->getMemoryVT().getStoreSize() ==
                                         MST1->getMemoryVT().getStoreSize()) ||
         ISD::isConstantSplatVectorAllOnes(Mask.getNode())) &&
        TypeSize::isKnownLE(MST1->getMemoryVT().getStoreSize(),
                            MST->getMemoryVT().getStoreSize())) {
      CombineTo(MST1, MST1->getChain());
      if (N->getOpcode() != ISD::DELETED_NODE)
        AddToWorklist(N);
      return SDValue(N, 0);
    }
  }
 
  // If this is a masked load with an all ones mask, we can use a unmasked load.
  // FIXME: Can we do this for indexed, compressing, or truncating stores?
  if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MST->isUnindexed() &&
      !MST->isCompressingStore() && !MST->isTruncatingStore())
    return DAG.getStore(MST->getChain(), SDLoc(N), MST->getValue(),
                        MST->getBasePtr(), MST->getPointerInfo(),
                        MST->getBaseAlign(), MST->getMemOperand()->getFlags(),
                        MST->getAAInfo());
 
  // Try transforming N to an indexed store.
  if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
    return SDValue(N, 0);
 
  if (MST->isTruncatingStore() && MST->isUnindexed() &&
      Value.getValueType().isInteger() &&
      (!isa<ConstantSDNode>(Value) ||
       !cast<ConstantSDNode>(Value)->isOpaque())) {
    APInt TruncDemandedBits =
        APInt::getLowBitsSet(Value.getScalarValueSizeInBits(),
                             MST->getMemoryVT().getScalarSizeInBits());
 
    // See if we can simplify the operation with
    // SimplifyDemandedBits, which only works if the value has a single use.
    if (SimplifyDemandedBits(Value, TruncDemandedBits)) {
      // Re-visit the store if anything changed and the store hasn't been merged
      // with another node (N is deleted) SimplifyDemandedBits will add Value's
      // node back to the worklist if necessary, but we also need to re-visit
      // the Store node itself.
      if (N->getOpcode() != ISD::DELETED_NODE)
        AddToWorklist(N);
      return SDValue(N, 0);
    }
  }
 
  // If this is a TRUNC followed by a masked store, fold this into a masked
  // truncating store.  We can do this even if this is already a masked
  // truncstore.
  // TODO: Try combine to masked compress store if possiable.
  if ((Value.getOpcode() == ISD::TRUNCATE) && Value->hasOneUse() &&
      MST->isUnindexed() && !MST->isCompressingStore() &&
      TLI.canCombineTruncStore(Value.getOperand(0).getValueType(),
                               MST->getMemoryVT(), MST->getAlign(),
                               MST->getAddressSpace(), LegalOperations)) {
    auto Mask = TLI.promoteTargetBoolean(DAG, MST->getMask(),
                                         Value.getOperand(0).getValueType());
    return DAG.getMaskedStore(Chain, SDLoc(N), Value.getOperand(0), Ptr,
                              MST->getOffset(), Mask, MST->getMemoryVT(),
                              MST->getMemOperand(), MST->getAddressingMode(),
                              /*IsTruncating=*/true);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitVP_STRIDED_STORE(SDNode *N) {
  auto *SST = cast<VPStridedStoreSDNode>(N);
  EVT EltVT = SST->getValue().getValueType().getVectorElementType();
  // Combine strided stores with unit-stride to a regular VP store.
  if (auto *CStride = dyn_cast<ConstantSDNode>(SST->getStride());
      CStride && CStride->getZExtValue() == EltVT.getStoreSize()) {
    return DAG.getStoreVP(SST->getChain(), SDLoc(N), SST->getValue(),
                          SST->getBasePtr(), SST->getOffset(), SST->getMask(),
                          SST->getVectorLength(), SST->getMemoryVT(),
                          SST->getMemOperand(), SST->getAddressingMode(),
                          SST->isTruncatingStore(), SST->isCompressingStore());
  }
  return SDValue();
}
 
SDValue DAGCombiner::visitVECTOR_COMPRESS(SDNode *N) {
  SDLoc DL(N);
  SDValue Vec = N->getOperand(0);
  SDValue Mask = N->getOperand(1);
  SDValue Passthru = N->getOperand(2);
  EVT VecVT = Vec.getValueType();
 
  bool HasPassthru = !Passthru.isUndef();
 
  APInt SplatVal;
  if (ISD::isConstantSplatVector(Mask.getNode(), SplatVal))
    return TLI.isConstTrueVal(Mask) ? Vec : Passthru;
 
  if (Vec.isUndef() || Mask.isUndef())
    return Passthru;
 
  // No need for potentially expensive compress if the mask is constant.
  if (ISD::isBuildVectorOfConstantSDNodes(Mask.getNode())) {
    SmallVector<SDValue, 16> Ops;
    EVT ScalarVT = VecVT.getVectorElementType();
    unsigned NumSelected = 0;
    unsigned NumElmts = VecVT.getVectorNumElements();
    for (unsigned I = 0; I < NumElmts; ++I) {
      SDValue MaskI = Mask.getOperand(I);
      // We treat undef mask entries as "false".
      if (MaskI.isUndef())
        continue;
 
      if (TLI.isConstTrueVal(MaskI)) {
        SDValue VecI = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Vec,
                                   DAG.getVectorIdxConstant(I, DL));
        Ops.push_back(VecI);
        NumSelected++;
      }
    }
    for (unsigned Rest = NumSelected; Rest < NumElmts; ++Rest) {
      SDValue Val =
          HasPassthru
              ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarVT, Passthru,
                            DAG.getVectorIdxConstant(Rest, DL))
              : DAG.getUNDEF(ScalarVT);
      Ops.push_back(Val);
    }
    return DAG.getBuildVector(VecVT, DL, Ops);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitVPGATHER(SDNode *N) {
  VPGatherSDNode *MGT = cast<VPGatherSDNode>(N);
  SDValue Mask = MGT->getMask();
  SDValue Chain = MGT->getChain();
  SDValue Index = MGT->getIndex();
  SDValue Scale = MGT->getScale();
  SDValue BasePtr = MGT->getBasePtr();
  SDValue VL = MGT->getVectorLength();
  ISD::MemIndexType IndexType = MGT->getIndexType();
  SDLoc DL(N);
 
  if (refineUniformBase(BasePtr, Index, MGT->isIndexScaled(), DAG, DL)) {
    SDValue Ops[] = {Chain, BasePtr, Index, Scale, Mask, VL};
    return DAG.getGatherVP(
        DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL,
        Ops, MGT->getMemOperand(), IndexType);
  }
 
  if (refineIndexType(Index, IndexType, N->getValueType(0), DAG)) {
    SDValue Ops[] = {Chain, BasePtr, Index, Scale, Mask, VL};
    return DAG.getGatherVP(
        DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL,
        Ops, MGT->getMemOperand(), IndexType);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMGATHER(SDNode *N) {
  MaskedGatherSDNode *MGT = cast<MaskedGatherSDNode>(N);
  SDValue Mask = MGT->getMask();
  SDValue Chain = MGT->getChain();
  SDValue Index = MGT->getIndex();
  SDValue Scale = MGT->getScale();
  SDValue PassThru = MGT->getPassThru();
  SDValue BasePtr = MGT->getBasePtr();
  ISD::MemIndexType IndexType = MGT->getIndexType();
  SDLoc DL(N);
 
  // Zap gathers with a zero mask.
  if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
    return CombineTo(N, PassThru, MGT->getChain());
 
  if (refineUniformBase(BasePtr, Index, MGT->isIndexScaled(), DAG, DL)) {
    SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
    return DAG.getMaskedGather(
        DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL,
        Ops, MGT->getMemOperand(), IndexType, MGT->getExtensionType());
  }
 
  if (refineIndexType(Index, IndexType, N->getValueType(0), DAG)) {
    SDValue Ops[] = {Chain, PassThru, Mask, BasePtr, Index, Scale};
    return DAG.getMaskedGather(
        DAG.getVTList(N->getValueType(0), MVT::Other), MGT->getMemoryVT(), DL,
        Ops, MGT->getMemOperand(), IndexType, MGT->getExtensionType());
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMLOAD(SDNode *N) {
  MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
  SDValue Mask = MLD->getMask();
 
  // Zap masked loads with a zero mask.
  if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
    return CombineTo(N, MLD->getPassThru(), MLD->getChain());
 
  // If this is a masked load with an all ones mask, we can use a unmasked load.
  // FIXME: Can we do this for indexed, expanding, or extending loads?
  if (ISD::isConstantSplatVectorAllOnes(Mask.getNode()) && MLD->isUnindexed() &&
      !MLD->isExpandingLoad() && MLD->getExtensionType() == ISD::NON_EXTLOAD) {
    SDValue NewLd = DAG.getLoad(
        N->getValueType(0), SDLoc(N), MLD->getChain(), MLD->getBasePtr(),
        MLD->getPointerInfo(), MLD->getBaseAlign(),
        MLD->getMemOperand()->getFlags(), MLD->getAAInfo(), MLD->getRanges());
    return CombineTo(N, NewLd, NewLd.getValue(1));
  }
 
  // Try transforming N to an indexed load.
  if (CombineToPreIndexedLoadStore(N) || CombineToPostIndexedLoadStore(N))
    return SDValue(N, 0);
 
  return SDValue();
}
 
SDValue DAGCombiner::visitMHISTOGRAM(SDNode *N) {
  MaskedHistogramSDNode *HG = cast<MaskedHistogramSDNode>(N);
  SDValue Chain = HG->getChain();
  SDValue Inc = HG->getInc();
  SDValue Mask = HG->getMask();
  SDValue BasePtr = HG->getBasePtr();
  SDValue Index = HG->getIndex();
  SDLoc DL(HG);
 
  EVT MemVT = HG->getMemoryVT();
  EVT DataVT = Index.getValueType();
  MachineMemOperand *MMO = HG->getMemOperand();
  ISD::MemIndexType IndexType = HG->getIndexType();
 
  if (ISD::isConstantSplatVectorAllZeros(Mask.getNode()))
    return Chain;
 
  if (refineUniformBase(BasePtr, Index, HG->isIndexScaled(), DAG, DL) ||
      refineIndexType(Index, IndexType, DataVT, DAG)) {
    SDValue Ops[] = {Chain,          Inc,           Mask, BasePtr, Index,
                     HG->getScale(), HG->getIntID()};
    return DAG.getMaskedHistogram(DAG.getVTList(MVT::Other), MemVT, DL, Ops,
                                  MMO, IndexType);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitPARTIAL_REDUCE_MLA(SDNode *N) {
  if (SDValue Res = foldPartialReduceMLAMulOp(N))
    return Res;
  if (SDValue Res = foldPartialReduceAdd(N))
    return Res;
  return SDValue();
}
 
// partial_reduce_*mla(acc, mul(*ext(a), *ext(b)), splat(1))
// -> partial_reduce_*mla(acc, a, b)
//
// partial_reduce_*mla(acc, mul(*ext(x), splat(C)), splat(1))
// -> partial_reduce_*mla(acc, x, splat(C))
//
// partial_reduce_*mla(acc, sel(p, mul(*ext(a), *ext(b)), splat(0)), splat(1))
// -> partial_reduce_*mla(acc, sel(p, a, splat(0)), b)
//
// partial_reduce_*mla(acc, sel(p, mul(*ext(a), splat(C)), splat(0)), splat(1))
// -> partial_reduce_*mla(acc, sel(p, a, splat(0)), splat(C))
SDValue DAGCombiner::foldPartialReduceMLAMulOp(SDNode *N) {
  SDLoc DL(N);
  auto *Context = DAG.getContext();
  SDValue Tmp;
  SDValue Acc = N->getOperand(0);
  SDValue Op1 = N->getOperand(1);
  SDValue Op2 = N->getOperand(2);
  unsigned Opc = Op1->getOpcode();
 
  // Handle predication by moving the SELECT into the operand of the MUL.
  SDValue Pred;
  if (Opc == ISD::VSELECT && (isZeroOrZeroSplat(Op1->getOperand(2)) ||
                              isZeroOrZeroSplatFP(Op1->getOperand(2)))) {
    Pred = Op1->getOperand(0);
    Op1 = Op1->getOperand(1);
    Opc = Op1->getOpcode();
  }
 
  // Handle negation (sub-reduction).
  bool IsMLS = false;
  if (sd_match(Op1, m_Neg(m_Value(Tmp)))) {
    Op1 = Tmp;
    Opc = Op1->getOpcode();
    IsMLS = true;
  }
 
  if (Opc != ISD::MUL && Opc != ISD::FMUL && Opc != ISD::SHL)
    return SDValue();
 
  SDValue LHS = Op1->getOperand(0);
  SDValue RHS = Op1->getOperand(1);
 
  // After instcombine, negation for FP operations is on the RHS, so implement:
  //   fmul(fpext(a), fneg(fpext(b)))
  //-> fmul(fpext(a), fpext(fneg(b)))
  if (sd_match(RHS, m_FNeg(m_Value(Tmp)))) {
    RHS = Tmp;
    IsMLS = true;
  }
 
  // Try to treat (shl %a, %c) as (mul %a, (1 << %c)) for constant %c.
  if (Opc == ISD::SHL) {
    APInt C;
    if (!ISD::isConstantSplatVector(RHS.getNode(), C))
      return SDValue();
 
    RHS =
        DAG.getSplatVector(RHS.getValueType(), DL,
                           DAG.getConstant(APInt(C.getBitWidth(), 1).shl(C), DL,
                                           RHS.getValueType().getScalarType()));
    Opc = ISD::MUL;
  }
 
  if (!(Opc == ISD::MUL && llvm::isOneOrOneSplat(Op2)) &&
      !(Opc == ISD::FMUL && llvm::isOneOrOneSplatFP(Op2)))
    return SDValue();
 
  auto IsIntOrFPExtOpcode = [](unsigned int Opcode) {
    return (ISD::isExtOpcode(Opcode) || Opcode == ISD::FP_EXTEND);
  };
 
  unsigned LHSOpcode = LHS->getOpcode();
  if (!IsIntOrFPExtOpcode(LHSOpcode))
    return SDValue();
 
  SDValue LHSExtOp = LHS->getOperand(0);
  EVT LHSExtOpVT = LHSExtOp.getValueType();
 
  // When Pred is non-zero, set Op = select(Pred, Op, splat(0)) and freeze
  // OtherOp to keep the same semantics when moving the selects into the MUL
  // operands.
  auto ApplyPredicate = [&](SDValue &Op, SDValue &OtherOp) {
    if (Pred) {
      EVT OpVT = Op.getValueType();
      SDValue Zero = OpVT.isFloatingPoint() ? DAG.getConstantFP(0.0, DL, OpVT)
                                            : DAG.getConstant(0, DL, OpVT);
      Op = DAG.getSelect(DL, OpVT, Pred, Op, Zero);
      OtherOp = DAG.getFreeze(OtherOp);
    }
  };
 
  // Generate an MLA or MLS.
  auto GetMLA = [&](unsigned Opc, SDValue Acc, SDValue LHS,
                    SDValue RHS) -> SDValue {
    EVT AccVT = Acc.getValueType();
    return IsMLS ? DAG.getPartialReduceMLS(Opc, DL, Acc, LHS, RHS)
                 : DAG.getNode(Opc, DL, AccVT, Acc, LHS, RHS);
  };
 
  // partial_reduce_*mla(acc, mul(ext(x), splat(C)), splat(1))
  // -> partial_reduce_*mla(acc, x, C)
  APInt C;
  if (ISD::isConstantSplatVector(RHS.getNode(), C)) {
    // TODO: Make use of partial_reduce_sumla here
    APInt CTrunc = C.trunc(LHSExtOpVT.getScalarSizeInBits());
    unsigned LHSBits = LHS.getValueType().getScalarSizeInBits();
    if ((LHSOpcode != ISD::ZERO_EXTEND || CTrunc.zext(LHSBits) != C) &&
        (LHSOpcode != ISD::SIGN_EXTEND || CTrunc.sext(LHSBits) != C))
      return SDValue();
 
    unsigned NewOpcode = LHSOpcode == ISD::SIGN_EXTEND
                             ? ISD::PARTIAL_REDUCE_SMLA
                             : ISD::PARTIAL_REDUCE_UMLA;
 
    // Only perform these combines if the target supports folding
    // the extends into the operation.
    if (!TLI.isPartialReduceMLALegalOrCustom(
            NewOpcode, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
            TLI.getTypeToTransformTo(*Context, LHSExtOpVT)))
      return SDValue();
 
    SDValue C = DAG.getConstant(CTrunc, DL, LHSExtOpVT);
    ApplyPredicate(C, LHSExtOp);
    return GetMLA(NewOpcode, Acc, LHSExtOp, C);
  }
 
  unsigned RHSOpcode = RHS->getOpcode();
  if (!IsIntOrFPExtOpcode(RHSOpcode))
    return SDValue();
 
  SDValue RHSExtOp = RHS->getOperand(0);
  if (LHSExtOpVT != RHSExtOp.getValueType())
    return SDValue();
 
  unsigned NewOpc;
  if (LHSOpcode == ISD::SIGN_EXTEND && RHSOpcode == ISD::SIGN_EXTEND)
    NewOpc = ISD::PARTIAL_REDUCE_SMLA;
  else if (LHSOpcode == ISD::ZERO_EXTEND && RHSOpcode == ISD::ZERO_EXTEND)
    NewOpc = ISD::PARTIAL_REDUCE_UMLA;
  else if (LHSOpcode == ISD::SIGN_EXTEND && RHSOpcode == ISD::ZERO_EXTEND)
    NewOpc = ISD::PARTIAL_REDUCE_SUMLA;
  else if (LHSOpcode == ISD::ZERO_EXTEND && RHSOpcode == ISD::SIGN_EXTEND) {
    NewOpc = ISD::PARTIAL_REDUCE_SUMLA;
    std::swap(LHSExtOp, RHSExtOp);
  } else if (LHSOpcode == ISD::FP_EXTEND && RHSOpcode == ISD::FP_EXTEND) {
    NewOpc = ISD::PARTIAL_REDUCE_FMLA;
  } else
    return SDValue();
  // For a 2-stage extend the signedness of both of the extends must match
  // If the mul has the same type, there is no outer extend, and thus we
  // can simply use the inner extends to pick the result node.
  // TODO: extend to handle nonneg zext as sext
  EVT AccElemVT = Acc.getValueType().getVectorElementType();
  if (Op1.getValueType().getVectorElementType() != AccElemVT &&
      NewOpc != N->getOpcode())
    return SDValue();
 
  // Only perform these combines if the target supports folding
  // the extends into the operation.
  if (!TLI.isPartialReduceMLALegalOrCustom(
          NewOpc, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
          TLI.getTypeToTransformTo(*Context, LHSExtOpVT)))
    return SDValue();
 
  ApplyPredicate(RHSExtOp, LHSExtOp);
  return GetMLA(NewOpc, Acc, LHSExtOp, RHSExtOp);
}
 
// partial.reduce.*mla(acc, *ext(op), splat(1))
// -> partial.reduce.*mla(acc, op, splat(trunc(1)))
// partial.reduce.sumla(acc, sext(op), splat(1))
// -> partial.reduce.smla(acc, op, splat(trunc(1)))
//
// partial.reduce.*mla(acc, sel(p, *ext(op), splat(0)), splat(1))
// -> partial.reduce.*mla(acc, sel(p, op, splat(0)), splat(trunc(1)))
SDValue DAGCombiner::foldPartialReduceAdd(SDNode *N) {
  SDLoc DL(N);
  SDValue Tmp;
  SDValue Acc = N->getOperand(0);
  SDValue Op1 = N->getOperand(1);
  SDValue Op2 = N->getOperand(2);
 
  if (!llvm::isOneOrOneSplat(Op2) && !llvm::isOneOrOneSplatFP(Op2))
    return SDValue();
 
  SDValue Pred;
  unsigned Op1Opcode = Op1.getOpcode();
  if (Op1Opcode == ISD::VSELECT && (isZeroOrZeroSplat(Op1->getOperand(2)) ||
                                    isZeroOrZeroSplatFP(Op1->getOperand(2)))) {
    Pred = Op1->getOperand(0);
    Op1 = Op1->getOperand(1);
    Op1Opcode = Op1->getOpcode();
  }
 
  // Handle negation (sub-reduction).
  bool IsMLS = false;
  if (sd_match(Op1, m_AnyOf(m_Neg(m_Value(Tmp)), m_FNeg(m_Value(Tmp))))) {
    Op1 = Tmp;
    Op1Opcode = Op1.getOpcode();
    IsMLS = true;
  }
 
  if (!ISD::isExtOpcode(Op1Opcode) && Op1Opcode != ISD::FP_EXTEND)
    return SDValue();
 
  bool Op1IsSigned =
      Op1Opcode == ISD::SIGN_EXTEND || Op1Opcode == ISD::FP_EXTEND;
  bool NodeIsSigned = N->getOpcode() != ISD::PARTIAL_REDUCE_UMLA;
  EVT AccElemVT = Acc.getValueType().getVectorElementType();
  if (Op1IsSigned != NodeIsSigned &&
      Op1.getValueType().getVectorElementType() != AccElemVT)
    return SDValue();
 
  unsigned NewOpcode = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
                           ? ISD::PARTIAL_REDUCE_FMLA
                       : Op1IsSigned ? ISD::PARTIAL_REDUCE_SMLA
                                     : ISD::PARTIAL_REDUCE_UMLA;
 
  SDValue UnextOp1 = Op1.getOperand(0);
  EVT UnextOp1VT = UnextOp1.getValueType();
  auto *Context = DAG.getContext();
  if (!TLI.isPartialReduceMLALegalOrCustom(
          NewOpcode, TLI.getTypeToTransformTo(*Context, N->getValueType(0)),
          TLI.getTypeToTransformTo(*Context, UnextOp1VT)))
    return SDValue();
 
  SDValue Constant = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
                         ? DAG.getConstantFP(1, DL, UnextOp1VT)
                         : DAG.getConstant(1, DL, UnextOp1VT);
 
  if (Pred) {
    SDValue Zero = N->getOpcode() == ISD::PARTIAL_REDUCE_FMLA
                       ? DAG.getConstantFP(0, DL, UnextOp1VT)
                       : DAG.getConstant(0, DL, UnextOp1VT);
    Constant = DAG.getSelect(DL, UnextOp1VT, Pred, Constant, Zero);
  }
  EVT AccVT = Acc.getValueType();
  return IsMLS ? DAG.getPartialReduceMLS(NewOpcode, DL, Acc, UnextOp1, Constant)
               : DAG.getNode(NewOpcode, DL, AccVT, Acc, UnextOp1, Constant);
}
 
SDValue DAGCombiner::visitVP_STRIDED_LOAD(SDNode *N) {
  auto *SLD = cast<VPStridedLoadSDNode>(N);
  EVT EltVT = SLD->getValueType(0).getVectorElementType();
  // Combine strided loads with unit-stride to a regular VP load.
  if (auto *CStride = dyn_cast<ConstantSDNode>(SLD->getStride());
      CStride && CStride->getZExtValue() == EltVT.getStoreSize()) {
    SDValue NewLd = DAG.getLoadVP(
        SLD->getAddressingMode(), SLD->getExtensionType(), SLD->getValueType(0),
        SDLoc(N), SLD->getChain(), SLD->getBasePtr(), SLD->getOffset(),
        SLD->getMask(), SLD->getVectorLength(), SLD->getMemoryVT(),
        SLD->getMemOperand(), SLD->isExpandingLoad());
    return CombineTo(N, NewLd, NewLd.getValue(1));
  }
  return SDValue();
}
 
/// A vector select of 2 constant vectors can be simplified to math/logic to
/// avoid a variable select instruction and possibly avoid constant loads.
SDValue DAGCombiner::foldVSelectOfConstants(SDNode *N) {
  SDValue Cond = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  EVT VT = N->getValueType(0);
  if (!Cond.hasOneUse() || Cond.getScalarValueSizeInBits() != 1 ||
      !shouldConvertSelectOfConstantsToMath(Cond, VT, TLI) ||
      !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()) ||
      !ISD::isBuildVectorOfConstantSDNodes(N2.getNode()))
    return SDValue();
 
  // Check if we can use the condition value to increment/decrement a single
  // constant value. This simplifies a select to an add and removes a constant
  // load/materialization from the general case.
  bool AllAddOne = true;
  bool AllSubOne = true;
  unsigned Elts = VT.getVectorNumElements();
  for (unsigned i = 0; i != Elts; ++i) {
    SDValue N1Elt = N1.getOperand(i);
    SDValue N2Elt = N2.getOperand(i);
    if (N1Elt.isUndef())
      continue;
    // N2 should not contain undef values since it will be reused in the fold.
    if (N2Elt.isUndef() || N1Elt.getValueType() != N2Elt.getValueType()) {
      AllAddOne = false;
      AllSubOne = false;
      break;
    }
 
    const APInt &C1 = N1Elt->getAsAPIntVal();
    const APInt &C2 = N2Elt->getAsAPIntVal();
    if (C1 != C2 + 1)
      AllAddOne = false;
    if (C1 != C2 - 1)
      AllSubOne = false;
  }
 
  // Further simplifications for the extra-special cases where the constants are
  // all 0 or all -1 should be implemented as folds of these patterns.
  SDLoc DL(N);
  if (AllAddOne || AllSubOne) {
    // vselect <N x i1> Cond, C+1, C --> add (zext Cond), C
    // vselect <N x i1> Cond, C-1, C --> add (sext Cond), C
    auto ExtendOpcode = AllAddOne ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND;
    SDValue ExtendedCond = DAG.getNode(ExtendOpcode, DL, VT, Cond);
    return DAG.getNode(ISD::ADD, DL, VT, ExtendedCond, N2);
  }
 
  // select Cond, Pow2C, 0 --> (zext Cond) << log2(Pow2C)
  APInt Pow2C;
  if (ISD::isConstantSplatVector(N1.getNode(), Pow2C) && Pow2C.isPowerOf2() &&
      isNullOrNullSplat(N2)) {
    SDValue ZextCond = DAG.getZExtOrTrunc(Cond, DL, VT);
    SDValue ShAmtC = DAG.getConstant(Pow2C.exactLogBase2(), DL, VT);
    return DAG.getNode(ISD::SHL, DL, VT, ZextCond, ShAmtC);
  }
 
  if (SDValue V = foldSelectOfConstantsUsingSra(N, DL, DAG))
    return V;
 
  // The general case for select-of-constants:
  // vselect <N x i1> Cond, C1, C2 --> xor (and (sext Cond), (C1^C2)), C2
  // ...but that only makes sense if a vselect is slower than 2 logic ops, so
  // leave that to a machine-specific pass.
  return SDValue();
}
 
SDValue DAGCombiner::visitVP_SELECT(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  SDLoc DL(N);
 
  if (SDValue V = DAG.simplifySelect(N0, N1, N2))
    return V;
 
  if (SDValue V = foldBoolSelectToLogic<VPMatchContext>(N, DL, DAG))
    return V;
 
  return SDValue();
}
 
static SDValue combineVSelectWithAllOnesOrZeros(SDValue Cond, SDValue TVal,
                                                SDValue FVal,
                                                const TargetLowering &TLI,
                                                SelectionDAG &DAG,
                                                const SDLoc &DL) {
  EVT VT = TVal.getValueType();
  if (!TLI.isTypeLegal(VT))
    return SDValue();
 
  EVT CondVT = Cond.getValueType();
  assert(CondVT.isVector() && "Vector select expects a vector selector!");
 
  bool IsTAllZero = ISD::isConstantSplatVectorAllZeros(TVal.getNode());
  bool IsTAllOne = ISD::isConstantSplatVectorAllOnes(TVal.getNode());
  bool IsFAllZero = ISD::isConstantSplatVectorAllZeros(FVal.getNode());
  bool IsFAllOne = ISD::isConstantSplatVectorAllOnes(FVal.getNode());
 
  // no vselect(cond, 0/-1, X) or vselect(cond, X, 0/-1), return
  if (!IsTAllZero && !IsTAllOne && !IsFAllZero && !IsFAllOne)
    return SDValue();
 
  // select Cond, 0, 0 → 0
  if (IsTAllZero && IsFAllZero) {
    return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, DL, VT)
                                : DAG.getConstant(0, DL, VT);
  }
 
  // check select(setgt lhs, -1), 1, -1 --> or (sra lhs, bitwidth - 1), 1
  APInt TValAPInt;
  if (Cond.getOpcode() == ISD::SETCC &&
      Cond.getOperand(2) == DAG.getCondCode(ISD::SETGT) &&
      Cond.getOperand(0).getValueType() == VT && VT.isSimple() &&
      ISD::isConstantSplatVector(TVal.getNode(), TValAPInt) &&
      TValAPInt.isOne() &&
      ISD::isConstantSplatVectorAllOnes(Cond.getOperand(1).getNode()) &&
      ISD::isConstantSplatVectorAllOnes(FVal.getNode()) &&
      !TLI.shouldAvoidTransformToShift(VT, VT.getScalarSizeInBits() - 1)) {
    SDValue LHS = Cond.getOperand(0);
    SDValue ShiftC =
        DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, DL);
    SDValue Shift = DAG.getNode(ISD::SRA, DL, VT, LHS, ShiftC);
    return DAG.getNode(ISD::OR, DL, VT, Shift, TVal);
  }
 
  // To use the condition operand as a bitwise mask, it must have elements that
  // are the same size as the select elements. i.e, the condition operand must
  // have already been promoted from the IR select condition type <N x i1>.
  // Don't check if the types themselves are equal because that excludes
  // vector floating-point selects.
  if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
    return SDValue();
 
  // Cond value must be 'sign splat' to be converted to a logical op.
  if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
    return SDValue();
 
  // Try inverting Cond and swapping T/F if it gives all-ones/all-zeros form
  if (!IsTAllOne && !IsFAllZero && Cond.hasOneUse() &&
      Cond.getOpcode() == ISD::SETCC &&
      TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
          CondVT) {
    if (IsTAllZero || IsFAllOne) {
      SDValue CC = Cond.getOperand(2);
      ISD::CondCode InverseCC = ISD::getSetCCInverse(
          cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
      Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
                          InverseCC);
      std::swap(TVal, FVal);
      std::swap(IsTAllOne, IsFAllOne);
      std::swap(IsTAllZero, IsFAllZero);
    }
  }
 
  assert(DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits() &&
         "Select condition no longer all-sign bits");
 
  // select Cond, -1, 0 → bitcast Cond
  if (IsTAllOne && IsFAllZero)
    return DAG.getBitcast(VT, Cond);
 
  // select Cond, -1, x → or Cond, x
  if (IsTAllOne) {
    SDValue X = DAG.getBitcast(CondVT, DAG.getFreeze(FVal));
    SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, X);
    return DAG.getBitcast(VT, Or);
  }
 
  // select Cond, x, 0 → and Cond, x
  if (IsFAllZero) {
    SDValue X = DAG.getBitcast(CondVT, DAG.getFreeze(TVal));
    SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, X);
    return DAG.getBitcast(VT, And);
  }
 
  // select Cond, 0, x -> and not(Cond), x
  if (IsTAllZero &&
      (isBitwiseNot(peekThroughBitcasts(Cond)) || TLI.hasAndNot(Cond))) {
    SDValue X = DAG.getBitcast(CondVT, DAG.getFreeze(FVal));
    SDValue And =
        DAG.getNode(ISD::AND, DL, CondVT, DAG.getNOT(DL, Cond, CondVT), X);
    return DAG.getBitcast(VT, And);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitVSELECT(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  if (SDValue V = DAG.simplifySelect(N0, N1, N2))
    return V;
 
  if (SDValue V = foldBoolSelectToLogic<EmptyMatchContext>(N, DL, DAG))
    return V;
 
  // vselect (not Cond), N1, N2 -> vselect Cond, N2, N1
  if (!TLI.isTargetCanonicalSelect(N))
    if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
      return DAG.getSelect(DL, VT, F, N2, N1, N->getFlags());
 
  // select (sext m), (add X, C), X --> (add X, (and C, (sext m))))
  if (N1.getOpcode() == ISD::ADD && N1.getOperand(0) == N2 && N1->hasOneUse() &&
      DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1)) &&
      N0.getScalarValueSizeInBits() == N1.getScalarValueSizeInBits() &&
      TLI.getBooleanContents(N0.getValueType()) ==
          TargetLowering::ZeroOrNegativeOneBooleanContent) {
    return DAG.getNode(
        ISD::ADD, DL, N1.getValueType(), N2,
        DAG.getNode(ISD::AND, DL, N0.getValueType(), N1.getOperand(1), N0));
  }
 
  // Canonicalize integer abs.
  // vselect (setg[te] X,  0),  X, -X ->
  // vselect (setgt    X, -1),  X, -X ->
  // vselect (setl[te] X,  0), -X,  X ->
  // Y = sra (X, size(X)-1); xor (add (X, Y), Y)
  if (N0.getOpcode() == ISD::SETCC) {
    SDValue LHS = N0.getOperand(0), RHS = N0.getOperand(1);
    ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
    bool isAbs = false;
    bool RHSIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
 
    if (((RHSIsAllZeros && (CC == ISD::SETGT || CC == ISD::SETGE)) ||
         (ISD::isBuildVectorAllOnes(RHS.getNode()) && CC == ISD::SETGT)) &&
        N1 == LHS && N2.getOpcode() == ISD::SUB && N1 == N2.getOperand(1))
      isAbs = ISD::isBuildVectorAllZeros(N2.getOperand(0).getNode());
    else if ((RHSIsAllZeros && (CC == ISD::SETLT || CC == ISD::SETLE)) &&
             N2 == LHS && N1.getOpcode() == ISD::SUB && N2 == N1.getOperand(1))
      isAbs = ISD::isBuildVectorAllZeros(N1.getOperand(0).getNode());
 
    if (isAbs) {
      if (TLI.isOperationLegalOrCustom(ISD::ABS, VT))
        return DAG.getNode(ISD::ABS, DL, VT, LHS);
 
      SDValue Shift = DAG.getNode(
          ISD::SRA, DL, VT, LHS,
          DAG.getShiftAmountConstant(VT.getScalarSizeInBits() - 1, VT, DL));
      SDValue Add = DAG.getNode(ISD::ADD, DL, VT, LHS, Shift);
      AddToWorklist(Shift.getNode());
      AddToWorklist(Add.getNode());
      return DAG.getNode(ISD::XOR, DL, VT, Add, Shift);
    }
 
    // vselect x, y (fcmp lt x, y) -> fminnum x, y
    // vselect x, y (fcmp gt x, y) -> fmaxnum x, y
    //
    // This is OK if we don't care about what happens if either operand is a
    // NaN.
    //
    if (N0.hasOneUse() &&
        isLegalToCombineMinNumMaxNum(DAG, LHS, RHS, N->getFlags(), TLI)) {
      if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, LHS, RHS, N1, N2, CC))
        return FMinMax;
    }
 
    if (SDValue S = PerformMinMaxFpToSatCombine(LHS, RHS, N1, N2, CC, DAG))
      return S;
    if (SDValue S = PerformUMinFpToSatCombine(LHS, RHS, N1, N2, CC, DAG))
      return S;
 
    // If this select has a condition (setcc) with narrower operands than the
    // select, try to widen the compare to match the select width.
    // TODO: This should be extended to handle any constant.
    // TODO: This could be extended to handle non-loading patterns, but that
    //       requires thorough testing to avoid regressions.
    if (isNullOrNullSplat(RHS)) {
      EVT NarrowVT = LHS.getValueType();
      EVT WideVT = N1.getValueType().changeVectorElementTypeToInteger();
      EVT SetCCVT = getSetCCResultType(LHS.getValueType());
      unsigned SetCCWidth = SetCCVT.getScalarSizeInBits();
      unsigned WideWidth = WideVT.getScalarSizeInBits();
      bool IsSigned = isSignedIntSetCC(CC);
      auto LoadExtOpcode = IsSigned ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
      if (LHS.getOpcode() == ISD::LOAD && LHS.hasOneUse() && SetCCWidth != 1 &&
          SetCCWidth < WideWidth &&
          TLI.isOperationLegalOrCustom(ISD::SETCC, WideVT)) {
        LoadSDNode *Ld = cast<LoadSDNode>(LHS);
 
        if (TLI.isLoadLegalOrCustom(WideVT, NarrowVT, Ld->getAlign(),
                                    Ld->getAddressSpace(), LoadExtOpcode,
                                    false)) {
          // Both compare operands can be widened for free. The LHS can use an
          // extended load, and the RHS is a constant:
          //   vselect (ext (setcc load(X), C)), N1, N2 -->
          //   vselect (setcc extload(X), C'), N1, N2
          auto ExtOpcode = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
          SDValue WideLHS = DAG.getNode(ExtOpcode, DL, WideVT, LHS);
          SDValue WideRHS = DAG.getNode(ExtOpcode, DL, WideVT, RHS);
          EVT WideSetCCVT = getSetCCResultType(WideVT);
          SDValue WideSetCC =
              DAG.getSetCC(DL, WideSetCCVT, WideLHS, WideRHS, CC);
          return DAG.getSelect(DL, N1.getValueType(), WideSetCC, N1, N2);
        }
      }
    }
 
    if (SDValue ABD = foldSelectToABD(LHS, RHS, N1, N2, CC, DL))
      return ABD;
 
    // Match VSELECTs into add with unsigned saturation.
    if (hasOperation(ISD::UADDSAT, VT)) {
      // Check if one of the arms of the VSELECT is vector with all bits set.
      // If it's on the left side invert the predicate to simplify logic below.
      SDValue Other;
      ISD::CondCode SatCC = CC;
      if (ISD::isConstantSplatVectorAllOnes(N1.getNode())) {
        Other = N2;
        SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType());
      } else if (ISD::isConstantSplatVectorAllOnes(N2.getNode())) {
        Other = N1;
      }
 
      if (Other && Other.getOpcode() == ISD::ADD) {
        SDValue CondLHS = LHS, CondRHS = RHS;
        SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
 
        // Canonicalize condition operands.
        if (SatCC == ISD::SETUGE) {
          std::swap(CondLHS, CondRHS);
          SatCC = ISD::SETULE;
        }
 
        // We can test against either of the addition operands.
        // x <= x+y ? x+y : ~0 --> uaddsat x, y
        // x+y >= x ? x+y : ~0 --> uaddsat x, y
        if (SatCC == ISD::SETULE && Other == CondRHS &&
            (OpLHS == CondLHS || OpRHS == CondLHS))
          return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
 
        if (OpRHS.getOpcode() == CondRHS.getOpcode() &&
            (OpRHS.getOpcode() == ISD::BUILD_VECTOR ||
             OpRHS.getOpcode() == ISD::SPLAT_VECTOR) &&
            CondLHS == OpLHS) {
          // If the RHS is a constant we have to reverse the const
          // canonicalization.
          // x >= ~C ? x+C : ~0 --> uaddsat x, C
          auto MatchUADDSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
            return Cond->getAPIntValue() == ~Op->getAPIntValue();
          };
          if (SatCC == ISD::SETULE &&
              ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT))
            return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
        }
      }
    }
 
    // Match VSELECTs into sub with unsigned saturation.
    if (hasOperation(ISD::USUBSAT, VT)) {
      // Check if one of the arms of the VSELECT is a zero vector. If it's on
      // the left side invert the predicate to simplify logic below.
      SDValue Other;
      ISD::CondCode SatCC = CC;
      if (ISD::isConstantSplatVectorAllZeros(N1.getNode())) {
        Other = N2;
        SatCC = ISD::getSetCCInverse(SatCC, VT.getScalarType());
      } else if (ISD::isConstantSplatVectorAllZeros(N2.getNode())) {
        Other = N1;
      }
 
      // zext(x) >= y ? trunc(zext(x) - y) : 0
      // --> usubsat(trunc(zext(x)),trunc(umin(y,SatLimit)))
      // zext(x) >  y ? trunc(zext(x) - y) : 0
      // --> usubsat(trunc(zext(x)),trunc(umin(y,SatLimit)))
      if (Other && Other.getOpcode() == ISD::TRUNCATE &&
          Other.getOperand(0).getOpcode() == ISD::SUB &&
          (SatCC == ISD::SETUGE || SatCC == ISD::SETUGT)) {
        SDValue OpLHS = Other.getOperand(0).getOperand(0);
        SDValue OpRHS = Other.getOperand(0).getOperand(1);
        if (LHS == OpLHS && RHS == OpRHS && LHS.getOpcode() == ISD::ZERO_EXTEND)
          if (SDValue R = getTruncatedUSUBSAT(VT, LHS.getValueType(), LHS, RHS,
                                              DAG, DL))
            return R;
      }
 
      if (Other && Other.getNumOperands() == 2) {
        SDValue CondRHS = RHS;
        SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
 
        if (OpLHS == LHS) {
          // Look for a general sub with unsigned saturation first.
          // x >= y ? x-y : 0 --> usubsat x, y
          // x >  y ? x-y : 0 --> usubsat x, y
          if ((SatCC == ISD::SETUGE || SatCC == ISD::SETUGT) &&
              Other.getOpcode() == ISD::SUB && OpRHS == CondRHS)
            return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
 
          if (OpRHS.getOpcode() == ISD::BUILD_VECTOR ||
              OpRHS.getOpcode() == ISD::SPLAT_VECTOR) {
            if (CondRHS.getOpcode() == ISD::BUILD_VECTOR ||
                CondRHS.getOpcode() == ISD::SPLAT_VECTOR) {
              // If the RHS is a constant we have to reverse the const
              // canonicalization.
              // x > C-1 ? x+-C : 0 --> usubsat x, C
              auto MatchUSUBSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
                return (!Op && !Cond) ||
                       (Op && Cond &&
                        Cond->getAPIntValue() == (-Op->getAPIntValue() - 1));
              };
              if (SatCC == ISD::SETUGT && Other.getOpcode() == ISD::ADD &&
                  ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT,
                                            /*AllowUndefs*/ true)) {
                OpRHS = DAG.getNegative(OpRHS, DL, VT);
                return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
              }
 
              // Another special case: If C was a sign bit, the sub has been
              // canonicalized into a xor.
              // FIXME: Would it be better to use computeKnownBits to
              // determine whether it's safe to decanonicalize the xor?
              // x s< 0 ? x^C : 0 --> usubsat x, C
              APInt SplatValue;
              if (SatCC == ISD::SETLT && Other.getOpcode() == ISD::XOR &&
                  ISD::isConstantSplatVector(OpRHS.getNode(), SplatValue) &&
                  ISD::isConstantSplatVectorAllZeros(CondRHS.getNode()) &&
                  SplatValue.isSignMask()) {
                // Note that we have to rebuild the RHS constant here to
                // ensure we don't rely on particular values of undef lanes.
                OpRHS = DAG.getConstant(SplatValue, DL, VT);
                return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
              }
            }
          }
        }
      }
    }
 
    // (vselect (ugt x, C), (add x, ~C), x) -> (umin (add x, ~C), x)
    // (vselect (ult x, C), x, (add x, -C)) -> (umin x, (add x, -C))
    if (SDValue UMin = foldSelectToUMin(LHS, RHS, N1, N2, CC, DL))
      return UMin;
  }
 
  if (SimplifySelectOps(N, N1, N2))
    return SDValue(N, 0);  // Don't revisit N.
 
  // Fold (vselect all_ones, N1, N2) -> N1
  if (ISD::isConstantSplatVectorAllOnes(N0.getNode()))
    return N1;
  // Fold (vselect all_zeros, N1, N2) -> N2
  if (ISD::isConstantSplatVectorAllZeros(N0.getNode()))
    return N2;
 
  // The ConvertSelectToConcatVector function is assuming both the above
  // checks for (vselect (build_vector all{ones,zeros) ...) have been made
  // and addressed.
  if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
      N2.getOpcode() == ISD::CONCAT_VECTORS &&
      ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
    if (SDValue CV = ConvertSelectToConcatVector(N, DAG))
      return CV;
  }
 
  if (SDValue V = foldVSelectOfConstants(N))
    return V;
 
  if (hasOperation(ISD::SRA, VT))
    if (SDValue V = foldVSelectToSignBitSplatMask(N, DAG))
      return V;
 
  if (SimplifyDemandedVectorElts(SDValue(N, 0)))
    return SDValue(N, 0);
 
  if (SDValue V = combineVSelectWithAllOnesOrZeros(N0, N1, N2, TLI, DAG, DL))
    return V;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSELECT_CC(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  SDValue N3 = N->getOperand(3);
  SDValue N4 = N->getOperand(4);
  ISD::CondCode CC = cast<CondCodeSDNode>(N4)->get();
  SDLoc DL(N);
 
  // fold select_cc lhs, rhs, x, x, cc -> x
  if (N2 == N3)
    return N2;
 
  // select_cc bool, 0, x, y, seteq -> select bool, y, x
  if (CC == ISD::SETEQ && !LegalTypes && N0.getValueType() == MVT::i1 &&
      isNullConstant(N1))
    return DAG.getSelect(DL, N2.getValueType(), N0, N3, N2);
 
  // Determine if the condition we're dealing with is constant
  if (SDValue SCC = SimplifySetCC(getSetCCResultType(N0.getValueType()), N0, N1,
                                  CC, DL, false)) {
    AddToWorklist(SCC.getNode());
 
    // cond always true -> true val
    // cond always false -> false val
    if (auto *SCCC = dyn_cast<ConstantSDNode>(SCC.getNode()))
      return SCCC->isZero() ? N3 : N2;
 
    // When the condition is UNDEF, just return the first operand. This is
    // coherent the DAG creation, no setcc node is created in this case
    if (SCC->isUndef())
      return N2;
 
    // Fold to a simpler select_cc
    if (SCC.getOpcode() == ISD::SETCC) {
      return DAG.getNode(ISD::SELECT_CC, DL, N2.getValueType(),
                         SCC.getOperand(0), SCC.getOperand(1), N2, N3,
                         SCC.getOperand(2), SCC->getFlags());
    }
  }
 
  // If we can fold this based on the true/false value, do so.
  if (SimplifySelectOps(N, N2, N3))
    return SDValue(N, 0); // Don't revisit N.
 
  // fold select_cc into other things, such as min/max/abs
  return SimplifySelectCC(DL, N0, N1, N2, N3, CC);
}
 
SDValue DAGCombiner::visitSETCC(SDNode *N) {
  // setcc is very commonly used as an argument to brcond or cond_loop. This
  // pattern also lend itself to numerous combines and, as a result, it is
  // desired we keep the argument to a brcond as a setcc as much as possible.
  bool PreferSetCC =
      N->hasOneUse() && (N->user_begin()->getOpcode() == ISD::BRCOND ||
                         N->user_begin()->getOpcode() == ISD::COND_LOOP);
 
  ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
  EVT VT = N->getValueType(0);
  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
  SDLoc DL(N);
 
  if (SDValue Combined = SimplifySetCC(VT, N0, N1, Cond, DL, !PreferSetCC)) {
    // If we prefer to have a setcc, and we don't, we'll try our best to
    // recreate one using rebuildSetCC.
    if (PreferSetCC && Combined.getOpcode() != ISD::SETCC) {
      SDValue NewSetCC = rebuildSetCC(Combined);
 
      // We don't have anything interesting to combine to.
      if (NewSetCC.getNode() == N)
        return SDValue();
 
      if (NewSetCC)
        return NewSetCC;
    }
    return Combined;
  }
 
  // Optimize
  //    1) (icmp eq/ne (and X, C0), (shift X, C1))
  // or
  //    2) (icmp eq/ne X, (rotate X, C1))
  // If C0 is a mask or shifted mask and the shift amt (C1) isolates the
  // remaining bits (i.e something like `(x64 & UINT32_MAX) == (x64 >> 32)`)
  // Then:
  // If C1 is a power of 2, then the rotate and shift+and versions are
  // equivilent, so we can interchange them depending on target preference.
  // Otherwise, if we have the shift+and version we can interchange srl/shl
  // which inturn affects the constant C0. We can use this to get better
  // constants again determined by target preference.
  if (Cond == ISD::SETNE || Cond == ISD::SETEQ) {
    auto IsAndWithShift = [](SDValue A, SDValue B) {
      return A.getOpcode() == ISD::AND &&
             (B.getOpcode() == ISD::SRL || B.getOpcode() == ISD::SHL) &&
             A.getOperand(0) == B.getOperand(0);
    };
    auto IsRotateWithOp = [](SDValue A, SDValue B) {
      return (B.getOpcode() == ISD::ROTL || B.getOpcode() == ISD::ROTR) &&
             B.getOperand(0) == A;
    };
    SDValue AndOrOp = SDValue(), ShiftOrRotate = SDValue();
    bool IsRotate = false;
 
    // Find either shift+and or rotate pattern.
    if (IsAndWithShift(N0, N1)) {
      AndOrOp = N0;
      ShiftOrRotate = N1;
    } else if (IsAndWithShift(N1, N0)) {
      AndOrOp = N1;
      ShiftOrRotate = N0;
    } else if (IsRotateWithOp(N0, N1)) {
      IsRotate = true;
      AndOrOp = N0;
      ShiftOrRotate = N1;
    } else if (IsRotateWithOp(N1, N0)) {
      IsRotate = true;
      AndOrOp = N1;
      ShiftOrRotate = N0;
    }
 
    if (AndOrOp && ShiftOrRotate && ShiftOrRotate.hasOneUse() &&
        (IsRotate || AndOrOp.hasOneUse())) {
      EVT OpVT = N0.getValueType();
      // Get constant shift/rotate amount and possibly mask (if its shift+and
      // variant).
      auto GetAPIntValue = [](SDValue Op) -> std::optional<APInt> {
        ConstantSDNode *CNode = isConstOrConstSplat(Op, /*AllowUndefs*/ false,
                                                    /*AllowTrunc*/ false);
        if (CNode == nullptr)
          return std::nullopt;
        return CNode->getAPIntValue();
      };
      std::optional<APInt> AndCMask =
          IsRotate ? std::nullopt : GetAPIntValue(AndOrOp.getOperand(1));
      std::optional<APInt> ShiftCAmt =
          GetAPIntValue(ShiftOrRotate.getOperand(1));
      unsigned NumBits = OpVT.getScalarSizeInBits();
 
      // We found constants.
      if (ShiftCAmt && (IsRotate || AndCMask) && ShiftCAmt->ult(NumBits)) {
        unsigned ShiftOpc = ShiftOrRotate.getOpcode();
        // Check that the constants meet the constraints.
        bool CanTransform = IsRotate;
        if (!CanTransform) {
          // Check that mask and shift compliment eachother
          CanTransform = *ShiftCAmt == (~*AndCMask).popcount();
          // Check that we are comparing all bits
          CanTransform &= (*ShiftCAmt + AndCMask->popcount()) == NumBits;
          // Check that the and mask is correct for the shift
          CanTransform &=
              ShiftOpc == ISD::SHL ? (~*AndCMask).isMask() : AndCMask->isMask();
        }
 
        // See if target prefers another shift/rotate opcode.
        unsigned NewShiftOpc = TLI.preferedOpcodeForCmpEqPiecesOfOperand(
            OpVT, ShiftOpc, ShiftCAmt->isPowerOf2(), *ShiftCAmt, AndCMask);
        // Transform is valid and we have a new preference.
        if (CanTransform && NewShiftOpc != ShiftOpc) {
          SDValue NewShiftOrRotate =
              DAG.getNode(NewShiftOpc, DL, OpVT, ShiftOrRotate.getOperand(0),
                          ShiftOrRotate.getOperand(1));
          SDValue NewAndOrOp = SDValue();
 
          if (NewShiftOpc == ISD::SHL || NewShiftOpc == ISD::SRL) {
            APInt NewMask =
                NewShiftOpc == ISD::SHL
                    ? APInt::getHighBitsSet(NumBits,
                                            NumBits - ShiftCAmt->getZExtValue())
                    : APInt::getLowBitsSet(NumBits,
                                           NumBits - ShiftCAmt->getZExtValue());
            NewAndOrOp =
                DAG.getNode(ISD::AND, DL, OpVT, ShiftOrRotate.getOperand(0),
                            DAG.getConstant(NewMask, DL, OpVT));
          } else {
            NewAndOrOp = ShiftOrRotate.getOperand(0);
          }
 
          return DAG.getSetCC(DL, VT, NewAndOrOp, NewShiftOrRotate, Cond);
        }
      }
    }
  }
  return SDValue();
}
 
SDValue DAGCombiner::visitSETCCCARRY(SDNode *N) {
  SDValue LHS = N->getOperand(0);
  SDValue RHS = N->getOperand(1);
  SDValue Carry = N->getOperand(2);
  SDValue Cond = N->getOperand(3);
 
  // If Carry is false, fold to a regular SETCC.
  if (isNullConstant(Carry))
    return DAG.getNode(ISD::SETCC, SDLoc(N), N->getVTList(), LHS, RHS, Cond);
 
  return SDValue();
}
 
/// Check if N satisfies:
///   N is used once.
///   N is a Load.
///   The load is compatible with ExtOpcode. It means
///     If load has explicit zero/sign extension, ExpOpcode must have the same
///     extension.
///     Otherwise returns true.
static bool isCompatibleLoad(SDValue N, unsigned ExtOpcode) {
  if (!N.hasOneUse())
    return false;
 
  if (!isa<LoadSDNode>(N))
    return false;
 
  LoadSDNode *Load = cast<LoadSDNode>(N);
  ISD::LoadExtType LoadExt = Load->getExtensionType();
  if (LoadExt == ISD::NON_EXTLOAD || LoadExt == ISD::EXTLOAD)
    return true;
 
  // Now LoadExt is either SEXTLOAD or ZEXTLOAD, ExtOpcode must have the same
  // extension.
  if ((LoadExt == ISD::SEXTLOAD && ExtOpcode != ISD::SIGN_EXTEND) ||
      (LoadExt == ISD::ZEXTLOAD && ExtOpcode != ISD::ZERO_EXTEND))
    return false;
 
  return true;
}
 
/// Fold
///   (sext (select c, load x, load y)) -> (select c, sextload x, sextload y)
///   (zext (select c, load x, load y)) -> (select c, zextload x, zextload y)
///   (aext (select c, load x, load y)) -> (select c, extload x, extload y)
/// This function is called by the DAGCombiner when visiting sext/zext/aext
/// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
static SDValue tryToFoldExtendSelectLoad(SDNode *N, const TargetLowering &TLI,
                                         SelectionDAG &DAG, const SDLoc &DL,
                                         CombineLevel Level) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND ||
          Opcode == ISD::ANY_EXTEND) &&
         "Expected EXTEND dag node in input!");
 
  SDValue Cond, Op1, Op2;
  if (!sd_match(N0, m_OneUse(m_SelectLike(m_Value(Cond), m_Value(Op1),
                                          m_Value(Op2)))))
    return SDValue();
 
  if (!isCompatibleLoad(Op1, Opcode) || !isCompatibleLoad(Op2, Opcode))
    return SDValue();
 
  auto ExtLoadOpcode = ISD::EXTLOAD;
  if (Opcode == ISD::SIGN_EXTEND)
    ExtLoadOpcode = ISD::SEXTLOAD;
  else if (Opcode == ISD::ZERO_EXTEND)
    ExtLoadOpcode = ISD::ZEXTLOAD;
 
  // Illegal VSELECT may ISel fail if happen after legalization (DAG
  // Combine2), so we should conservatively check the OperationAction.
  LoadSDNode *Load1 = cast<LoadSDNode>(Op1);
  LoadSDNode *Load2 = cast<LoadSDNode>(Op2);
  if (!TLI.isLoadLegal(VT, Load1->getMemoryVT(), Load1->getAlign(),
                       Load1->getAddressSpace(), ExtLoadOpcode, false) ||
      !TLI.isLoadLegal(VT, Load2->getMemoryVT(), Load2->getAlign(),
                       Load2->getAddressSpace(), ExtLoadOpcode, false) ||
      (N0->getOpcode() == ISD::VSELECT && Level >= AfterLegalizeTypes &&
       TLI.getOperationAction(ISD::VSELECT, VT) != TargetLowering::Legal))
    return SDValue();
 
  SDValue Ext1 = DAG.getNode(Opcode, DL, VT, Op1);
  SDValue Ext2 = DAG.getNode(Opcode, DL, VT, Op2);
  return DAG.getSelect(DL, VT, Cond, Ext1, Ext2);
}
 
/// Try to fold a sext/zext/aext dag node into a ConstantSDNode or
/// a build_vector of constants.
/// This function is called by the DAGCombiner when visiting sext/zext/aext
/// dag nodes (see for example method DAGCombiner::visitSIGN_EXTEND).
/// Vector extends are not folded if operations are legal; this is to
/// avoid introducing illegal build_vector dag nodes.
static SDValue tryToFoldExtendOfConstant(SDNode *N, const SDLoc &DL,
                                         const TargetLowering &TLI,
                                         SelectionDAG &DAG, bool LegalTypes) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
 
  assert((ISD::isExtOpcode(Opcode) || ISD::isExtVecInRegOpcode(Opcode)) &&
         "Expected EXTEND dag node in input!");
 
  // fold (sext c1) -> c1
  // fold (zext c1) -> c1
  // fold (aext c1) -> c1
  if (isa<ConstantSDNode>(N0))
    return DAG.getNode(Opcode, DL, VT, N0);
 
  // fold (sext (select cond, c1, c2)) -> (select cond, sext c1, sext c2)
  // fold (zext (select cond, c1, c2)) -> (select cond, zext c1, zext c2)
  // fold (aext (select cond, c1, c2)) -> (select cond, sext c1, sext c2)
  if (N0->getOpcode() == ISD::SELECT) {
    SDValue Op1 = N0->getOperand(1);
    SDValue Op2 = N0->getOperand(2);
    if (isa<ConstantSDNode>(Op1) && isa<ConstantSDNode>(Op2) &&
        (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) {
      // For any_extend, choose sign extension of the constants to allow a
      // possible further transform to sign_extend_inreg.i.e.
      //
      // t1: i8 = select t0, Constant:i8<-1>, Constant:i8<0>
      // t2: i64 = any_extend t1
      // -->
      // t3: i64 = select t0, Constant:i64<-1>, Constant:i64<0>
      // -->
      // t4: i64 = sign_extend_inreg t3
      unsigned FoldOpc = Opcode;
      if (FoldOpc == ISD::ANY_EXTEND)
        FoldOpc = ISD::SIGN_EXTEND;
      return DAG.getSelect(DL, VT, N0->getOperand(0),
                           DAG.getNode(FoldOpc, DL, VT, Op1),
                           DAG.getNode(FoldOpc, DL, VT, Op2));
    }
  }
 
  // fold (sext (build_vector AllConstants) -> (build_vector AllConstants)
  // fold (zext (build_vector AllConstants) -> (build_vector AllConstants)
  // fold (aext (build_vector AllConstants) -> (build_vector AllConstants)
  EVT SVT = VT.getScalarType();
  if (!(VT.isVector() && (!LegalTypes || TLI.isTypeLegal(SVT)) &&
      ISD::isBuildVectorOfConstantSDNodes(N0.getNode())))
    return SDValue();
 
  // We can fold this node into a build_vector.
  unsigned VTBits = SVT.getSizeInBits();
  unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits();
  SmallVector<SDValue, 8> Elts;
  unsigned NumElts = VT.getVectorNumElements();
 
  for (unsigned i = 0; i != NumElts; ++i) {
    SDValue Op = N0.getOperand(i);
    if (Op.isUndef()) {
      if (Opcode == ISD::ANY_EXTEND || Opcode == ISD::ANY_EXTEND_VECTOR_INREG)
        Elts.push_back(DAG.getUNDEF(SVT));
      else
        Elts.push_back(DAG.getConstant(0, DL, SVT));
      continue;
    }
 
    SDLoc DL(Op);
    // Get the constant value and if needed trunc it to the size of the type.
    // Nodes like build_vector might have constants wider than the scalar type.
    APInt C = Op->getAsAPIntVal().zextOrTrunc(EVTBits);
    if (Opcode == ISD::SIGN_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG)
      Elts.push_back(DAG.getConstant(C.sext(VTBits), DL, SVT));
    else
      Elts.push_back(DAG.getConstant(C.zext(VTBits), DL, SVT));
  }
 
  return DAG.getBuildVector(VT, DL, Elts);
}
 
// ExtendUsesToFormExtLoad - Trying to extend uses of a load to enable this:
// "fold ({s|z|a}ext (load x)) -> ({s|z|a}ext (truncate ({s|z|a}extload x)))"
// transformation. Returns true if extension are possible and the above
// mentioned transformation is profitable.
static bool ExtendUsesToFormExtLoad(EVT VT, SDNode *N, SDValue N0,
                                    unsigned ExtOpc,
                                    SmallVectorImpl<SDNode *> &ExtendNodes,
                                    const TargetLowering &TLI) {
  bool HasCopyToRegUses = false;
  bool isTruncFree = TLI.isTruncateFree(VT, N0.getValueType());
  for (SDUse &Use : N0->uses()) {
    SDNode *User = Use.getUser();
    if (User == N)
      continue;
    if (Use.getResNo() != N0.getResNo())
      continue;
    // FIXME: Only extend SETCC N, N and SETCC N, c for now.
    if (ExtOpc != ISD::ANY_EXTEND && User->getOpcode() == ISD::SETCC) {
      ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
      if (ExtOpc == ISD::ZERO_EXTEND && ISD::isSignedIntSetCC(CC))
        // Sign bits will be lost after a zext.
        return false;
      bool Add = false;
      for (unsigned i = 0; i != 2; ++i) {
        SDValue UseOp = User->getOperand(i);
        if (UseOp == N0)
          continue;
        if (!isa<ConstantSDNode>(UseOp))
          return false;
        Add = true;
      }
      if (Add)
        ExtendNodes.push_back(User);
      continue;
    }
    // If truncates aren't free and there are users we can't
    // extend, it isn't worthwhile.
    if (!isTruncFree)
      return false;
    // Remember if this value is live-out.
    if (User->getOpcode() == ISD::CopyToReg)
      HasCopyToRegUses = true;
  }
 
  if (HasCopyToRegUses) {
    bool BothLiveOut = false;
    for (SDUse &Use : N->uses()) {
      if (Use.getResNo() == 0 && Use.getUser()->getOpcode() == ISD::CopyToReg) {
        BothLiveOut = true;
        break;
      }
    }
    if (BothLiveOut)
      // Both unextended and extended values are live out. There had better be
      // a good reason for the transformation.
      return !ExtendNodes.empty();
  }
  return true;
}
 
void DAGCombiner::ExtendSetCCUses(const SmallVectorImpl<SDNode *> &SetCCs,
                                  SDValue OrigLoad, SDValue ExtLoad,
                                  ISD::NodeType ExtType) {
  // Extend SetCC uses if necessary.
  SDLoc DL(ExtLoad);
  for (SDNode *SetCC : SetCCs) {
    SmallVector<SDValue, 4> Ops;
 
    for (unsigned j = 0; j != 2; ++j) {
      SDValue SOp = SetCC->getOperand(j);
      if (SOp == OrigLoad)
        Ops.push_back(ExtLoad);
      else
        Ops.push_back(DAG.getNode(ExtType, DL, ExtLoad->getValueType(0), SOp));
    }
 
    Ops.push_back(SetCC->getOperand(2));
    CombineTo(SetCC, DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
  }
}
 
// FIXME: Bring more similar combines here, common to sext/zext (maybe aext?).
SDValue DAGCombiner::CombineExtLoad(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT DstVT = N->getValueType(0);
  EVT SrcVT = N0.getValueType();
 
  assert((N->getOpcode() == ISD::SIGN_EXTEND ||
          N->getOpcode() == ISD::ZERO_EXTEND) &&
         "Unexpected node type (not an extend)!");
 
  // fold (sext (load x)) to multiple smaller sextloads; same for zext.
  // For example, on a target with legal v4i32, but illegal v8i32, turn:
  //   (v8i32 (sext (v8i16 (load x))))
  // into:
  //   (v8i32 (concat_vectors (v4i32 (sextload x)),
  //                          (v4i32 (sextload (x + 16)))))
  // Where uses of the original load, i.e.:
  //   (v8i16 (load x))
  // are replaced with:
  //   (v8i16 (truncate
  //     (v8i32 (concat_vectors (v4i32 (sextload x)),
  //                            (v4i32 (sextload (x + 16)))))))
  //
  // This combine is only applicable to illegal, but splittable, vectors.
  // All legal types, and illegal non-vector types, are handled elsewhere.
  // This combine is controlled by TargetLowering::isVectorLoadExtDesirable.
  //
  if (N0->getOpcode() != ISD::LOAD)
    return SDValue();
 
  LoadSDNode *LN0 = cast<LoadSDNode>(N0);
 
  if (!ISD::isNON_EXTLoad(LN0) || !ISD::isUNINDEXEDLoad(LN0) ||
      !N0.hasOneUse() || !LN0->isSimple() ||
      !DstVT.isVector() || !DstVT.isPow2VectorType() ||
      !TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
    return SDValue();
 
  SmallVector<SDNode *, 4> SetCCs;
  if (!ExtendUsesToFormExtLoad(DstVT, N, N0, N->getOpcode(), SetCCs, TLI))
    return SDValue();
 
  ISD::LoadExtType ExtType =
      N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
 
  // Try to split the vector types to get down to legal types.
  EVT SplitSrcVT = SrcVT;
  EVT SplitDstVT = DstVT;
  while (!TLI.isLoadLegalOrCustom(SplitDstVT, SplitSrcVT, LN0->getAlign(),
                                  LN0->getAddressSpace(), ExtType, false) &&
         SplitSrcVT.getVectorNumElements() > 1) {
    SplitDstVT = DAG.GetSplitDestVTs(SplitDstVT).first;
    SplitSrcVT = DAG.GetSplitDestVTs(SplitSrcVT).first;
  }
 
  if (!TLI.isLoadLegalOrCustom(SplitDstVT, SplitSrcVT, LN0->getAlign(),
                               LN0->getAddressSpace(), ExtType, false))
    return SDValue();
 
  assert(!DstVT.isScalableVector() && "Unexpected scalable vector type");
 
  SDLoc DL(N);
  const unsigned NumSplits =
      DstVT.getVectorNumElements() / SplitDstVT.getVectorNumElements();
  const unsigned Stride = SplitSrcVT.getStoreSize();
  SmallVector<SDValue, 4> Loads;
  SmallVector<SDValue, 4> Chains;
 
  SDValue BasePtr = LN0->getBasePtr();
  for (unsigned Idx = 0; Idx < NumSplits; Idx++) {
    const unsigned Offset = Idx * Stride;
 
    SDValue SplitLoad =
        DAG.getExtLoad(ExtType, SDLoc(LN0), SplitDstVT, LN0->getChain(),
                       BasePtr, LN0->getPointerInfo().getWithOffset(Offset),
                       SplitSrcVT, LN0->getBaseAlign(),
                       LN0->getMemOperand()->getFlags(), LN0->getAAInfo());
 
    BasePtr = DAG.getMemBasePlusOffset(BasePtr, TypeSize::getFixed(Stride), DL);
 
    Loads.push_back(SplitLoad.getValue(0));
    Chains.push_back(SplitLoad.getValue(1));
  }
 
  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
  SDValue NewValue = DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Loads);
 
  // Simplify TF.
  AddToWorklist(NewChain.getNode());
 
  CombineTo(N, NewValue);
 
  // Replace uses of the original load (before extension)
  // with a truncate of the concatenated sextloaded vectors.
  SDValue Trunc =
      DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), NewValue);
  ExtendSetCCUses(SetCCs, N0, NewValue, (ISD::NodeType)N->getOpcode());
  CombineTo(N0.getNode(), Trunc, NewChain);
  return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
 
// fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
//      (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) {
  assert(N->getOpcode() == ISD::ZERO_EXTEND);
  EVT VT = N->getValueType(0);
  EVT OrigVT = N->getOperand(0).getValueType();
  if (TLI.isZExtFree(OrigVT, VT))
    return SDValue();
 
  // and/or/xor
  SDValue N0 = N->getOperand(0);
  if (!ISD::isBitwiseLogicOp(N0.getOpcode()) ||
      N0.getOperand(1).getOpcode() != ISD::Constant ||
      (LegalOperations && !TLI.isOperationLegal(N0.getOpcode(), VT)))
    return SDValue();
 
  // shl/shr
  SDValue N1 = N0->getOperand(0);
  if (!(N1.getOpcode() == ISD::SHL || N1.getOpcode() == ISD::SRL) ||
      N1.getOperand(1).getOpcode() != ISD::Constant ||
      (LegalOperations && !TLI.isOperationLegal(N1.getOpcode(), VT)))
    return SDValue();
 
  // load
  if (!isa<LoadSDNode>(N1.getOperand(0)))
    return SDValue();
  LoadSDNode *Load = cast<LoadSDNode>(N1.getOperand(0));
  EVT MemVT = Load->getMemoryVT();
  if (!TLI.isLoadLegal(VT, MemVT, Load->getAlign(), Load->getAddressSpace(),
                       ISD::ZEXTLOAD, false) ||
      Load->getExtensionType() == ISD::SEXTLOAD || Load->isIndexed())
    return SDValue();
 
 
  // If the shift op is SHL, the logic op must be AND, otherwise the result
  // will be wrong.
  if (N1.getOpcode() == ISD::SHL && N0.getOpcode() != ISD::AND)
    return SDValue();
 
  if (!N0.hasOneUse() || !N1.hasOneUse())
    return SDValue();
 
  SmallVector<SDNode*, 4> SetCCs;
  if (!ExtendUsesToFormExtLoad(VT, N1.getNode(), N1.getOperand(0),
                               ISD::ZERO_EXTEND, SetCCs, TLI))
    return SDValue();
 
  // Actually do the transformation.
  SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(Load), VT,
                                   Load->getChain(), Load->getBasePtr(),
                                   Load->getMemoryVT(), Load->getMemOperand());
 
  SDLoc DL1(N1);
  SDValue Shift = DAG.getNode(N1.getOpcode(), DL1, VT, ExtLoad,
                              N1.getOperand(1));
 
  APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
  SDLoc DL0(N0);
  SDValue And = DAG.getNode(N0.getOpcode(), DL0, VT, Shift,
                            DAG.getConstant(Mask, DL0, VT));
 
  ExtendSetCCUses(SetCCs, N1.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
  CombineTo(N, And);
  if (SDValue(Load, 0).hasOneUse()) {
    DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
  } else {
    SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(Load),
                                Load->getValueType(0), ExtLoad);
    CombineTo(Load, Trunc, ExtLoad.getValue(1));
  }
 
  // N0 is dead at this point.
  recursivelyDeleteUnusedNodes(N0.getNode());
 
  return SDValue(N,0); // Return N so it doesn't get rechecked!
}
 
/// If we're narrowing or widening the result of a vector select and the final
/// size is the same size as a setcc (compare) feeding the select, then try to
/// apply the cast operation to the select's operands because matching vector
/// sizes for a select condition and other operands should be more efficient.
SDValue DAGCombiner::matchVSelectOpSizesWithSetCC(SDNode *Cast) {
  unsigned CastOpcode = Cast->getOpcode();
  assert((CastOpcode == ISD::SIGN_EXTEND || CastOpcode == ISD::ZERO_EXTEND ||
          CastOpcode == ISD::TRUNCATE || CastOpcode == ISD::FP_EXTEND ||
          CastOpcode == ISD::FP_ROUND) &&
         "Unexpected opcode for vector select narrowing/widening");
 
  // We only do this transform before legal ops because the pattern may be
  // obfuscated by target-specific operations after legalization. Do not create
  // an illegal select op, however, because that may be difficult to lower.
  EVT VT = Cast->getValueType(0);
  if (LegalOperations || !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
    return SDValue();
 
  SDValue VSel = Cast->getOperand(0);
  if (VSel.getOpcode() != ISD::VSELECT || !VSel.hasOneUse() ||
      VSel.getOperand(0).getOpcode() != ISD::SETCC)
    return SDValue();
 
  // Does the setcc have the same vector size as the casted select?
  SDValue SetCC = VSel.getOperand(0);
  EVT SetCCVT = getSetCCResultType(SetCC.getOperand(0).getValueType());
  if (SetCCVT.getSizeInBits() != VT.getSizeInBits())
    return SDValue();
 
  // cast (vsel (setcc X), A, B) --> vsel (setcc X), (cast A), (cast B)
  SDValue A = VSel.getOperand(1);
  SDValue B = VSel.getOperand(2);
  SDValue CastA, CastB;
  SDLoc DL(Cast);
  if (CastOpcode == ISD::FP_ROUND) {
    // FP_ROUND (fptrunc) has an extra flag operand to pass along.
    CastA = DAG.getNode(CastOpcode, DL, VT, A, Cast->getOperand(1));
    CastB = DAG.getNode(CastOpcode, DL, VT, B, Cast->getOperand(1));
  } else {
    CastA = DAG.getNode(CastOpcode, DL, VT, A);
    CastB = DAG.getNode(CastOpcode, DL, VT, B);
  }
  return DAG.getNode(ISD::VSELECT, DL, VT, SetCC, CastA, CastB);
}
 
// fold ([s|z]ext ([s|z]extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
// fold ([s|z]ext (     extload x)) -> ([s|z]ext (truncate ([s|z]extload x)))
static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner,
                                     const TargetLowering &TLI, EVT VT,
                                     bool LegalOperations, SDNode *N,
                                     SDValue N0, ISD::LoadExtType ExtLoadType) {
  bool Frozen = N0.getOpcode() == ISD::FREEZE;
  auto *OldExtLoad = dyn_cast<LoadSDNode>(Frozen ? N0.getOperand(0) : N0);
  if (!OldExtLoad)
    return SDValue();
 
  bool isAExtLoad = (ExtLoadType == ISD::SEXTLOAD)
                        ? ISD::isSEXTLoad(OldExtLoad)
                        : ISD::isZEXTLoad(OldExtLoad);
  if ((!isAExtLoad && !ISD::isEXTLoad(OldExtLoad)) ||
      !ISD::isUNINDEXEDLoad(OldExtLoad) || !OldExtLoad->hasNUsesOfValue(1, 0))
    return SDValue();
 
  EVT MemVT = OldExtLoad->getMemoryVT();
  if ((LegalOperations || !OldExtLoad->isSimple() || VT.isVector()) &&
      !TLI.isLoadLegal(VT, MemVT, OldExtLoad->getAlign(),
                       OldExtLoad->getAddressSpace(), ExtLoadType, false))
    return SDValue();
 
  SDLoc DL(OldExtLoad);
  SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, DL, VT, OldExtLoad->getChain(),
                                   OldExtLoad->getBasePtr(), MemVT,
                                   OldExtLoad->getMemOperand());
  SDValue Res = ExtLoad;
  if (Frozen) {
    Res = DAG.getFreeze(ExtLoad);
    Res = DAG.getNode(
        ExtLoadType == ISD::SEXTLOAD ? ISD::AssertSext : ISD::AssertZext, DL,
        Res.getValueType(), Res,
        DAG.getValueType(OldExtLoad->getValueType(0).getScalarType()));
  }
  Combiner.CombineTo(N, Res);
  DAG.ReplaceAllUsesOfValueWith(SDValue(OldExtLoad, 1), ExtLoad.getValue(1));
  if (N0->use_empty())
    Combiner.recursivelyDeleteUnusedNodes(N0.getNode());
  return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
 
// fold ([s|z]ext (load x)) -> ([s|z]ext (truncate ([s|z]extload x)))
// Only generate vector extloads when 1) they're legal, and 2) they are
// deemed desirable by the target. NonNegZExt can be set to true if a zero
// extend has the nonneg flag to allow use of sextload if profitable.
static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner,
                                  const TargetLowering &TLI, EVT VT,
                                  bool LegalOperations, SDNode *N, SDValue N0,
                                  ISD::LoadExtType ExtLoadType,
                                  ISD::NodeType ExtOpc,
                                  bool NonNegZExt = false) {
 
  bool Frozen = N0.getOpcode() == ISD::FREEZE;
  SDValue Freeze = Frozen ? N0 : SDValue();
  auto *Load = dyn_cast<LoadSDNode>(Frozen ? N0.getOperand(0) : N0);
  // TODO: Support multiple uses of the load when frozen.
  if (!Load || !ISD::isNON_EXTLoad(Load) || !ISD::isUNINDEXEDLoad(Load) ||
      (Frozen && !Load->hasNUsesOfValue(1, 0)))
    return {};
 
  // If this is zext nneg, see if it would make sense to treat it as a sext.
  if (NonNegZExt) {
    assert(ExtLoadType == ISD::ZEXTLOAD && ExtOpc == ISD::ZERO_EXTEND &&
           "Unexpected load type or opcode");
    for (SDNode *User : Load->users()) {
      if (User->getOpcode() == ISD::SETCC) {
        ISD::CondCode CC = cast<CondCodeSDNode>(User->getOperand(2))->get();
        if (ISD::isSignedIntSetCC(CC)) {
          ExtLoadType = ISD::SEXTLOAD;
          ExtOpc = ISD::SIGN_EXTEND;
          break;
        }
      }
    }
  }
 
  // TODO: isFixedLengthVector() should be removed and any negative effects on
  // code generation being the result of that target's implementation of
  // isVectorLoadExtDesirable().
  if ((LegalOperations || VT.isFixedLengthVector() || !Load->isSimple()) &&
      !TLI.isLoadLegal(VT, Load->getValueType(0), Load->getAlign(),
                       Load->getAddressSpace(), ExtLoadType, false))
    return {};
 
  bool DoXform = true;
  SmallVector<SDNode *, 4> SetCCs;
  if (!N0->hasOneUse())
    DoXform = ExtendUsesToFormExtLoad(VT, N, Frozen ? Freeze : SDValue(Load, 0),
                                      ExtOpc, SetCCs, TLI);
  if (VT.isVector())
    DoXform &= TLI.isVectorLoadExtDesirable(SDValue(N, 0));
  if (!DoXform)
    return {};
 
  SDLoc DL(Load);
 
  auto SalvageDbgValue = [&](SDDbgValue *Dbg, SDValue Old, SDValue New,
                             unsigned OldBits, unsigned NewBits,
                             bool IsSigned) {
    SmallVector<SDDbgOperand> Locs = Dbg->copyLocationOps();
    bool Changed = false;
 
    bool IsVariadic = Dbg->isVariadic();
    SmallVector<unsigned, 2> AffectedArgs;
 
    for (unsigned I = 0, E = Locs.size(); I != E; ++I) {
      SDDbgOperand &Op = Locs[I];
      if (Op.getKind() != SDDbgOperand::SDNODE)
        continue;
 
      if (Op.getSDNode() == Old.getNode() && Op.getResNo() == Old.getResNo()) {
        Op = SDDbgOperand::fromNode(New.getNode(), New.getResNo());
        Changed = true;
 
        if (IsVariadic)
          AffectedArgs.push_back(I);
      }
    }
 
    if (!Changed)
      return;
 
    const DIExpression *OldExpr = Dbg->getExpression();
    const DIExpression *NewExpr = nullptr;
 
    if (!IsVariadic) {
      // Do not introduce DW_OP_LLVM_arg into ordinary single-location
      // DBG_VALUEs.
      NewExpr = DIExpression::appendExt(OldExpr, NewBits, OldBits, IsSigned);
    } else {
      auto ExtOps = DIExpression::getExtOps(NewBits, OldBits, IsSigned);
 
      NewExpr = DIExpression::convertToVariadicExpression(OldExpr);
 
      for (unsigned ArgNo : AffectedArgs)
        NewExpr = DIExpression::appendOpsToArg(NewExpr, ExtOps, ArgNo,
                                               /*StackValue=*/false);
    }
 
    SDDbgValue *NewDV = DAG.getDbgValueList(
        Dbg->getVariable(), const_cast<DIExpression *>(NewExpr), Locs,
        Dbg->getAdditionalDependencies(), Dbg->isIndirect(), Dbg->getDebugLoc(),
        Dbg->getOrder(), Dbg->isVariadic());
 
    Dbg->setIsInvalidated();
    Dbg->setIsEmitted();
    DAG.AddDbgValue(NewDV, /*isParameter=*/false);
  };
 
  // Because we are replacing a load and a s|z ext with a load-s|z ext
  // instruction, the dbg_value attached to the load will be of a smaller bit
  // width, and we have to add a DW_OP_LLVM_convert expression to get the
  // correct size.
  auto SalvageToOldLoadSize = [&](SDValue Old, SDValue New, bool IsSigned) {
    SmallVector<SDDbgValue *, 4> DbgVals(
        DAG.GetDbgValues(Old.getNode()).begin(),
        DAG.GetDbgValues(Old.getNode()).end());
 
    unsigned VarBitsOld = Old.getValueSizeInBits();
    unsigned VarBitsNew = New.getValueSizeInBits();
 
    for (SDDbgValue *Dbg : DbgVals) {
      if (Dbg->isInvalidated())
        continue;
 
      SalvageDbgValue(Dbg, Old, New, VarBitsOld, VarBitsNew, IsSigned);
    }
  };
 
  SDValue ExtLoad =
      DAG.getExtLoad(ExtLoadType, DL, VT, Load->getChain(), Load->getBasePtr(),
                     Load->getValueType(0), Load->getMemOperand());
  SDValue Res = ExtLoad;
  if (Frozen) {
    Res = DAG.getFreeze(ExtLoad);
    Res = DAG.getNode(ExtLoadType == ISD::SEXTLOAD ? ISD::AssertSext
                                                   : ISD::AssertZext,
                      DL, Res.getValueType(), Res,
                      DAG.getValueType(Load->getValueType(0).getScalarType()));
  }
  Combiner.ExtendSetCCUses(SetCCs, N0, Res, ExtOpc);
  // If the load value is used only by N, replace it via CombineTo N.
  bool NoReplaceTrunc = N0.hasOneUse();
  if (N->getHasDebugValue()) {
    SDValue OldExtValue(N, 0);
    DAG.transferDbgValues(OldExtValue, ExtLoad);
  }
  if (NoReplaceTrunc) {
    bool IsSigned = N->getOpcode() == ISD::SIGN_EXTEND;
    if (Load->getHasDebugValue()) {
      SDValue OldLoadVal(Load, 0);
      SalvageToOldLoadSize(OldLoadVal, ExtLoad, IsSigned);
    }
    DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
    Combiner.CombineTo(N, Res);
    Combiner.recursivelyDeleteUnusedNodes(N0.getNode());
  } else {
    Combiner.CombineTo(N, Res);
    SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, Load->getValueType(0), Res);
    if (Frozen) {
      Combiner.CombineTo(Freeze.getNode(), Trunc);
      DAG.ReplaceAllUsesOfValueWith(SDValue(Load, 1), ExtLoad.getValue(1));
    } else {
      Combiner.CombineTo(Load, Trunc, ExtLoad.getValue(1));
    }
  }
  return SDValue(N, 0); // Return N so it doesn't get rechecked!
}
 
static SDValue
tryToFoldExtOfMaskedLoad(SelectionDAG &DAG, const TargetLowering &TLI, EVT VT,
                         bool LegalOperations, SDNode *N, SDValue N0,
                         ISD::LoadExtType ExtLoadType, ISD::NodeType ExtOpc) {
  if (!N0.hasOneUse())
    return SDValue();
 
  MaskedLoadSDNode *Ld = dyn_cast<MaskedLoadSDNode>(N0);
  if (!Ld || Ld->getExtensionType() != ISD::NON_EXTLOAD)
    return SDValue();
 
  if ((LegalOperations || !cast<MaskedLoadSDNode>(N0)->isSimple()) &&
      !TLI.isLoadLegalOrCustom(VT, Ld->getValueType(0), Ld->getAlign(),
                               Ld->getAddressSpace(), ExtLoadType, false))
    return SDValue();
 
  if (!TLI.isVectorLoadExtDesirable(SDValue(N, 0)))
    return SDValue();
 
  SDLoc dl(Ld);
  SDValue PassThru = DAG.getNode(ExtOpc, dl, VT, Ld->getPassThru());
  SDValue NewLoad = DAG.getMaskedLoad(
      VT, dl, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(), Ld->getMask(),
      PassThru, Ld->getMemoryVT(), Ld->getMemOperand(), Ld->getAddressingMode(),
      ExtLoadType, Ld->isExpandingLoad());
  DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), SDValue(NewLoad.getNode(), 1));
  return NewLoad;
}
 
// fold ([s|z]ext (atomic_load)) -> ([s|z]ext (truncate ([s|z]ext atomic_load)))
static SDValue tryToFoldExtOfAtomicLoad(SelectionDAG &DAG,
                                        const TargetLowering &TLI, EVT VT,
                                        SDValue N0,
                                        ISD::LoadExtType ExtLoadType) {
  auto *ALoad = dyn_cast<AtomicSDNode>(N0);
  if (!ALoad || ALoad->getOpcode() != ISD::ATOMIC_LOAD)
    return {};
  EVT MemoryVT = ALoad->getMemoryVT();
  if (!TLI.isLoadLegal(VT, MemoryVT, ALoad->getAlign(),
                       ALoad->getAddressSpace(), ExtLoadType, true))
    return {};
  // Can't fold into ALoad if it is already extending differently.
  ISD::LoadExtType ALoadExtTy = ALoad->getExtensionType();
  if ((ALoadExtTy == ISD::ZEXTLOAD && ExtLoadType == ISD::SEXTLOAD) ||
      (ALoadExtTy == ISD::SEXTLOAD && ExtLoadType == ISD::ZEXTLOAD))
    return {};
 
  EVT OrigVT = ALoad->getValueType(0);
  assert(OrigVT.getSizeInBits() < VT.getSizeInBits() && "VT should be wider.");
  auto *NewALoad = cast<AtomicSDNode>(DAG.getAtomicLoad(
      ExtLoadType, SDLoc(ALoad), MemoryVT, VT, ALoad->getChain(),
      ALoad->getBasePtr(), ALoad->getMemOperand()));
  DAG.ReplaceAllUsesOfValueWith(
      SDValue(ALoad, 0),
      DAG.getNode(ISD::TRUNCATE, SDLoc(ALoad), OrigVT, SDValue(NewALoad, 0)));
  // Update the chain uses.
  DAG.ReplaceAllUsesOfValueWith(SDValue(ALoad, 1), SDValue(NewALoad, 1));
  return SDValue(NewALoad, 0);
}
 
static SDValue foldExtendedSignBitTest(SDNode *N, SelectionDAG &DAG,
                                       bool LegalOperations) {
  assert((N->getOpcode() == ISD::SIGN_EXTEND ||
          N->getOpcode() == ISD::ZERO_EXTEND) && "Expected sext or zext");
 
  SDValue SetCC = N->getOperand(0);
  if (LegalOperations || SetCC.getOpcode() != ISD::SETCC ||
      !SetCC.hasOneUse() || SetCC.getValueType() != MVT::i1)
    return SDValue();
 
  SDValue X = SetCC.getOperand(0);
  SDValue Ones = SetCC.getOperand(1);
  ISD::CondCode CC = cast<CondCodeSDNode>(SetCC.getOperand(2))->get();
  EVT VT = N->getValueType(0);
  EVT XVT = X.getValueType();
  // setge X, C is canonicalized to setgt, so we do not need to match that
  // pattern. The setlt sibling is folded in SimplifySelectCC() because it does
  // not require the 'not' op.
  if (CC == ISD::SETGT && isAllOnesConstant(Ones) && VT == XVT) {
    // Invert and smear/shift the sign bit:
    // sext i1 (setgt iN X, -1) --> sra (not X), (N - 1)
    // zext i1 (setgt iN X, -1) --> srl (not X), (N - 1)
    SDLoc DL(N);
    unsigned ShCt = VT.getSizeInBits() - 1;
    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
    if (!TLI.shouldAvoidTransformToShift(VT, ShCt)) {
      SDValue NotX = DAG.getNOT(DL, X, VT);
      SDValue ShiftAmount = DAG.getConstant(ShCt, DL, VT);
      auto ShiftOpcode =
        N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SRA : ISD::SRL;
      return DAG.getNode(ShiftOpcode, DL, VT, NotX, ShiftAmount);
    }
  }
  return SDValue();
}
 
SDValue DAGCombiner::foldSextSetcc(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  if (N0.getOpcode() != ISD::SETCC)
    return SDValue();
 
  SDValue N00 = N0.getOperand(0);
  SDValue N01 = N0.getOperand(1);
  ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
  EVT VT = N->getValueType(0);
  EVT N00VT = N00.getValueType();
  SDLoc DL(N);
 
  // Propagate fast-math-flags.
  SDNodeFlags Flags = N0->getFlags();
 
  // On some architectures (such as SSE/NEON/etc) the SETCC result type is
  // the same size as the compared operands. Try to optimize sext(setcc())
  // if this is the case.
  if (VT.isVector() && !LegalOperations &&
      TLI.getBooleanContents(N00VT) ==
          TargetLowering::ZeroOrNegativeOneBooleanContent) {
    EVT SVT = getSetCCResultType(N00VT);
 
    // If we already have the desired type, don't change it.
    if (SVT != N0.getValueType()) {
      // We know that the # elements of the results is the same as the
      // # elements of the compare (and the # elements of the compare result
      // for that matter).  Check to see that they are the same size.  If so,
      // we know that the element size of the sext'd result matches the
      // element size of the compare operands.
      if (VT.getSizeInBits() == SVT.getSizeInBits())
        return DAG.getSetCC(DL, VT, N00, N01, CC, /*Chain=*/{},
                            /*Signaling=*/false, Flags);
 
      // If the desired elements are smaller or larger than the source
      // elements, we can use a matching integer vector type and then
      // truncate/sign extend.
      EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
      if (SVT == MatchingVecType) {
        SDValue VsetCC = DAG.getSetCC(DL, MatchingVecType, N00, N01, CC,
                                      /*Chain=*/{}, /*Signaling=*/false, Flags);
        return DAG.getSExtOrTrunc(VsetCC, DL, VT);
      }
    }
 
    // Try to eliminate the sext of a setcc by zexting the compare operands.
    if (N0.hasOneUse() && TLI.isOperationLegalOrCustom(ISD::SETCC, VT) &&
        !TLI.isOperationLegalOrCustom(ISD::SETCC, SVT)) {
      bool IsSignedCmp = ISD::isSignedIntSetCC(CC);
      unsigned LoadOpcode = IsSignedCmp ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
      unsigned ExtOpcode = IsSignedCmp ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
 
      // We have an unsupported narrow vector compare op that would be legal
      // if extended to the destination type. See if the compare operands
      // can be freely extended to the destination type.
      auto IsFreeToExtend = [&](SDValue V) {
        if (isConstantOrConstantVector(V, /*NoOpaques*/ true))
          return true;
        // Match a simple, non-extended load that can be converted to a
        // legal {z/s}ext-load.
        // TODO: Allow widening of an existing {z/s}ext-load?
        if (!(ISD::isNON_EXTLoad(V.getNode()) &&
              ISD::isUNINDEXEDLoad(V.getNode())))
          return false;
 
        LoadSDNode *Ld = cast<LoadSDNode>(V.getNode());
 
        if (!Ld->isSimple() ||
            !TLI.isLoadLegal(VT, V.getValueType(), Ld->getAlign(),
                             Ld->getAddressSpace(), LoadOpcode, false))
          return false;
 
        // Non-chain users of this value must either be the setcc in this
        // sequence or extends that can be folded into the new {z/s}ext-load.
        for (SDUse &Use : V->uses()) {
          // Skip uses of the chain and the setcc.
          SDNode *User = Use.getUser();
          if (Use.getResNo() != 0 || User == N0.getNode())
            continue;
          // Extra users must have exactly the same cast we are about to create.
          // TODO: This restriction could be eased if ExtendUsesToFormExtLoad()
          //       is enhanced similarly.
          if (User->getOpcode() != ExtOpcode || User->getValueType(0) != VT)
            return false;
        }
        return true;
      };
 
      if (IsFreeToExtend(N00) && IsFreeToExtend(N01)) {
        SDValue Ext0 = DAG.getNode(ExtOpcode, DL, VT, N00);
        SDValue Ext1 = DAG.getNode(ExtOpcode, DL, VT, N01);
        return DAG.getSetCC(DL, VT, Ext0, Ext1, CC, /*Chain=*/{},
                            /*Signaling=*/false, Flags);
      }
    }
  }
 
  // sext(setcc x, y, cc) -> (select (setcc x, y, cc), T, 0)
  // Here, T can be 1 or -1, depending on the type of the setcc and
  // getBooleanContents().
  unsigned SetCCWidth = N0.getScalarValueSizeInBits();
 
  // To determine the "true" side of the select, we need to know the high bit
  // of the value returned by the setcc if it evaluates to true.
  // If the type of the setcc is i1, then the true case of the select is just
  // sext(i1 1), that is, -1.
  // If the type of the setcc is larger (say, i8) then the value of the high
  // bit depends on getBooleanContents(), so ask TLI for a real "true" value
  // of the appropriate width.
  SDValue ExtTrueVal = (SetCCWidth == 1)
                           ? DAG.getAllOnesConstant(DL, VT)
                           : DAG.getBoolConstant(true, DL, VT, N00VT);
  SDValue Zero = DAG.getConstant(0, DL, VT);
  if (SDValue SCC = SimplifySelectCC(DL, N00, N01, ExtTrueVal, Zero, CC, true))
    return SCC;
 
  if (!VT.isVector() && !shouldConvertSelectOfConstantsToMath(N0, VT, TLI)) {
    EVT SetCCVT = getSetCCResultType(N00VT);
    // Don't do this transform for i1 because there's a select transform
    // that would reverse it.
    // TODO: We should not do this transform at all without a target hook
    // because a sext is likely cheaper than a select?
    if (SetCCVT.getScalarSizeInBits() != 1 &&
        (!LegalOperations || TLI.isOperationLegal(ISD::SETCC, N00VT))) {
      SDValue SetCC = DAG.getSetCC(DL, SetCCVT, N00, N01, CC, /*Chain=*/{},
                                   /*Signaling=*/false, Flags);
      return DAG.getSelect(DL, VT, SetCC, ExtTrueVal, Zero, Flags);
    }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVCastOp(N, DL))
      return FoldedVOp;
 
  // sext(undef) = 0 because the top bit will all be the same.
  if (N0.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
    return Res;
 
  // fold (sext (sext x)) -> (sext x)
  // fold (sext (aext x)) -> (sext x)
  if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND)
    return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N0.getOperand(0));
 
  // fold (sext (aext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
  // fold (sext (sext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
  if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
      N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG)
    return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, SDLoc(N), VT,
                       N0.getOperand(0));
 
  if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
    SDValue N00 = N0.getOperand(0);
    EVT ExtVT = cast<VTSDNode>(N0->getOperand(1))->getVT();
    if (N00.getOpcode() == ISD::TRUNCATE || TLI.isTruncateFree(N00, ExtVT)) {
      // fold (sext (sext_inreg x)) -> (sext (trunc x))
      if ((!LegalTypes || TLI.isTypeLegal(ExtVT))) {
        SDValue T = DAG.getNode(ISD::TRUNCATE, DL, ExtVT, N00);
        return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, T);
      }
 
      // If the trunc wasn't legal, try to fold to (sext_inreg (anyext x))
      if (!LegalTypes || TLI.isTypeLegal(VT)) {
        SDValue ExtSrc = DAG.getAnyExtOrTrunc(N00, DL, VT);
        return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, ExtSrc,
                           N0->getOperand(1));
      }
    }
  }
 
  if (N0.getOpcode() == ISD::TRUNCATE) {
    // fold (sext (truncate (load x))) -> (sext (smaller load x))
    // fold (sext (truncate (srl (load x), c))) -> (sext (smaller load (x+c/n)))
    if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) {
      SDNode *oye = N0.getOperand(0).getNode();
      if (NarrowLoad.getNode() != N0.getNode()) {
        CombineTo(N0.getNode(), NarrowLoad);
        // CombineTo deleted the truncate, if needed, but not what's under it.
        AddToWorklist(oye);
      }
      return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    }
 
    // See if the value being truncated is already sign extended.  If so, just
    // eliminate the trunc/sext pair.
    SDValue Op = N0.getOperand(0);
    unsigned OpBits   = Op.getScalarValueSizeInBits();
    unsigned MidBits  = N0.getScalarValueSizeInBits();
    unsigned DestBits = VT.getScalarSizeInBits();
 
    if (N0->getFlags().hasNoSignedWrap() ||
        DAG.ComputeNumSignBits(Op) > OpBits - MidBits) {
      if (OpBits == DestBits) {
        // Op is i32, Mid is i8, and Dest is i32.  If Op has more than 24 sign
        // bits, it is already ready.
        return Op;
      }
 
      if (OpBits < DestBits) {
        // Op is i32, Mid is i8, and Dest is i64.  If Op has more than 24 sign
        // bits, just sext from i32.
        return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
      }
 
      // Op is i64, Mid is i8, and Dest is i32.  If Op has more than 56 sign
      // bits, just truncate to i32.
      SDNodeFlags Flags;
      Flags.setNoSignedWrap(true);
      Flags.setNoUnsignedWrap(N0->getFlags().hasNoUnsignedWrap());
      return DAG.getNode(ISD::TRUNCATE, DL, VT, Op, Flags);
    }
 
    // fold (sext (truncate x)) -> (sextinreg x).
    if (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND_INREG,
                                                 N0.getValueType())) {
      if (OpBits < DestBits)
        Op = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N0), VT, Op);
      else if (OpBits > DestBits)
        Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), VT, Op);
      return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
                         DAG.getValueType(N0.getValueType()));
    }
  }
 
  // Try to simplify (sext (load x)).
  if (SDValue foldedExt =
          tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
                             ISD::SEXTLOAD, ISD::SIGN_EXTEND))
    return foldedExt;
 
  if (SDValue foldedExt =
          tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0,
                                   ISD::SEXTLOAD, ISD::SIGN_EXTEND))
    return foldedExt;
 
  // fold (sext (load x)) to multiple smaller sextloads.
  // Only on illegal but splittable vectors.
  if (SDValue ExtLoad = CombineExtLoad(N))
    return ExtLoad;
 
  // Try to simplify (sext (sextload x)).
  if (SDValue foldedExt = tryToFoldExtOfExtload(
          DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::SEXTLOAD))
    return foldedExt;
 
  // Try to simplify (sext (atomic_load x)).
  if (SDValue foldedExt =
          tryToFoldExtOfAtomicLoad(DAG, TLI, VT, N0, ISD::SEXTLOAD))
    return foldedExt;
 
  // fold (sext (and/or/xor (load x), cst)) ->
  //      (and/or/xor (sextload x), (sext cst))
  if (ISD::isBitwiseLogicOp(N0.getOpcode()) &&
      isa<LoadSDNode>(N0.getOperand(0)) &&
      N0.getOperand(1).getOpcode() == ISD::Constant &&
      (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
    LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
    EVT MemVT = LN00->getMemoryVT();
    if (TLI.isLoadLegal(VT, MemVT, LN00->getAlign(), LN00->getAddressSpace(),
                        ISD::SEXTLOAD, false) &&
        LN00->getExtensionType() != ISD::ZEXTLOAD && LN00->isUnindexed()) {
      SmallVector<SDNode*, 4> SetCCs;
      bool DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
                                             ISD::SIGN_EXTEND, SetCCs, TLI);
      if (DoXform) {
        SDValue ExtLoad = DAG.getExtLoad(ISD::SEXTLOAD, SDLoc(LN00), VT,
                                         LN00->getChain(), LN00->getBasePtr(),
                                         LN00->getMemoryVT(),
                                         LN00->getMemOperand());
        APInt Mask = N0.getConstantOperandAPInt(1).sext(VT.getSizeInBits());
        SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
                                  ExtLoad, DAG.getConstant(Mask, DL, VT));
        ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::SIGN_EXTEND);
        bool NoReplaceTruncAnd = !N0.hasOneUse();
        bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
        CombineTo(N, And);
        // If N0 has multiple uses, change other uses as well.
        if (NoReplaceTruncAnd) {
          SDValue TruncAnd =
              DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
          CombineTo(N0.getNode(), TruncAnd);
        }
        if (NoReplaceTrunc) {
          DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
        } else {
          SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
                                      LN00->getValueType(0), ExtLoad);
          CombineTo(LN00, Trunc, ExtLoad.getValue(1));
        }
        return SDValue(N,0); // Return N so it doesn't get rechecked!
      }
    }
  }
 
  if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
    return V;
 
  if (SDValue V = foldSextSetcc(N))
    return V;
 
  // fold (sext x) -> (zext x) if the sign bit is known zero.
  if (!TLI.isSExtCheaperThanZExt(N0.getValueType(), VT) &&
      (!LegalOperations || TLI.isOperationLegal(ISD::ZERO_EXTEND, VT)) &&
      DAG.SignBitIsZero(N0))
    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0, SDNodeFlags::NonNeg);
 
  if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    return NewVSel;
 
  // Eliminate this sign extend by doing a negation in the destination type:
  // sext i32 (0 - (zext i8 X to i32)) to i64 --> 0 - (zext i8 X to i64)
  if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() &&
      isNullOrNullSplat(N0.getOperand(0)) &&
      N0.getOperand(1).getOpcode() == ISD::ZERO_EXTEND &&
      TLI.isOperationLegalOrCustom(ISD::SUB, VT)) {
    SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT);
    return DAG.getNegative(Zext, DL, VT);
  }
  // Eliminate this sign extend by doing a decrement in the destination type:
  // sext i32 ((zext i8 X to i32) + (-1)) to i64 --> (zext i8 X to i64) + (-1)
  if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() &&
      isAllOnesOrAllOnesSplat(N0.getOperand(1)) &&
      N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
      TLI.isOperationLegalOrCustom(ISD::ADD, VT)) {
    SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT);
    return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
  }
 
  // fold sext (not i1 X) -> add (zext i1 X), -1
  // TODO: This could be extended to handle bool vectors.
  if (N0.getValueType() == MVT::i1 && isBitwiseNot(N0) && N0.hasOneUse() &&
      (!LegalOperations || (TLI.isOperationLegal(ISD::ZERO_EXTEND, VT) &&
                            TLI.isOperationLegal(ISD::ADD, VT)))) {
    // If we can eliminate the 'not', the sext form should be better
    if (SDValue NewXor = visitXOR(N0.getNode())) {
      // Returning N0 is a form of in-visit replacement that may have
      // invalidated N0.
      if (NewXor.getNode() == N0.getNode()) {
        // Return SDValue here as the xor should have already been replaced in
        // this sext.
        return SDValue();
      }
 
      // Return a new sext with the new xor.
      return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, NewXor);
    }
 
    SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
    return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT));
  }
 
  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
    return Res;
 
  return SDValue();
}
 
/// Given an extending node with a pop-count operand, if the target does not
/// support a pop-count in the narrow source type but does support it in the
/// destination type, widen the pop-count to the destination type.
static SDValue widenCtPop(SDNode *Extend, SelectionDAG &DAG, const SDLoc &DL) {
  assert((Extend->getOpcode() == ISD::ZERO_EXTEND ||
          Extend->getOpcode() == ISD::ANY_EXTEND) &&
         "Expected extend op");
 
  SDValue CtPop = Extend->getOperand(0);
  if (CtPop.getOpcode() != ISD::CTPOP || !CtPop.hasOneUse())
    return SDValue();
 
  EVT VT = Extend->getValueType(0);
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  if (TLI.isOperationLegalOrCustom(ISD::CTPOP, CtPop.getValueType()) ||
      !TLI.isOperationLegalOrCustom(ISD::CTPOP, VT))
    return SDValue();
 
  // zext (ctpop X) --> ctpop (zext X)
  SDValue NewZext = DAG.getZExtOrTrunc(CtPop.getOperand(0), DL, VT);
  return DAG.getNode(ISD::CTPOP, DL, VT, NewZext);
}
 
// If we have (zext (abs X)) where X is a type that will be promoted by type
// legalization, convert to (abs_min_poison (sext X)). But do not extend
// past a legal type.
static SDValue widenAbs(SDNode *Extend, SelectionDAG &DAG) {
  assert(Extend->getOpcode() == ISD::ZERO_EXTEND && "Expected zero extend.");
 
  EVT VT = Extend->getValueType(0);
  if (VT.isVector())
    return SDValue();
 
  SDValue Abs = Extend->getOperand(0);
  if (!ISD::isAbsOpcode(Abs.getOpcode()) || !Abs.hasOneUse())
    return SDValue();
 
  EVT AbsVT = Abs.getValueType();
  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
  if (TLI.getTypeAction(*DAG.getContext(), AbsVT) !=
      TargetLowering::TypePromoteInteger)
    return SDValue();
 
  EVT LegalVT = TLI.getTypeToTransformTo(*DAG.getContext(), AbsVT);
 
  SDValue SExt =
      DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Abs), LegalVT, Abs.getOperand(0));
  SDValue NewAbs = DAG.getNode(ISD::ABS_MIN_POISON, SDLoc(Abs), LegalVT, SExt);
  return DAG.getZExtOrTrunc(NewAbs, SDLoc(Extend), VT);
}
 
SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVCastOp(N, DL))
      return FoldedVOp;
 
  // zext(undef) = 0
  if (N0.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
    return Res;
 
  // fold (zext (zext x)) -> (zext x)
  // fold (zext (aext x)) -> (zext x)
  if (N0.getOpcode() == ISD::ZERO_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
    SDNodeFlags Flags;
    if (N0.getOpcode() == ISD::ZERO_EXTEND)
      Flags.setNonNeg(N0->getFlags().hasNonNeg());
    return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0), Flags);
  }
 
  // fold (zext (aext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
  // fold (zext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
  if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
      N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG)
    return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, N0.getOperand(0));
 
  // fold (zext (truncate x)) -> (zext x) or
  //      (zext (truncate x)) -> (truncate x)
  // This is valid when the truncated bits of x are already zero.
  SDValue Op;
  KnownBits Known;
  if (isTruncateOf(DAG, N0, Op, Known)) {
    APInt TruncatedBits =
      (Op.getScalarValueSizeInBits() == N0.getScalarValueSizeInBits()) ?
      APInt(Op.getScalarValueSizeInBits(), 0) :
      APInt::getBitsSet(Op.getScalarValueSizeInBits(),
                        N0.getScalarValueSizeInBits(),
                        std::min(Op.getScalarValueSizeInBits(),
                                 VT.getScalarSizeInBits()));
    if (TruncatedBits.isSubsetOf(Known.Zero)) {
      SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, DL, VT);
      DAG.salvageDebugInfo(*N0.getNode());
 
      return ZExtOrTrunc;
    }
  }
 
  // fold (zext (truncate x)) -> (and x, mask)
  if (N0.getOpcode() == ISD::TRUNCATE) {
    // fold (zext (truncate (load x))) -> (zext (smaller load x))
    // fold (zext (truncate (srl (load x), c))) -> (zext (smaller load (x+c/n)))
    if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) {
      SDNode *oye = N0.getOperand(0).getNode();
      if (NarrowLoad.getNode() != N0.getNode()) {
        CombineTo(N0.getNode(), NarrowLoad);
        // CombineTo deleted the truncate, if needed, but not what's under it.
        AddToWorklist(oye);
      }
      return SDValue(N, 0); // Return N so it doesn't get rechecked!
    }
 
    EVT SrcVT = N0.getOperand(0).getValueType();
    EVT MinVT = N0.getValueType();
 
    if (N->getFlags().hasNonNeg()) {
      SDValue Op = N0.getOperand(0);
      unsigned OpBits = SrcVT.getScalarSizeInBits();
      unsigned MidBits = MinVT.getScalarSizeInBits();
      unsigned DestBits = VT.getScalarSizeInBits();
 
      if (N0->getFlags().hasNoSignedWrap() ||
          DAG.ComputeNumSignBits(Op) > OpBits - MidBits) {
        if (OpBits == DestBits) {
          // Op is i32, Mid is i8, and Dest is i32.  If Op has more than 24 sign
          // bits, it is already ready.
          return Op;
        }
 
        if (OpBits < DestBits) {
          // Op is i32, Mid is i8, and Dest is i64.  If Op has more than 24 sign
          // bits, just sext from i32.
          // FIXME: This can probably be ZERO_EXTEND nneg?
          return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op);
        }
 
        // Op is i64, Mid is i8, and Dest is i32.  If Op has more than 56 sign
        // bits, just truncate to i32.
        SDNodeFlags Flags;
        Flags.setNoSignedWrap(true);
        Flags.setNoUnsignedWrap(true);
        return DAG.getNode(ISD::TRUNCATE, DL, VT, Op, Flags);
      }
    }
 
    // Try to mask before the extension to avoid having to generate a larger mask,
    // possibly over several sub-vectors.
    if (SrcVT.bitsLT(VT) && VT.isVector()) {
      if (!LegalOperations || (TLI.isOperationLegal(ISD::AND, SrcVT) &&
                               TLI.isOperationLegal(ISD::ZERO_EXTEND, VT))) {
        SDValue Op = N0.getOperand(0);
        Op = DAG.getZeroExtendInReg(Op, DL, MinVT);
        AddToWorklist(Op.getNode());
        SDValue ZExtOrTrunc = DAG.getZExtOrTrunc(Op, DL, VT);
        // Transfer the debug info; the new node is equivalent to N0.
        DAG.transferDbgValues(N0, ZExtOrTrunc);
        return ZExtOrTrunc;
      }
    }
 
    if (!LegalOperations || TLI.isOperationLegal(ISD::AND, VT)) {
      SDValue Op = DAG.getAnyExtOrTrunc(N0.getOperand(0), DL, VT);
      AddToWorklist(Op.getNode());
      SDValue And = DAG.getZeroExtendInReg(Op, DL, MinVT);
      // We may safely transfer the debug info describing the truncate node over
      // to the equivalent and operation.
      DAG.transferDbgValues(N0, And);
      return And;
    }
  }
 
  // Fold (zext (and (trunc x), cst)) -> (and x, cst),
  // if either of the casts is not free.
  // Also handles (zext (and (bitcast (extract_subvector vNi1, 0)) cst))
  // by treating the bitcast+extract as equivalent to a truncate of the
  // wider bitcast, e.g. on AVX512DQ where v8i1 extract replaces truncate.
  if (N0.getOpcode() == ISD::AND &&
      N0.getOperand(1).getOpcode() == ISD::Constant) {
    SDValue AndSrc = N0.getOperand(0);
    SDValue X;
    if (AndSrc.getOpcode() == ISD::TRUNCATE) {
      X = AndSrc.getOperand(0);
    } else if (AndSrc.getOpcode() == ISD::BITCAST &&
               AndSrc.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
               AndSrc.getOperand(0).getConstantOperandVal(1) == 0) {
      // (bitcast (extract_subvector vNi1, 0) -> iK) is equivalent to
      // (truncate (bitcast vNi1 -> iN) -> iK); use the wider vNi1 as X.
      SDValue Src = AndSrc.getOperand(0).getOperand(0);
      EVT SrcVT = Src.getValueType();
      if (SrcVT.isFixedLengthVectorOf(MVT::i1)) {
        EVT WideIntVT =
            EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits());
        if (TLI.isTypeLegal(WideIntVT))
          X = DAG.getBitcast(WideIntVT, Src);
      }
    }
    if (X && (!TLI.isTruncateFree(X, N0.getValueType()) ||
              !TLI.isZExtFree(N0.getValueType(), VT))) {
      X = DAG.getAnyExtOrTrunc(X, SDLoc(X), VT);
      APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
      return DAG.getNode(ISD::AND, DL, VT, X, DAG.getConstant(Mask, DL, VT));
    }
  }
 
  // Try to simplify (zext (load x)).
  if (SDValue foldedExt = tryToFoldExtOfLoad(
          DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD,
          ISD::ZERO_EXTEND, N->getFlags().hasNonNeg()))
    return foldedExt;
 
  if (SDValue foldedExt =
          tryToFoldExtOfMaskedLoad(DAG, TLI, VT, LegalOperations, N, N0,
                                   ISD::ZEXTLOAD, ISD::ZERO_EXTEND))
    return foldedExt;
 
  // fold (zext (load x)) to multiple smaller zextloads.
  // Only on illegal but splittable vectors.
  if (SDValue ExtLoad = CombineExtLoad(N))
    return ExtLoad;
 
  // Try to simplify (zext (atomic_load x)).
  if (SDValue foldedExt =
          tryToFoldExtOfAtomicLoad(DAG, TLI, VT, N0, ISD::ZEXTLOAD))
    return foldedExt;
 
  // fold (zext (and/or/xor (load x), cst)) ->
  //      (and/or/xor (zextload x), (zext cst))
  // Unless (and (load x) cst) will match as a zextload already and has
  // additional users, or the zext is already free.
  if (ISD::isBitwiseLogicOp(N0.getOpcode()) && !TLI.isZExtFree(N0, VT) &&
      isa<LoadSDNode>(N0.getOperand(0)) &&
      N0.getOperand(1).getOpcode() == ISD::Constant &&
      (!LegalOperations && TLI.isOperationLegal(N0.getOpcode(), VT))) {
    LoadSDNode *LN00 = cast<LoadSDNode>(N0.getOperand(0));
    EVT MemVT = LN00->getMemoryVT();
    if (TLI.isLoadLegal(VT, MemVT, LN00->getAlign(), LN00->getAddressSpace(),
                        ISD::ZEXTLOAD, false) &&
        LN00->getExtensionType() != ISD::SEXTLOAD && LN00->isUnindexed()) {
      bool DoXform = true;
      SmallVector<SDNode*, 4> SetCCs;
      if (!N0.hasOneUse()) {
        if (N0.getOpcode() == ISD::AND) {
          auto *AndC = cast<ConstantSDNode>(N0.getOperand(1));
          EVT LoadResultTy = AndC->getValueType(0);
          EVT ExtVT;
          if (isAndLoadExtLoad(AndC, LN00, LoadResultTy, ExtVT))
            DoXform = false;
        }
      }
      if (DoXform)
        DoXform = ExtendUsesToFormExtLoad(VT, N0.getNode(), N0.getOperand(0),
                                          ISD::ZERO_EXTEND, SetCCs, TLI);
      if (DoXform) {
        SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(LN00), VT,
                                         LN00->getChain(), LN00->getBasePtr(),
                                         LN00->getMemoryVT(),
                                         LN00->getMemOperand());
        APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
        SDValue And = DAG.getNode(N0.getOpcode(), DL, VT,
                                  ExtLoad, DAG.getConstant(Mask, DL, VT));
        ExtendSetCCUses(SetCCs, N0.getOperand(0), ExtLoad, ISD::ZERO_EXTEND);
        bool NoReplaceTruncAnd = !N0.hasOneUse();
        bool NoReplaceTrunc = SDValue(LN00, 0).hasOneUse();
        CombineTo(N, And);
        // If N0 has multiple uses, change other uses as well.
        if (NoReplaceTruncAnd) {
          SDValue TruncAnd =
              DAG.getNode(ISD::TRUNCATE, DL, N0.getValueType(), And);
          CombineTo(N0.getNode(), TruncAnd);
        }
        if (NoReplaceTrunc) {
          DAG.ReplaceAllUsesOfValueWith(SDValue(LN00, 1), ExtLoad.getValue(1));
        } else {
          SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(LN00),
                                      LN00->getValueType(0), ExtLoad);
          CombineTo(LN00, Trunc, ExtLoad.getValue(1));
        }
        return SDValue(N,0); // Return N so it doesn't get rechecked!
      }
    }
  }
 
  // fold (zext (and/or/xor (shl/shr (load x), cst), cst)) ->
  //      (and/or/xor (shl/shr (zextload x), (zext cst)), (zext cst))
  if (SDValue ZExtLoad = CombineZExtLogicopShiftLoad(N))
    return ZExtLoad;
 
  // Try to simplify (zext (zextload x)).
  if (SDValue foldedExt = tryToFoldExtOfExtload(
          DAG, *this, TLI, VT, LegalOperations, N, N0, ISD::ZEXTLOAD))
    return foldedExt;
 
  if (SDValue V = foldExtendedSignBitTest(N, DAG, LegalOperations))
    return V;
 
  if (N0.getOpcode() == ISD::SETCC) {
    // Propagate fast-math-flags.
    SelectionDAG::FlagInserter FlagsInserter(DAG, N0->getFlags());
 
    // Only do this before legalize for now.
    if (!LegalOperations && VT.isVector() &&
        N0.getValueType().getVectorElementType() == MVT::i1) {
      EVT N00VT = N0.getOperand(0).getValueType();
      if (getSetCCResultType(N00VT) == N0.getValueType())
        return SDValue();
 
      // We know that the # elements of the results is the same as the #
      // elements of the compare (and the # elements of the compare result for
      // that matter). Check to see that they are the same size. If so, we know
      // that the element size of the sext'd result matches the element size of
      // the compare operands.
      if (VT.getSizeInBits() == N00VT.getSizeInBits()) {
        // zext(setcc) -> zext_in_reg(vsetcc) for vectors.
        SDValue VSetCC = DAG.getNode(ISD::SETCC, DL, VT, N0.getOperand(0),
                                     N0.getOperand(1), N0.getOperand(2));
        return DAG.getZeroExtendInReg(VSetCC, DL, N0.getValueType());
      }
 
      // If the desired elements are smaller or larger than the source
      // elements we can use a matching integer vector type and then
      // truncate/any extend followed by zext_in_reg.
      EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
      SDValue VsetCC =
          DAG.getNode(ISD::SETCC, DL, MatchingVectorType, N0.getOperand(0),
                      N0.getOperand(1), N0.getOperand(2));
      return DAG.getZeroExtendInReg(DAG.getAnyExtOrTrunc(VsetCC, DL, VT), DL,
                                    N0.getValueType());
    }
 
    // zext(setcc x,y,cc) -> zext(select x, y, true, false, cc)
    EVT N0VT = N0.getValueType();
    EVT N00VT = N0.getOperand(0).getValueType();
    if (SDValue SCC = SimplifySelectCC(
            DL, N0.getOperand(0), N0.getOperand(1),
            DAG.getBoolConstant(true, DL, N0VT, N00VT),
            DAG.getBoolConstant(false, DL, N0VT, N00VT),
            cast<CondCodeSDNode>(N0.getOperand(2))->get(), true))
      return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, SCC);
  }
 
  // (zext (shl (zext x), cst)) -> (shl (zext x), cst)
  if ((N0.getOpcode() == ISD::SHL || N0.getOpcode() == ISD::SRL) &&
      !TLI.isZExtFree(N0, VT)) {
    SDValue ShVal = N0.getOperand(0);
    SDValue ShAmt = N0.getOperand(1);
    if (auto *ShAmtC = dyn_cast<ConstantSDNode>(ShAmt)) {
      if (ShVal.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse()) {
        if (N0.getOpcode() == ISD::SHL) {
          // If the original shl may be shifting out bits, do not perform this
          // transformation.
          unsigned KnownZeroBits = ShVal.getValueSizeInBits() -
                                   ShVal.getOperand(0).getValueSizeInBits();
          if (ShAmtC->getAPIntValue().ugt(KnownZeroBits)) {
            // If the shift is too large, then see if we can deduce that the
            // shift is safe anyway.
 
            // Check if the bits being shifted out are known to be zero.
            KnownBits KnownShVal = DAG.computeKnownBits(ShVal);
            if (ShAmtC->getAPIntValue().ugt(KnownShVal.countMinLeadingZeros()))
              return SDValue();
          }
        }
 
        // Ensure that the shift amount is wide enough for the shifted value.
        if (Log2_32_Ceil(VT.getSizeInBits()) > ShAmt.getValueSizeInBits())
          ShAmt = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, ShAmt);
 
        return DAG.getNode(N0.getOpcode(), DL, VT,
                           DAG.getNode(ISD::ZERO_EXTEND, DL, VT, ShVal), ShAmt);
      }
    }
  }
 
  if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    return NewVSel;
 
  if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
    return NewCtPop;
 
  if (SDValue V = widenAbs(N, DAG))
    return V;
 
  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
    return Res;
 
  // CSE zext nneg with sext if the zext is not free.
  if (N->getFlags().hasNonNeg() && !TLI.isZExtFree(N0.getValueType(), VT)) {
    SDNode *CSENode = DAG.getNodeIfExists(ISD::SIGN_EXTEND, N->getVTList(), N0);
    if (CSENode)
      return SDValue(CSENode, 0);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // aext(undef) = undef
  if (N0.isUndef())
    return DAG.getUNDEF(VT);
 
  if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
    return Res;
 
  // fold (aext (aext x)) -> (aext x)
  // fold (aext (zext x)) -> (zext x)
  // fold (aext (sext x)) -> (sext x)
  if (N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::ZERO_EXTEND ||
      N0.getOpcode() == ISD::SIGN_EXTEND) {
    SDNodeFlags Flags;
    if (N0.getOpcode() == ISD::ZERO_EXTEND)
      Flags.setNonNeg(N0->getFlags().hasNonNeg());
    return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Flags);
  }
 
  // fold (aext (aext_extend_vector_inreg x)) -> (aext_extend_vector_inreg x)
  // fold (aext (zext_extend_vector_inreg x)) -> (zext_extend_vector_inreg x)
  // fold (aext (sext_extend_vector_inreg x)) -> (sext_extend_vector_inreg x)
  if (N0.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG ||
      N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
      N0.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG)
    return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0));
 
  // fold (aext (truncate (load x))) -> (aext (smaller load x))
  // fold (aext (truncate (srl (load x), c))) -> (aext (small load (x+c/n)))
  if (N0.getOpcode() == ISD::TRUNCATE) {
    if (SDValue NarrowLoad = reduceLoadWidth(N0.getNode())) {
      SDNode *oye = N0.getOperand(0).getNode();
      if (NarrowLoad.getNode() != N0.getNode()) {
        CombineTo(N0.getNode(), NarrowLoad);
        // CombineTo deleted the truncate, if needed, but not what's under it.
        AddToWorklist(oye);
      }
      return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    }
  }
 
  // fold (aext (truncate x))
  if (N0.getOpcode() == ISD::TRUNCATE)
    return DAG.getAnyExtOrTrunc(N0.getOperand(0), DL, VT);
 
  // Fold (aext (and (trunc x), cst)) -> (and x, cst)
  // if either of the casts is not free, and sign-extending the narrow type is
  // not cheaper than zero-extending it (which would indicate the target prefers
  // to keep operations at the narrower width).
  // Also handles (aext (and (bitcast (extract_subvector vNi1, 0)) cst))
  // which arises on AVX512DQ where v8i1 extract replaces truncate.
  if (N0.getOpcode() == ISD::AND &&
      N0.getOperand(1).getOpcode() == ISD::Constant) {
    SDValue AndSrc = N0.getOperand(0);
    SDValue X;
    if (AndSrc.getOpcode() == ISD::TRUNCATE) {
      X = AndSrc.getOperand(0);
    } else if (AndSrc.getOpcode() == ISD::BITCAST &&
               AndSrc.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
               AndSrc.getOperand(0).getConstantOperandVal(1) == 0) {
      SDValue Src = AndSrc.getOperand(0).getOperand(0);
      EVT SrcVT = Src.getValueType();
      if (SrcVT.isFixedLengthVectorOf(MVT::i1)) {
        EVT WideIntVT =
            EVT::getIntegerVT(*DAG.getContext(), SrcVT.getSizeInBits());
        if (TLI.isTypeLegal(WideIntVT))
          X = DAG.getBitcast(WideIntVT, Src);
      }
    }
    if (X && (!TLI.isTruncateFree(X, N0.getValueType()) ||
              (!TLI.isZExtFree(N0.getValueType(), VT) &&
               !TLI.isSExtCheaperThanZExt(N0.getValueType(), VT)))) {
      X = DAG.getAnyExtOrTrunc(X, DL, VT);
      APInt Mask = N0.getConstantOperandAPInt(1).zext(VT.getSizeInBits());
      return DAG.getNode(ISD::AND, DL, VT, X, DAG.getConstant(Mask, DL, VT));
    }
  }
 
  // fold (aext (load x)) -> (aext (truncate (extload x)))
  // None of the supported targets knows how to perform load and any_ext
  // on vectors in one instruction, so attempt to fold to zext instead.
  if (VT.isVector()) {
    // Try to simplify (zext (load x)).
    if (SDValue foldedExt =
            tryToFoldExtOfLoad(DAG, *this, TLI, VT, LegalOperations, N, N0,
                               ISD::ZEXTLOAD, ISD::ZERO_EXTEND))
      return foldedExt;
  } else if (ISD::isNON_EXTLoad(N0.getNode()) &&
             ISD::isUNINDEXEDLoad(N0.getNode())) {
    LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    if (TLI.isLoadLegalOrCustom(VT, N0.getValueType(), LN0->getAlign(),
                                LN0->getAddressSpace(), ISD::EXTLOAD, false)) {
      bool DoXform = true;
      SmallVector<SDNode *, 4> SetCCs;
      if (!N0.hasOneUse())
        DoXform =
            ExtendUsesToFormExtLoad(VT, N, N0, ISD::ANY_EXTEND, SetCCs, TLI);
      if (DoXform) {
        SDValue ExtLoad = DAG.getExtLoad(ISD::EXTLOAD, DL, VT, LN0->getChain(),
                                         LN0->getBasePtr(), N0.getValueType(),
                                         LN0->getMemOperand());
        ExtendSetCCUses(SetCCs, N0, ExtLoad, ISD::ANY_EXTEND);
        // If the load value is used only by N, replace it via CombineTo N.
        bool NoReplaceTrunc = N0.hasOneUse();
        CombineTo(N, ExtLoad);
        if (NoReplaceTrunc) {
          DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
          recursivelyDeleteUnusedNodes(LN0);
        } else {
          SDValue Trunc =
              DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad);
          CombineTo(LN0, Trunc, ExtLoad.getValue(1));
        }
        return SDValue(N, 0); // Return N so it doesn't get rechecked!
      }
    }
  }
 
  // fold (aext (zextload x)) -> (aext (truncate (zextload x)))
  // fold (aext (sextload x)) -> (aext (truncate (sextload x)))
  // fold (aext ( extload x)) -> (aext (truncate (extload  x)))
  if (N0.getOpcode() == ISD::LOAD && !ISD::isNON_EXTLoad(N0.getNode()) &&
      ISD::isUNINDEXEDLoad(N0.getNode()) && N0.hasOneUse()) {
    LoadSDNode *LN0 = cast<LoadSDNode>(N0);
    ISD::LoadExtType ExtType = LN0->getExtensionType();
    EVT MemVT = LN0->getMemoryVT();
    if (!LegalOperations ||
        TLI.isLoadLegal(VT, MemVT, LN0->getAlign(), LN0->getAddressSpace(),
                        ExtType, false)) {
      SDValue ExtLoad =
          DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), LN0->getBasePtr(),
                         MemVT, LN0->getMemOperand());
      CombineTo(N, ExtLoad);
      DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1));
      recursivelyDeleteUnusedNodes(LN0);
      return SDValue(N, 0);   // Return N so it doesn't get rechecked!
    }
  }
 
  if (N0.getOpcode() == ISD::SETCC) {
    // Propagate fast-math-flags.
    SDNodeFlags Flags = N0->getFlags();
    SelectionDAG::FlagInserter FlagsInserter(DAG, Flags);
 
    // For vectors:
    // aext(setcc) -> vsetcc
    // aext(setcc) -> truncate(vsetcc)
    // aext(setcc) -> aext(vsetcc)
    // Only do this before legalize for now.
    if (VT.isVector() && !LegalOperations) {
      EVT N00VT = N0.getOperand(0).getValueType();
      if (getSetCCResultType(N00VT) == N0.getValueType())
        return SDValue();
 
      // We know that the # elements of the results is the same as the
      // # elements of the compare (and the # elements of the compare result
      // for that matter).  Check to see that they are the same size.  If so,
      // we know that the element size of the sext'd result matches the
      // element size of the compare operands.
      if (VT.getSizeInBits() == N00VT.getSizeInBits())
        return DAG.getSetCC(DL, VT, N0.getOperand(0), N0.getOperand(1),
                            cast<CondCodeSDNode>(N0.getOperand(2))->get(),
                            /*Chain=*/{}, /*Signaling=*/false, Flags);
 
      // If the desired elements are smaller or larger than the source
      // elements we can use a matching integer vector type and then
      // truncate/any extend
      EVT MatchingVectorType = N00VT.changeVectorElementTypeToInteger();
      SDValue VsetCC = DAG.getSetCC(
          DL, MatchingVectorType, N0.getOperand(0), N0.getOperand(1),
          cast<CondCodeSDNode>(N0.getOperand(2))->get(), /*Chain=*/{},
          /*Signaling=*/false, Flags);
      return DAG.getAnyExtOrTrunc(VsetCC, DL, VT);
    }
 
    // aext(setcc x,y,cc) -> select_cc x, y, 1, 0, cc
    if (SDValue SCC = SimplifySelectCC(
            DL, N0.getOperand(0), N0.getOperand(1), DAG.getConstant(1, DL, VT),
            DAG.getConstant(0, DL, VT),
            cast<CondCodeSDNode>(N0.getOperand(2))->get(), true))
      return SCC;
  }
 
  if (SDValue NewCtPop = widenCtPop(N, DAG, DL))
    return NewCtPop;
 
  if (SDValue Res = tryToFoldExtendSelectLoad(N, TLI, DAG, DL, Level))
    return Res;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitAssertExt(SDNode *N) {
  unsigned Opcode = N->getOpcode();
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT AssertVT = cast<VTSDNode>(N1)->getVT();
 
  // fold (assert?ext (assert?ext x, vt), vt) -> (assert?ext x, vt)
  if (N0.getOpcode() == Opcode &&
      AssertVT == cast<VTSDNode>(N0.getOperand(1))->getVT())
    return N0;
 
  // fold (assert?ext c, vt) -> c
  if (isa<ConstantSDNode>(N0))
    return N0;
 
  if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
      N0.getOperand(0).getOpcode() == Opcode) {
    // We have an assert, truncate, assert sandwich. Make one stronger assert
    // by asserting on the smallest asserted type to the larger source type.
    // This eliminates the later assert:
    // assert (trunc (assert X, i8) to iN), i1 --> trunc (assert X, i1) to iN
    // assert (trunc (assert X, i1) to iN), i8 --> trunc (assert X, i1) to iN
    SDLoc DL(N);
    SDValue BigA = N0.getOperand(0);
    EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
    EVT MinAssertVT = AssertVT.bitsLT(BigA_AssertVT) ? AssertVT : BigA_AssertVT;
    SDValue MinAssertVTVal = DAG.getValueType(MinAssertVT);
    SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
                                    BigA.getOperand(0), MinAssertVTVal);
    return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
  }
 
  // If we have (AssertZext (truncate (AssertSext X, iX)), iY) and Y is smaller
  // than X. Just move the AssertZext in front of the truncate and drop the
  // AssertSExt.
  if (N0.getOpcode() == ISD::TRUNCATE && N0.hasOneUse() &&
      N0.getOperand(0).getOpcode() == ISD::AssertSext &&
      Opcode == ISD::AssertZext) {
    SDValue BigA = N0.getOperand(0);
    EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
    if (AssertVT.bitsLT(BigA_AssertVT)) {
      SDLoc DL(N);
      SDValue NewAssert = DAG.getNode(Opcode, DL, BigA.getValueType(),
                                      BigA.getOperand(0), N1);
      return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewAssert);
    }
  }
 
  if (Opcode == ISD::AssertZext && N0.getOpcode() == ISD::AND &&
      isa<ConstantSDNode>(N0.getOperand(1))) {
    const APInt &Mask = N0.getConstantOperandAPInt(1);
 
    // If we have (AssertZext (and (AssertSext X, iX), M), iY) and Y is smaller
    // than X, and the And doesn't change the lower iX bits, we can move the
    // AssertZext in front of the And and drop the AssertSext.
    if (N0.getOperand(0).getOpcode() == ISD::AssertSext && N0.hasOneUse()) {
      SDValue BigA = N0.getOperand(0);
      EVT BigA_AssertVT = cast<VTSDNode>(BigA.getOperand(1))->getVT();
      if (AssertVT.bitsLT(BigA_AssertVT) &&
          Mask.countr_one() >= BigA_AssertVT.getScalarSizeInBits()) {
        SDLoc DL(N);
        SDValue NewAssert =
            DAG.getNode(Opcode, DL, N->getValueType(0), BigA.getOperand(0), N1);
        return DAG.getNode(ISD::AND, DL, N->getValueType(0), NewAssert,
                           N0.getOperand(1));
      }
    }
 
    // Remove AssertZext entirely if the mask guarantees the assertion cannot
    // fail.
    // TODO: Use KB countMinLeadingZeros to handle non-constant masks?
    if (Mask.isIntN(AssertVT.getScalarSizeInBits()))
      return N0;
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitAssertAlign(SDNode *N) {
  SDLoc DL(N);
 
  Align AL = cast<AssertAlignSDNode>(N)->getAlign();
  SDValue N0 = N->getOperand(0);
 
  // Fold (assertalign (assertalign x, AL0), AL1) ->
  // (assertalign x, max(AL0, AL1))
  if (auto *AAN = dyn_cast<AssertAlignSDNode>(N0))
    return DAG.getAssertAlign(DL, N0.getOperand(0),
                              std::max(AL, AAN->getAlign()));
 
  // In rare cases, there are trivial arithmetic ops in source operands. Sink
  // this assert down to source operands so that those arithmetic ops could be
  // exposed to the DAG combining.
  switch (N0.getOpcode()) {
  default:
    break;
  case ISD::ADD:
  case ISD::PTRADD:
  case ISD::SUB: {
    unsigned AlignShift = Log2(AL);
    SDValue LHS = N0.getOperand(0);
    SDValue RHS = N0.getOperand(1);
    unsigned LHSAlignShift = DAG.computeKnownBits(LHS).countMinTrailingZeros();
    unsigned RHSAlignShift = DAG.computeKnownBits(RHS).countMinTrailingZeros();
    if (LHSAlignShift >= AlignShift || RHSAlignShift >= AlignShift) {
      if (LHSAlignShift < AlignShift)
        LHS = DAG.getAssertAlign(DL, LHS, AL);
      if (RHSAlignShift < AlignShift)
        RHS = DAG.getAssertAlign(DL, RHS, AL);
      return DAG.getNode(N0.getOpcode(), DL, N0.getValueType(), LHS, RHS);
    }
    break;
  }
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitIS_FPCLASS(SDNode *N) {
  SDValue Src = N->getOperand(0);
  FPClassTest Mask = static_cast<FPClassTest>(N->getConstantOperandVal(1));
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // is.fpclass(poison, mask) -> poison
  if (Src.getOpcode() == ISD::POISON)
    return DAG.getPOISON(VT);
 
  KnownFPClass Known = DAG.computeKnownFPClass(Src, Mask);
 
  // All possible classes are within the mask: result is always true.
  if ((~Mask & Known.KnownFPClasses) == fcNone)
    return DAG.getBoolConstant(true, DL, VT, Src.getValueType());
 
  // Clear test bits we know must be false from the source value.
  // fp_class (nnan x), qnan|snan|other -> fp_class (nnan x), other
  // fp_class (ninf x), ninf|pinf|other -> fp_class (ninf x), other
  if ((Mask & Known.KnownFPClasses) != Mask) {
    return DAG.getNode(
        ISD::IS_FPCLASS, DL, VT, Src,
        DAG.getTargetConstant(Mask & Known.KnownFPClasses, DL, MVT::i32),
        N->getFlags());
  }
 
  return SDValue();
}
 
/// If the result of a load is shifted/masked/truncated to an effectively
/// narrower type, try to transform the load to a narrower type and/or
/// use an extending load.
SDValue DAGCombiner::reduceLoadWidth(SDNode *N) {
  unsigned Opc = N->getOpcode();
 
  ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  EVT ExtVT = VT;
 
  // This transformation isn't valid for vector loads.
  if (VT.isVector())
    return SDValue();
 
  // The ShAmt variable is used to indicate that we've consumed a right
  // shift. I.e. we want to narrow the width of the load by skipping to load the
  // ShAmt least significant bits.
  unsigned ShAmt = 0;
  // A special case is when the least significant bits from the load are masked
  // away, but using an AND rather than a right shift. HasShiftedOffset is used
  // to indicate that the narrowed load should be left-shifted ShAmt bits to get
  // the result.
  unsigned ShiftedOffset = 0;
  // Special case: SIGN_EXTEND_INREG is basically truncating to ExtVT then
  // extended to VT.
  if (Opc == ISD::SIGN_EXTEND_INREG) {
    ExtType = ISD::SEXTLOAD;
    ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT();
  } else if (Opc == ISD::SRL || Opc == ISD::SRA) {
    // Another special-case: SRL/SRA is basically zero/sign-extending a narrower
    // value, or it may be shifting a higher subword, half or byte into the
    // lowest bits.
 
    // Only handle shift with constant shift amount, and the shiftee must be a
    // load.
    auto *LN = dyn_cast<LoadSDNode>(N0);
    auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
    if (!N1C || !LN)
      return SDValue();
    // If the shift amount is larger than the memory type then we're not
    // accessing any of the loaded bytes.
    ShAmt = N1C->getZExtValue();
    uint64_t MemoryWidth = LN->getMemoryVT().getScalarSizeInBits();
    if (MemoryWidth <= ShAmt)
      return SDValue();
    // Attempt to fold away the SRL by using ZEXTLOAD and SRA by using SEXTLOAD.
    ExtType = Opc == ISD::SRL ? ISD::ZEXTLOAD : ISD::SEXTLOAD;
    ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShAmt);
    // If original load is a SEXTLOAD then we can't simply replace it by a
    // ZEXTLOAD (we could potentially replace it by a more narrow SEXTLOAD
    // followed by a ZEXT, but that is not handled at the moment). Similarly if
    // the original load is a ZEXTLOAD and we want to use a SEXTLOAD.
    if ((LN->getExtensionType() == ISD::SEXTLOAD ||
         LN->getExtensionType() == ISD::ZEXTLOAD) &&
        LN->getExtensionType() != ExtType)
      return SDValue();
  } else if (Opc == ISD::AND) {
    // An AND with a constant mask is the same as a truncate + zero-extend.
    auto AndC = dyn_cast<ConstantSDNode>(N->getOperand(1));
    if (!AndC)
      return SDValue();
 
    const APInt &Mask = AndC->getAPIntValue();
    unsigned ActiveBits = 0;
    if (Mask.isMask()) {
      ActiveBits = Mask.countr_one();
    } else if (Mask.isShiftedMask(ShAmt, ActiveBits)) {
      ShiftedOffset = ShAmt;
    } else {
      return SDValue();
    }
 
    ExtType = ISD::ZEXTLOAD;
    ExtVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
  }
 
  // In case Opc==SRL we've already prepared ExtVT/ExtType/ShAmt based on doing
  // a right shift. Here we redo some of those checks, to possibly adjust the
  // ExtVT even further based on "a masking AND". We could also end up here for
  // other reasons (e.g. based on Opc==TRUNCATE) and that is why some checks
  // need to be done here as well.
  if (Opc == ISD::SRL || N0.getOpcode() == ISD::SRL) {
    SDValue SRL = Opc == ISD::SRL ? SDValue(N, 0) : N0;
    // Bail out when the SRL has more than one use. This is done for historical
    // (undocumented) reasons. Maybe intent was to guard the AND-masking below
    // check below? And maybe it could be non-profitable to do the transform in
    // case the SRL has multiple uses and we get here with Opc!=ISD::SRL?
    // FIXME: Can't we just skip this check for the Opc==ISD::SRL case.
    if (!SRL.hasOneUse())
      return SDValue();
 
    // Only handle shift with constant shift amount, and the shiftee must be a
    // load.
    auto *LN = dyn_cast<LoadSDNode>(SRL.getOperand(0));
    auto *SRL1C = dyn_cast<ConstantSDNode>(SRL.getOperand(1));
    if (!SRL1C || !LN)
      return SDValue();
 
    // If the shift amount is larger than the input type then we're not
    // accessing any of the loaded bytes.  If the load was a zextload/extload
    // then the result of the shift+trunc is zero/undef (handled elsewhere).
    ShAmt = SRL1C->getZExtValue();
    uint64_t MemoryWidth = LN->getMemoryVT().getSizeInBits();
    if (ShAmt >= MemoryWidth)
      return SDValue();
 
    // Because a SRL must be assumed to *need* to zero-extend the high bits
    // (as opposed to anyext the high bits), we can't combine the zextload
    // lowering of SRL and an sextload.
    if (LN->getExtensionType() == ISD::SEXTLOAD)
      return SDValue();
 
    // Avoid reading outside the memory accessed by the original load (could
    // happened if we only adjust the load base pointer by ShAmt). Instead we
    // try to narrow the load even further. The typical scenario here is:
    //   (i64 (truncate (i96 (srl (load x), 64)))) ->
    //     (i64 (truncate (i96 (zextload (load i32 + offset) from i32))))
    if (ExtVT.getScalarSizeInBits() > MemoryWidth - ShAmt) {
      // Don't replace sextload by zextload.
      if (ExtType == ISD::SEXTLOAD)
        return SDValue();
      // Narrow the load.
      ExtType = ISD::ZEXTLOAD;
      ExtVT = EVT::getIntegerVT(*DAG.getContext(), MemoryWidth - ShAmt);
    }
 
    // If the SRL is only used by a masking AND, we may be able to adjust
    // the ExtVT to make the AND redundant.
    SDNode *Mask = *(SRL->user_begin());
    if (SRL.hasOneUse() && Mask->getOpcode() == ISD::AND &&
        isa<ConstantSDNode>(Mask->getOperand(1))) {
      unsigned Offset, ActiveBits;
      const APInt& ShiftMask = Mask->getConstantOperandAPInt(1);
      if (ShiftMask.isMask()) {
        EVT MaskedVT =
            EVT::getIntegerVT(*DAG.getContext(), ShiftMask.countr_one());
        // If the mask is smaller, recompute the type.
        if ((ExtVT.getScalarSizeInBits() > MaskedVT.getScalarSizeInBits()) &&
            TLI.isLoadLegal(SRL.getValueType(), MaskedVT, LN->getAlign(),
                            LN->getAddressSpace(), ExtType, false))
          ExtVT = MaskedVT;
      } else if (ExtType == ISD::ZEXTLOAD &&
                 ShiftMask.isShiftedMask(Offset, ActiveBits) &&
                 (Offset + ShAmt) < VT.getScalarSizeInBits()) {
        EVT MaskedVT = EVT::getIntegerVT(*DAG.getContext(), ActiveBits);
        // If the mask is shifted we can use a narrower load and a shl to insert
        // the trailing zeros.
        if (((Offset + ActiveBits) <= ExtVT.getScalarSizeInBits()) &&
            TLI.isLoadLegal(SRL.getValueType(), MaskedVT, LN->getAlign(),
                            LN->getAddressSpace(), ExtType, false)) {
          ExtVT = MaskedVT;
          ShAmt = Offset + ShAmt;
          ShiftedOffset = Offset;
        }
      }
    }
 
    N0 = SRL.getOperand(0);
  }
 
  // If the load is shifted left (and the result isn't shifted back right), we
  // can fold a truncate through the shift. The typical scenario is that N
  // points at a TRUNCATE here so the attempted fold is:
  //   (truncate (shl (load x), c))) -> (shl (narrow load x), c)
  // ShLeftAmt will indicate how much a narrowed load should be shifted left.
  unsigned ShLeftAmt = 0;
  if (ShAmt == 0 && N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
      ExtVT == VT && TLI.isNarrowingProfitable(N, N0.getValueType(), VT)) {
    if (ConstantSDNode *N01 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) {
      ShLeftAmt = N01->getZExtValue();
      N0 = N0.getOperand(0);
    }
  }
 
  // Look through a freeze if present between the operation and the load.
  // The freeze will be preserved on the narrowed result.
  SDValue FreezeNode;
  if (N0.getOpcode() == ISD::FREEZE) {
    FreezeNode = N0;
    N0 = N0.getOperand(0);
  }
 
  // If we haven't found a load, we can't narrow it.
  if (!isa<LoadSDNode>(N0))
    return SDValue();
 
  LoadSDNode *LN0 = cast<LoadSDNode>(N0);
  // Reducing the width of a volatile load is illegal.  For atomics, we may be
  // able to reduce the width provided we never widen again. (see D66309)
  if (!LN0->isSimple() ||
      !isLegalNarrowLdSt(LN0, ExtType, ExtVT, ShAmt))
    return SDValue();
 
  // Bail early when looking through a multi-use freeze, since other users of
  // the freeze can depend on the full load value. But its still safe to change
  // the extension type from anyext to zext.
  if (FreezeNode && !FreezeNode.hasOneUse() &&
      (LN0->getMemoryVT().bitsGT(ExtVT) || ExtType != ISD::ZEXTLOAD ||
       (LN0->getExtensionType() != ISD::EXTLOAD &&
        LN0->getExtensionType() != ISD::ZEXTLOAD)))
    return SDValue();
 
  auto AdjustBigEndianShift = [&](unsigned ShAmt) {
    unsigned LVTStoreBits =
        LN0->getMemoryVT().getStoreSizeInBits().getFixedValue();
    unsigned EVTStoreBits = ExtVT.getStoreSizeInBits().getFixedValue();
    return LVTStoreBits - EVTStoreBits - ShAmt;
  };
 
  // We need to adjust the pointer to the load by ShAmt bits in order to load
  // the correct bytes.
  unsigned PtrAdjustmentInBits =
      DAG.getDataLayout().isBigEndian() ? AdjustBigEndianShift(ShAmt) : ShAmt;
 
  uint64_t PtrOff = PtrAdjustmentInBits / 8;
  SDLoc DL(LN0);
  // The original load itself didn't wrap, so an offset within it doesn't.
  SDValue NewPtr =
      DAG.getMemBasePlusOffset(LN0->getBasePtr(), TypeSize::getFixed(PtrOff),
                               DL, SDNodeFlags::NoUnsignedWrap);
  AddToWorklist(NewPtr.getNode());
 
  SDValue Load;
  if (ExtType == ISD::NON_EXTLOAD) {
    const MDNode *OldRanges = LN0->getRanges();
    const MDNode *NewRanges = nullptr;
    // If LSBs are loaded and the truncated ConstantRange for the OldRanges
    // metadata is not the full-set for the new width then create a NewRanges
    // metadata for the truncated load
    if (ShAmt == 0 && OldRanges) {
      ConstantRange CR = getConstantRangeFromMetadata(*OldRanges);
      unsigned BitSize = VT.getScalarSizeInBits();
 
      // It is possible for an 8-bit extending load with 8-bit range
      // metadata to be narrowed to an 8-bit load.  This guard is necessary to
      // ensure that truncation is strictly smaller.
      if (CR.getBitWidth() > BitSize) {
        ConstantRange TruncatedCR = CR.truncate(BitSize);
        if (!TruncatedCR.isFullSet()) {
          Metadata *Bounds[2] = {
              ConstantAsMetadata::get(
                  ConstantInt::get(*DAG.getContext(), TruncatedCR.getLower())),
              ConstantAsMetadata::get(
                  ConstantInt::get(*DAG.getContext(), TruncatedCR.getUpper()))};
          NewRanges = MDNode::get(*DAG.getContext(), Bounds);
        }
      } else if (CR.getBitWidth() == BitSize)
        NewRanges = OldRanges;
    }
    Load = DAG.getLoad(VT, DL, LN0->getChain(), NewPtr,
                       LN0->getPointerInfo().getWithOffset(PtrOff),
                       LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
                       LN0->getAAInfo(), NewRanges);
  } else
    Load = DAG.getExtLoad(ExtType, DL, VT, LN0->getChain(), NewPtr,
                          LN0->getPointerInfo().getWithOffset(PtrOff), ExtVT,
                          LN0->getBaseAlign(), LN0->getMemOperand()->getFlags(),
                          LN0->getAAInfo());
 
  // Replace the old load's chain with the new load's chain.
  WorklistRemover DeadNodes(*this);
  DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
 
  // Replace old load value for multi-use freeze so all users benefit.
  if (FreezeNode && !FreezeNode.hasOneUse())
    DAG.ReplaceAllUsesOfValueWith(N0.getValue(0), Load.getValue(0));
 
  // If we looked through a freeze, rewrap the narrowed result and add an
  // Assert node so downstream analyses can see the range.
  SDValue Result = Load;
  if (FreezeNode) {
    Result = DAG.getNode(ISD::FREEZE, DL, VT, Result);
    if (ExtType == ISD::ZEXTLOAD)
      Result =
          DAG.getNode(ISD::AssertZext, DL, VT, Result, DAG.getValueType(ExtVT));
    else if (ExtType == ISD::SEXTLOAD)
      Result =
          DAG.getNode(ISD::AssertSext, DL, VT, Result, DAG.getValueType(ExtVT));
  }
 
  // Shift the result left, if we've swallowed a left shift.
  if (ShLeftAmt != 0) {
    // If the shift amount is as large as the result size (but, presumably,
    // no larger than the source) then the useful bits of the result are
    // zero; we can't simply return the shortened shift, because the result
    // of that operation is undefined.
    if (ShLeftAmt >= VT.getScalarSizeInBits())
      Result = DAG.getConstant(0, DL, VT);
    else
      Result = DAG.getNode(ISD::SHL, DL, VT, Result,
                           DAG.getShiftAmountConstant(ShLeftAmt, VT, DL));
  }
 
  if (ShiftedOffset != 0) {
    // We're using a shifted mask, so the load now has an offset. This means
    // that data has been loaded into the lower bytes than it would have been
    // before, so we need to shl the loaded data into the correct position in the
    // register.
    SDValue ShiftC = DAG.getConstant(ShiftedOffset, DL, VT);
    Result = DAG.getNode(ISD::SHL, DL, VT, Result, ShiftC);
    DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
  }
 
  // Return the new loaded value.
  return Result;
}
 
SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  EVT ExtVT = cast<VTSDNode>(N1)->getVT();
  unsigned VTBits = VT.getScalarSizeInBits();
  unsigned ExtVTBits = ExtVT.getScalarSizeInBits();
  SDLoc DL(N);
 
  // sext_vector_inreg(undef) = 0 because the top bit will all be the same.
  if (N0.isUndef())
    return DAG.getConstant(0, DL, VT);
 
  // fold (sext_in_reg c1) -> c1
  if (SDValue C =
          DAG.FoldConstantArithmetic(ISD::SIGN_EXTEND_INREG, DL, VT, {N0, N1}))
    return C;
 
  // If the input is already sign extended, just drop the extension.
  if (ExtVTBits >= DAG.ComputeMaxSignificantBits(N0))
    return N0;
 
  // fold (sext_in_reg (sext_in_reg x, VT2), VT1) -> (sext_in_reg x, minVT) pt2
  if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
      ExtVT.bitsLT(cast<VTSDNode>(N0.getOperand(1))->getVT()))
    return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N0.getOperand(0), N1);
 
  // fold (sext_in_reg (sext x)) -> (sext x)
  // fold (sext_in_reg (aext x)) -> (sext x)
  // if x is small enough or if we know that x has more than 1 sign bit and the
  // sign_extend_inreg is extending from one of them.
  if (N0.getOpcode() == ISD::SIGN_EXTEND || N0.getOpcode() == ISD::ANY_EXTEND) {
    SDValue N00 = N0.getOperand(0);
    unsigned N00Bits = N00.getScalarValueSizeInBits();
    if ((N00Bits <= ExtVTBits ||
         DAG.ComputeMaxSignificantBits(N00) <= ExtVTBits) &&
        (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
      return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N00);
  }
 
  // fold (sext_in_reg (*_extend_vector_inreg x)) -> (sext_vector_inreg x)
  // if x is small enough or if we know that x has more than 1 sign bit and the
  // sign_extend_inreg is extending from one of them.
  if (ISD::isExtVecInRegOpcode(N0.getOpcode())) {
    SDValue N00 = N0.getOperand(0);
    unsigned N00Bits = N00.getScalarValueSizeInBits();
    bool IsZext = N0.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG;
    if ((N00Bits == ExtVTBits ||
         (!IsZext && (N00Bits < ExtVTBits ||
                      DAG.ComputeMaxSignificantBits(N00) <= ExtVTBits))) &&
        (!LegalOperations ||
         TLI.isOperationLegal(ISD::SIGN_EXTEND_VECTOR_INREG, VT)))
      return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, N00);
  }
 
  // fold (sext_in_reg (zext x)) -> (sext x)
  // iff we are extending the source sign bit.
  if (N0.getOpcode() == ISD::ZERO_EXTEND) {
    SDValue N00 = N0.getOperand(0);
    if (N00.getScalarValueSizeInBits() == ExtVTBits &&
        (!LegalOperations || TLI.isOperationLegal(ISD::SIGN_EXTEND, VT)))
      return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N00);
  }
 
  // fold (sext_in_reg x) -> (zext_in_reg x) if the sign bit is known zero.
  if (DAG.MaskedValueIsZero(N0, APInt::getOneBitSet(VTBits, ExtVTBits - 1)))
    return DAG.getZeroExtendInReg(N0, DL, ExtVT);
 
  // fold operands of sext_in_reg based on knowledge that the top bits are not
  // demanded.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  // fold (sext_in_reg (load x)) -> (smaller sextload x)
  // fold (sext_in_reg (srl (load x), c)) -> (smaller sextload (x+c/evtbits))
  if (SDValue NarrowLoad = reduceLoadWidth(N))
    return NarrowLoad;
 
  // fold (sext_in_reg (srl X, 24), i8) -> (sra X, 24)
  // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible.
  // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above.
  if (N0.getOpcode() == ISD::SRL) {
    if (auto *ShAmt = dyn_cast<ConstantSDNode>(N0.getOperand(1)))
      if (ShAmt->getAPIntValue().ule(VTBits - ExtVTBits)) {
        // We can turn this into an SRA iff the input to the SRL is already sign
        // extended enough.
        unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0));
        if (((VTBits - ExtVTBits) - ShAmt->getZExtValue()) < InSignBits)
          return DAG.getNode(ISD::SRA, DL, VT, N0.getOperand(0),
                             N0.getOperand(1));
      }
  }
 
  // fold (sext_inreg (extload x)) -> (sextload x)
  // If sextload is not supported by target, we can only do the combine when
  // load has one use. Doing otherwise can block folding the extload with other
  // extends that the target does support.
  if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
    auto *LN0 = cast<LoadSDNode>(N0);
    if (ExtVT == LN0->getMemoryVT() &&
        ((!LegalOperations && LN0->isSimple() && N0.hasOneUse()) ||
         TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(), LN0->getAddressSpace(),
                         ISD::SEXTLOAD, false))) {
      SDValue ExtLoad =
          DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
                         LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
      CombineTo(N, ExtLoad);
      CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
      AddToWorklist(ExtLoad.getNode());
      return SDValue(N, 0); // Return N so it doesn't get rechecked!
    }
  }
 
  // fold (sext_inreg (zextload x)) -> (sextload x) iff load has one use
  if (ISD::isZEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) {
    auto *LN0 = cast<LoadSDNode>(N0);
 
    if (N0.hasOneUse() && ExtVT == LN0->getMemoryVT() &&
        ((!LegalOperations && LN0->isSimple()) &&
         TLI.isLoadLegal(VT, ExtVT, LN0->getAlign(), LN0->getAddressSpace(),
                         ISD::SEXTLOAD, false))) {
      SDValue ExtLoad =
          DAG.getExtLoad(ISD::SEXTLOAD, DL, VT, LN0->getChain(),
                         LN0->getBasePtr(), ExtVT, LN0->getMemOperand());
      CombineTo(N, ExtLoad);
      CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
      return SDValue(N, 0); // Return N so it doesn't get rechecked!
    }
  }
 
  // fold (sext_inreg (masked_load x)) -> (sext_masked_load x)
  // ignore it if the masked load is already sign extended
  bool Frozen = N0.getOpcode() == ISD::FREEZE && N0.hasOneUse();
  if (auto *Ld = dyn_cast<MaskedLoadSDNode>(Frozen ? N0.getOperand(0) : N0)) {
    if (ExtVT == Ld->getMemoryVT() && Ld->hasNUsesOfValue(1, 0) &&
        Ld->getExtensionType() != ISD::LoadExtType::NON_EXTLOAD &&
        TLI.isLoadLegal(VT, ExtVT, Ld->getAlign(), Ld->getAddressSpace(),
                        ISD::SEXTLOAD, false)) {
      SDValue ExtMaskedLoad = DAG.getMaskedLoad(
          VT, DL, Ld->getChain(), Ld->getBasePtr(), Ld->getOffset(),
          Ld->getMask(), Ld->getPassThru(), ExtVT, Ld->getMemOperand(),
          Ld->getAddressingMode(), ISD::SEXTLOAD, Ld->isExpandingLoad());
      CombineTo(N, Frozen ? N0 : ExtMaskedLoad);
      CombineTo(Ld, ExtMaskedLoad, ExtMaskedLoad.getValue(1));
      return SDValue(N, 0); // Return N so it doesn't get rechecked!
    }
  }
 
  // fold (sext_inreg (masked_gather x)) -> (sext_masked_gather x)
  if (auto *GN0 = dyn_cast<MaskedGatherSDNode>(N0)) {
    if (SDValue(GN0, 0).hasOneUse() && ExtVT == GN0->getMemoryVT() &&
        TLI.isVectorLoadExtDesirable(SDValue(N, 0))) {
      SDValue Ops[] = {GN0->getChain(),   GN0->getPassThru(), GN0->getMask(),
                       GN0->getBasePtr(), GN0->getIndex(),    GN0->getScale()};
 
      SDValue ExtLoad = DAG.getMaskedGather(
          DAG.getVTList(VT, MVT::Other), ExtVT, DL, Ops, GN0->getMemOperand(),
          GN0->getIndexType(), ISD::SEXTLOAD);
 
      CombineTo(N, ExtLoad);
      CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1));
      AddToWorklist(ExtLoad.getNode());
      return SDValue(N, 0); // Return N so it doesn't get rechecked!
    }
  }
 
  // Form (sext_inreg (bswap >> 16)) or (sext_inreg (rotl (bswap) 16))
  if (ExtVTBits <= 16 && N0.getOpcode() == ISD::OR) {
    if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0),
                                           N0.getOperand(1), false))
      return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, BSwap, N1);
  }
 
  // Fold (iM_signext_inreg
  //        (extract_subvector (zext|anyext|sext iN_v to _) _)
  //        from iN)
  //      -> (extract_subvector (signext iN_v to iM))
  if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR && N0.hasOneUse() &&
      ISD::isExtOpcode(N0.getOperand(0).getOpcode())) {
    SDValue InnerExt = N0.getOperand(0);
    EVT InnerExtVT = InnerExt->getValueType(0);
    SDValue Extendee = InnerExt->getOperand(0);
 
    if (ExtVTBits == Extendee.getValueType().getScalarSizeInBits() &&
        (!LegalOperations ||
         TLI.isOperationLegal(ISD::SIGN_EXTEND, InnerExtVT))) {
      SDValue SignExtExtendee =
          DAG.getNode(ISD::SIGN_EXTEND, DL, InnerExtVT, Extendee);
      return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SignExtExtendee,
                         N0.getOperand(1));
    }
  }
 
  return SDValue();
}
 
static SDValue foldExtendVectorInregToExtendOfSubvector(
    SDNode *N, const SDLoc &DL, const TargetLowering &TLI, SelectionDAG &DAG,
    bool LegalOperations) {
  unsigned InregOpcode = N->getOpcode();
  unsigned Opcode = DAG.getOpcode_EXTEND(InregOpcode);
 
  SDValue Src = N->getOperand(0);
  EVT VT = N->getValueType(0);
  EVT SrcVT = VT.changeVectorElementType(
      *DAG.getContext(), Src.getValueType().getVectorElementType());
 
  assert(ISD::isExtVecInRegOpcode(InregOpcode) &&
         "Expected EXTEND_VECTOR_INREG dag node in input!");
 
  // Profitability check: our operand must be an one-use CONCAT_VECTORS.
  // FIXME: one-use check may be overly restrictive
  if (!Src.hasOneUse() || Src.getOpcode() != ISD::CONCAT_VECTORS)
    return SDValue();
 
  // Profitability check: we must be extending exactly one of it's operands.
  // FIXME: this is probably overly restrictive.
  Src = Src.getOperand(0);
  if (Src.getValueType() != SrcVT)
    return SDValue();
 
  if (LegalOperations && !TLI.isOperationLegal(Opcode, VT))
    return SDValue();
 
  return DAG.getNode(Opcode, DL, VT, Src);
}
 
SDValue DAGCombiner::visitEXTEND_VECTOR_INREG(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  if (N0.isUndef()) {
    // aext_vector_inreg(undef) = undef because the top bits are undefined.
    // {s/z}ext_vector_inreg(undef) = 0 because the top bits must be the same.
    return N->getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG
               ? DAG.getUNDEF(VT)
               : DAG.getConstant(0, DL, VT);
  }
 
  if (SDValue Res = tryToFoldExtendOfConstant(N, DL, TLI, DAG, LegalTypes))
    return Res;
 
  if (SimplifyDemandedVectorElts(SDValue(N, 0)))
    return SDValue(N, 0);
 
  if (SDValue R = foldExtendVectorInregToExtendOfSubvector(N, DL, TLI, DAG,
                                                           LegalOperations))
    return R;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitTRUNCATE_USAT_U(SDNode *N) {
  EVT VT = N->getValueType(0);
  SDValue N0 = N->getOperand(0);
 
  SDValue FPVal;
  if (sd_match(N0, m_FPToUI(m_Value(FPVal))) &&
      DAG.getTargetLoweringInfo().shouldConvertFpToSat(
          ISD::FP_TO_UINT_SAT, FPVal.getValueType(), VT))
    return DAG.getNode(ISD::FP_TO_UINT_SAT, SDLoc(N0), VT, FPVal,
                       DAG.getValueType(VT.getScalarType()));
 
  return SDValue();
}
 
/// Detect patterns of truncation with unsigned saturation:
///
/// (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type).
/// Return the source value x to be truncated or SDValue() if the pattern was
/// not matched.
///
static SDValue detectUSatUPattern(SDValue In, EVT VT) {
  unsigned NumDstBits = VT.getScalarSizeInBits();
  unsigned NumSrcBits = In.getScalarValueSizeInBits();
  // Saturation with truncation. We truncate from InVT to VT.
  assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
 
  SDValue Min;
  APInt UnsignedMax = APInt::getMaxValue(NumDstBits).zext(NumSrcBits);
  if (sd_match(In, m_UMin(m_Value(Min), m_SpecificInt(UnsignedMax))))
    return Min;
 
  return SDValue();
}
 
/// Detect patterns of truncation with signed saturation:
/// (truncate (smin (smax (x, signed_min_of_dest_type),
///                  signed_max_of_dest_type)) to dest_type)
/// or:
/// (truncate (smax (smin (x, signed_max_of_dest_type),
///                  signed_min_of_dest_type)) to dest_type).
///
/// Return the source value to be truncated or SDValue() if the pattern was not
/// matched.
static SDValue detectSSatSPattern(SDValue In, EVT VT) {
  unsigned NumDstBits = VT.getScalarSizeInBits();
  unsigned NumSrcBits = In.getScalarValueSizeInBits();
  // Saturation with truncation. We truncate from InVT to VT.
  assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
 
  SDValue Val;
  APInt SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
  APInt SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
 
  if (sd_match(In, m_SMin(m_SMax(m_Value(Val), m_SpecificInt(SignedMin)),
                          m_SpecificInt(SignedMax))))
    return Val;
 
  if (sd_match(In, m_SMax(m_SMin(m_Value(Val), m_SpecificInt(SignedMax)),
                          m_SpecificInt(SignedMin))))
    return Val;
 
  return SDValue();
}
 
/// Detect patterns of truncation with unsigned saturation:
static SDValue detectSSatUPattern(SDValue In, EVT VT, SelectionDAG &DAG,
                                  const SDLoc &DL) {
  unsigned NumDstBits = VT.getScalarSizeInBits();
  unsigned NumSrcBits = In.getScalarValueSizeInBits();
  // Saturation with truncation. We truncate from InVT to VT.
  assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation");
 
  SDValue Val;
  APInt UnsignedMax = APInt::getMaxValue(NumDstBits).zext(NumSrcBits);
  // Min == 0, Max is unsigned max of destination type.
  if (sd_match(In, m_SMax(m_SMin(m_Value(Val), m_SpecificInt(UnsignedMax)),
                          m_Zero())))
    return Val;
 
  if (sd_match(In, m_SMin(m_SMax(m_Value(Val), m_Zero()),
                          m_SpecificInt(UnsignedMax))))
    return Val;
 
  if (sd_match(In, m_UMin(m_SMax(m_Value(Val), m_Zero()),
                          m_SpecificInt(UnsignedMax))))
    return Val;
 
  return SDValue();
}
 
static SDValue foldToSaturated(SDNode *N, EVT &VT, SDValue &Src, EVT &SrcVT,
                               SDLoc &DL, const TargetLowering &TLI,
                               SelectionDAG &DAG) {
  auto AllowedTruncateSat = [&](unsigned Opc, EVT SrcVT, EVT VT) -> bool {
    return (TLI.isOperationLegalOrCustom(Opc, SrcVT) &&
            TLI.isTypeDesirableForOp(Opc, VT));
  };
 
  if (Src.getOpcode() == ISD::SMIN || Src.getOpcode() == ISD::SMAX) {
    if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_S, SrcVT, VT))
      if (SDValue SSatVal = detectSSatSPattern(Src, VT))
        return DAG.getNode(ISD::TRUNCATE_SSAT_S, DL, VT, SSatVal);
    if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_U, SrcVT, VT))
      if (SDValue SSatVal = detectSSatUPattern(Src, VT, DAG, DL))
        return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, VT, SSatVal);
  } else if (Src.getOpcode() == ISD::UMIN) {
    if (AllowedTruncateSat(ISD::TRUNCATE_SSAT_U, SrcVT, VT))
      if (SDValue SSatVal = detectSSatUPattern(Src, VT, DAG, DL))
        return DAG.getNode(ISD::TRUNCATE_SSAT_U, DL, VT, SSatVal);
    if (AllowedTruncateSat(ISD::TRUNCATE_USAT_U, SrcVT, VT))
      if (SDValue USatVal = detectUSatUPattern(Src, VT))
        return DAG.getNode(ISD::TRUNCATE_USAT_U, DL, VT, USatVal);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
  EVT SrcVT = N0.getValueType();
  bool isLE = DAG.getDataLayout().isLittleEndian();
  SDLoc DL(N);
 
  // trunc(undef) = undef
  if (N0.isUndef())
    return DAG.getUNDEF(VT);
 
  // fold (truncate (truncate x)) -> (truncate x)
  if (N0.getOpcode() == ISD::TRUNCATE)
    return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
 
  // fold saturated truncate
  if (SDValue SaturatedTR = foldToSaturated(N, VT, N0, SrcVT, DL, TLI, DAG))
    return SaturatedTR;
 
  // fold (truncate c1) -> c1
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::TRUNCATE, DL, VT, {N0}))
    return C;
 
  // fold (truncate (ext x)) -> (ext x) or (truncate x) or x
  if (N0.getOpcode() == ISD::ZERO_EXTEND ||
      N0.getOpcode() == ISD::SIGN_EXTEND ||
      N0.getOpcode() == ISD::ANY_EXTEND) {
    // if the source is smaller than the dest, we still need an extend.
    if (N0.getOperand(0).getValueType().bitsLT(VT)) {
      SDNodeFlags Flags;
      if (N0.getOpcode() == ISD::ZERO_EXTEND)
        Flags.setNonNeg(N0->getFlags().hasNonNeg());
      return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Flags);
    }
    // if the source is larger than the dest, than we just need the truncate.
    if (N0.getOperand(0).getValueType().bitsGT(VT))
      return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
    // if the source and dest are the same type, we can drop both the extend
    // and the truncate.
    return N0.getOperand(0);
  }
 
  // Try to narrow a truncate-of-sext_in_reg to the destination type:
  // trunc (sign_ext_inreg X, iM) to iN --> sign_ext_inreg (trunc X to iN), iM
  if (!LegalTypes && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
      N0.hasOneUse()) {
    SDValue X = N0.getOperand(0);
    SDValue ExtVal = N0.getOperand(1);
    EVT ExtVT = cast<VTSDNode>(ExtVal)->getVT();
    if (ExtVT.bitsLT(VT) && TLI.preferSextInRegOfTruncate(VT, SrcVT, ExtVT)) {
      SDValue TrX = DAG.getNode(ISD::TRUNCATE, DL, VT, X);
      return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, TrX, ExtVal);
    }
  }
 
  // If this is anyext(trunc), don't fold it, allow ourselves to be folded.
  if (N->hasOneUse() && (N->user_begin()->getOpcode() == ISD::ANY_EXTEND))
    return SDValue();
 
  // Fold extract-and-trunc into a narrow extract. For example:
  //   i64 x = EXTRACT_VECTOR_ELT(v2i64 val, i32 1)
  //   i32 y = TRUNCATE(i64 x)
  //        -- becomes --
  //   v16i8 b = BITCAST (v2i64 val)
  //   i8 x = EXTRACT_VECTOR_ELT(v16i8 b, i32 8)
  //
  // Note: We only run this optimization after type legalization (which often
  // creates this pattern) and before operation legalization after which
  // we need to be more careful about the vector instructions that we generate.
  if (LegalTypes && !LegalOperations && VT.isScalarInteger() && VT != MVT::i1 &&
      N0->hasOneUse()) {
    EVT TrTy = N->getValueType(0);
    SDValue Src = N0;
 
    // Check for cases where we shift down an upper element before truncation.
    int EltOffset = 0;
    if (Src.getOpcode() == ISD::SRL && Src.getOperand(0)->hasOneUse()) {
      if (auto ShAmt = DAG.getValidShiftAmount(Src)) {
        if ((*ShAmt % TrTy.getSizeInBits()) == 0) {
          Src = Src.getOperand(0);
          EltOffset = *ShAmt / TrTy.getSizeInBits();
        }
      }
    }
 
    if (Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
      EVT VecTy = Src.getOperand(0).getValueType();
      EVT ExTy = Src.getValueType();
 
      auto EltCnt = VecTy.getVectorElementCount();
      unsigned SizeRatio = ExTy.getSizeInBits() / TrTy.getSizeInBits();
      auto NewEltCnt = EltCnt * SizeRatio;
 
      EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, NewEltCnt);
      assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
 
      SDValue EltNo = Src->getOperand(1);
      if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
        int Elt = EltNo->getAsZExtVal();
        int Index = isLE ? (Elt * SizeRatio + EltOffset)
                         : (Elt * SizeRatio + (SizeRatio - 1) - EltOffset);
        return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TrTy,
                           DAG.getBitcast(NVT, Src.getOperand(0)),
                           DAG.getVectorIdxConstant(Index, DL));
      }
    }
  }
 
  // trunc (select c, a, b) -> select c, (trunc a), (trunc b)
  if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse() &&
      TLI.isTruncateFree(SrcVT, VT)) {
    if (!LegalOperations ||
        (TLI.isOperationLegal(ISD::SELECT, SrcVT) &&
         TLI.isNarrowingProfitable(N0.getNode(), SrcVT, VT))) {
      SDLoc SL(N0);
      SDValue Cond = N0.getOperand(0);
      SDValue TruncOp0 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1));
      SDValue TruncOp1 = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(2));
      return DAG.getNode(ISD::SELECT, DL, VT, Cond, TruncOp0, TruncOp1);
    }
  }
 
  // trunc (shl x, K) -> shl (trunc x), K => K < VT.getScalarSizeInBits()
  if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() &&
      (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) &&
      TLI.isTypeDesirableForOp(ISD::SHL, VT)) {
    SDValue Amt = N0.getOperand(1);
    KnownBits Known = DAG.computeKnownBits(Amt);
    unsigned Size = VT.getScalarSizeInBits();
    if (Known.countMaxActiveBits() <= Log2_32(Size)) {
      EVT AmtVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
      SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
      if (AmtVT != Amt.getValueType()) {
        Amt = DAG.getZExtOrTrunc(Amt, DL, AmtVT);
        AddToWorklist(Amt.getNode());
      }
      return DAG.getNode(ISD::SHL, DL, VT, Trunc, Amt);
    }
  }
 
  if (SDValue V = foldSubToUSubSat(VT, N0.getNode(), DL))
    return V;
 
  if (SDValue ABD = foldABSToABD(N, DL))
    return ABD;
 
  // Attempt to pre-truncate BUILD_VECTOR sources.
  if (N0.getOpcode() == ISD::BUILD_VECTOR && !LegalOperations &&
      N0.hasOneUse() &&
      // Avoid creating illegal types if running after type legalizer.
      (!LegalTypes || TLI.isTypeLegal(VT.getScalarType()))) {
    if (TLI.isTruncateFree(SrcVT.getScalarType(), VT.getScalarType()))
      return DAG.UnrollVectorOp(N);
 
    // trunc(build_vector(ext(x), ext(x)) -> build_vector(x,x)
    if (SDValue SplatVal = DAG.getSplatValue(N0)) {
      if (ISD::isExtOpcode(SplatVal.getOpcode()) &&
          SrcVT.getScalarType() == SplatVal.getValueType())
        return DAG.UnrollVectorOp(N);
    }
  }
 
  // trunc (splat_vector x) -> splat_vector (trunc x)
  if (N0.getOpcode() == ISD::SPLAT_VECTOR &&
      (!LegalTypes || TLI.isTypeLegal(VT.getScalarType())) &&
      (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, VT))) {
    EVT SVT = VT.getScalarType();
    return DAG.getSplatVector(
        VT, DL, DAG.getNode(ISD::TRUNCATE, DL, SVT, N0->getOperand(0)));
  }
 
  // Fold a series of buildvector, bitcast, and truncate if possible.
  // For example fold
  //   (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to
  //   (2xi32 (buildvector x, y)).
  if (Level == AfterLegalizeVectorOps && VT.isVector() &&
      N0.getOpcode() == ISD::BITCAST && N0.hasOneUse() &&
      N0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR &&
      N0.getOperand(0).hasOneUse()) {
    SDValue BuildVect = N0.getOperand(0);
    EVT BuildVectEltTy = BuildVect.getValueType().getVectorElementType();
    EVT TruncVecEltTy = VT.getVectorElementType();
 
    // Check that the element types match.
    if (BuildVectEltTy == TruncVecEltTy) {
      // Now we only need to compute the offset of the truncated elements.
      unsigned BuildVecNumElts =  BuildVect.getNumOperands();
      unsigned TruncVecNumElts = VT.getVectorNumElements();
      unsigned TruncEltOffset = BuildVecNumElts / TruncVecNumElts;
      unsigned FirstElt = isLE ? 0 : (TruncEltOffset - 1);
 
      assert((BuildVecNumElts % TruncVecNumElts) == 0 &&
             "Invalid number of elements");
 
      SmallVector<SDValue, 8> Opnds;
      for (unsigned i = FirstElt, e = BuildVecNumElts; i < e;
           i += TruncEltOffset)
        Opnds.push_back(BuildVect.getOperand(i));
 
      return DAG.getBuildVector(VT, DL, Opnds);
    }
  }
 
  // fold (truncate (load x)) -> (smaller load x)
  // fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
  if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
    if (SDValue Reduced = reduceLoadWidth(N))
      return Reduced;
 
    // Handle the case where the truncated result is at least as wide as the
    // loaded type.
    if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) {
      auto *LN0 = cast<LoadSDNode>(N0);
      if (LN0->isSimple() && LN0->getMemoryVT().bitsLE(VT)) {
        SDValue NewLoad = DAG.getExtLoad(
            LN0->getExtensionType(), SDLoc(LN0), VT, LN0->getChain(),
            LN0->getBasePtr(), LN0->getMemoryVT(), LN0->getMemOperand());
        DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1));
        return NewLoad;
      }
    }
  }
 
  // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)),
  // where ... are all 'undef'.
  if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) {
    SmallVector<EVT, 8> VTs;
    SDValue V;
    unsigned Idx = 0;
    unsigned NumDefs = 0;
 
    for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
      SDValue X = N0.getOperand(i);
      if (!X.isUndef()) {
        V = X;
        Idx = i;
        NumDefs++;
      }
      // Stop if more than one members are non-undef.
      if (NumDefs > 1)
        break;
 
      VTs.push_back(EVT::getVectorVT(*DAG.getContext(),
                                     VT.getVectorElementType(),
                                     X.getValueType().getVectorElementCount()));
    }
 
    if (NumDefs == 0)
      return DAG.getUNDEF(VT);
 
    if (NumDefs == 1) {
      assert(V.getNode() && "The single defined operand is empty!");
      SmallVector<SDValue, 8> Opnds;
      for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
        if (i != Idx) {
          Opnds.push_back(DAG.getUNDEF(VTs[i]));
          continue;
        }
        SDValue NV = DAG.getNode(ISD::TRUNCATE, SDLoc(V), VTs[i], V);
        AddToWorklist(NV.getNode());
        Opnds.push_back(NV);
      }
      return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Opnds);
    }
  }
 
  // Fold truncate of a bitcast of a vector to an extract of the low vector
  // element.
  //
  // e.g. trunc (i64 (bitcast v2i32:x)) -> extract_vector_elt v2i32:x, idx
  if (N0.getOpcode() == ISD::BITCAST && !VT.isVector()) {
    SDValue VecSrc = N0.getOperand(0);
    EVT VecSrcVT = VecSrc.getValueType();
    if (VecSrcVT.isVectorOf(VT) &&
        (!LegalOperations ||
         TLI.isOperationLegal(ISD::EXTRACT_VECTOR_ELT, VecSrcVT))) {
      unsigned Idx = isLE ? 0 : VecSrcVT.getVectorNumElements() - 1;
      return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VecSrc,
                         DAG.getVectorIdxConstant(Idx, DL));
    }
  }
 
  // Simplify the operands using demanded-bits information.
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  // fold (truncate (extract_subvector(ext x))) ->
  //      (extract_subvector x)
  // TODO: This can be generalized to cover cases where the truncate and extract
  // do not fully cancel each other out.
  if (!LegalTypes && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
    SDValue N00 = N0.getOperand(0);
    if (N00.getOpcode() == ISD::SIGN_EXTEND ||
        N00.getOpcode() == ISD::ZERO_EXTEND ||
        N00.getOpcode() == ISD::ANY_EXTEND) {
      if (N00.getOperand(0)->getValueType(0).getVectorElementType() ==
          VT.getVectorElementType())
        return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N0->getOperand(0)), VT,
                           N00.getOperand(0), N0.getOperand(1));
    }
  }
 
  if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N))
    return NewVSel;
 
  // Narrow a suitable binary operation with a non-opaque constant operand by
  // moving it ahead of the truncate. This is limited to pre-legalization
  // because targets may prefer a wider type during later combines and invert
  // this transform.
  switch (N0.getOpcode()) {
  case ISD::ADD:
  case ISD::SUB:
  case ISD::MUL:
  case ISD::AND:
  case ISD::OR:
  case ISD::XOR:
    if (!LegalOperations && N0.hasOneUse() &&
        (N0.getOperand(0) == N0.getOperand(1) ||
         isConstantOrConstantVector(N0.getOperand(0), true) ||
         isConstantOrConstantVector(N0.getOperand(1), true))) {
      // TODO: We already restricted this to pre-legalization, but for vectors
      // we are extra cautious to not create an unsupported operation.
      // Target-specific changes are likely needed to avoid regressions here.
      if (VT.isScalarInteger() || TLI.isOperationLegal(N0.getOpcode(), VT)) {
        SDValue NarrowL = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
        SDValue NarrowR = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1));
        SDNodeFlags Flags;
        // Propagate nuw for sub.
        if (N0->getOpcode() == ISD::SUB && N0->getFlags().hasNoUnsignedWrap() &&
            DAG.MaskedValueIsZero(
                N0->getOperand(0),
                APInt::getBitsSetFrom(SrcVT.getScalarSizeInBits(),
                                      VT.getScalarSizeInBits())))
          Flags.setNoUnsignedWrap(true);
        return DAG.getNode(N0.getOpcode(), DL, VT, NarrowL, NarrowR, Flags);
      }
    }
    break;
  case ISD::ADDE:
  case ISD::UADDO_CARRY:
    // (trunc adde(X, Y, Carry)) -> (adde trunc(X), trunc(Y), Carry)
    // (trunc uaddo_carry(X, Y, Carry)) ->
    //     (uaddo_carry trunc(X), trunc(Y), Carry)
    // When the adde's carry is not used.
    // We only do for uaddo_carry before legalize operation
    if (((!LegalOperations && N0.getOpcode() == ISD::UADDO_CARRY) ||
         TLI.isOperationLegal(N0.getOpcode(), VT)) &&
        N0.hasOneUse() && !N0->hasAnyUseOfValue(1)) {
      SDValue X = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
      SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(1));
      SDVTList VTs = DAG.getVTList(VT, N0->getValueType(1));
      return DAG.getNode(N0.getOpcode(), DL, VTs, X, Y, N0.getOperand(2));
    }
    break;
  case ISD::USUBSAT:
    // Truncate the USUBSAT only if LHS is a known zero-extension, its not
    // enough to know that the upper bits are zero we must ensure that we don't
    // introduce an extra truncate.
    if (!LegalOperations && N0.hasOneUse() &&
        N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND &&
        N0.getOperand(0).getOperand(0).getScalarValueSizeInBits() <=
            VT.getScalarSizeInBits() &&
        hasOperation(N0.getOpcode(), VT)) {
      return getTruncatedUSUBSAT(VT, SrcVT, N0.getOperand(0), N0.getOperand(1),
                                 DAG, DL);
    }
    break;
  case ISD::AVGCEILS:
  case ISD::AVGCEILU:
    // trunc (avgceilu (sext (x), sext (y))) -> avgceils(x, y)
    // trunc (avgceils (zext (x), zext (y))) -> avgceilu(x, y)
    if (N0.hasOneUse()) {
      SDValue Op0 = N0.getOperand(0);
      SDValue Op1 = N0.getOperand(1);
      if (N0.getOpcode() == ISD::AVGCEILU) {
        if (TLI.isOperationLegalOrCustom(ISD::AVGCEILS, VT) &&
            Op0.getOpcode() == ISD::SIGN_EXTEND &&
            Op1.getOpcode() == ISD::SIGN_EXTEND &&
            Op0.getOperand(0).getValueType() == VT &&
            Op1.getOperand(0).getValueType() == VT)
          return DAG.getNode(ISD::AVGCEILS, DL, VT, Op0.getOperand(0),
                             Op1.getOperand(0));
      } else {
        if (TLI.isOperationLegalOrCustom(ISD::AVGCEILU, VT) &&
            Op0.getOpcode() == ISD::ZERO_EXTEND &&
            Op1.getOpcode() == ISD::ZERO_EXTEND &&
            Op0.getOperand(0).getValueType() == VT &&
            Op1.getOperand(0).getValueType() == VT)
          return DAG.getNode(ISD::AVGCEILU, DL, VT, Op0.getOperand(0),
                             Op1.getOperand(0));
      }
    }
    [[fallthrough]];
  case ISD::AVGFLOORS:
  case ISD::AVGFLOORU:
  case ISD::ABDS:
  case ISD::ABDU:
    // (trunc (avg a, b)) -> (avg (trunc a), (trunc b))
    // (trunc (abdu/abds a, b)) -> (abdu/abds (trunc a), (trunc b))
    if (!LegalOperations && N0.hasOneUse() &&
        TLI.isOperationLegal(N0.getOpcode(), VT)) {
      EVT TruncVT = VT;
      unsigned SrcBits = SrcVT.getScalarSizeInBits();
      unsigned TruncBits = TruncVT.getScalarSizeInBits();
 
      SDValue A = N0.getOperand(0);
      SDValue B = N0.getOperand(1);
      bool CanFold = false;
 
      if (N0.getOpcode() == ISD::AVGFLOORU || N0.getOpcode() == ISD::AVGCEILU ||
          N0.getOpcode() == ISD::ABDU) {
        APInt UpperBits = APInt::getBitsSetFrom(SrcBits, TruncBits);
        CanFold = DAG.MaskedValueIsZero(B, UpperBits) &&
                  DAG.MaskedValueIsZero(A, UpperBits);
      } else {
        unsigned NeededBits = SrcBits - TruncBits;
        CanFold = DAG.ComputeNumSignBits(B) > NeededBits &&
                  DAG.ComputeNumSignBits(A) > NeededBits;
      }
 
      if (CanFold) {
        SDValue NewA = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, A);
        SDValue NewB = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, B);
        return DAG.getNode(N0.getOpcode(), DL, TruncVT, NewA, NewB);
      }
    }
    break;
  }
 
  return SDValue();
}
 
static SDNode *getBuildPairElt(SDNode *N, unsigned i) {
  SDValue Elt = N->getOperand(i);
  if (Elt.getOpcode() != ISD::MERGE_VALUES)
    return Elt.getNode();
  return Elt.getOperand(Elt.getResNo()).getNode();
}
 
/// build_pair (load, load) -> load
/// if load locations are consecutive.
SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
  assert(N->getOpcode() == ISD::BUILD_PAIR);
 
  auto *LD1 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 0));
  auto *LD2 = dyn_cast<LoadSDNode>(getBuildPairElt(N, 1));
 
  // A BUILD_PAIR is always having the least significant part in elt 0 and the
  // most significant part in elt 1. So when combining into one large load, we
  // need to consider the endianness.
  if (DAG.getDataLayout().isBigEndian())
    std::swap(LD1, LD2);
 
  if (!LD1 || !LD2 || !ISD::isNON_EXTLoad(LD1) || !ISD::isNON_EXTLoad(LD2) ||
      !LD1->hasOneUse() || !LD2->hasOneUse() ||
      LD1->getAddressSpace() != LD2->getAddressSpace())
    return SDValue();
 
  unsigned LD1Fast = 0;
  EVT LD1VT = LD1->getValueType(0);
  unsigned LD1Bytes = LD1VT.getStoreSize();
  if ((!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT)) &&
      DAG.areNonVolatileConsecutiveLoads(LD2, LD1, LD1Bytes, 1) &&
      TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
                             *LD1->getMemOperand(), &LD1Fast) && LD1Fast)
    return DAG.getLoad(VT, SDLoc(N), LD1->getChain(), LD1->getBasePtr(),
                       LD1->getPointerInfo(), LD1->getAlign());
 
  return SDValue();
}
 
static unsigned getPPCf128HiElementSelector(const SelectionDAG &DAG) {
  // On little-endian machines, bitcasting from ppcf128 to i128 does swap the Hi
  // and Lo parts; on big-endian machines it doesn't.
  return DAG.getDataLayout().isBigEndian() ? 1 : 0;
}
 
SDValue DAGCombiner::foldBitcastedFPLogic(SDNode *N, SelectionDAG &DAG,
                                          const TargetLowering &TLI) {
  // If this is not a bitcast to an FP type or if the target doesn't have
  // IEEE754-compliant FP logic, we're done.
  EVT VT = N->getValueType(0);
  SDValue N0 = N->getOperand(0);
  EVT SourceVT = N0.getValueType();
 
  if (!VT.isFloatingPoint())
    return SDValue();
 
  // TODO: Handle cases where the integer constant is a different scalar
  // bitwidth to the FP.
  if (VT.getScalarSizeInBits() != SourceVT.getScalarSizeInBits())
    return SDValue();
 
  unsigned FPOpcode;
  APInt SignMask;
  switch (N0.getOpcode()) {
  case ISD::AND:
    FPOpcode = ISD::FABS;
    SignMask = ~APInt::getSignMask(SourceVT.getScalarSizeInBits());
    break;
  case ISD::XOR:
    FPOpcode = ISD::FNEG;
    SignMask = APInt::getSignMask(SourceVT.getScalarSizeInBits());
    break;
  case ISD::OR:
    FPOpcode = ISD::FABS;
    SignMask = APInt::getSignMask(SourceVT.getScalarSizeInBits());
    break;
  default:
    return SDValue();
  }
 
  if (LegalOperations && !TLI.isOperationLegal(FPOpcode, VT))
    return SDValue();
 
  // This needs to be the inverse of logic in foldSignChangeInBitcast.
  // FIXME: I don't think looking for bitcast intrinsically makes sense, but
  // removing this would require more changes.
  auto IsBitCastOrFree = [&TLI, FPOpcode](SDValue Op, EVT VT) {
    if (sd_match(Op, m_BitCast(m_SpecificVT(VT))))
      return true;
 
    return FPOpcode == ISD::FABS ? TLI.isFAbsFree(VT) : TLI.isFNegFree(VT);
  };
 
  // Fold (bitcast int (and (bitcast fp X to int), 0x7fff...) to fp) -> fabs X
  // Fold (bitcast int (xor (bitcast fp X to int), 0x8000...) to fp) -> fneg X
  // Fold (bitcast int (or (bitcast fp X to int), 0x8000...) to fp) ->
  //   fneg (fabs X)
  SDValue LogicOp0 = N0.getOperand(0);
  ConstantSDNode *LogicOp1 = isConstOrConstSplat(N0.getOperand(1), true);
  if (LogicOp1 && LogicOp1->getAPIntValue() == SignMask &&
      IsBitCastOrFree(LogicOp0, VT)) {
    SDValue CastOp0 = DAG.getNode(ISD::BITCAST, SDLoc(N), VT, LogicOp0);
    SDValue FPOp = DAG.getNode(FPOpcode, SDLoc(N), VT, CastOp0);
    NumFPLogicOpsConv++;
    if (N0.getOpcode() == ISD::OR)
      return DAG.getNode(ISD::FNEG, SDLoc(N), VT, FPOp);
    return FPOp;
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitBITCAST(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  EVT VT = N->getValueType(0);
 
  if (N0.isUndef())
    return DAG.getUNDEF(VT);
 
  // If the input is a BUILD_VECTOR with all constant elements, fold this now.
  // Only do this before legalize types, unless both types are integer and the
  // scalar type is legal. Only do this before legalize ops, since the target
  // maybe depending on the bitcast.
  // First check to see if this is all constant.
  // TODO: Support FP bitcasts after legalize types.
  if (VT.isVector() &&
      (!LegalTypes ||
       (!LegalOperations && VT.isInteger() && N0.getValueType().isInteger() &&
        TLI.isTypeLegal(VT.getVectorElementType()))) &&
      N0.getOpcode() == ISD::BUILD_VECTOR && N0->hasOneUse() &&
      cast<BuildVectorSDNode>(N0)->isConstant())
    return DAG.FoldConstantBuildVector(cast<BuildVectorSDNode>(N0), SDLoc(N),
                                       VT.getVectorElementType());
 
  // If the input is a constant, let getNode fold it.
  if (isIntOrFPConstant(N0)) {
    // If we can't allow illegal operations, we need to check that this is just
    // a fp -> int or int -> conversion and that the resulting operation will
    // be legal.
    if (!LegalOperations ||
        (isa<ConstantSDNode>(N0) && VT.isFloatingPoint() && !VT.isVector() &&
         TLI.isOperationLegal(ISD::ConstantFP, VT)) ||
        (isa<ConstantFPSDNode>(N0) && VT.isInteger() && !VT.isVector() &&
         TLI.isOperationLegal(ISD::Constant, VT))) {
      SDValue C = DAG.getBitcast(VT, N0);
      if (C.getNode() != N)
        return C;
    }
  }
 
  // (conv (conv x, t1), t2) -> (conv x, t2)
  if (N0.getOpcode() == ISD::BITCAST)
    return DAG.getBitcast(VT, N0.getOperand(0));
 
  // fold (conv (logicop (conv x), (c))) -> (logicop x, (conv c))
  // iff the current bitwise logicop type isn't legal
  if (ISD::isBitwiseLogicOp(N0.getOpcode()) && VT.isInteger() &&
      !TLI.isTypeLegal(N0.getOperand(0).getValueType())) {
    auto IsFreeBitcast = [VT](SDValue V) {
      return (V.getOpcode() == ISD::BITCAST &&
              V.getOperand(0).getValueType() == VT) ||
             (ISD::isBuildVectorOfConstantSDNodes(V.getNode()) &&
              V->hasOneUse());
    };
    if (IsFreeBitcast(N0.getOperand(0)) && IsFreeBitcast(N0.getOperand(1)))
      return DAG.getNode(N0.getOpcode(), SDLoc(N), VT,
                         DAG.getBitcast(VT, N0.getOperand(0)),
                         DAG.getBitcast(VT, N0.getOperand(1)));
  }
 
  // fold (conv (load x)) -> (load (conv*)x)
  // fold (conv (freeze (load x))) -> (freeze (load (conv*)x))
  // If the resultant load doesn't need a higher alignment than the original!
  auto CastLoad = [this, &VT](SDValue N0, const SDLoc &DL) {
    // Peek through scalar_to_vector if the scalar is same size as VT - often a
    // leftover from legalization.
    if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && N0.hasOneUse() &&
        N0.getOperand(0).getValueSizeInBits() == VT.getSizeInBits())
      N0 = N0.getOperand(0);
    if (N0.getOpcode() == ISD::AssertNoFPClass)
      N0 = N0.getOperand(0);
    if (!ISD::isNormalLoad(N0.getNode()) || !N0.hasOneUse())
      return SDValue();
 
    // Do not remove the cast if the types differ in endian layout.
    if (TLI.hasBigEndianPartOrdering(N0.getValueType(), DAG.getDataLayout()) !=
        TLI.hasBigEndianPartOrdering(VT, DAG.getDataLayout()))
      return SDValue();
 
    // If the load is volatile, we only want to change the load type if the
    // resulting load is legal. Otherwise we might increase the number of
    // memory accesses. We don't care if the original type was legal or not
    // as we assume software couldn't rely on the number of accesses of an
    // illegal type.
    auto *LN0 = cast<LoadSDNode>(N0);
    if ((LegalOperations || !LN0->isSimple()) &&
        !TLI.isOperationLegal(ISD::LOAD, VT))
      return SDValue();
 
    if (!TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG,
                                     *LN0->getMemOperand()))
      return SDValue();
 
    // If the range metadata type does not match the new memory
    // operation type, remove the range metadata.
    if (const MDNode *MD = LN0->getRanges()) {
      ConstantInt *Lower = mdconst::extract<ConstantInt>(MD->getOperand(0));
      if (Lower->getBitWidth() != VT.getScalarSizeInBits() || !VT.isInteger()) {
        LN0->getMemOperand()->clearRanges();
      }
    }
    SDValue Load = DAG.getLoad(VT, DL, LN0->getChain(), LN0->getBasePtr(),
                               LN0->getMemOperand());
    DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
    return Load;
  };
 
  if (SDValue NewLd = CastLoad(N0, SDLoc(N)))
    return NewLd;
 
  if (N0.getOpcode() == ISD::FREEZE && N0.hasOneUse())
    if (SDValue NewLd = CastLoad(N0.getOperand(0), SDLoc(N)))
      return DAG.getFreeze(NewLd);
 
  if (SDValue V = foldBitcastedFPLogic(N, DAG, TLI))
    return V;
 
  // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
  // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
  //
  // For ppc_fp128:
  // fold (bitcast (fneg x)) ->
  //     flipbit = signbit
  //     (xor (bitcast x) (build_pair flipbit, flipbit))
  //
  // fold (bitcast (fabs x)) ->
  //     flipbit = (and (extract_element (bitcast x), 0), signbit)
  //     (xor (bitcast x) (build_pair flipbit, flipbit))
  // This often reduces constant pool loads.
  if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(N0.getValueType())) ||
       (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(N0.getValueType()))) &&
      N0->hasOneUse() && VT.isInteger() && !VT.isVector() &&
      !N0.getValueType().isVector()) {
    SDValue NewConv = DAG.getBitcast(VT, N0.getOperand(0));
    AddToWorklist(NewConv.getNode());
 
    SDLoc DL(N);
    if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) {
      assert(VT.getSizeInBits() == 128);
      SDValue SignBit = DAG.getConstant(
          APInt::getSignMask(VT.getSizeInBits() / 2), SDLoc(N0), MVT::i64);
      SDValue FlipBit;
      if (N0.getOpcode() == ISD::FNEG) {
        FlipBit = SignBit;
        AddToWorklist(FlipBit.getNode());
      } else {
        assert(N0.getOpcode() == ISD::FABS);
        SDValue Hi =
            DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(NewConv), MVT::i64, NewConv,
                        DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
                                              SDLoc(NewConv)));
        AddToWorklist(Hi.getNode());
        FlipBit = DAG.getNode(ISD::AND, SDLoc(N0), MVT::i64, Hi, SignBit);
        AddToWorklist(FlipBit.getNode());
      }
      SDValue FlipBits =
          DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
      AddToWorklist(FlipBits.getNode());
      return DAG.getNode(ISD::XOR, DL, VT, NewConv, FlipBits);
    }
    APInt SignBit = APInt::getSignMask(VT.getSizeInBits());
    if (N0.getOpcode() == ISD::FNEG)
      return DAG.getNode(ISD::XOR, DL, VT,
                         NewConv, DAG.getConstant(SignBit, DL, VT));
    assert(N0.getOpcode() == ISD::FABS);
    return DAG.getNode(ISD::AND, DL, VT,
                       NewConv, DAG.getConstant(~SignBit, DL, VT));
  }
 
  // fold (bitconvert (fcopysign cst, x)) ->
  //         (or (and (bitconvert x), sign), (and cst, (not sign)))
  // Note that we don't handle (copysign x, cst) because this can always be
  // folded to an fneg or fabs.
  //
  // For ppc_fp128:
  // fold (bitcast (fcopysign cst, x)) ->
  //     flipbit = (and (extract_element
  //                     (xor (bitcast cst), (bitcast x)), 0),
  //                    signbit)
  //     (xor (bitcast cst) (build_pair flipbit, flipbit))
  if (N0.getOpcode() == ISD::FCOPYSIGN && N0->hasOneUse() &&
      isa<ConstantFPSDNode>(N0.getOperand(0)) && VT.isInteger() &&
      !VT.isVector()) {
    unsigned OrigXWidth = N0.getOperand(1).getValueSizeInBits();
    EVT IntXVT = EVT::getIntegerVT(*DAG.getContext(), OrigXWidth);
    if (isTypeLegal(IntXVT)) {
      SDValue X = DAG.getBitcast(IntXVT, N0.getOperand(1));
      AddToWorklist(X.getNode());
 
      // If X has a different width than the result/lhs, sext it or truncate it.
      unsigned VTWidth = VT.getSizeInBits();
      if (OrigXWidth < VTWidth) {
        X = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(N), VT, X);
        AddToWorklist(X.getNode());
      } else if (OrigXWidth > VTWidth) {
        // To get the sign bit in the right place, we have to shift it right
        // before truncating.
        SDLoc DL(X);
        X = DAG.getNode(ISD::SRL, DL,
                        X.getValueType(), X,
                        DAG.getConstant(OrigXWidth-VTWidth, DL,
                                        X.getValueType()));
        AddToWorklist(X.getNode());
        X = DAG.getNode(ISD::TRUNCATE, SDLoc(X), VT, X);
        AddToWorklist(X.getNode());
      }
 
      if (N0.getValueType() == MVT::ppcf128 && !LegalTypes) {
        APInt SignBit = APInt::getSignMask(VT.getSizeInBits() / 2);
        SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
        AddToWorklist(Cst.getNode());
        SDValue X = DAG.getBitcast(VT, N0.getOperand(1));
        AddToWorklist(X.getNode());
        SDValue XorResult = DAG.getNode(ISD::XOR, SDLoc(N0), VT, Cst, X);
        AddToWorklist(XorResult.getNode());
        SDValue XorResult64 = DAG.getNode(
            ISD::EXTRACT_ELEMENT, SDLoc(XorResult), MVT::i64, XorResult,
            DAG.getIntPtrConstant(getPPCf128HiElementSelector(DAG),
                                  SDLoc(XorResult)));
        AddToWorklist(XorResult64.getNode());
        SDValue FlipBit =
            DAG.getNode(ISD::AND, SDLoc(XorResult64), MVT::i64, XorResult64,
                        DAG.getConstant(SignBit, SDLoc(XorResult64), MVT::i64));
        AddToWorklist(FlipBit.getNode());
        SDValue FlipBits =
            DAG.getNode(ISD::BUILD_PAIR, SDLoc(N0), VT, FlipBit, FlipBit);
        AddToWorklist(FlipBits.getNode());
        return DAG.getNode(ISD::XOR, SDLoc(N), VT, Cst, FlipBits);
      }
      APInt SignBit = APInt::getSignMask(VT.getSizeInBits());
      X = DAG.getNode(ISD::AND, SDLoc(X), VT,
                      X, DAG.getConstant(SignBit, SDLoc(X), VT));
      AddToWorklist(X.getNode());
 
      SDValue Cst = DAG.getBitcast(VT, N0.getOperand(0));
      Cst = DAG.getNode(ISD::AND, SDLoc(Cst), VT,
                        Cst, DAG.getConstant(~SignBit, SDLoc(Cst), VT));
      AddToWorklist(Cst.getNode());
 
      return DAG.getNode(ISD::OR, SDLoc(N), VT, X, Cst);
    }
  }
 
  // bitconvert(build_pair(ld, ld)) -> ld iff load locations are consecutive.
  if (N0.getOpcode() == ISD::BUILD_PAIR)
    if (SDValue CombineLD = CombineConsecutiveLoads(N0.getNode(), VT))
      return CombineLD;
 
  // int_vt (bitcast (vec_vt (scalar_to_vector elt_vt:x)))
  //   => int_vt (any_extend elt_vt:x)
  if (N0.getOpcode() == ISD::SCALAR_TO_VECTOR && VT.isScalarInteger()) {
    SDValue SrcScalar = N0.getOperand(0);
    if (SrcScalar.getValueType().isScalarInteger())
      return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, SrcScalar);
  }
 
  // Remove double bitcasts from shuffles - this is often a legacy of
  // XformToShuffleWithZero being used to combine bitmaskings (of
  // float vectors bitcast to integer vectors) into shuffles.
  // bitcast(shuffle(bitcast(s0),bitcast(s1))) -> shuffle(s0,s1)
  if (Level < AfterLegalizeDAG && TLI.isTypeLegal(VT) && VT.isVector() &&
      N0->getOpcode() == ISD::VECTOR_SHUFFLE && N0.hasOneUse() &&
      VT.getVectorNumElements() >= N0.getValueType().getVectorNumElements() &&
      !(VT.getVectorNumElements() % N0.getValueType().getVectorNumElements())) {
    ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N0);
 
    // If operands are a bitcast, peek through if it casts the original VT.
    // If operands are a constant, just bitcast back to original VT.
    auto PeekThroughBitcast = [&](SDValue Op) {
      if (Op.getOpcode() == ISD::BITCAST &&
          Op.getOperand(0).getValueType() == VT)
        return SDValue(Op.getOperand(0));
      if (Op.isUndef() || isAnyConstantBuildVector(Op))
        return DAG.getBitcast(VT, Op);
      return SDValue();
    };
 
    // FIXME: If either input vector is bitcast, try to convert the shuffle to
    // the result type of this bitcast. This would eliminate at least one
    // bitcast. See the transform in InstCombine.
    SDValue SV0 = PeekThroughBitcast(N0->getOperand(0));
    SDValue SV1 = PeekThroughBitcast(N0->getOperand(1));
    if (!(SV0 && SV1))
      return SDValue();
 
    int MaskScale =
        VT.getVectorNumElements() / N0.getValueType().getVectorNumElements();
    SmallVector<int, 8> NewMask;
    for (int M : SVN->getMask())
      for (int i = 0; i != MaskScale; ++i)
        NewMask.push_back(M < 0 ? -1 : M * MaskScale + i);
 
    SDValue LegalShuffle =
        TLI.buildLegalVectorShuffle(VT, SDLoc(N), SV0, SV1, NewMask, DAG);
    if (LegalShuffle)
      return LegalShuffle;
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitBUILD_PAIR(SDNode *N) {
  EVT VT = N->getValueType(0);
  return CombineConsecutiveLoads(N, VT);
}
 
SDValue DAGCombiner::visitFREEZE(SDNode *N) {
  SDValue N0 = N->getOperand(0);
 
  if (DAG.isGuaranteedNotToBeUndefOrPoison(N0, UndefPoisonKind::UndefOrPoison))
    return N0;
 
  // If we have frozen and unfrozen users of N0, update so everything uses N.
  if (!N0.isUndef() && !N0.hasOneUse()) {
    SDValue FrozenN0(N, 0);
    // Unfreeze all (possibly nested) uses of N to avoid double deleting N from
    // the CSE map.
    while (!N->use_empty())
      DAG.ReplaceAllUsesOfValueWith(FrozenN0, N0);
    DAG.ReplaceAllUsesOfValueWith(N0, FrozenN0);
    // ReplaceAllUsesOfValueWith will have also updated the use in N, thus
    // creating a cycle in a DAG. Let's undo that by mutating the freeze.
    assert(N->getOperand(0) == FrozenN0 && "Expected cycle in DAG");
    DAG.UpdateNodeOperands(N, N0);
    // Revisit the node.
    AddToWorklist(N);
    return FrozenN0;
  }
 
  // We currently avoid folding freeze over SRA/SRL, due to the problems seen
  // with (freeze (assert ext)) blocking simplifications of SRA/SRL. See for
  // example https://reviews.llvm.org/D136529#4120959.
  if (N0.getOpcode() == ISD::SRA || N0.getOpcode() == ISD::SRL)
    return SDValue();
 
  // Fold freeze(op(x, ...)) -> op(freeze(x), ...).
  // Try to push freeze through instructions that propagate but don't produce
  // poison as far as possible. If an operand of freeze follows three
  // conditions 1) one-use, 2) does not produce poison, and 3) has all but one
  // guaranteed-non-poison operands (or is a BUILD_VECTOR or similar) then push
  // the freeze through to the operands that are not guaranteed non-poison.
  // NOTE: we will strip poison-generating flags, so ignore them here.
  if (DAG.canCreateUndefOrPoison(N0, UndefPoisonKind::UndefOrPoison,
                                 /*ConsiderFlags*/ false) ||
      N0->getNumValues() != 1 || !N0->hasOneUse())
    return SDValue();
 
  // TOOD: we should always allow multiple operands, however this increases the
  // likelihood of infinite loops due to the ReplaceAllUsesOfValueWith call
  // below causing later nodes that share frozen operands to fold again and no
  // longer being able to confirm other operands are not poison due to recursion
  // depth limits on isGuaranteedNotToBeUndefOrPoison.
  bool AllowMultipleMaybePoisonOperands =
      N0.getOpcode() == ISD::SELECT_CC || N0.getOpcode() == ISD::SETCC ||
      N0.getOpcode() == ISD::BUILD_VECTOR ||
      N0.getOpcode() == ISD::INSERT_SUBVECTOR ||
      N0.getOpcode() == ISD::BUILD_PAIR ||
      N0.getOpcode() == ISD::VECTOR_SHUFFLE ||
      N0.getOpcode() == ISD::CONCAT_VECTORS || N0.getOpcode() == ISD::FMUL;
 
  // Avoid turning a BUILD_VECTOR that can be recognized as "all zeros", "all
  // ones" or "constant" into something that depends on FrozenUndef. We can
  // instead pick undef values to keep those properties, while at the same time
  // folding away the freeze.
  // If we implement a more general solution for folding away freeze(undef) in
  // the future, then this special handling can be removed.
  if (N0.getOpcode() == ISD::BUILD_VECTOR) {
    SDLoc DL(N0);
    EVT VT = N0.getValueType();
    if (llvm::ISD::isBuildVectorAllOnes(N0.getNode()) && VT.isInteger())
      return DAG.getAllOnesConstant(DL, VT);
    if (llvm::ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
      SmallVector<SDValue, 8> NewVecC;
      for (const SDValue &Op : N0->op_values())
        NewVecC.push_back(
            Op.isUndef() ? DAG.getConstant(0, DL, Op.getValueType()) : Op);
      return DAG.getBuildVector(VT, DL, NewVecC);
    }
  }
 
  SmallSet<SDValue, 8> MaybePoisonOperands;
  SmallVector<unsigned, 8> MaybePoisonOperandNumbers;
  for (auto [OpNo, Op] : enumerate(N0->ops())) {
    if (DAG.isGuaranteedNotToBeUndefOrPoison(Op,
                                             UndefPoisonKind::UndefOrPoison))
      continue;
    bool HadMaybePoisonOperands = !MaybePoisonOperands.empty();
    bool IsNewMaybePoisonOperand = MaybePoisonOperands.insert(Op).second;
    if (IsNewMaybePoisonOperand)
      MaybePoisonOperandNumbers.push_back(OpNo);
    if (!HadMaybePoisonOperands)
      continue;
    if (IsNewMaybePoisonOperand && !AllowMultipleMaybePoisonOperands) {
      // Multiple maybe-poison ops when not allowed - bail out.
      return SDValue();
    }
  }
  // NOTE: the whole op may be not guaranteed to not be undef or poison because
  // it could create undef or poison due to it's poison-generating flags.
  // So not finding any maybe-poison operands is fine.
 
  for (unsigned OpNo : MaybePoisonOperandNumbers) {
    // N0 can mutate during iteration, so make sure to refetch the maybe poison
    // operands via the operand numbers. The typical scenario is that we have
    // something like this
    //   t262: i32 = freeze t181
    //   t150: i32 = ctlz_zero_poison t262
    //   t184: i32 = ctlz_zero_poison t181
    //   t268: i32 = select_cc t181, Constant:i32<0>, t184, t186, setne:ch
    // When freezing the t181 operand we get t262 back, and then the
    // ReplaceAllUsesOfValueWith call will not only replace t181 by t262, but
    // also recursively replace t184 by t150.
    SDValue MaybePoisonOperand = N->getOperand(0).getOperand(OpNo);
    // Don't replace every single UNDEF everywhere with frozen UNDEF, though.
    if (MaybePoisonOperand.isUndef())
      continue;
    // First, freeze each offending operand.
    SDValue FrozenMaybePoisonOperand = DAG.getFreeze(MaybePoisonOperand);
    // Then, change all other uses of unfrozen operand to use frozen operand.
    DAG.ReplaceAllUsesOfValueWith(MaybePoisonOperand, FrozenMaybePoisonOperand);
    if (FrozenMaybePoisonOperand.getOpcode() == ISD::FREEZE &&
        FrozenMaybePoisonOperand.getOperand(0) == FrozenMaybePoisonOperand) {
      // But, that also updated the use in the freeze we just created, thus
      // creating a cycle in a DAG. Let's undo that by mutating the freeze.
      DAG.UpdateNodeOperands(FrozenMaybePoisonOperand.getNode(),
                             MaybePoisonOperand);
    }
 
    // This node has been merged with another.
    if (N->getOpcode() == ISD::DELETED_NODE)
      return SDValue(N, 0);
  }
 
  assert(N->getOpcode() != ISD::DELETED_NODE && "Node was deleted!");
 
  // The whole node may have been updated, so the value we were holding
  // may no longer be valid. Re-fetch the operand we're `freeze`ing.
  N0 = N->getOperand(0);
 
  // Finally, recreate the node, it's operands were updated to use
  // frozen operands, so we just need to use it's "original" operands.
  SmallVector<SDValue> Ops(N0->ops());
  // TODO: ISD::UNDEF and ISD::POISON should get separate handling, but best
  // leave for a future patch.
  for (SDValue &Op : Ops) {
    if (Op.isUndef())
      Op = DAG.getFreeze(Op);
  }
 
  SDLoc DL(N0);
 
  // Special case handling for ShuffleVectorSDNode nodes.
  if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(N0))
    return DAG.getVectorShuffle(N0.getValueType(), DL, Ops[0], Ops[1],
                                SVN->getMask());
 
  // NOTE: this strips poison generating flags.
  // Folding freeze(op(x, ...)) -> op(freeze(x), ...) does not require nnan,
  // ninf, nsz, or fast.
  // However, contract, reassoc, afn, and arcp should be preserved,
  // as these fast-math flags do not introduce poison values.
  SDNodeFlags SrcFlags = N0->getFlags();
  SDNodeFlags SafeFlags;
  SafeFlags.setAllowContract(SrcFlags.hasAllowContract());
  SafeFlags.setAllowReassociation(SrcFlags.hasAllowReassociation());
  SafeFlags.setApproximateFuncs(SrcFlags.hasApproximateFuncs());
  SafeFlags.setAllowReciprocal(SrcFlags.hasAllowReciprocal());
  return DAG.getNode(N0.getOpcode(), DL, N0->getVTList(), Ops, SafeFlags);
}
 
// Returns true if floating point contraction is allowed on the FMUL-SDValue
// `N`
static bool isContractableFMUL(const TargetOptions &Options, SDValue N) {
  assert(N.getOpcode() == ISD::FMUL);
 
  return Options.AllowFPOpFusion == FPOpFusion::Fast ||
         N->getFlags().hasAllowContract();
}
 
/// Try to perform FMA combining on a given FADD node.
SDValue DAGCombiner::visitFADDForFMACombine(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc SL(N);
  const TargetOptions &Options = DAG.getTarget().Options;
 
  // Floating-point multiply-add with intermediate rounding.
  bool HasFMAD = (LegalOperations && TLI.isFMADLegal(DAG, N));
 
  // Floating-point multiply-add without intermediate rounding.
  bool HasFMA =
      (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)) &&
      TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT);
 
  // No valid opcode, do not combine.
  if (!HasFMAD && !HasFMA)
    return SDValue();
 
  bool AllowFusionGlobally =
      Options.AllowFPOpFusion == FPOpFusion::Fast || HasFMAD;
  // If the addition is not contractable, do not combine.
  if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
    return SDValue();
 
  // Folding fadd (fmul x, y), (fmul x, y) -> fma x, y, (fmul x, y) is never
  // beneficial. It does not reduce latency. It increases register pressure. It
  // replaces an fadd with an fma which is a more complex instruction, so is
  // likely to have a larger encoding, use more functional units, etc.
  if (N0 == N1)
    return SDValue();
 
  if (TLI.generateFMAsInMachineCombiner(VT, OptLevel))
    return SDValue();
 
  // Always prefer FMAD to FMA for precision.
  unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
  bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
 
  auto isFusedOp = [&](SDValue N) {
    unsigned Opcode = N.getOpcode();
    return Opcode == ISD::FMA || Opcode == ISD::FMAD;
  };
 
  // Is the node an FMUL and contractable either due to global flags or
  // SDNodeFlags.
  auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
    if (N.getOpcode() != ISD::FMUL)
      return false;
    return AllowFusionGlobally || N->getFlags().hasAllowContract();
  };
  // If we have two choices trying to fold (fadd (fmul u, v), (fmul x, y)),
  // prefer to fold the multiply with fewer uses.
  if (Aggressive && isContractableFMUL(N0) && isContractableFMUL(N1)) {
    if (N0->use_size() > N1->use_size())
      std::swap(N0, N1);
  }
 
  // fold (fadd (fmul x, y), z) -> (fma x, y, z)
  if (isContractableFMUL(N0) && (Aggressive || N0->hasOneUse())) {
    return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0),
                       N0.getOperand(1), N1);
  }
 
  // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
  // Note: Commutes FADD operands.
  if (isContractableFMUL(N1) && (Aggressive || N1->hasOneUse())) {
    return DAG.getNode(PreferredFusedOpcode, SL, VT, N1.getOperand(0),
                       N1.getOperand(1), N0);
  }
 
  // fadd (fma A, B, (fmul C, D)), E --> fma A, B, (fma C, D, E)
  // fadd E, (fma A, B, (fmul C, D)) --> fma A, B, (fma C, D, E)
  // This also works with nested fma instructions:
  // fadd (fma A, B, (fma (C, D, (fmul (E, F))))), G -->
  // fma A, B, (fma C, D, fma (E, F, G))
  // fadd (G, (fma A, B, (fma (C, D, (fmul (E, F)))))) -->
  // fma A, B, (fma C, D, fma (E, F, G)).
  // This requires reassociation because it changes the order of operations.
  bool CanReassociate = N->getFlags().hasAllowReassociation();
  if (CanReassociate) {
    SDValue FMA, E;
    if (isFusedOp(N0) && N0.hasOneUse()) {
      FMA = N0;
      E = N1;
    } else if (isFusedOp(N1) && N1.hasOneUse()) {
      FMA = N1;
      E = N0;
    }
 
    SDValue TmpFMA = FMA;
    while (E && isFusedOp(TmpFMA) && TmpFMA.hasOneUse()) {
      SDValue FMul = TmpFMA->getOperand(2);
      if (FMul.getOpcode() == ISD::FMUL && FMul.hasOneUse()) {
        SDValue C = FMul.getOperand(0);
        SDValue D = FMul.getOperand(1);
        SDValue CDE = DAG.getNode(PreferredFusedOpcode, SL, VT, C, D, E);
        DAG.ReplaceAllUsesOfValueWith(FMul, CDE);
        // Replacing the inner FMul could cause the outer FMA to be simplified
        // away.
        return FMA.getOpcode() == ISD::DELETED_NODE ? SDValue(N, 0) : FMA;
      }
 
      TmpFMA = TmpFMA->getOperand(2);
    }
  }
 
  // Look through FP_EXTEND nodes to do more combining.
 
  // fold (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z)
  if (N0.getOpcode() == ISD::FP_EXTEND) {
    SDValue N00 = N0.getOperand(0);
    if (isContractableFMUL(N00) &&
        TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                            N00.getValueType())) {
      return DAG.getNode(PreferredFusedOpcode, SL, VT,
                         DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
                         DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
                         N1);
    }
  }
 
  // fold (fadd x, (fpext (fmul y, z))) -> (fma (fpext y), (fpext z), x)
  // Note: Commutes FADD operands.
  if (N1.getOpcode() == ISD::FP_EXTEND) {
    SDValue N10 = N1.getOperand(0);
    if (isContractableFMUL(N10) &&
        TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                            N10.getValueType())) {
      return DAG.getNode(PreferredFusedOpcode, SL, VT,
                         DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0)),
                         DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)),
                         N0);
    }
  }
 
  // More folding opportunities when target permits.
  if (Aggressive) {
    // fold (fadd (fma x, y, (fpext (fmul u, v))), z)
    //   -> (fma x, y, (fma (fpext u), (fpext v), z))
    auto FoldFAddFMAFPExtFMul = [&](SDValue X, SDValue Y, SDValue U, SDValue V,
                                    SDValue Z) {
      return DAG.getNode(PreferredFusedOpcode, SL, VT, X, Y,
                         DAG.getNode(PreferredFusedOpcode, SL, VT,
                                     DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
                                     DAG.getNode(ISD::FP_EXTEND, SL, VT, V),
                                     Z));
    };
    if (isFusedOp(N0)) {
      SDValue N02 = N0.getOperand(2);
      if (N02.getOpcode() == ISD::FP_EXTEND) {
        SDValue N020 = N02.getOperand(0);
        if (isContractableFMUL(N020) &&
            TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                                N020.getValueType())) {
          return FoldFAddFMAFPExtFMul(N0.getOperand(0), N0.getOperand(1),
                                      N020.getOperand(0), N020.getOperand(1),
                                      N1);
        }
      }
    }
 
    // fold (fadd (fpext (fma x, y, (fmul u, v))), z)
    //   -> (fma (fpext x), (fpext y), (fma (fpext u), (fpext v), z))
    // FIXME: This turns two single-precision and one double-precision
    // operation into two double-precision operations, which might not be
    // interesting for all targets, especially GPUs.
    auto FoldFAddFPExtFMAFMul = [&](SDValue X, SDValue Y, SDValue U, SDValue V,
                                    SDValue Z) {
      return DAG.getNode(
          PreferredFusedOpcode, SL, VT, DAG.getNode(ISD::FP_EXTEND, SL, VT, X),
          DAG.getNode(ISD::FP_EXTEND, SL, VT, Y),
          DAG.getNode(PreferredFusedOpcode, SL, VT,
                      DAG.getNode(ISD::FP_EXTEND, SL, VT, U),
                      DAG.getNode(ISD::FP_EXTEND, SL, VT, V), Z));
    };
    if (N0.getOpcode() == ISD::FP_EXTEND) {
      SDValue N00 = N0.getOperand(0);
      if (isFusedOp(N00)) {
        SDValue N002 = N00.getOperand(2);
        if (isContractableFMUL(N002) &&
            TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                                N00.getValueType())) {
          return FoldFAddFPExtFMAFMul(N00.getOperand(0), N00.getOperand(1),
                                      N002.getOperand(0), N002.getOperand(1),
                                      N1);
        }
      }
    }
 
    // fold (fadd x, (fma y, z, (fpext (fmul u, v)))
    //   -> (fma y, z, (fma (fpext u), (fpext v), x))
    if (isFusedOp(N1)) {
      SDValue N12 = N1.getOperand(2);
      if (N12.getOpcode() == ISD::FP_EXTEND) {
        SDValue N120 = N12.getOperand(0);
        if (isContractableFMUL(N120) &&
            TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                                N120.getValueType())) {
          return FoldFAddFMAFPExtFMul(N1.getOperand(0), N1.getOperand(1),
                                      N120.getOperand(0), N120.getOperand(1),
                                      N0);
        }
      }
    }
 
    // fold (fadd x, (fpext (fma y, z, (fmul u, v)))
    //   -> (fma (fpext y), (fpext z), (fma (fpext u), (fpext v), x))
    // FIXME: This turns two single-precision and one double-precision
    // operation into two double-precision operations, which might not be
    // interesting for all targets, especially GPUs.
    if (N1.getOpcode() == ISD::FP_EXTEND) {
      SDValue N10 = N1.getOperand(0);
      if (isFusedOp(N10)) {
        SDValue N102 = N10.getOperand(2);
        if (isContractableFMUL(N102) &&
            TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                                N10.getValueType())) {
          return FoldFAddFPExtFMAFMul(N10.getOperand(0), N10.getOperand(1),
                                      N102.getOperand(0), N102.getOperand(1),
                                      N0);
        }
      }
    }
  }
 
  return SDValue();
}
 
/// Try to perform FMA combining on a given FSUB node.
SDValue DAGCombiner::visitFSUBForFMACombine(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc SL(N);
 
  const TargetOptions &Options = DAG.getTarget().Options;
  // Floating-point multiply-add with intermediate rounding.
  bool HasFMAD = (LegalOperations && TLI.isFMADLegal(DAG, N));
 
  // Floating-point multiply-add without intermediate rounding.
  bool HasFMA =
      (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)) &&
      TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT);
 
  // No valid opcode, do not combine.
  if (!HasFMAD && !HasFMA)
    return SDValue();
 
  const SDNodeFlags Flags = N->getFlags();
  bool AllowFusionGlobally =
      (Options.AllowFPOpFusion == FPOpFusion::Fast || HasFMAD);
 
  // If the subtraction is not contractable, do not combine.
  if (!AllowFusionGlobally && !N->getFlags().hasAllowContract())
    return SDValue();
 
  if (TLI.generateFMAsInMachineCombiner(VT, OptLevel))
    return SDValue();
 
  // Always prefer FMAD to FMA for precision.
  unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
  bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
  bool NoSignedZero = Flags.hasNoSignedZeros();
 
  // Is the node an FMUL and contractable either due to global flags or
  // SDNodeFlags.
  auto isContractableFMUL = [AllowFusionGlobally](SDValue N) {
    if (N.getOpcode() != ISD::FMUL)
      return false;
    return AllowFusionGlobally || N->getFlags().hasAllowContract();
  };
 
  // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
  auto tryToFoldXYSubZ = [&](SDValue XY, SDValue Z) {
    if (isContractableFMUL(XY) && (Aggressive || XY->hasOneUse())) {
      return DAG.getNode(PreferredFusedOpcode, SL, VT, XY.getOperand(0),
                         XY.getOperand(1), DAG.getNode(ISD::FNEG, SL, VT, Z));
    }
    return SDValue();
  };
 
  // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
  // Note: Commutes FSUB operands.
  auto tryToFoldXSubYZ = [&](SDValue X, SDValue YZ) {
    if (isContractableFMUL(YZ) && (Aggressive || YZ->hasOneUse())) {
      return DAG.getNode(PreferredFusedOpcode, SL, VT,
                         DAG.getNode(ISD::FNEG, SL, VT, YZ.getOperand(0)),
                         YZ.getOperand(1), X);
    }
    return SDValue();
  };
 
  // If we have two choices trying to fold (fsub (fmul u, v), (fmul x, y)),
  // prefer to fold the multiply with fewer uses.
  if (isContractableFMUL(N0) && isContractableFMUL(N1) &&
      (N0->use_size() > N1->use_size())) {
    // fold (fsub (fmul a, b), (fmul c, d)) -> (fma (fneg c), d, (fmul a, b))
    if (SDValue V = tryToFoldXSubYZ(N0, N1))
      return V;
    // fold (fsub (fmul a, b), (fmul c, d)) -> (fma a, b, (fneg (fmul c, d)))
    if (SDValue V = tryToFoldXYSubZ(N0, N1))
      return V;
  } else {
    // fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
    if (SDValue V = tryToFoldXYSubZ(N0, N1))
      return V;
    // fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
    if (SDValue V = tryToFoldXSubYZ(N0, N1))
      return V;
  }
 
  // fold (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z))
  if (N0.getOpcode() == ISD::FNEG && isContractableFMUL(N0.getOperand(0)) &&
      (Aggressive || (N0->hasOneUse() && N0.getOperand(0).hasOneUse()))) {
    SDValue N00 = N0.getOperand(0).getOperand(0);
    SDValue N01 = N0.getOperand(0).getOperand(1);
    return DAG.getNode(PreferredFusedOpcode, SL, VT,
                       DAG.getNode(ISD::FNEG, SL, VT, N00), N01,
                       DAG.getNode(ISD::FNEG, SL, VT, N1));
  }
 
  // Look through FP_EXTEND nodes to do more combining.
 
  // fold (fsub (fpext (fmul x, y)), z)
  //   -> (fma (fpext x), (fpext y), (fneg z))
  if (N0.getOpcode() == ISD::FP_EXTEND) {
    SDValue N00 = N0.getOperand(0);
    if (isContractableFMUL(N00) &&
        TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                            N00.getValueType())) {
      return DAG.getNode(PreferredFusedOpcode, SL, VT,
                         DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
                         DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
                         DAG.getNode(ISD::FNEG, SL, VT, N1));
    }
  }
 
  // fold (fsub x, (fpext (fmul y, z)))
  //   -> (fma (fneg (fpext y)), (fpext z), x)
  // Note: Commutes FSUB operands.
  if (N1.getOpcode() == ISD::FP_EXTEND) {
    SDValue N10 = N1.getOperand(0);
    if (isContractableFMUL(N10) &&
        TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                            N10.getValueType())) {
      return DAG.getNode(
          PreferredFusedOpcode, SL, VT,
          DAG.getNode(ISD::FNEG, SL, VT,
                      DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(0))),
          DAG.getNode(ISD::FP_EXTEND, SL, VT, N10.getOperand(1)), N0);
    }
  }
 
  // fold (fsub (fpext (fneg (fmul, x, y))), z)
  //   -> (fneg (fma (fpext x), (fpext y), z))
  // Note: This could be removed with appropriate canonicalization of the
  // input expression into (fneg (fadd (fpext (fmul, x, y)), z)). However, the
  // command line flag -fp-contract=fast and fast-math flag contract prevent
  // from implementing the canonicalization in visitFSUB.
  if (N0.getOpcode() == ISD::FP_EXTEND) {
    SDValue N00 = N0.getOperand(0);
    if (N00.getOpcode() == ISD::FNEG) {
      SDValue N000 = N00.getOperand(0);
      if (isContractableFMUL(N000) &&
          TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                              N00.getValueType())) {
        return DAG.getNode(
            ISD::FNEG, SL, VT,
            DAG.getNode(PreferredFusedOpcode, SL, VT,
                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)),
                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)),
                        N1));
      }
    }
  }
 
  // fold (fsub (fneg (fpext (fmul, x, y))), z)
  //   -> (fneg (fma (fpext x)), (fpext y), z)
  // Note: This could be removed with appropriate canonicalization of the
  // input expression into (fneg (fadd (fpext (fmul, x, y)), z). However, the
  // command line flag -fp-contract=fast and fast-math flag contract prevent
  // from implementing the canonicalization in visitFSUB.
  if (N0.getOpcode() == ISD::FNEG) {
    SDValue N00 = N0.getOperand(0);
    if (N00.getOpcode() == ISD::FP_EXTEND) {
      SDValue N000 = N00.getOperand(0);
      if (isContractableFMUL(N000) &&
          TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                              N000.getValueType())) {
        return DAG.getNode(
            ISD::FNEG, SL, VT,
            DAG.getNode(PreferredFusedOpcode, SL, VT,
                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(0)),
                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N000.getOperand(1)),
                        N1));
      }
    }
  }
 
  auto isContractableAndReassociableFMUL = [&isContractableFMUL](SDValue N) {
    return isContractableFMUL(N) && N->getFlags().hasAllowReassociation();
  };
 
  auto isFusedOp = [&](SDValue N) {
    unsigned Opcode = N.getOpcode();
    return Opcode == ISD::FMA || Opcode == ISD::FMAD;
  };
 
  // More folding opportunities when target permits.
  if (Aggressive && N->getFlags().hasAllowReassociation()) {
    bool CanFuse = N->getFlags().hasAllowContract();
    // fold (fsub (fma x, y, (fmul u, v)), z)
    //   -> (fma x, y (fma u, v, (fneg z)))
    if (CanFuse && isFusedOp(N0) &&
        isContractableAndReassociableFMUL(N0.getOperand(2)) &&
        N0->hasOneUse() && N0.getOperand(2)->hasOneUse()) {
      return DAG.getNode(PreferredFusedOpcode, SL, VT, N0.getOperand(0),
                         N0.getOperand(1),
                         DAG.getNode(PreferredFusedOpcode, SL, VT,
                                     N0.getOperand(2).getOperand(0),
                                     N0.getOperand(2).getOperand(1),
                                     DAG.getNode(ISD::FNEG, SL, VT, N1)));
    }
 
    // fold (fsub x, (fma y, z, (fmul u, v)))
    //   -> (fma (fneg y), z, (fma (fneg u), v, x))
    if (CanFuse && isFusedOp(N1) &&
        isContractableAndReassociableFMUL(N1.getOperand(2)) &&
        N1->hasOneUse() && NoSignedZero) {
      SDValue N20 = N1.getOperand(2).getOperand(0);
      SDValue N21 = N1.getOperand(2).getOperand(1);
      return DAG.getNode(
          PreferredFusedOpcode, SL, VT,
          DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1),
          DAG.getNode(PreferredFusedOpcode, SL, VT,
                      DAG.getNode(ISD::FNEG, SL, VT, N20), N21, N0));
    }
 
    // fold (fsub (fma x, y, (fpext (fmul u, v))), z)
    //   -> (fma x, y (fma (fpext u), (fpext v), (fneg z)))
    if (isFusedOp(N0) && N0->hasOneUse()) {
      SDValue N02 = N0.getOperand(2);
      if (N02.getOpcode() == ISD::FP_EXTEND) {
        SDValue N020 = N02.getOperand(0);
        if (isContractableAndReassociableFMUL(N020) &&
            TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                                N020.getValueType())) {
          return DAG.getNode(
              PreferredFusedOpcode, SL, VT, N0.getOperand(0), N0.getOperand(1),
              DAG.getNode(
                  PreferredFusedOpcode, SL, VT,
                  DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(0)),
                  DAG.getNode(ISD::FP_EXTEND, SL, VT, N020.getOperand(1)),
                  DAG.getNode(ISD::FNEG, SL, VT, N1)));
        }
      }
    }
 
    // fold (fsub (fpext (fma x, y, (fmul u, v))), z)
    //   -> (fma (fpext x), (fpext y),
    //           (fma (fpext u), (fpext v), (fneg z)))
    // FIXME: This turns two single-precision and one double-precision
    // operation into two double-precision operations, which might not be
    // interesting for all targets, especially GPUs.
    if (N0.getOpcode() == ISD::FP_EXTEND) {
      SDValue N00 = N0.getOperand(0);
      if (isFusedOp(N00)) {
        SDValue N002 = N00.getOperand(2);
        if (isContractableAndReassociableFMUL(N002) &&
            TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                                N00.getValueType())) {
          return DAG.getNode(
              PreferredFusedOpcode, SL, VT,
              DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(0)),
              DAG.getNode(ISD::FP_EXTEND, SL, VT, N00.getOperand(1)),
              DAG.getNode(
                  PreferredFusedOpcode, SL, VT,
                  DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(0)),
                  DAG.getNode(ISD::FP_EXTEND, SL, VT, N002.getOperand(1)),
                  DAG.getNode(ISD::FNEG, SL, VT, N1)));
        }
      }
    }
 
    // fold (fsub x, (fma y, z, (fpext (fmul u, v))))
    //   -> (fma (fneg y), z, (fma (fneg (fpext u)), (fpext v), x))
    if (isFusedOp(N1) && N1.getOperand(2).getOpcode() == ISD::FP_EXTEND &&
        N1->hasOneUse()) {
      SDValue N120 = N1.getOperand(2).getOperand(0);
      if (isContractableAndReassociableFMUL(N120) &&
          TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                              N120.getValueType())) {
        SDValue N1200 = N120.getOperand(0);
        SDValue N1201 = N120.getOperand(1);
        return DAG.getNode(
            PreferredFusedOpcode, SL, VT,
            DAG.getNode(ISD::FNEG, SL, VT, N1.getOperand(0)), N1.getOperand(1),
            DAG.getNode(PreferredFusedOpcode, SL, VT,
                        DAG.getNode(ISD::FNEG, SL, VT,
                                    DAG.getNode(ISD::FP_EXTEND, SL, VT, N1200)),
                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N1201), N0));
      }
    }
 
    // fold (fsub x, (fpext (fma y, z, (fmul u, v))))
    //   -> (fma (fneg (fpext y)), (fpext z),
    //           (fma (fneg (fpext u)), (fpext v), x))
    // FIXME: This turns two single-precision and one double-precision
    // operation into two double-precision operations, which might not be
    // interesting for all targets, especially GPUs.
    if (N1.getOpcode() == ISD::FP_EXTEND && isFusedOp(N1.getOperand(0))) {
      SDValue CvtSrc = N1.getOperand(0);
      SDValue N100 = CvtSrc.getOperand(0);
      SDValue N101 = CvtSrc.getOperand(1);
      SDValue N102 = CvtSrc.getOperand(2);
      if (isContractableAndReassociableFMUL(N102) &&
          TLI.isFPExtFoldable(DAG, PreferredFusedOpcode, VT,
                              CvtSrc.getValueType())) {
        SDValue N1020 = N102.getOperand(0);
        SDValue N1021 = N102.getOperand(1);
        return DAG.getNode(
            PreferredFusedOpcode, SL, VT,
            DAG.getNode(ISD::FNEG, SL, VT,
                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N100)),
            DAG.getNode(ISD::FP_EXTEND, SL, VT, N101),
            DAG.getNode(PreferredFusedOpcode, SL, VT,
                        DAG.getNode(ISD::FNEG, SL, VT,
                                    DAG.getNode(ISD::FP_EXTEND, SL, VT, N1020)),
                        DAG.getNode(ISD::FP_EXTEND, SL, VT, N1021), N0));
      }
    }
  }
 
  return SDValue();
}
 
/// Try to perform FMA combining on a given FMUL node based on the distributive
/// law x * (y + 1) = x * y + x and variants thereof (commuted versions,
/// subtraction instead of addition).
SDValue DAGCombiner::visitFMULForFMADistributiveCombine(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc SL(N);
 
  assert(N->getOpcode() == ISD::FMUL && "Expected FMUL Operation");
 
  const TargetOptions &Options = DAG.getTarget().Options;
 
  // The transforms below are incorrect when x == 0 and y == inf, because the
  // intermediate multiplication produces a nan.
  SDValue FAdd = N0.getOpcode() == ISD::FADD ? N0 : N1;
  if (!FAdd->getFlags().hasNoInfs())
    return SDValue();
 
  // Floating-point multiply-add without intermediate rounding.
  bool HasFMA =
      isContractableFMUL(Options, SDValue(N, 0)) &&
      (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FMA, VT)) &&
      TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT);
 
  // Floating-point multiply-add with intermediate rounding. This can result
  // in a less precise result due to the changed rounding order.
  bool HasFMAD = LegalOperations && TLI.isFMADLegal(DAG, N);
 
  // No valid opcode, do not combine.
  if (!HasFMAD && !HasFMA)
    return SDValue();
 
  // Always prefer FMAD to FMA for precision.
  unsigned PreferredFusedOpcode = HasFMAD ? ISD::FMAD : ISD::FMA;
  bool Aggressive = TLI.enableAggressiveFMAFusion(VT);
 
  // fold (fmul (fadd x0, +1.0), y) -> (fma x0, y, y)
  // fold (fmul (fadd x0, -1.0), y) -> (fma x0, y, (fneg y))
  auto FuseFADD = [&](SDValue X, SDValue Y) {
    if (X.getOpcode() == ISD::FADD && (Aggressive || X->hasOneUse())) {
      if (auto *C = isConstOrConstSplatFP(X.getOperand(1), true)) {
        if (C->isExactlyValue(+1.0))
          return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
                             Y);
        if (C->isExactlyValue(-1.0))
          return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
                             DAG.getNode(ISD::FNEG, SL, VT, Y));
      }
    }
    return SDValue();
  };
 
  if (SDValue FMA = FuseFADD(N0, N1))
    return FMA;
  if (SDValue FMA = FuseFADD(N1, N0))
    return FMA;
 
  // fold (fmul (fsub +1.0, x1), y) -> (fma (fneg x1), y, y)
  // fold (fmul (fsub -1.0, x1), y) -> (fma (fneg x1), y, (fneg y))
  // fold (fmul (fsub x0, +1.0), y) -> (fma x0, y, (fneg y))
  // fold (fmul (fsub x0, -1.0), y) -> (fma x0, y, y)
  auto FuseFSUB = [&](SDValue X, SDValue Y) {
    if (X.getOpcode() == ISD::FSUB && (Aggressive || X->hasOneUse())) {
      if (auto *C0 = isConstOrConstSplatFP(X.getOperand(0), true)) {
        if (C0->isExactlyValue(+1.0))
          return DAG.getNode(PreferredFusedOpcode, SL, VT,
                             DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
                             Y);
        if (C0->isExactlyValue(-1.0))
          return DAG.getNode(PreferredFusedOpcode, SL, VT,
                             DAG.getNode(ISD::FNEG, SL, VT, X.getOperand(1)), Y,
                             DAG.getNode(ISD::FNEG, SL, VT, Y));
      }
      if (auto *C1 = isConstOrConstSplatFP(X.getOperand(1), true)) {
        if (C1->isExactlyValue(+1.0))
          return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
                             DAG.getNode(ISD::FNEG, SL, VT, Y));
        if (C1->isExactlyValue(-1.0))
          return DAG.getNode(PreferredFusedOpcode, SL, VT, X.getOperand(0), Y,
                             Y);
      }
    }
    return SDValue();
  };
 
  if (SDValue FMA = FuseFSUB(N0, N1))
    return FMA;
  if (SDValue FMA = FuseFSUB(N1, N0))
    return FMA;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitFADD(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  bool N0CFP = DAG.isConstantFPBuildVectorOrConstantFP(N0);
  bool N1CFP = DAG.isConstantFPBuildVectorOrConstantFP(N1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  SDNodeFlags Flags = N->getFlags();
  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
  if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
    return R;
 
  // fold (fadd c1, c2) -> c1 + c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FADD, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS
  if (N0CFP && !N1CFP)
    return DAG.getNode(ISD::FADD, DL, VT, N1, N0);
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  // N0 + -0.0 --> N0 (also allowed with +0.0 and fast-math)
  ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, true);
  if (N1C && N1C->isZero())
    if (N1C->isNegative() || DAG.canIgnoreSignBitOfZero(SDValue(N, 0)))
      return N0;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // fold (fadd A, (fneg B)) -> (fsub A, B)
  if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
    if (SDValue NegN1 = TLI.getCheaperNegatedExpression(
            N1, DAG, LegalOperations, ForCodeSize))
      return DAG.getNode(ISD::FSUB, DL, VT, N0, NegN1);
 
  // fold (fadd (fneg A), B) -> (fsub B, A)
  if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
    if (SDValue NegN0 = TLI.getCheaperNegatedExpression(
            N0, DAG, LegalOperations, ForCodeSize))
      return DAG.getNode(ISD::FSUB, DL, VT, N1, NegN0);
 
  auto isFMulNegTwo = [](SDValue FMul) {
    if (!FMul.hasOneUse() || FMul.getOpcode() != ISD::FMUL)
      return false;
    auto *C = isConstOrConstSplatFP(FMul.getOperand(1), true);
    return C && C->isExactlyValue(-2.0);
  };
 
  // fadd (fmul B, -2.0), A --> fsub A, (fadd B, B)
  if (isFMulNegTwo(N0)) {
    SDValue B = N0.getOperand(0);
    SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B);
    return DAG.getNode(ISD::FSUB, DL, VT, N1, Add);
  }
  // fadd A, (fmul B, -2.0) --> fsub A, (fadd B, B)
  if (isFMulNegTwo(N1)) {
    SDValue B = N1.getOperand(0);
    SDValue Add = DAG.getNode(ISD::FADD, DL, VT, B, B);
    return DAG.getNode(ISD::FSUB, DL, VT, N0, Add);
  }
 
  // No FP constant should be created after legalization as Instruction
  // Selection pass has a hard time dealing with FP constants.
  bool AllowNewConst = (Level < AfterLegalizeDAG);
 
  // If nnan is enabled, fold lots of things.
  if (Flags.hasNoNaNs() && AllowNewConst) {
    // If allowed, fold (fadd (fneg x), x) -> 0.0
    if (N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1)
      return DAG.getConstantFP(0.0, DL, VT);
 
    // If allowed, fold (fadd x, (fneg x)) -> 0.0
    if (N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0)
      return DAG.getConstantFP(0.0, DL, VT);
  }
 
  // If reassoc and nsz, fold lots of things.
  // TODO: break out portions of the transformations below for which Unsafe is
  //       considered and which do not require both nsz and reassoc
  if (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros() &&
      AllowNewConst) {
    // fadd (fadd x, c1), c2 -> fadd x, c1 + c2
    if (N1CFP && N0.getOpcode() == ISD::FADD &&
        DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(1))) {
      SDValue NewC = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1), N1);
      return DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(0), NewC);
    }
 
    // We can fold chains of FADD's of the same value into multiplications.
    // This transform is not safe in general because we are reducing the number
    // of rounding steps.
    if (TLI.isOperationLegalOrCustom(ISD::FMUL, VT) && !N0CFP && !N1CFP) {
      if (N0.getOpcode() == ISD::FMUL) {
        bool CFP00 = DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(0));
        bool CFP01 = DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(1));
 
        // (fadd (fmul x, c), x) -> (fmul x, c+1)
        if (CFP01 && !CFP00 && N0.getOperand(0) == N1) {
          SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
                                       DAG.getConstantFP(1.0, DL, VT));
          return DAG.getNode(ISD::FMUL, DL, VT, N1, NewCFP);
        }
 
        // (fadd (fmul x, c), (fadd x, x)) -> (fmul x, c+2)
        if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD &&
            N1.getOperand(0) == N1.getOperand(1) &&
            N0.getOperand(0) == N1.getOperand(0)) {
          SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N0.getOperand(1),
                                       DAG.getConstantFP(2.0, DL, VT));
          return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), NewCFP);
        }
      }
 
      if (N1.getOpcode() == ISD::FMUL) {
        bool CFP10 = DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(0));
        bool CFP11 = DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(1));
 
        // (fadd x, (fmul x, c)) -> (fmul x, c+1)
        if (CFP11 && !CFP10 && N1.getOperand(0) == N0) {
          SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
                                       DAG.getConstantFP(1.0, DL, VT));
          return DAG.getNode(ISD::FMUL, DL, VT, N0, NewCFP);
        }
 
        // (fadd (fadd x, x), (fmul x, c)) -> (fmul x, c+2)
        if (CFP11 && !CFP10 && N0.getOpcode() == ISD::FADD &&
            N0.getOperand(0) == N0.getOperand(1) &&
            N1.getOperand(0) == N0.getOperand(0)) {
          SDValue NewCFP = DAG.getNode(ISD::FADD, DL, VT, N1.getOperand(1),
                                       DAG.getConstantFP(2.0, DL, VT));
          return DAG.getNode(ISD::FMUL, DL, VT, N1.getOperand(0), NewCFP);
        }
      }
 
      if (N0.getOpcode() == ISD::FADD) {
        bool CFP00 = DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(0));
        // (fadd (fadd x, x), x) -> (fmul x, 3.0)
        if (!CFP00 && N0.getOperand(0) == N0.getOperand(1) &&
            (N0.getOperand(0) == N1)) {
          return DAG.getNode(ISD::FMUL, DL, VT, N1,
                             DAG.getConstantFP(3.0, DL, VT));
        }
      }
 
      if (N1.getOpcode() == ISD::FADD) {
        bool CFP10 = DAG.isConstantFPBuildVectorOrConstantFP(N1.getOperand(0));
        // (fadd x, (fadd x, x)) -> (fmul x, 3.0)
        if (!CFP10 && N1.getOperand(0) == N1.getOperand(1) &&
            N1.getOperand(0) == N0) {
          return DAG.getNode(ISD::FMUL, DL, VT, N0,
                             DAG.getConstantFP(3.0, DL, VT));
        }
      }
 
      // (fadd (fadd x, x), (fadd x, x)) -> (fmul x, 4.0)
      if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD &&
          N0.getOperand(0) == N0.getOperand(1) &&
          N1.getOperand(0) == N1.getOperand(1) &&
          N0.getOperand(0) == N1.getOperand(0)) {
        return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0),
                           DAG.getConstantFP(4.0, DL, VT));
      }
    }
  } // reassoc && nsz && AllowNewConst
 
  if (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros()) {
    // Fold fadd(vecreduce(x), vecreduce(y)) -> vecreduce(fadd(x, y))
    if (SDValue SD = reassociateReduction(ISD::VECREDUCE_FADD, ISD::FADD, DL,
                                          VT, N0, N1, Flags))
      return SD;
  }
 
  // FADD -> FMA combines:
  if (SDValue Fused = visitFADDForFMACombine(N)) {
    if (Fused.getOpcode() != ISD::DELETED_NODE)
      AddToWorklist(Fused.getNode());
    return Fused;
  }
  return SDValue();
}
 
SDValue DAGCombiner::visitSTRICT_FADD(SDNode *N) {
  SDValue Chain = N->getOperand(0);
  SDValue N0 = N->getOperand(1);
  SDValue N1 = N->getOperand(2);
  EVT VT = N->getValueType(0);
  EVT ChainVT = N->getValueType(1);
  SDLoc DL(N);
  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
  // fold (strict_fadd A, (fneg B)) -> (strict_fsub A, B)
  if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT))
    if (SDValue NegN1 = TLI.getCheaperNegatedExpression(
            N1, DAG, LegalOperations, ForCodeSize)) {
      return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT),
                         {Chain, N0, NegN1});
    }
 
  // fold (strict_fadd (fneg A), B) -> (strict_fsub B, A)
  if (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::STRICT_FSUB, VT))
    if (SDValue NegN0 = TLI.getCheaperNegatedExpression(
            N0, DAG, LegalOperations, ForCodeSize)) {
      return DAG.getNode(ISD::STRICT_FSUB, DL, DAG.getVTList(VT, ChainVT),
                         {Chain, N1, NegN0});
    }
  return SDValue();
}
 
SDValue DAGCombiner::visitFSUB(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0, true);
  ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, true);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  const SDNodeFlags Flags = N->getFlags();
  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
  if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
    return R;
 
  // fold (fsub c1, c2) -> c1-c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FSUB, DL, VT, {N0, N1}))
    return C;
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // (fsub A, 0) -> A
  if (N1CFP && N1CFP->isZero()) {
    if (!N1CFP->isNegative() || DAG.canIgnoreSignBitOfZero(SDValue(N, 0))) {
      return N0;
    }
  }
 
  if (N0 == N1) {
    // (fsub x, x) -> 0.0
    if (Flags.hasNoNaNs())
      return DAG.getConstantFP(0.0f, DL, VT);
  }
 
  // (fsub -0.0, N1) -> -N1
  if (N0CFP && N0CFP->isZero()) {
    if (N0CFP->isNegative() || DAG.canIgnoreSignBitOfZero(SDValue(N, 0))) {
      // We cannot replace an FSUB(+-0.0,X) with FNEG(X) when denormals are
      // flushed to zero, unless all users treat denorms as zero (DAZ).
      // FIXME: This transform will change the sign of a NaN and the behavior
      // of a signaling NaN. It is only valid when a NoNaN flag is present.
      DenormalMode DenormMode = DAG.getDenormalMode(VT);
      if (DenormMode == DenormalMode::getIEEE()) {
        if (SDValue NegN1 =
                TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize))
          return NegN1;
        if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
          return DAG.getNode(ISD::FNEG, DL, VT, N1);
      }
    }
  }
 
  if (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros() &&
      N1.getOpcode() == ISD::FADD) {
    // X - (X + Y) -> -Y
    if (N0 == N1->getOperand(0))
      return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1));
    // X - (Y + X) -> -Y
    if (N0 == N1->getOperand(1))
      return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(0));
  }
 
  // fold (fsub A, (fneg B)) -> (fadd A, B)
  if (SDValue NegN1 =
          TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize))
    return DAG.getNode(ISD::FADD, DL, VT, N0, NegN1);
 
  // FSUB -> FMA combines:
  if (SDValue Fused = visitFSUBForFMACombine(N)) {
    AddToWorklist(Fused.getNode());
    return Fused;
  }
 
  return SDValue();
}
 
// Transform IEEE Floats:
//      (fmul C, (uitofp Pow2))
//          -> (bitcast_to_FP (add (bitcast_to_INT C), Log2(Pow2) << mantissa))
//      (fdiv C, (uitofp Pow2))
//          -> (bitcast_to_FP (sub (bitcast_to_INT C), Log2(Pow2) << mantissa))
//
// The rationale is fmul/fdiv by a power of 2 is just change the exponent, so
// there is no need for more than an add/sub.
//
// This is valid under the following circumstances:
// 1) We are dealing with IEEE floats
// 2) C is normal
// 3) The fmul/fdiv add/sub will not go outside of min/max exponent bounds.
// TODO: Much of this could also be used for generating `ldexp` on targets the
// prefer it.
SDValue DAGCombiner::combineFMulOrFDivWithIntPow2(SDNode *N) {
  EVT VT = N->getValueType(0);
  if (!APFloat::isIEEELikeFP(VT.getFltSemantics()))
    return SDValue();
 
  SDValue ConstOp, Pow2Op;
 
  std::optional<int> Mantissa;
  auto GetConstAndPow2Ops = [&](unsigned ConstOpIdx) {
    if (ConstOpIdx == 1 && N->getOpcode() == ISD::FDIV)
      return false;
 
    ConstOp = peekThroughBitcasts(N->getOperand(ConstOpIdx));
    Pow2Op = N->getOperand(1 - ConstOpIdx);
    unsigned Pow2Opc = Pow2Op.getOpcode();
    if (Pow2Opc != ISD::UINT_TO_FP && Pow2Opc != ISD::SINT_TO_FP)
      return false;
 
    Pow2Op = Pow2Op.getOperand(0);
 
    KnownBits Pow2OpKnownBits = DAG.computeKnownBits(Pow2Op);
    if (Pow2Opc == ISD::SINT_TO_FP && !Pow2OpKnownBits.isNonNegative())
      return false;
 
    int MaxExpChange = Pow2OpKnownBits.countMaxActiveBits();
 
    auto IsFPConstValid = [N, MaxExpChange, &Mantissa](ConstantFPSDNode *CFP) {
      if (CFP == nullptr)
        return false;
 
      const APFloat &APF = CFP->getValueAPF();
 
      // Make sure we have normal constant.
      if (!APF.isNormal())
        return false;
 
      // Make sure the floats exponent is within the bounds that this transform
      // produces bitwise equals value.
      int CurExp = ilogb(APF);
      // FMul by pow2 will only increase exponent.
      int MinExp =
          N->getOpcode() == ISD::FMUL ? CurExp : (CurExp - MaxExpChange);
      // FDiv by pow2 will only decrease exponent.
      int MaxExp =
          N->getOpcode() == ISD::FDIV ? CurExp : (CurExp + MaxExpChange);
      if (MinExp <= APFloat::semanticsMinExponent(APF.getSemantics()) ||
          MaxExp >= APFloat::semanticsMaxExponent(APF.getSemantics()))
        return false;
 
      // Finally make sure we actually know the mantissa for the float type.
      int ThisMantissa = APFloat::semanticsPrecision(APF.getSemantics()) - 1;
      if (!Mantissa)
        Mantissa = ThisMantissa;
 
      return *Mantissa == ThisMantissa && ThisMantissa > 0;
    };
 
    // TODO: We may be able to include undefs.
    return ISD::matchUnaryFpPredicate(ConstOp, IsFPConstValid);
  };
 
  if (!GetConstAndPow2Ops(0) && !GetConstAndPow2Ops(1))
    return SDValue();
 
  if (!TLI.optimizeFMulOrFDivAsShiftAddBitcast(N, ConstOp, Pow2Op))
    return SDValue();
 
  // Get log2 after all other checks have taken place. This is because
  // BuildLogBase2 may create a new node.
  SDLoc DL(N);
  // Get Log2 type with same bitwidth as the float type (VT).
  EVT NewIntVT = VT.changeElementType(
      *DAG.getContext(),
      EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits()));
 
  SDValue Log2 = BuildLogBase2(Pow2Op, DL, DAG.isKnownNeverZero(Pow2Op),
                               /*InexpensiveOnly*/ true, NewIntVT);
  if (!Log2)
    return SDValue();
 
  // Perform actual transform.
  SDValue MantissaShiftCnt =
      DAG.getShiftAmountConstant(*Mantissa, NewIntVT, DL);
  // TODO: Sometimes Log2 is of form `(X + C)`. `(X + C) << C1` should fold to
  // `(X << C1) + (C << C1)`, but that isn't always the case because of the
  // cast. We could implement that by handle here to handle the casts.
  SDValue Shift = DAG.getNode(ISD::SHL, DL, NewIntVT, Log2, MantissaShiftCnt);
  SDValue ResAsInt =
      DAG.getNode(N->getOpcode() == ISD::FMUL ? ISD::ADD : ISD::SUB, DL,
                  NewIntVT, DAG.getBitcast(NewIntVT, ConstOp), Shift);
  SDValue ResAsFP = DAG.getBitcast(VT, ResAsInt);
  return ResAsFP;
}
 
SDValue DAGCombiner::visitFMUL(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, true);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  const SDNodeFlags Flags = N->getFlags();
  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
  if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
    return R;
 
  // fold (fmul c1, c2) -> c1*c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMUL, DL, VT, {N0, N1}))
    return C;
 
  // canonicalize constant to RHS
  if (DAG.isConstantFPBuildVectorOrConstantFP(N0) &&
     !DAG.isConstantFPBuildVectorOrConstantFP(N1))
    return DAG.getNode(ISD::FMUL, DL, VT, N1, N0);
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  if (Flags.hasAllowReassociation()) {
    // fmul (fmul X, C1), C2 -> fmul X, C1 * C2
    if (DAG.isConstantFPBuildVectorOrConstantFP(N1) &&
        N0.getOpcode() == ISD::FMUL) {
      SDValue N00 = N0.getOperand(0);
      SDValue N01 = N0.getOperand(1);
      // Avoid an infinite loop by making sure that N00 is not a constant
      // (the inner multiply has not been constant folded yet).
      if (DAG.isConstantFPBuildVectorOrConstantFP(N01) &&
          !DAG.isConstantFPBuildVectorOrConstantFP(N00)) {
        SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, N01, N1);
        return DAG.getNode(ISD::FMUL, DL, VT, N00, MulConsts);
      }
    }
 
    // Match a special-case: we convert X * 2.0 into fadd.
    // fmul (fadd X, X), C -> fmul X, 2.0 * C
    if (N0.getOpcode() == ISD::FADD && N0.hasOneUse() &&
        N0.getOperand(0) == N0.getOperand(1)) {
      const SDValue Two = DAG.getConstantFP(2.0, DL, VT);
      SDValue MulConsts = DAG.getNode(ISD::FMUL, DL, VT, Two, N1);
      return DAG.getNode(ISD::FMUL, DL, VT, N0.getOperand(0), MulConsts);
    }
 
    // Fold fmul(vecreduce(x), vecreduce(y)) -> vecreduce(fmul(x, y))
    if (SDValue SD = reassociateReduction(ISD::VECREDUCE_FMUL, ISD::FMUL, DL,
                                          VT, N0, N1, Flags))
      return SD;
  }
 
  // fold (fmul X, 2.0) -> (fadd X, X)
  if (N1CFP && N1CFP->isExactlyValue(+2.0))
    return DAG.getNode(ISD::FADD, DL, VT, N0, N0);
 
  // fold (fmul X, -1.0) -> (fsub -0.0, X)
  if (N1CFP && N1CFP->isExactlyValue(-1.0)) {
    if (!LegalOperations || TLI.isOperationLegal(ISD::FSUB, VT)) {
      return DAG.getNode(ISD::FSUB, DL, VT,
                         DAG.getConstantFP(-0.0, DL, VT), N0, Flags);
    }
  }
 
  // -N0 * -N1 --> N0 * N1
  TargetLowering::NegatibleCost CostN0 =
      TargetLowering::NegatibleCost::Expensive;
  TargetLowering::NegatibleCost CostN1 =
      TargetLowering::NegatibleCost::Expensive;
  SDValue NegN0 =
      TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize, CostN0);
  if (NegN0) {
    HandleSDNode NegN0Handle(NegN0);
    SDValue NegN1 =
        TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize, CostN1);
    if (NegN1 && (CostN0 == TargetLowering::NegatibleCost::Cheaper ||
                  CostN1 == TargetLowering::NegatibleCost::Cheaper))
      return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1);
  }
 
  // fold (fmul X, (select (fcmp X > 0.0), -1.0, 1.0)) -> (fneg (fabs X))
  // fold (fmul X, (select (fcmp X > 0.0), 1.0, -1.0)) -> (fabs X)
  if (Flags.hasNoNaNs() && Flags.hasNoSignedZeros() &&
      (N0.getOpcode() == ISD::SELECT || N1.getOpcode() == ISD::SELECT) &&
      TLI.isOperationLegal(ISD::FABS, VT)) {
    SDValue Select = N0, X = N1;
    if (Select.getOpcode() != ISD::SELECT)
      std::swap(Select, X);
 
    SDValue Cond = Select.getOperand(0);
    auto TrueOpnd  = dyn_cast<ConstantFPSDNode>(Select.getOperand(1));
    auto FalseOpnd = dyn_cast<ConstantFPSDNode>(Select.getOperand(2));
 
    if (TrueOpnd && FalseOpnd && Cond.getOpcode() == ISD::SETCC &&
        Cond.getOperand(0) == X && isa<ConstantFPSDNode>(Cond.getOperand(1)) &&
        cast<ConstantFPSDNode>(Cond.getOperand(1))->isPosZero()) {
      ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
      switch (CC) {
      default: break;
      case ISD::SETOLT:
      case ISD::SETULT:
      case ISD::SETOLE:
      case ISD::SETULE:
      case ISD::SETLT:
      case ISD::SETLE:
        std::swap(TrueOpnd, FalseOpnd);
        [[fallthrough]];
      case ISD::SETOGT:
      case ISD::SETUGT:
      case ISD::SETOGE:
      case ISD::SETUGE:
      case ISD::SETGT:
      case ISD::SETGE:
        if (TrueOpnd->isExactlyValue(-1.0) && FalseOpnd->isExactlyValue(1.0) &&
            TLI.isOperationLegal(ISD::FNEG, VT))
          return DAG.getNode(ISD::FNEG, DL, VT,
                   DAG.getNode(ISD::FABS, DL, VT, X));
        if (TrueOpnd->isExactlyValue(1.0) && FalseOpnd->isExactlyValue(-1.0))
          return DAG.getNode(ISD::FABS, DL, VT, X);
 
        break;
      }
    }
  }
 
  // FMUL -> FMA combines:
  if (SDValue Fused = visitFMULForFMADistributiveCombine(N)) {
    AddToWorklist(Fused.getNode());
    return Fused;
  }
 
  // Don't do `combineFMulOrFDivWithIntPow2` until after FMUL -> FMA has been
  // able to run.
  if (SDValue R = combineFMulOrFDivWithIntPow2(N))
    return R;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitFMA(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
  ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
  ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  // FMA nodes have flags that propagate to the created nodes.
  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
  // Constant fold FMA.
  if (SDValue C =
          DAG.FoldConstantArithmetic(N->getOpcode(), DL, VT, {N0, N1, N2}))
    return C;
 
  // (-N0 * -N1) + N2 --> (N0 * N1) + N2
  TargetLowering::NegatibleCost CostN0 =
      TargetLowering::NegatibleCost::Expensive;
  TargetLowering::NegatibleCost CostN1 =
      TargetLowering::NegatibleCost::Expensive;
  SDValue NegN0 =
      TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize, CostN0);
  if (NegN0) {
    HandleSDNode NegN0Handle(NegN0);
    SDValue NegN1 =
        TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize, CostN1);
    if (NegN1 && (CostN0 == TargetLowering::NegatibleCost::Cheaper ||
                  CostN1 == TargetLowering::NegatibleCost::Cheaper))
      return DAG.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2);
  }
 
  if (N->getFlags().hasNoNaNs() && N->getFlags().hasNoInfs()) {
    if (N->getFlags().hasNoSignedZeros() || (N2CFP && !N2CFP->isNegZero())) {
      if (N0CFP && N0CFP->isZero())
        return N2;
      if (N1CFP && N1CFP->isZero())
        return N2;
    }
  }
 
  if (N0CFP && N0CFP->isExactlyValue(1.0))
    return DAG.getNode(ISD::FADD, DL, VT, N1, N2);
  if (N1CFP && N1CFP->isExactlyValue(1.0))
    return DAG.getNode(ISD::FADD, DL, VT, N0, N2);
 
  // Canonicalize (fma c, x, y) -> (fma x, c, y)
  if (DAG.isConstantFPBuildVectorOrConstantFP(N0) &&
     !DAG.isConstantFPBuildVectorOrConstantFP(N1))
    return DAG.getNode(ISD::FMA, DL, VT, N1, N0, N2);
 
  bool CanReassociate = N->getFlags().hasAllowReassociation();
  if (CanReassociate) {
    // (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2)
    if (N2.getOpcode() == ISD::FMUL && N0 == N2.getOperand(0) &&
        DAG.isConstantFPBuildVectorOrConstantFP(N1) &&
        DAG.isConstantFPBuildVectorOrConstantFP(N2.getOperand(1))) {
      return DAG.getNode(ISD::FMUL, DL, VT, N0,
                         DAG.getNode(ISD::FADD, DL, VT, N1, N2.getOperand(1)));
    }
 
    // (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y)
    if (N0.getOpcode() == ISD::FMUL &&
        DAG.isConstantFPBuildVectorOrConstantFP(N1) &&
        DAG.isConstantFPBuildVectorOrConstantFP(N0.getOperand(1))) {
      return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0),
                         DAG.getNode(ISD::FMUL, DL, VT, N1, N0.getOperand(1)),
                         N2);
    }
  }
 
  // (fma x, -1, y) -> (fadd (fneg x), y)
  if (N1CFP) {
    if (N1CFP->isExactlyValue(1.0))
      return DAG.getNode(ISD::FADD, DL, VT, N0, N2);
 
    if (N1CFP->isExactlyValue(-1.0) &&
        (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) {
      SDValue RHSNeg = DAG.getNode(ISD::FNEG, DL, VT, N0);
      AddToWorklist(RHSNeg.getNode());
      return DAG.getNode(ISD::FADD, DL, VT, N2, RHSNeg);
    }
 
    // fma (fneg x), K, y -> fma x -K, y
    if (N0.getOpcode() == ISD::FNEG &&
        (TLI.isOperationLegal(ISD::ConstantFP, VT) ||
         (N1.hasOneUse() &&
          !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT, ForCodeSize)))) {
      return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0),
                         DAG.getNode(ISD::FNEG, DL, VT, N1), N2);
    }
  }
 
  if (CanReassociate) {
    // (fma x, c, x) -> (fmul x, (c+1))
    if (N1CFP && N0 == N2) {
      return DAG.getNode(
          ISD::FMUL, DL, VT, N0,
          DAG.getNode(ISD::FADD, DL, VT, N1, DAG.getConstantFP(1.0, DL, VT)));
    }
 
    // (fma x, c, (fneg x)) -> (fmul x, (c-1))
    if (N1CFP && N2.getOpcode() == ISD::FNEG && N2.getOperand(0) == N0) {
      return DAG.getNode(
          ISD::FMUL, DL, VT, N0,
          DAG.getNode(ISD::FADD, DL, VT, N1, DAG.getConstantFP(-1.0, DL, VT)));
    }
  }
 
  // fold ((fma (fneg X), Y, (fneg Z)) -> fneg (fma X, Y, Z))
  // fold ((fma X, (fneg Y), (fneg Z)) -> fneg (fma X, Y, Z))
  if (!TLI.isFNegFree(VT))
    if (SDValue Neg = TLI.getCheaperNegatedExpression(
            SDValue(N, 0), DAG, LegalOperations, ForCodeSize))
      return DAG.getNode(ISD::FNEG, DL, VT, Neg);
  return SDValue();
}
 
SDValue DAGCombiner::visitFMAD(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // Constant fold FMAD.
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FMAD, DL, VT, {N0, N1, N2}))
    return C;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitFMULADD(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  SDValue N2 = N->getOperand(2);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // Constant fold FMULADD.
  if (SDValue C =
          DAG.FoldConstantArithmetic(ISD::FMULADD, DL, VT, {N0, N1, N2}))
    return C;
 
  return SDValue();
}
 
// Combine multiple FDIVs with the same divisor into multiple FMULs by the
// reciprocal.
// E.g., (a / D; b / D;) -> (recip = 1.0 / D; a * recip; b * recip)
// Notice that this is not always beneficial. One reason is different targets
// may have different costs for FDIV and FMUL, so sometimes the cost of two
// FDIVs may be lower than the cost of one FDIV and two FMULs. Another reason
// is the critical path is increased from "one FDIV" to "one FDIV + one FMUL".
SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) {
  // TODO: Limit this transform based on optsize/minsize - it always creates at
  //       least 1 extra instruction. But the perf win may be substantial enough
  //       that only minsize should restrict this.
  const SDNodeFlags Flags = N->getFlags();
  if (LegalDAG || !Flags.hasAllowReciprocal())
    return SDValue();
 
  // Skip if current node is a reciprocal/fneg-reciprocal.
  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
  ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0, /* AllowUndefs */ true);
  if (N0CFP && (N0CFP->isExactlyValue(1.0) || N0CFP->isExactlyValue(-1.0)))
    return SDValue();
 
  // Exit early if the target does not want this transform or if there can't
  // possibly be enough uses of the divisor to make the transform worthwhile.
  unsigned MinUses = TLI.combineRepeatedFPDivisors();
 
  // For splat vectors, scale the number of uses by the splat factor. If we can
  // convert the division into a scalar op, that will likely be much faster.
  unsigned NumElts = 1;
  EVT VT = N->getValueType(0);
  if (VT.isVector() && DAG.isSplatValue(N1))
    NumElts = VT.getVectorMinNumElements();
 
  if (!MinUses || (N1->use_size() * NumElts) < MinUses)
    return SDValue();
 
  // Find all FDIV users of the same divisor.
  // Use a set because duplicates may be present in the user list.
  SetVector<SDNode *> Users;
  for (auto *U : N1->users()) {
    if (U->getOpcode() == ISD::FDIV && U->getOperand(1) == N1) {
      // Skip X/sqrt(X) that has not been simplified to sqrt(X) yet.
      if (U->getOperand(1).getOpcode() == ISD::FSQRT &&
          U->getOperand(0) == U->getOperand(1).getOperand(0) &&
          U->getFlags().hasAllowReassociation() &&
          U->getFlags().hasNoSignedZeros())
        continue;
 
      // This division is eligible for optimization only if global unsafe math
      // is enabled or if this division allows reciprocal formation.
      if (U->getFlags().hasAllowReciprocal())
        Users.insert(U);
    }
  }
 
  // Now that we have the actual number of divisor uses, make sure it meets
  // the minimum threshold specified by the target.
  if ((Users.size() * NumElts) < MinUses)
    return SDValue();
 
  SDLoc DL(N);
  SDValue FPOne = DAG.getConstantFP(1.0, DL, VT);
  SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags);
 
  // Dividend / Divisor -> Dividend * Reciprocal
  for (auto *U : Users) {
    SDValue Dividend = U->getOperand(0);
    if (Dividend != FPOne) {
      SDValue NewNode = DAG.getNode(ISD::FMUL, SDLoc(U), VT, Dividend,
                                    Reciprocal, Flags);
      CombineTo(U, NewNode);
    } else if (U != Reciprocal.getNode()) {
      // In the absence of fast-math-flags, this user node is always the
      // same node as Reciprocal, but with FMF they may be different nodes.
      CombineTo(U, Reciprocal);
    }
  }
  return SDValue(N, 0);  // N was replaced.
}
 
SDValue DAGCombiner::visitFDIV(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
  SDNodeFlags Flags = N->getFlags();
  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
  if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
    return R;
 
  // fold (fdiv c1, c2) -> c1/c2
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FDIV, DL, VT, {N0, N1}))
    return C;
 
  // fold vector ops
  if (VT.isVector())
    if (SDValue FoldedVOp = SimplifyVBinOp(N, DL))
      return FoldedVOp;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  if (SDValue V = combineRepeatedFPDivisors(N))
    return V;
 
  // fold (fdiv X, c2) -> (fmul X, 1/c2) if there is no loss in precision, or
  // the loss is acceptable with AllowReciprocal.
  if (auto *N1CFP = isConstOrConstSplatFP(N1, true)) {
    // Compute the reciprocal 1.0 / c2.
    const APFloat &N1APF = N1CFP->getValueAPF();
    APFloat Recip = APFloat::getOne(N1APF.getSemantics());
    APFloat::opStatus st = Recip.divide(N1APF, APFloat::rmNearestTiesToEven);
    // Only do the transform if the reciprocal is a legal fp immediate that
    // isn't too nasty (eg NaN, denormal, ...).
    if (((st == APFloat::opOK && !Recip.isDenormal()) ||
         (st == APFloat::opInexact && Flags.hasAllowReciprocal())) &&
        (!LegalOperations ||
         // FIXME: custom lowering of ConstantFP might fail (see e.g. ARM
         // backend)... we should handle this gracefully after Legalize.
         // TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT) ||
         TLI.isOperationLegal(ISD::ConstantFP, VT) ||
         TLI.isFPImmLegal(Recip, VT, ForCodeSize)))
      return DAG.getNode(ISD::FMUL, DL, VT, N0,
                         DAG.getConstantFP(Recip, DL, VT));
  }
 
  if (Flags.hasAllowReciprocal()) {
    // If this FDIV is part of a reciprocal square root, it may be folded
    // into a target-specific square root estimate instruction.
    bool N1AllowReciprocal = N1->getFlags().hasAllowReciprocal();
    if (N1.getOpcode() == ISD::FSQRT) {
      if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0), N1->getFlags()))
        return DAG.getNode(ISD::FMUL, DL, VT, N0, RV);
    } else if (N1.getOpcode() == ISD::FP_EXTEND &&
               N1.getOperand(0).getOpcode() == ISD::FSQRT &&
               N1AllowReciprocal) {
      if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0).getOperand(0),
                                          N1.getOperand(0)->getFlags())) {
        RV = DAG.getNode(ISD::FP_EXTEND, SDLoc(N1), VT, RV);
        AddToWorklist(RV.getNode());
        return DAG.getNode(ISD::FMUL, DL, VT, N0, RV);
      }
    } else if (N1.getOpcode() == ISD::FP_ROUND &&
               N1.getOperand(0).getOpcode() == ISD::FSQRT) {
      if (SDValue RV = buildRsqrtEstimate(N1.getOperand(0).getOperand(0),
                                          N1.getOperand(0)->getFlags())) {
        RV = DAG.getNode(ISD::FP_ROUND, SDLoc(N1), VT, RV, N1.getOperand(1));
        AddToWorklist(RV.getNode());
        return DAG.getNode(ISD::FMUL, DL, VT, N0, RV);
      }
    } else if (N1.getOpcode() == ISD::FMUL) {
      // Look through an FMUL. Even though this won't remove the FDIV directly,
      // it's still worthwhile to get rid of the FSQRT if possible.
      SDValue Sqrt, Y;
      if (N1.getOperand(0).getOpcode() == ISD::FSQRT) {
        Sqrt = N1.getOperand(0);
        Y = N1.getOperand(1);
      } else if (N1.getOperand(1).getOpcode() == ISD::FSQRT) {
        Sqrt = N1.getOperand(1);
        Y = N1.getOperand(0);
      }
      if (Sqrt.getNode()) {
        // If the other multiply operand is known positive, pull it into the
        // sqrt. That will eliminate the division if we convert to an estimate.
        if (Flags.hasAllowReassociation() && N1.hasOneUse() &&
            N1->getFlags().hasAllowReassociation() && Sqrt.hasOneUse()) {
          SDValue A;
          if (Y.getOpcode() == ISD::FABS && Y.hasOneUse())
            A = Y.getOperand(0);
          else if (Y == Sqrt.getOperand(0))
            A = Y;
          if (A) {
            // X / (fabs(A) * sqrt(Z)) --> X / sqrt(A*A*Z) --> X * rsqrt(A*A*Z)
            // X / (A * sqrt(A))       --> X / sqrt(A*A*A) --> X * rsqrt(A*A*A)
            SDValue AA = DAG.getNode(ISD::FMUL, DL, VT, A, A);
            SDValue AAZ =
                DAG.getNode(ISD::FMUL, DL, VT, AA, Sqrt.getOperand(0));
            if (SDValue Rsqrt = buildRsqrtEstimate(AAZ, Sqrt->getFlags()))
              return DAG.getNode(ISD::FMUL, DL, VT, N0, Rsqrt);
 
            // Estimate creation failed. Clean up speculatively created nodes.
            recursivelyDeleteUnusedNodes(AAZ.getNode());
          }
        }
 
        // We found a FSQRT, so try to make this fold:
        // X / (Y * sqrt(Z)) -> X * (rsqrt(Z) / Y)
        if (SDValue Rsqrt =
                buildRsqrtEstimate(Sqrt.getOperand(0), Sqrt->getFlags())) {
          SDValue Div = DAG.getNode(ISD::FDIV, SDLoc(N1), VT, Rsqrt, Y);
          AddToWorklist(Div.getNode());
          return DAG.getNode(ISD::FMUL, DL, VT, N0, Div);
        }
      }
    }
 
    // Fold into a reciprocal estimate and multiply instead of a real divide.
    if (Flags.hasNoInfs())
      if (SDValue RV = BuildDivEstimate(N0, N1, Flags))
        return RV;
  }
 
  // Fold X/Sqrt(X) -> Sqrt(X)
  if (DAG.canIgnoreSignBitOfZero(SDValue(N, 0)) &&
      Flags.hasAllowReassociation())
    if (N1.getOpcode() == ISD::FSQRT && N0 == N1.getOperand(0))
      return N1;
 
  // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y)
  TargetLowering::NegatibleCost CostN0 =
      TargetLowering::NegatibleCost::Expensive;
  TargetLowering::NegatibleCost CostN1 =
      TargetLowering::NegatibleCost::Expensive;
  SDValue NegN0 =
      TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize, CostN0);
  if (NegN0) {
    HandleSDNode NegN0Handle(NegN0);
    SDValue NegN1 =
        TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize, CostN1);
    if (NegN1 && (CostN0 == TargetLowering::NegatibleCost::Cheaper ||
                  CostN1 == TargetLowering::NegatibleCost::Cheaper))
      return DAG.getNode(ISD::FDIV, DL, VT, NegN0, NegN1);
  }
 
  if (SDValue R = combineFMulOrFDivWithIntPow2(N))
    return R;
 
  return SDValue();
}
 
SDValue DAGCombiner::visitFREM(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDNodeFlags Flags = N->getFlags();
  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
  SDLoc DL(N);
 
  if (SDValue R = DAG.simplifyFPBinop(N->getOpcode(), N0, N1, Flags))
    return R;
 
  // fold (frem c1, c2) -> fmod(c1,c2)
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FREM, DL, VT, {N0, N1}))
    return C;
 
  if (SDValue NewSel = foldBinOpIntoSelect(N))
    return NewSel;
 
  // Lower frem N0, N1 => x - trunc(N0 / N1) * N1, providing N1 is an integer
  // power of 2.
  if (!TLI.isOperationLegal(ISD::FREM, VT) &&
      TLI.isOperationLegalOrCustom(ISD::FMUL, VT) &&
      TLI.isOperationLegalOrCustom(ISD::FDIV, VT) &&
      TLI.isOperationLegalOrCustom(ISD::FTRUNC, VT) &&
      DAG.isKnownToBeAPowerOfTwoFP(N1)) {
    bool NeedsCopySign = !DAG.canIgnoreSignBitOfZero(SDValue(N, 0)) &&
                         !DAG.cannotBeOrderedNegativeFP(N0);
    SDValue Div = DAG.getNode(ISD::FDIV, DL, VT, N0, N1);
    SDValue Rnd = DAG.getNode(ISD::FTRUNC, DL, VT, Div);
    SDValue MLA;
    if (TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) {
      MLA = DAG.getNode(ISD::FMA, DL, VT, DAG.getNode(ISD::FNEG, DL, VT, Rnd),
                        N1, N0);
    } else {
      SDValue Mul = DAG.getNode(ISD::FMUL, DL, VT, Rnd, N1);
      MLA = DAG.getNode(ISD::FSUB, DL, VT, N0, Mul);
    }
    return NeedsCopySign ? DAG.getNode(ISD::FCOPYSIGN, DL, VT, MLA, N0) : MLA;
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitFSQRT(SDNode *N) {
  SDNodeFlags Flags = N->getFlags();
 
  // Require 'ninf' flag since sqrt(+Inf) = +Inf, but the estimation goes as:
  // sqrt(+Inf) == rsqrt(+Inf) * +Inf = 0 * +Inf = NaN
  if (!Flags.hasApproximateFuncs() || !Flags.hasNoInfs())
    return SDValue();
 
  SDValue N0 = N->getOperand(0);
  if (TLI.isFsqrtCheap(N0, DAG))
    return SDValue();
 
  // FSQRT nodes have flags that propagate to the created nodes.
  SelectionDAG::FlagInserter FlagInserter(DAG, Flags);
  // TODO: If this is N0/sqrt(N0), and we reach this node before trying to
  //       transform the fdiv, we may produce a sub-optimal estimate sequence
  //       because the reciprocal calculation may not have to filter out a
  //       0.0 input.
  return buildSqrtEstimate(N0, Flags);
}
 
/// copysign(x, fp_extend(y)) -> copysign(x, y)
/// copysign(x, fp_round(y)) -> copysign(x, y)
/// Operands to the functions are the type of X and Y respectively.
static inline bool CanCombineFCOPYSIGN_EXTEND_ROUND(EVT XTy, EVT YTy) {
  // Always fold no-op FP casts.
  if (XTy == YTy)
    return true;
 
  // Do not optimize out type conversion of f128 type yet.
  // For some targets like x86_64, configuration is changed to keep one f128
  // value in one SSE register, but instruction selection cannot handle
  // FCOPYSIGN on SSE registers yet.
  if (YTy == MVT::f128)
    return false;
 
  // Avoid mismatched vector operand types, for better instruction selection.
  return !YTy.isVector();
}
 
static inline bool CanCombineFCOPYSIGN_EXTEND_ROUND(SDNode *N) {
  SDValue N1 = N->getOperand(1);
  if (N1.getOpcode() != ISD::FP_EXTEND &&
      N1.getOpcode() != ISD::FP_ROUND)
    return false;
  EVT N1VT = N1->getValueType(0);
  EVT N1Op0VT = N1->getOperand(0).getValueType();
  return CanCombineFCOPYSIGN_EXTEND_ROUND(N1VT, N1Op0VT);
}
 
SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
  SDValue N0 = N->getOperand(0);
  SDValue N1 = N->getOperand(1);
  EVT VT = N->getValueType(0);
  SDLoc DL(N);
 
  // fold (fcopysign c1, c2) -> fcopysign(c1,c2)
  if (SDValue C = DAG.FoldConstantArithmetic(ISD::FCOPYSIGN, DL, VT, {N0, N1}))
    return C;
 
  // copysign(x, fp_extend(y)) -> copysign(x, y)
  // copysign(x, fp_round(y)) -> copysign(x, y)
  if (CanCombineFCOPYSIGN_EXTEND_ROUND(N))
    return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N0, N1.getOperand(0));
 
  if (SimplifyDemandedBits(SDValue(N, 0)))
    return SDValue(N, 0);
 
  if (VT != N1.getValueType())
    return SDValue();
 
  // If this is equivalent to a disjoint or, replace it with one. This can
  // happen if the sign operand is a sign mask (i.e., x << sign_bit_position).
  if (DAG.SignBitIsZeroFP(N0) &&
      DAG.computeKnownBits(N1).Zero.isMaxSignedValue()) {
    // TODO: Just directly match the shift pattern. computeKnownBits is heavy
    // for a such a narrowly targeted case.
    EVT IntVT = VT.changeTypeToInteger();
    // TODO: It appears to be profitable in some situations to unconditionally
    // emit a fabs(n0) to perform this combine.
    SDValue CastSrc0 = DAG.getNode(ISD::BITCAST, DL, IntVT, N0);
    SDValue CastSrc1 = DAG.getNode(ISD::BITCAST, DL, IntVT, N1);
 
    SDValue SignOr = DAG.getNode(ISD::OR, DL, IntVT, CastSrc0, CastSrc1,
                                 SDNodeFlags::Disjoint);
    return DAG.getNode(ISD::BITCAST, DL, VT, SignOr);
  }
 
  return SDValue();
}
 
SDValue DAGCombiner::visitFPOW(SDNode *N) {
  ConstantFPSDNode *ExponentC = isConstOrConstSplatFP(N->getOperand(1));
  if (!ExponentC)
    return SDValue();
  SelectionDAG::FlagInserter FlagsInserter(DAG, N);
 
  // Try to convert x ** (1/3) into cube root.
  // TODO: Handle the various flavors of long double.
  // TODO: Since we're approximating, we don't need an exact 1/3 exponent.
  //       Some range near 1/3 should be fine.
  EVT VT = N->getValueType(0);
  EVT ScalarVT = VT.getScalarType();
  if ((ScalarVT == MVT::f32 &&
       ExponentC->getValueAPF().isExactlyValue(1.0f / 3.0f)) ||
      (ScalarVT == MVT::f64 &&
       ExponentC->getValueAPF().isExactlyValue(1.0 / 3.0))) {
    // pow(-0.0, 1/3) = +0.0; cbrt(-0.0) = -0.0.
    // pow(-inf, 1/3) = +inf; cbrt(-inf) = -inf.
    // pow(-val, 1/3) =  nan; cbrt(-val) = -num.
    // For regular numbers, rounding may cause the results to differ.
    // Therefore, we require { nsz ninf nnan afn } for this transform.
    // TODO: We could select out the special cases if we don't have nsz/ninf.
    SDNodeFlags Flags = N->getFlags();
    if (!Flags.hasNoSignedZeros() || !Flags.hasNoInfs() || !Flags.hasNoNaNs() ||
        !Flags.hasApproximateFuncs())
      return SDValue();
 
    // Do not create a cbrt() libcall if the target does not have it, and do not
    // turn a pow that has lowering support into a cbrt() libcall.
    RTLIB::Libcall LC = RTLIB::getCBRT(VT);
    bool HasLibCall =
        DAG.getLibcalls().getLibcallImpl(LC) != RTLIB::Unsupported;
    if (!HasLibCall ||
        (!DAG.getTargetLoweringInfo().isOperationExpand(ISD::FPOW, VT) &&
         DAG.getTargetLoweringInfo().isOperationExpand(ISD::FCBRT, VT)))
      return SDValue();
 
    return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0));
  }
 
  // Try to convert x ** (1/4) and x ** (3/4) into square roots.
  // x ** (1/2) is canonicalized to sqrt, so we do not bother with that case.
  // TODO: This could be extended (using a target hook) to handle smaller
  // power-of-2 fractional exponents.
  bool ExponentIs025 = ExponentC->getValueAPF().isExactlyValue(0.25);
  bool ExponentIs075 = ExponentC->getValueAPF().isExactlyValue(0.75);
  if (ExponentIs025 || ExponentIs075) {
    // pow(-0.0, 0.25) = +0.0; sqrt(sqrt(-0.0)) = -0.0.
    // pow(-inf, 0.25) = +inf; sqrt(sqrt(-inf)) =  NaN.
    // pow(-0.0, 0.75) = +0.0; sqrt(-0.0) * sqrt(sqrt(-0.0)) = +0.0.
    // pow(-inf, 0.75) = +inf; sqrt(-inf) * sqrt(sqrt(-inf)) =  NaN.
    // For regular numbers, rounding may cause the results to differ.
    // Therefore, we require { nsz ninf afn } for this transform.
    // TODO: We could select out the special cases if we don't have nsz/ninf.
    SDNodeFlags Flags = N->getFlags();
 
    // We only need no signed zeros for the 0.25 case.
    if ((!Flags.hasNoSignedZeros() && ExponentIs025) || !Flags.hasNoInfs() ||
        !Flags.hasApproximateFuncs())
      return SDValue();
 
    // Don't double the number of libcalls. We are trying to inline fast code.
    if (!DAG.getTargetLoweringInfo().isOperationLegalOrCustom(ISD::FSQRT, VT))
      return SDValue();
 
    // Assume that libcalls are the smallest code.
    // TODO: This restriction should probably be lifted for vectors.
    if (ForCodeSize)
      return SDValue();
 
    // pow(X, 0.25) --> sqrt(sqrt(X))
    SDLoc DL(N);
    SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0));
    SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt);
    if (ExponentIs025)
      return SqrtSqrt;
    // pow(X, 0.75) --> sqrt(X) * sqrt(sqrt(X))
    return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt);
  }
 
  return SDValue();
}
 
static SDValue foldFPToIntToFP(SDNode *N, const SDLoc &DL, SelectionDAG &DAG,
                               const TargetLowering &TLI) {
  // We can fold the fpto[us]i -> [us]itofp pattern into a single ftrunc.
  // Additionally, if there are clamps ([us]min or [us]max) around
  // the fpto[us]i, we can fold those into fminnum/fmaxnum around the ftrunc.
  // If NoSignedZerosFPMath is enabled, this is a direct replacement.
  // Otherwise, for strict math, we must handle edge cases:
  // 1. For unsigned conversions, use FABS to handle negative cases. Take -0.0
  // as example, it first becomes integer 0, and is converted back to +0.0.
  // FTRUNC on its own could produce -0.0.
 
  // FIXME: We should be able to use node-level FMF here.
  EVT VT = N->getValueType(0);
  if (!TLI.isOperationLegalOrCustom(ISD::FTRUNC, VT))
    return SDValue();
 
  bool IsUnsigned = N->getOpcode() == ISD::UINT_TO_FP;
  bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP;
  assert