AArch64ISelLowering.cpp

Go to the documentation of this file.

//===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation  ----===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the AArch64TargetLowering class.
//
//===----------------------------------------------------------------------===//
 
#include "AArch64ISelLowering.h"
#include "AArch64CallingConvention.h"
#include "AArch64ExpandImm.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/OperandTraits.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MachineValueType.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <tuple>
#include <utility>
#include <vector>
 
using namespace llvm;
using namespace llvm::PatternMatch;
 
#define DEBUG_TYPE "aarch64-lower"
 
STATISTIC(NumTailCalls, "Number of tail calls");
STATISTIC(NumShiftInserts, "Number of vector shift inserts");
STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
 
// FIXME: The necessary dtprel relocations don't seem to be supported
// well in the GNU bfd and gold linkers at the moment. Therefore, by
// default, for now, fall back to GeneralDynamic code generation.
cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
    "aarch64-elf-ldtls-generation", cl::Hidden,
    cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
    cl::init(false));
 
static cl::opt<bool>
EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
                         cl::desc("Enable AArch64 logical imm instruction "
                                  "optimization"),
                         cl::init(true));
 
// Temporary option added for the purpose of testing functionality added
// to DAGCombiner.cpp in D92230. It is expected that this can be removed
// in future when both implementations will be based off MGATHER rather
// than the GLD1 nodes added for the SVE gather load intrinsics.
static cl::opt<bool>
EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden,
                                cl::desc("Combine extends of AArch64 masked "
                                         "gather intrinsics"),
                                cl::init(true));
 
/// Value type used for condition codes.
static const MVT MVT_CC = MVT::i32;
 
static inline EVT getPackedSVEVectorVT(EVT VT) {
  switch (VT.getSimpleVT().SimpleTy) {
  default:
    llvm_unreachable("unexpected element type for vector");
  case MVT::i8:
    return MVT::nxv16i8;
  case MVT::i16:
    return MVT::nxv8i16;
  case MVT::i32:
    return MVT::nxv4i32;
  case MVT::i64:
    return MVT::nxv2i64;
  case MVT::f16:
    return MVT::nxv8f16;
  case MVT::f32:
    return MVT::nxv4f32;
  case MVT::f64:
    return MVT::nxv2f64;
  case MVT::bf16:
    return MVT::nxv8bf16;
  }
}
 
// NOTE: Currently there's only a need to return integer vector types. If this
// changes then just add an extra "type" parameter.
static inline EVT getPackedSVEVectorVT(ElementCount EC) {
  switch (EC.getKnownMinValue()) {
  default:
    llvm_unreachable("unexpected element count for vector");
  case 16:
    return MVT::nxv16i8;
  case 8:
    return MVT::nxv8i16;
  case 4:
    return MVT::nxv4i32;
  case 2:
    return MVT::nxv2i64;
  }
}
 
static inline EVT getPromotedVTForPredicate(EVT VT) {
  assert(VT.isScalableVector() && (VT.getVectorElementType() == MVT::i1) &&
         "Expected scalable predicate vector type!");
  switch (VT.getVectorMinNumElements()) {
  default:
    llvm_unreachable("unexpected element count for vector");
  case 2:
    return MVT::nxv2i64;
  case 4:
    return MVT::nxv4i32;
  case 8:
    return MVT::nxv8i16;
  case 16:
    return MVT::nxv16i8;
  }
}
 
/// Returns true if VT's elements occupy the lowest bit positions of its
/// associated register class without any intervening space.
///
/// For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the
/// same register class, but only nxv8f16 can be treated as a packed vector.
static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
  assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
         "Expected legal vector type!");
  return VT.isFixedLengthVector() ||
         VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock;
}
 
// Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
// predicate and end with a passthru value matching the result type.
static bool isMergePassthruOpcode(unsigned Opc) {
  switch (Opc) {
  default:
    return false;
  case AArch64ISD::BITREVERSE_MERGE_PASSTHRU:
  case AArch64ISD::BSWAP_MERGE_PASSTHRU:
  case AArch64ISD::REVH_MERGE_PASSTHRU:
  case AArch64ISD::REVW_MERGE_PASSTHRU:
  case AArch64ISD::REVD_MERGE_PASSTHRU:
  case AArch64ISD::CTLZ_MERGE_PASSTHRU:
  case AArch64ISD::CTPOP_MERGE_PASSTHRU:
  case AArch64ISD::DUP_MERGE_PASSTHRU:
  case AArch64ISD::ABS_MERGE_PASSTHRU:
  case AArch64ISD::NEG_MERGE_PASSTHRU:
  case AArch64ISD::FNEG_MERGE_PASSTHRU:
  case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
  case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
  case AArch64ISD::FCEIL_MERGE_PASSTHRU:
  case AArch64ISD::FFLOOR_MERGE_PASSTHRU:
  case AArch64ISD::FNEARBYINT_MERGE_PASSTHRU:
  case AArch64ISD::FRINT_MERGE_PASSTHRU:
  case AArch64ISD::FROUND_MERGE_PASSTHRU:
  case AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU:
  case AArch64ISD::FTRUNC_MERGE_PASSTHRU:
  case AArch64ISD::FP_ROUND_MERGE_PASSTHRU:
  case AArch64ISD::FP_EXTEND_MERGE_PASSTHRU:
  case AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU:
  case AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU:
  case AArch64ISD::FCVTZU_MERGE_PASSTHRU:
  case AArch64ISD::FCVTZS_MERGE_PASSTHRU:
  case AArch64ISD::FSQRT_MERGE_PASSTHRU:
  case AArch64ISD::FRECPX_MERGE_PASSTHRU:
  case AArch64ISD::FABS_MERGE_PASSTHRU:
    return true;
  }
}
 
// Returns true if inactive lanes are known to be zeroed by construction.
static bool isZeroingInactiveLanes(SDValue Op) {
  switch (Op.getOpcode()) {
  default:
    // We guarantee i1 splat_vectors to zero the other lanes by
    // implementing it with ptrue and possibly a punpklo for nxv1i1.
    if (ISD::isConstantSplatVectorAllOnes(Op.getNode()))
      return true;
    return false;
  case AArch64ISD::PTRUE:
  case AArch64ISD::SETCC_MERGE_ZERO:
    return true;
  case ISD::INTRINSIC_WO_CHAIN:
    switch (Op.getConstantOperandVal(0)) {
    default:
      return false;
    case Intrinsic::aarch64_sve_ptrue:
    case Intrinsic::aarch64_sve_pnext:
    case Intrinsic::aarch64_sve_cmpeq:
    case Intrinsic::aarch64_sve_cmpne:
    case Intrinsic::aarch64_sve_cmpge:
    case Intrinsic::aarch64_sve_cmpgt:
    case Intrinsic::aarch64_sve_cmphs:
    case Intrinsic::aarch64_sve_cmphi:
    case Intrinsic::aarch64_sve_cmpeq_wide:
    case Intrinsic::aarch64_sve_cmpne_wide:
    case Intrinsic::aarch64_sve_cmpge_wide:
    case Intrinsic::aarch64_sve_cmpgt_wide:
    case Intrinsic::aarch64_sve_cmplt_wide:
    case Intrinsic::aarch64_sve_cmple_wide:
    case Intrinsic::aarch64_sve_cmphs_wide:
    case Intrinsic::aarch64_sve_cmphi_wide:
    case Intrinsic::aarch64_sve_cmplo_wide:
    case Intrinsic::aarch64_sve_cmpls_wide:
    case Intrinsic::aarch64_sve_fcmpeq:
    case Intrinsic::aarch64_sve_fcmpne:
    case Intrinsic::aarch64_sve_fcmpge:
    case Intrinsic::aarch64_sve_fcmpgt:
    case Intrinsic::aarch64_sve_fcmpuo:
      return true;
    }
  }
}
 
AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
                                             const AArch64Subtarget &STI)
    : TargetLowering(TM), Subtarget(&STI) {
  // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
  // we have to make something up. Arbitrarily, choose ZeroOrOne.
  setBooleanContents(ZeroOrOneBooleanContent);
  // When comparing vectors the result sets the different elements in the
  // vector to all-one or all-zero.
  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
  // Set up the register classes.
  addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
  addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
 
  if (Subtarget->hasLS64()) {
    addRegisterClass(MVT::i64x8, &AArch64::GPR64x8ClassRegClass);
    setOperationAction(ISD::LOAD, MVT::i64x8, Custom);
    setOperationAction(ISD::STORE, MVT::i64x8, Custom);
  }
 
  if (Subtarget->hasFPARMv8()) {
    addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
    addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
    addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
    addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
    addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
  }
 
  if (Subtarget->hasNEON()) {
    addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
    addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
    // Someone set us up the NEON.
    addDRTypeForNEON(MVT::v2f32);
    addDRTypeForNEON(MVT::v8i8);
    addDRTypeForNEON(MVT::v4i16);
    addDRTypeForNEON(MVT::v2i32);
    addDRTypeForNEON(MVT::v1i64);
    addDRTypeForNEON(MVT::v1f64);
    addDRTypeForNEON(MVT::v4f16);
    if (Subtarget->hasBF16())
      addDRTypeForNEON(MVT::v4bf16);
 
    addQRTypeForNEON(MVT::v4f32);
    addQRTypeForNEON(MVT::v2f64);
    addQRTypeForNEON(MVT::v16i8);
    addQRTypeForNEON(MVT::v8i16);
    addQRTypeForNEON(MVT::v4i32);
    addQRTypeForNEON(MVT::v2i64);
    addQRTypeForNEON(MVT::v8f16);
    if (Subtarget->hasBF16())
      addQRTypeForNEON(MVT::v8bf16);
  }
 
  if (Subtarget->hasSVE() || Subtarget->hasSME()) {
    // Add legal sve predicate types
    addRegisterClass(MVT::nxv1i1, &AArch64::PPRRegClass);
    addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
    addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
    addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
    addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
 
    // Add legal sve data types
    addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
 
    addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
    addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
 
    if (Subtarget->hasBF16()) {
      addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
      addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
      addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
    }
 
    if (Subtarget->useSVEForFixedLengthVectors()) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
        if (useSVEForFixedLengthVectorVT(VT))
          addRegisterClass(VT, &AArch64::ZPRRegClass);
 
      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
        if (useSVEForFixedLengthVectorVT(VT))
          addRegisterClass(VT, &AArch64::ZPRRegClass);
    }
  }
 
  // Compute derived properties from the register classes
  computeRegisterProperties(Subtarget->getRegisterInfo());
 
  // Provide all sorts of operation actions
  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
  setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
  setOperationAction(ISD::SETCC, MVT::i32, Custom);
  setOperationAction(ISD::SETCC, MVT::i64, Custom);
  setOperationAction(ISD::SETCC, MVT::f16, Custom);
  setOperationAction(ISD::SETCC, MVT::f32, Custom);
  setOperationAction(ISD::SETCC, MVT::f64, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
  setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
  setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
  setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
  setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
  setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
  setOperationAction(ISD::BRCOND, MVT::Other, Custom);
  setOperationAction(ISD::BR_CC, MVT::i32, Custom);
  setOperationAction(ISD::BR_CC, MVT::i64, Custom);
  setOperationAction(ISD::BR_CC, MVT::f16, Custom);
  setOperationAction(ISD::BR_CC, MVT::f32, Custom);
  setOperationAction(ISD::BR_CC, MVT::f64, Custom);
  setOperationAction(ISD::SELECT, MVT::i32, Custom);
  setOperationAction(ISD::SELECT, MVT::i64, Custom);
  setOperationAction(ISD::SELECT, MVT::f16, Custom);
  setOperationAction(ISD::SELECT, MVT::bf16, Custom);
  setOperationAction(ISD::SELECT, MVT::f32, Custom);
  setOperationAction(ISD::SELECT, MVT::f64, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand);
  setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
  setOperationAction(ISD::BR_JT, MVT::Other, Custom);
  setOperationAction(ISD::JumpTable, MVT::i64, Custom);
 
  setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
  setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
  setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
 
  setOperationAction(ISD::FREM, MVT::f32, Expand);
  setOperationAction(ISD::FREM, MVT::f64, Expand);
  setOperationAction(ISD::FREM, MVT::f80, Expand);
 
  setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
 
  // Custom lowering hooks are needed for XOR
  // to fold it into CSINC/CSINV.
  setOperationAction(ISD::XOR, MVT::i32, Custom);
  setOperationAction(ISD::XOR, MVT::i64, Custom);
 
  // Virtually no operation on f128 is legal, but LLVM can't expand them when
  // there's a valid register class, so we need custom operations in most cases.
  setOperationAction(ISD::FABS, MVT::f128, Expand);
  setOperationAction(ISD::FADD, MVT::f128, LibCall);
  setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
  setOperationAction(ISD::FCOS, MVT::f128, Expand);
  setOperationAction(ISD::FDIV, MVT::f128, LibCall);
  setOperationAction(ISD::FMA, MVT::f128, Expand);
  setOperationAction(ISD::FMUL, MVT::f128, LibCall);
  setOperationAction(ISD::FNEG, MVT::f128, Expand);
  setOperationAction(ISD::FPOW, MVT::f128, Expand);
  setOperationAction(ISD::FREM, MVT::f128, Expand);
  setOperationAction(ISD::FRINT, MVT::f128, Expand);
  setOperationAction(ISD::FSIN, MVT::f128, Expand);
  setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
  setOperationAction(ISD::FSQRT, MVT::f128, Expand);
  setOperationAction(ISD::FSUB, MVT::f128, LibCall);
  setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
  setOperationAction(ISD::SETCC, MVT::f128, Custom);
  setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
  setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom);
  setOperationAction(ISD::BR_CC, MVT::f128, Custom);
  setOperationAction(ISD::SELECT, MVT::f128, Custom);
  setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
  setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
  // FIXME: f128 FMINIMUM and FMAXIMUM (including STRICT versions) currently
  // aren't handled.
 
  // Lowering for many of the conversions is actually specified by the non-f128
  // type. The LowerXXX function will be trivial when f128 isn't involved.
  setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
  setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
  setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
  setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
  setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
  setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
  setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
  setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
  setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
  setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
  setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
 
  setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
  setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom);
  setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
 
  // Variable arguments.
  setOperationAction(ISD::VASTART, MVT::Other, Custom);
  setOperationAction(ISD::VAARG, MVT::Other, Custom);
  setOperationAction(ISD::VACOPY, MVT::Other, Custom);
  setOperationAction(ISD::VAEND, MVT::Other, Expand);
 
  // Variable-sized objects.
  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
 
  if (Subtarget->isTargetWindows())
    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
  else
    setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
 
  // Constant pool entries
  setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
 
  // BlockAddress
  setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
 
  // AArch64 lacks both left-rotate and popcount instructions.
  setOperationAction(ISD::ROTL, MVT::i32, Expand);
  setOperationAction(ISD::ROTL, MVT::i64, Expand);
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    setOperationAction(ISD::ROTL, VT, Expand);
    setOperationAction(ISD::ROTR, VT, Expand);
  }
 
  // AArch64 doesn't have i32 MULH{S|U}.
  setOperationAction(ISD::MULHU, MVT::i32, Expand);
  setOperationAction(ISD::MULHS, MVT::i32, Expand);
 
  // AArch64 doesn't have {U|S}MUL_LOHI.
  setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
  setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
 
  setOperationAction(ISD::CTPOP, MVT::i32, Custom);
  setOperationAction(ISD::CTPOP, MVT::i64, Custom);
  setOperationAction(ISD::CTPOP, MVT::i128, Custom);
 
  setOperationAction(ISD::PARITY, MVT::i64, Custom);
  setOperationAction(ISD::PARITY, MVT::i128, Custom);
 
  setOperationAction(ISD::ABS, MVT::i32, Custom);
  setOperationAction(ISD::ABS, MVT::i64, Custom);
 
  setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
  setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
    setOperationAction(ISD::SDIVREM, VT, Expand);
    setOperationAction(ISD::UDIVREM, VT, Expand);
  }
  setOperationAction(ISD::SREM, MVT::i32, Expand);
  setOperationAction(ISD::SREM, MVT::i64, Expand);
  setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
  setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
  setOperationAction(ISD::UREM, MVT::i32, Expand);
  setOperationAction(ISD::UREM, MVT::i64, Expand);
 
  // Custom lower Add/Sub/Mul with overflow.
  setOperationAction(ISD::SADDO, MVT::i32, Custom);
  setOperationAction(ISD::SADDO, MVT::i64, Custom);
  setOperationAction(ISD::UADDO, MVT::i32, Custom);
  setOperationAction(ISD::UADDO, MVT::i64, Custom);
  setOperationAction(ISD::SSUBO, MVT::i32, Custom);
  setOperationAction(ISD::SSUBO, MVT::i64, Custom);
  setOperationAction(ISD::USUBO, MVT::i32, Custom);
  setOperationAction(ISD::USUBO, MVT::i64, Custom);
  setOperationAction(ISD::SMULO, MVT::i32, Custom);
  setOperationAction(ISD::SMULO, MVT::i64, Custom);
  setOperationAction(ISD::UMULO, MVT::i32, Custom);
  setOperationAction(ISD::UMULO, MVT::i64, Custom);
 
  setOperationAction(ISD::ADDCARRY, MVT::i32, Custom);
  setOperationAction(ISD::ADDCARRY, MVT::i64, Custom);
  setOperationAction(ISD::SUBCARRY, MVT::i32, Custom);
  setOperationAction(ISD::SUBCARRY, MVT::i64, Custom);
  setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::SADDO_CARRY, MVT::i64, Custom);
  setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom);
  setOperationAction(ISD::SSUBO_CARRY, MVT::i64, Custom);
 
  setOperationAction(ISD::FSIN, MVT::f32, Expand);
  setOperationAction(ISD::FSIN, MVT::f64, Expand);
  setOperationAction(ISD::FCOS, MVT::f32, Expand);
  setOperationAction(ISD::FCOS, MVT::f64, Expand);
  setOperationAction(ISD::FPOW, MVT::f32, Expand);
  setOperationAction(ISD::FPOW, MVT::f64, Expand);
  setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
  setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
  if (Subtarget->hasFullFP16())
    setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
  else
    setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
 
  for (auto Op : {ISD::FREM,        ISD::FPOW,         ISD::FPOWI,
                  ISD::FCOS,        ISD::FSIN,         ISD::FSINCOS,
                  ISD::FEXP,        ISD::FEXP2,        ISD::FLOG,
                  ISD::FLOG2,       ISD::FLOG10,       ISD::STRICT_FREM,
                  ISD::STRICT_FPOW, ISD::STRICT_FPOWI, ISD::STRICT_FCOS,
                  ISD::STRICT_FSIN, ISD::STRICT_FEXP,  ISD::STRICT_FEXP2,
                  ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10}) {
    setOperationAction(Op, MVT::f16, Promote);
    setOperationAction(Op, MVT::v4f16, Expand);
    setOperationAction(Op, MVT::v8f16, Expand);
  }
 
  if (!Subtarget->hasFullFP16()) {
    for (auto Op :
         {ISD::SETCC,          ISD::SELECT_CC,
          ISD::BR_CC,          ISD::FADD,           ISD::FSUB,
          ISD::FMUL,           ISD::FDIV,           ISD::FMA,
          ISD::FNEG,           ISD::FABS,           ISD::FCEIL,
          ISD::FSQRT,          ISD::FFLOOR,         ISD::FNEARBYINT,
          ISD::FRINT,          ISD::FROUND,         ISD::FROUNDEVEN,
          ISD::FTRUNC,         ISD::FMINNUM,        ISD::FMAXNUM,
          ISD::FMINIMUM,       ISD::FMAXIMUM,       ISD::STRICT_FADD,
          ISD::STRICT_FSUB,    ISD::STRICT_FMUL,    ISD::STRICT_FDIV,
          ISD::STRICT_FMA,     ISD::STRICT_FCEIL,   ISD::STRICT_FFLOOR,
          ISD::STRICT_FSQRT,   ISD::STRICT_FRINT,   ISD::STRICT_FNEARBYINT,
          ISD::STRICT_FROUND,  ISD::STRICT_FTRUNC,  ISD::STRICT_FROUNDEVEN,
          ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM,
          ISD::STRICT_FMAXIMUM})
      setOperationAction(Op, MVT::f16, Promote);
 
    // Round-to-integer need custom lowering for fp16, as Promote doesn't work
    // because the result type is integer.
    for (auto Op : {ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT,
                    ISD::STRICT_LLRINT})
      setOperationAction(Op, MVT::f16, Custom);
 
    // promote v4f16 to v4f32 when that is known to be safe.
    setOperationPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32);
    setOperationPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32);
    setOperationPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32);
    setOperationPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32);
 
    setOperationAction(ISD::FABS,        MVT::v4f16, Expand);
    setOperationAction(ISD::FNEG,        MVT::v4f16, Expand);
    setOperationAction(ISD::FROUND,      MVT::v4f16, Expand);
    setOperationAction(ISD::FROUNDEVEN,  MVT::v4f16, Expand);
    setOperationAction(ISD::FMA,         MVT::v4f16, Expand);
    setOperationAction(ISD::SETCC,       MVT::v4f16, Expand);
    setOperationAction(ISD::BR_CC,       MVT::v4f16, Expand);
    setOperationAction(ISD::SELECT,      MVT::v4f16, Expand);
    setOperationAction(ISD::SELECT_CC,   MVT::v4f16, Expand);
    setOperationAction(ISD::FTRUNC,      MVT::v4f16, Expand);
    setOperationAction(ISD::FCOPYSIGN,   MVT::v4f16, Expand);
    setOperationAction(ISD::FFLOOR,      MVT::v4f16, Expand);
    setOperationAction(ISD::FCEIL,       MVT::v4f16, Expand);
    setOperationAction(ISD::FRINT,       MVT::v4f16, Expand);
    setOperationAction(ISD::FNEARBYINT,  MVT::v4f16, Expand);
    setOperationAction(ISD::FSQRT,       MVT::v4f16, Expand);
 
    setOperationAction(ISD::FABS,        MVT::v8f16, Expand);
    setOperationAction(ISD::FADD,        MVT::v8f16, Expand);
    setOperationAction(ISD::FCEIL,       MVT::v8f16, Expand);
    setOperationAction(ISD::FCOPYSIGN,   MVT::v8f16, Expand);
    setOperationAction(ISD::FDIV,        MVT::v8f16, Expand);
    setOperationAction(ISD::FFLOOR,      MVT::v8f16, Expand);
    setOperationAction(ISD::FMA,         MVT::v8f16, Expand);
    setOperationAction(ISD::FMUL,        MVT::v8f16, Expand);
    setOperationAction(ISD::FNEARBYINT,  MVT::v8f16, Expand);
    setOperationAction(ISD::FNEG,        MVT::v8f16, Expand);
    setOperationAction(ISD::FROUND,      MVT::v8f16, Expand);
    setOperationAction(ISD::FROUNDEVEN,  MVT::v8f16, Expand);
    setOperationAction(ISD::FRINT,       MVT::v8f16, Expand);
    setOperationAction(ISD::FSQRT,       MVT::v8f16, Expand);
    setOperationAction(ISD::FSUB,        MVT::v8f16, Expand);
    setOperationAction(ISD::FTRUNC,      MVT::v8f16, Expand);
    setOperationAction(ISD::SETCC,       MVT::v8f16, Expand);
    setOperationAction(ISD::BR_CC,       MVT::v8f16, Expand);
    setOperationAction(ISD::SELECT,      MVT::v8f16, Expand);
    setOperationAction(ISD::SELECT_CC,   MVT::v8f16, Expand);
    setOperationAction(ISD::FP_EXTEND,   MVT::v8f16, Expand);
  }
 
  // AArch64 has implementations of a lot of rounding-like FP operations.
  for (auto Op :
       {ISD::FFLOOR,          ISD::FNEARBYINT,      ISD::FCEIL,
        ISD::FRINT,           ISD::FTRUNC,          ISD::FROUND,
        ISD::FROUNDEVEN,      ISD::FMINNUM,         ISD::FMAXNUM,
        ISD::FMINIMUM,        ISD::FMAXIMUM,        ISD::LROUND,
        ISD::LLROUND,         ISD::LRINT,           ISD::LLRINT,
        ISD::STRICT_FFLOOR,   ISD::STRICT_FCEIL,    ISD::STRICT_FNEARBYINT,
        ISD::STRICT_FRINT,    ISD::STRICT_FTRUNC,   ISD::STRICT_FROUNDEVEN,
        ISD::STRICT_FROUND,   ISD::STRICT_FMINNUM,  ISD::STRICT_FMAXNUM,
        ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_LROUND,
        ISD::STRICT_LLROUND,  ISD::STRICT_LRINT,    ISD::STRICT_LLRINT}) {
    for (MVT Ty : {MVT::f32, MVT::f64})
      setOperationAction(Op, Ty, Legal);
    if (Subtarget->hasFullFP16())
      setOperationAction(Op, MVT::f16, Legal);
  }
 
  // Basic strict FP operations are legal
  for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
                  ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT}) {
    for (MVT Ty : {MVT::f32, MVT::f64})
      setOperationAction(Op, Ty, Legal);
    if (Subtarget->hasFullFP16())
      setOperationAction(Op, MVT::f16, Legal);
  }
 
  // Strict conversion to a larger type is legal
  for (auto VT : {MVT::f32, MVT::f64})
    setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
 
  setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
  setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
  setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
 
  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
  setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
  setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
  setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
  setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
 
  // Generate outline atomics library calls only if LSE was not specified for
  // subtarget
  if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) {
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall);
    setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall);
#define LCALLNAMES(A, B, N)                                                    \
  setLibcallName(A##N##_RELAX, #B #N "_relax");                                \
  setLibcallName(A##N##_ACQ, #B #N "_acq");                                    \
  setLibcallName(A##N##_REL, #B #N "_rel");                                    \
  setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
#define LCALLNAME4(A, B)                                                       \
  LCALLNAMES(A, B, 1)                                                          \
  LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
#define LCALLNAME5(A, B)                                                       \
  LCALLNAMES(A, B, 1)                                                          \
  LCALLNAMES(A, B, 2)                                                          \
  LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
    LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas)
    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp)
    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd)
    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset)
    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr)
    LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor)
#undef LCALLNAMES
#undef LCALLNAME4
#undef LCALLNAME5
  }
 
  // 128-bit loads and stores can be done without expanding
  setOperationAction(ISD::LOAD, MVT::i128, Custom);
  setOperationAction(ISD::STORE, MVT::i128, Custom);
 
  // Aligned 128-bit loads and stores are single-copy atomic according to the
  // v8.4a spec.
  if (Subtarget->hasLSE2()) {
    setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
    setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
  }
 
  // 256 bit non-temporal stores can be lowered to STNP. Do this as part of the
  // custom lowering, as there are no un-paired non-temporal stores and
  // legalization will break up 256 bit inputs.
  setOperationAction(ISD::STORE, MVT::v32i8, Custom);
  setOperationAction(ISD::STORE, MVT::v16i16, Custom);
  setOperationAction(ISD::STORE, MVT::v16f16, Custom);
  setOperationAction(ISD::STORE, MVT::v8i32, Custom);
  setOperationAction(ISD::STORE, MVT::v8f32, Custom);
  setOperationAction(ISD::STORE, MVT::v4f64, Custom);
  setOperationAction(ISD::STORE, MVT::v4i64, Custom);
 
  // 256 bit non-temporal loads can be lowered to LDNP. This is done using
  // custom lowering, as there are no un-paired non-temporal loads legalization
  // will break up 256 bit inputs.
  setOperationAction(ISD::LOAD, MVT::v32i8, Custom);
  setOperationAction(ISD::LOAD, MVT::v16i16, Custom);
  setOperationAction(ISD::LOAD, MVT::v16f16, Custom);
  setOperationAction(ISD::LOAD, MVT::v8i32, Custom);
  setOperationAction(ISD::LOAD, MVT::v8f32, Custom);
  setOperationAction(ISD::LOAD, MVT::v4f64, Custom);
  setOperationAction(ISD::LOAD, MVT::v4i64, Custom);
 
  // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0.
  // This requires the Performance Monitors extension.
  if (Subtarget->hasPerfMon())
    setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
 
  if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
      getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
    // Issue __sincos_stret if available.
    setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
    setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
  } else {
    setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
    setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
  }
 
  if (Subtarget->getTargetTriple().isOSMSVCRT()) {
    // MSVCRT doesn't have powi; fall back to pow
    setLibcallName(RTLIB::POWI_F32, nullptr);
    setLibcallName(RTLIB::POWI_F64, nullptr);
  }
 
  // Make floating-point constants legal for the large code model, so they don't
  // become loads from the constant pool.
  if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
    setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
    setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
  }
 
  // AArch64 does not have floating-point extending loads, i1 sign-extending
  // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
  for (MVT VT : MVT::fp_valuetypes()) {
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
    setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
  }
  for (MVT VT : MVT::integer_valuetypes())
    setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
 
  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
  setTruncStoreAction(MVT::f128, MVT::f80, Expand);
  setTruncStoreAction(MVT::f128, MVT::f64, Expand);
  setTruncStoreAction(MVT::f128, MVT::f32, Expand);
  setTruncStoreAction(MVT::f128, MVT::f16, Expand);
 
  setOperationAction(ISD::BITCAST, MVT::i16, Custom);
  setOperationAction(ISD::BITCAST, MVT::f16, Custom);
  setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
 
  // Indexed loads and stores are supported.
  for (unsigned im = (unsigned)ISD::PRE_INC;
       im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
    setIndexedLoadAction(im, MVT::i8, Legal);
    setIndexedLoadAction(im, MVT::i16, Legal);
    setIndexedLoadAction(im, MVT::i32, Legal);
    setIndexedLoadAction(im, MVT::i64, Legal);
    setIndexedLoadAction(im, MVT::f64, Legal);
    setIndexedLoadAction(im, MVT::f32, Legal);
    setIndexedLoadAction(im, MVT::f16, Legal);
    setIndexedLoadAction(im, MVT::bf16, Legal);
    setIndexedStoreAction(im, MVT::i8, Legal);
    setIndexedStoreAction(im, MVT::i16, Legal);
    setIndexedStoreAction(im, MVT::i32, Legal);
    setIndexedStoreAction(im, MVT::i64, Legal);
    setIndexedStoreAction(im, MVT::f64, Legal);
    setIndexedStoreAction(im, MVT::f32, Legal);
    setIndexedStoreAction(im, MVT::f16, Legal);
    setIndexedStoreAction(im, MVT::bf16, Legal);
  }
 
  // Trap.
  setOperationAction(ISD::TRAP, MVT::Other, Legal);
  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
  setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
 
  // We combine OR nodes for bitfield operations.
  setTargetDAGCombine(ISD::OR);
  // Try to create BICs for vector ANDs.
  setTargetDAGCombine(ISD::AND);
 
  // Vector add and sub nodes may conceal a high-half opportunity.
  // Also, try to fold ADD into CSINC/CSINV..
  setTargetDAGCombine({ISD::ADD, ISD::ABS, ISD::SUB, ISD::XOR, ISD::SINT_TO_FP,
                       ISD::UINT_TO_FP});
 
  setTargetDAGCombine({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
                       ISD::FP_TO_UINT_SAT, ISD::FDIV});
 
  // Try and combine setcc with csel
  setTargetDAGCombine(ISD::SETCC);
 
  setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
 
  setTargetDAGCombine({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND,
                       ISD::VECTOR_SPLICE, ISD::SIGN_EXTEND_INREG,
                       ISD::CONCAT_VECTORS, ISD::EXTRACT_SUBVECTOR,
                       ISD::INSERT_SUBVECTOR, ISD::STORE, ISD::BUILD_VECTOR});
  if (Subtarget->supportsAddressTopByteIgnored())
    setTargetDAGCombine(ISD::LOAD);
 
  setTargetDAGCombine(ISD::MSTORE);
 
  setTargetDAGCombine(ISD::MUL);
 
  setTargetDAGCombine({ISD::SELECT, ISD::VSELECT});
 
  setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
                       ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
                       ISD::VECREDUCE_ADD, ISD::STEP_VECTOR});
 
  setTargetDAGCombine({ISD::MGATHER, ISD::MSCATTER});
 
  setTargetDAGCombine(ISD::FP_EXTEND);
 
  setTargetDAGCombine(ISD::GlobalAddress);
 
  // In case of strict alignment, avoid an excessive number of byte wide stores.
  MaxStoresPerMemsetOptSize = 8;
  MaxStoresPerMemset =
      Subtarget->requiresStrictAlign() ? MaxStoresPerMemsetOptSize : 32;
 
  MaxGluedStoresPerMemcpy = 4;
  MaxStoresPerMemcpyOptSize = 4;
  MaxStoresPerMemcpy =
      Subtarget->requiresStrictAlign() ? MaxStoresPerMemcpyOptSize : 16;
 
  MaxStoresPerMemmoveOptSize = 4;
  MaxStoresPerMemmove = 4;
 
  MaxLoadsPerMemcmpOptSize = 4;
  MaxLoadsPerMemcmp =
      Subtarget->requiresStrictAlign() ? MaxLoadsPerMemcmpOptSize : 8;
 
  setStackPointerRegisterToSaveRestore(AArch64::SP);
 
  setSchedulingPreference(Sched::Hybrid);
 
  EnableExtLdPromotion = true;
 
  // Set required alignment.
  setMinFunctionAlignment(Align(4));
  // Set preferred alignments.
  setPrefLoopAlignment(Align(1ULL << STI.getPrefLoopLogAlignment()));
  setMaxBytesForAlignment(STI.getMaxBytesForLoopAlignment());
  setPrefFunctionAlignment(Align(1ULL << STI.getPrefFunctionLogAlignment()));
 
  // Only change the limit for entries in a jump table if specified by
  // the sub target, but not at the command line.
  unsigned MaxJT = STI.getMaximumJumpTableSize();
  if (MaxJT && getMaximumJumpTableSize() == UINT_MAX)
    setMaximumJumpTableSize(MaxJT);
 
  setHasExtractBitsInsn(true);
 
  setMaxDivRemBitWidthSupported(128);
 
  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 
  if (Subtarget->hasNEON()) {
    // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
    // silliness like this:
    for (auto Op :
         {ISD::SELECT,         ISD::SELECT_CC,      ISD::SETCC,
          ISD::BR_CC,          ISD::FADD,           ISD::FSUB,
          ISD::FMUL,           ISD::FDIV,           ISD::FMA,
          ISD::FNEG,           ISD::FABS,           ISD::FCEIL,
          ISD::FSQRT,          ISD::FFLOOR,         ISD::FNEARBYINT,
          ISD::FRINT,          ISD::FROUND,         ISD::FROUNDEVEN,
          ISD::FTRUNC,         ISD::FMINNUM,        ISD::FMAXNUM,
          ISD::FMINIMUM,       ISD::FMAXIMUM,       ISD::STRICT_FADD,
          ISD::STRICT_FSUB,    ISD::STRICT_FMUL,    ISD::STRICT_FDIV,
          ISD::STRICT_FMA,     ISD::STRICT_FCEIL,   ISD::STRICT_FFLOOR,
          ISD::STRICT_FSQRT,   ISD::STRICT_FRINT,   ISD::STRICT_FNEARBYINT,
          ISD::STRICT_FROUND,  ISD::STRICT_FTRUNC,  ISD::STRICT_FROUNDEVEN,
          ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM,
          ISD::STRICT_FMAXIMUM})
      setOperationAction(Op, MVT::v1f64, Expand);
 
    for (auto Op :
         {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP,
          ISD::FP_ROUND, ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, ISD::MUL,
          ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT,
          ISD::STRICT_SINT_TO_FP, ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_ROUND})
      setOperationAction(Op, MVT::v1i64, Expand);
 
    // AArch64 doesn't have a direct vector ->f32 conversion instructions for
    // elements smaller than i32, so promote the input to i32 first.
    setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
    setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
 
    // Similarly, there is no direct i32 -> f64 vector conversion instruction.
    // Or, direct i32 -> f16 vector conversion.  Set it so custom, so the
    // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
    for (auto Op : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
                    ISD::STRICT_UINT_TO_FP})
      for (auto VT : {MVT::v2i32, MVT::v2i64, MVT::v4i32})
        setOperationAction(Op, VT, Custom);
 
    if (Subtarget->hasFullFP16()) {
      setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
 
      setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v16i8, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
      setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
      setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
    } else {
      // when AArch64 doesn't have fullfp16 support, promote the input
      // to i32 first.
      setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
      setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
      setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v16i8, MVT::v16i32);
      setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v16i8, MVT::v16i32);
      setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
      setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
      setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
      setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
    }
 
    setOperationAction(ISD::CTLZ,       MVT::v1i64, Expand);
    setOperationAction(ISD::CTLZ,       MVT::v2i64, Expand);
    setOperationAction(ISD::BITREVERSE, MVT::v8i8, Legal);
    setOperationAction(ISD::BITREVERSE, MVT::v16i8, Legal);
    setOperationAction(ISD::BITREVERSE, MVT::v2i32, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::v4i32, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
    setOperationAction(ISD::BITREVERSE, MVT::v2i64, Custom);
    for (auto VT : {MVT::v1i64, MVT::v2i64}) {
      setOperationAction(ISD::UMAX, VT, Custom);
      setOperationAction(ISD::SMAX, VT, Custom);
      setOperationAction(ISD::UMIN, VT, Custom);
      setOperationAction(ISD::SMIN, VT, Custom);
    }
 
    // AArch64 doesn't have MUL.2d:
    setOperationAction(ISD::MUL, MVT::v2i64, Expand);
    // Custom handling for some quad-vector types to detect MULL.
    setOperationAction(ISD::MUL, MVT::v8i16, Custom);
    setOperationAction(ISD::MUL, MVT::v4i32, Custom);
    setOperationAction(ISD::MUL, MVT::v2i64, Custom);
 
    // Saturates
    for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
                    MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
      setOperationAction(ISD::SADDSAT, VT, Legal);
      setOperationAction(ISD::UADDSAT, VT, Legal);
      setOperationAction(ISD::SSUBSAT, VT, Legal);
      setOperationAction(ISD::USUBSAT, VT, Legal);
    }
 
    for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
                   MVT::v4i32}) {
      setOperationAction(ISD::AVGFLOORS, VT, Legal);
      setOperationAction(ISD::AVGFLOORU, VT, Legal);
      setOperationAction(ISD::AVGCEILS, VT, Legal);
      setOperationAction(ISD::AVGCEILU, VT, Legal);
      setOperationAction(ISD::ABDS, VT, Legal);
      setOperationAction(ISD::ABDU, VT, Legal);
    }
 
    // Vector reductions
    for (MVT VT : { MVT::v4f16, MVT::v2f32,
                    MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
      if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) {
        setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
        setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
 
        setOperationAction(ISD::VECREDUCE_FADD, VT, Legal);
      }
    }
    for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
                    MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
      setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
    }
    setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom);
 
    setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
    setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
    // Likewise, narrowing and extending vector loads/stores aren't handled
    // directly.
    for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
      setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
 
      if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
        setOperationAction(ISD::MULHS, VT, Legal);
        setOperationAction(ISD::MULHU, VT, Legal);
      } else {
        setOperationAction(ISD::MULHS, VT, Expand);
        setOperationAction(ISD::MULHU, VT, Expand);
      }
      setOperationAction(ISD::SMUL_LOHI, VT, Expand);
      setOperationAction(ISD::UMUL_LOHI, VT, Expand);
 
      setOperationAction(ISD::BSWAP, VT, Expand);
      setOperationAction(ISD::CTTZ, VT, Expand);
 
      for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
        setTruncStoreAction(VT, InnerVT, Expand);
        setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
      }
    }
 
    // AArch64 has implementations of a lot of rounding-like FP operations.
    for (auto Op :
         {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL, ISD::FRINT, ISD::FTRUNC,
          ISD::FROUND, ISD::FROUNDEVEN, ISD::STRICT_FFLOOR,
          ISD::STRICT_FNEARBYINT, ISD::STRICT_FCEIL, ISD::STRICT_FRINT,
          ISD::STRICT_FTRUNC, ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN}) {
      for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64})
        setOperationAction(Op, Ty, Legal);
      if (Subtarget->hasFullFP16())
        for (MVT Ty : {MVT::v4f16, MVT::v8f16})
          setOperationAction(Op, Ty, Legal);
    }
 
    setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
 
    setLoadExtAction(ISD::EXTLOAD,  MVT::v4i16, MVT::v4i8, Custom);
    setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
    setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
    setLoadExtAction(ISD::EXTLOAD,  MVT::v4i32, MVT::v4i8, Custom);
    setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
    setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
 
    // ADDP custom lowering
    for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
      setOperationAction(ISD::ADD, VT, Custom);
    // FADDP custom lowering
    for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 })
      setOperationAction(ISD::FADD, VT, Custom);
  }
 
  if (Subtarget->hasSME()) {
    setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
  }
 
  // FIXME: Move lowering for more nodes here if those are common between
  // SVE and SME.
  if (Subtarget->hasSVE() || Subtarget->hasSME()) {
    for (auto VT :
         {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
      setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
      setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
    }
  }
 
  if (Subtarget->hasSME())
    setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
 
  if (Subtarget->hasSVE()) {
    for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
      setOperationAction(ISD::BITREVERSE, VT, Custom);
      setOperationAction(ISD::BSWAP, VT, Custom);
      setOperationAction(ISD::CTLZ, VT, Custom);
      setOperationAction(ISD::CTPOP, VT, Custom);
      setOperationAction(ISD::CTTZ, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::UINT_TO_FP, VT, Custom);
      setOperationAction(ISD::SINT_TO_FP, VT, Custom);
      setOperationAction(ISD::FP_TO_UINT, VT, Custom);
      setOperationAction(ISD::FP_TO_SINT, VT, Custom);
      setOperationAction(ISD::MGATHER, VT, Custom);
      setOperationAction(ISD::MSCATTER, VT, Custom);
      setOperationAction(ISD::MLOAD, VT, Custom);
      setOperationAction(ISD::MUL, VT, Custom);
      setOperationAction(ISD::MULHS, VT, Custom);
      setOperationAction(ISD::MULHU, VT, Custom);
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SETCC, VT, Custom);
      setOperationAction(ISD::SDIV, VT, Custom);
      setOperationAction(ISD::UDIV, VT, Custom);
      setOperationAction(ISD::SMIN, VT, Custom);
      setOperationAction(ISD::UMIN, VT, Custom);
      setOperationAction(ISD::SMAX, VT, Custom);
      setOperationAction(ISD::UMAX, VT, Custom);
      setOperationAction(ISD::SHL, VT, Custom);
      setOperationAction(ISD::SRL, VT, Custom);
      setOperationAction(ISD::SRA, VT, Custom);
      setOperationAction(ISD::ABS, VT, Custom);
      setOperationAction(ISD::ABDS, VT, Custom);
      setOperationAction(ISD::ABDU, VT, Custom);
      setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
 
      setOperationAction(ISD::UMUL_LOHI, VT, Expand);
      setOperationAction(ISD::SMUL_LOHI, VT, Expand);
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction(ISD::ROTL, VT, Expand);
      setOperationAction(ISD::ROTR, VT, Expand);
 
      setOperationAction(ISD::SADDSAT, VT, Legal);
      setOperationAction(ISD::UADDSAT, VT, Legal);
      setOperationAction(ISD::SSUBSAT, VT, Legal);
      setOperationAction(ISD::USUBSAT, VT, Legal);
      setOperationAction(ISD::UREM, VT, Expand);
      setOperationAction(ISD::SREM, VT, Expand);
      setOperationAction(ISD::SDIVREM, VT, Expand);
      setOperationAction(ISD::UDIVREM, VT, Expand);
    }
 
    // Illegal unpacked integer vector types.
    for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) {
      setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
    }
 
    // Legalize unpacked bitcasts to REINTERPRET_CAST.
    for (auto VT : {MVT::nxv2i16, MVT::nxv4i16, MVT::nxv2i32, MVT::nxv2bf16,
                    MVT::nxv4bf16, MVT::nxv2f16, MVT::nxv4f16, MVT::nxv2f32})
      setOperationAction(ISD::BITCAST, VT, Custom);
 
    for (auto VT :
         { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8,
           MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 })
      setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
 
    for (auto VT :
         {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::SETCC, VT, Custom);
      setOperationAction(ISD::TRUNCATE, VT, Custom);
      setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
      setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
 
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
 
      // There are no legal MVT::nxv16f## based types.
      if (VT != MVT::nxv16i1) {
        setOperationAction(ISD::SINT_TO_FP, VT, Custom);
        setOperationAction(ISD::UINT_TO_FP, VT, Custom);
      }
    }
 
    // NEON doesn't support masked loads/stores/gathers/scatters, but SVE does
    for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v1f64,
                    MVT::v2f64, MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
                    MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
      setOperationAction(ISD::MLOAD, VT, Custom);
      setOperationAction(ISD::MSTORE, VT, Custom);
      setOperationAction(ISD::MGATHER, VT, Custom);
      setOperationAction(ISD::MSCATTER, VT, Custom);
    }
 
    // Firstly, exclude all scalable vector extending loads/truncating stores,
    // include both integer and floating scalable vector.
    for (MVT VT : MVT::scalable_vector_valuetypes()) {
      for (MVT InnerVT : MVT::scalable_vector_valuetypes()) {
        setTruncStoreAction(VT, InnerVT, Expand);
        setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
        setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
      }
    }
 
    // Then, selectively enable those which we directly support.
    setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i8, Legal);
    setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i16, Legal);
    setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i32, Legal);
    setTruncStoreAction(MVT::nxv4i32, MVT::nxv4i8, Legal);
    setTruncStoreAction(MVT::nxv4i32, MVT::nxv4i16, Legal);
    setTruncStoreAction(MVT::nxv8i16, MVT::nxv8i8, Legal);
    for (auto Op : {ISD::ZEXTLOAD, ISD::SEXTLOAD, ISD::EXTLOAD}) {
      setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i8, Legal);
      setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i16, Legal);
      setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i32, Legal);
      setLoadExtAction(Op, MVT::nxv4i32, MVT::nxv4i8, Legal);
      setLoadExtAction(Op, MVT::nxv4i32, MVT::nxv4i16, Legal);
      setLoadExtAction(Op, MVT::nxv8i16, MVT::nxv8i8, Legal);
    }
 
    // SVE supports truncating stores of 64 and 128-bit vectors
    setTruncStoreAction(MVT::v2i64, MVT::v2i8, Custom);
    setTruncStoreAction(MVT::v2i64, MVT::v2i16, Custom);
    setTruncStoreAction(MVT::v2i64, MVT::v2i32, Custom);
    setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom);
    setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom);
 
    for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
                    MVT::nxv4f32, MVT::nxv2f64}) {
      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::MGATHER, VT, Custom);
      setOperationAction(ISD::MSCATTER, VT, Custom);
      setOperationAction(ISD::MLOAD, VT, Custom);
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
      setOperationAction(ISD::SELECT, VT, Custom);
      setOperationAction(ISD::FADD, VT, Custom);
      setOperationAction(ISD::FCOPYSIGN, VT, Custom);
      setOperationAction(ISD::FDIV, VT, Custom);
      setOperationAction(ISD::FMA, VT, Custom);
      setOperationAction(ISD::FMAXIMUM, VT, Custom);
      setOperationAction(ISD::FMAXNUM, VT, Custom);
      setOperationAction(ISD::FMINIMUM, VT, Custom);
      setOperationAction(ISD::FMINNUM, VT, Custom);
      setOperationAction(ISD::FMUL, VT, Custom);
      setOperationAction(ISD::FNEG, VT, Custom);
      setOperationAction(ISD::FSUB, VT, Custom);
      setOperationAction(ISD::FCEIL, VT, Custom);
      setOperationAction(ISD::FFLOOR, VT, Custom);
      setOperationAction(ISD::FNEARBYINT, VT, Custom);
      setOperationAction(ISD::FRINT, VT, Custom);
      setOperationAction(ISD::FROUND, VT, Custom);
      setOperationAction(ISD::FROUNDEVEN, VT, Custom);
      setOperationAction(ISD::FTRUNC, VT, Custom);
      setOperationAction(ISD::FSQRT, VT, Custom);
      setOperationAction(ISD::FABS, VT, Custom);
      setOperationAction(ISD::FP_EXTEND, VT, Custom);
      setOperationAction(ISD::FP_ROUND, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
      setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
      setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
      setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
 
      setOperationAction(ISD::SELECT_CC, VT, Expand);
      setOperationAction(ISD::FREM, VT, Expand);
      setOperationAction(ISD::FPOW, VT, Expand);
      setOperationAction(ISD::FPOWI, VT, Expand);
      setOperationAction(ISD::FCOS, VT, Expand);
      setOperationAction(ISD::FSIN, VT, Expand);
      setOperationAction(ISD::FSINCOS, VT, Expand);
      setOperationAction(ISD::FEXP, VT, Expand);
      setOperationAction(ISD::FEXP2, VT, Expand);
      setOperationAction(ISD::FLOG, VT, Expand);
      setOperationAction(ISD::FLOG2, VT, Expand);
      setOperationAction(ISD::FLOG10, VT, Expand);
 
      setCondCodeAction(ISD::SETO, VT, Expand);
      setCondCodeAction(ISD::SETOLT, VT, Expand);
      setCondCodeAction(ISD::SETLT, VT, Expand);
      setCondCodeAction(ISD::SETOLE, VT, Expand);
      setCondCodeAction(ISD::SETLE, VT, Expand);
      setCondCodeAction(ISD::SETULT, VT, Expand);
      setCondCodeAction(ISD::SETULE, VT, Expand);
      setCondCodeAction(ISD::SETUGE, VT, Expand);
      setCondCodeAction(ISD::SETUGT, VT, Expand);
      setCondCodeAction(ISD::SETUEQ, VT, Expand);
      setCondCodeAction(ISD::SETONE, VT, Expand);
    }
 
    for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
      setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
      setOperationAction(ISD::MGATHER, VT, Custom);
      setOperationAction(ISD::MSCATTER, VT, Custom);
      setOperationAction(ISD::MLOAD, VT, Custom);
      setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
      setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
    }
 
    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
    setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
 
    // NEON doesn't support integer divides, but SVE does
    for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
                    MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
      setOperationAction(ISD::SDIV, VT, Custom);
      setOperationAction(ISD::UDIV, VT, Custom);
    }
 
    // NEON doesn't support 64-bit vector integer muls, but SVE does.
    setOperationAction(ISD::MUL, MVT::v1i64, Custom);
    setOperationAction(ISD::MUL, MVT::v2i64, Custom);
 
    // NEON doesn't support across-vector reductions, but SVE does.
    for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v2f64})
      setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
 
    // NOTE: Currently this has to happen after computeRegisterProperties rather
    // than the preferred option of combining it with the addRegisterClass call.
    if (Subtarget->useSVEForFixedLengthVectors()) {
      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
        if (useSVEForFixedLengthVectorVT(VT))
          addTypeForFixedLengthSVE(VT);
      for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
        if (useSVEForFixedLengthVectorVT(VT))
          addTypeForFixedLengthSVE(VT);
 
      // 64bit results can mean a bigger than NEON input.
      for (auto VT : {MVT::v8i8, MVT::v4i16})
        setOperationAction(ISD::TRUNCATE, VT, Custom);
      setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
 
      // 128bit results imply a bigger than NEON input.
      for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
        setOperationAction(ISD::TRUNCATE, VT, Custom);
      for (auto VT : {MVT::v8f16, MVT::v4f32})
        setOperationAction(ISD::FP_ROUND, VT, Custom);
 
      // These operations are not supported on NEON but SVE can do them.
      setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
      setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
      setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
      setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
      setOperationAction(ISD::MULHS, MVT::v1i64, Custom);
      setOperationAction(ISD::MULHS, MVT::v2i64, Custom);
      setOperationAction(ISD::MULHU, MVT::v1i64, Custom);
      setOperationAction(ISD::MULHU, MVT::v2i64, Custom);
      setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
      setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
      setOperationAction(ISD::SMIN, MVT::v1i64, Custom);
      setOperationAction(ISD::SMIN, MVT::v2i64, Custom);
      setOperationAction(ISD::UMAX, MVT::v1i64, Custom);
      setOperationAction(ISD::UMAX, MVT::v2i64, Custom);
      setOperationAction(ISD::UMIN, MVT::v1i64, Custom);
      setOperationAction(ISD::UMIN, MVT::v2i64, Custom);
      setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom);
      setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom);
      setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom);
      setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
 
      // Int operations with no NEON support.
      for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
                      MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
        setOperationAction(ISD::BITREVERSE, VT, Custom);
        setOperationAction(ISD::CTTZ, VT, Custom);
        setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
        setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
        setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
      }
 
 
      // Use SVE for vectors with more than 2 elements.
      for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
        setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
    }
 
    setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv2i1, MVT::nxv2i64);
    setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv4i1, MVT::nxv4i32);
    setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv8i1, MVT::nxv8i16);
    setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv16i1, MVT::nxv16i8);
 
    setOperationAction(ISD::VSCALE, MVT::i32, Custom);
  }
 
  if (Subtarget->hasMOPS() && Subtarget->hasMTE()) {
    // Only required for llvm.aarch64.mops.memset.tag
    setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
  }
 
  PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
 
  IsStrictFPEnabled = true;
}
 
void AArch64TargetLowering::addTypeForNEON(MVT VT) {
  assert(VT.isVector() && "VT should be a vector type");
 
  if (VT.isFloatingPoint()) {
    MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
    setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
    setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
  }
 
  // Mark vector float intrinsics as expand.
  if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
    setOperationAction(ISD::FSIN, VT, Expand);
    setOperationAction(ISD::FCOS, VT, Expand);
    setOperationAction(ISD::FPOW, VT, Expand);
    setOperationAction(ISD::FLOG, VT, Expand);
    setOperationAction(ISD::FLOG2, VT, Expand);
    setOperationAction(ISD::FLOG10, VT, Expand);
    setOperationAction(ISD::FEXP, VT, Expand);
    setOperationAction(ISD::FEXP2, VT, Expand);
  }
 
  // But we do support custom-lowering for FCOPYSIGN.
  if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
      ((VT == MVT::v4f16 || VT == MVT::v8f16) && Subtarget->hasFullFP16()))
    setOperationAction(ISD::FCOPYSIGN, VT, Custom);
 
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
  setOperationAction(ISD::SRA, VT, Custom);
  setOperationAction(ISD::SRL, VT, Custom);
  setOperationAction(ISD::SHL, VT, Custom);
  setOperationAction(ISD::OR, VT, Custom);
  setOperationAction(ISD::SETCC, VT, Custom);
  setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
 
  setOperationAction(ISD::SELECT, VT, Expand);
  setOperationAction(ISD::SELECT_CC, VT, Expand);
  setOperationAction(ISD::VSELECT, VT, Expand);
  for (MVT InnerVT : MVT::all_valuetypes())
    setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
 
  // CNT supports only B element sizes, then use UADDLP to widen.
  if (VT != MVT::v8i8 && VT != MVT::v16i8)
    setOperationAction(ISD::CTPOP, VT, Custom);
 
  setOperationAction(ISD::UDIV, VT, Expand);
  setOperationAction(ISD::SDIV, VT, Expand);
  setOperationAction(ISD::UREM, VT, Expand);
  setOperationAction(ISD::SREM, VT, Expand);
  setOperationAction(ISD::FREM, VT, Expand);
 
  for (unsigned Opcode :
       {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
        ISD::FP_TO_UINT_SAT, ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
    setOperationAction(Opcode, VT, Custom);
 
  if (!VT.isFloatingPoint())
    setOperationAction(ISD::ABS, VT, Legal);
 
  // [SU][MIN|MAX] are available for all NEON types apart from i64.
  if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
    for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
      setOperationAction(Opcode, VT, Legal);
 
  // F[MIN|MAX][NUM|NAN] and simple strict operations are available for all FP
  // NEON types.
  if (VT.isFloatingPoint() &&
      VT.getVectorElementType() != MVT::bf16 &&
      (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
    for (unsigned Opcode :
         {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM,
          ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_FMINNUM,
          ISD::STRICT_FMAXNUM, ISD::STRICT_FADD, ISD::STRICT_FSUB,
          ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FMA,
          ISD::STRICT_FSQRT})
      setOperationAction(Opcode, VT, Legal);
 
  // Strict fp extend and trunc are legal
  if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 16)
    setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
  if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 64)
    setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal);
 
  // FIXME: We could potentially make use of the vector comparison instructions
  // for STRICT_FSETCC and STRICT_FSETCSS, but there's a number of
  // complications:
  //  * FCMPEQ/NE are quiet comparisons, the rest are signalling comparisons,
  //    so we would need to expand when the condition code doesn't match the
  //    kind of comparison.
  //  * Some kinds of comparison require more than one FCMXY instruction so
  //    would need to be expanded instead.
  //  * The lowering of the non-strict versions involves target-specific ISD
  //    nodes so we would likely need to add strict versions of all of them and
  //    handle them appropriately.
  setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
  setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
 
  if (Subtarget->isLittleEndian()) {
    for (unsigned im = (unsigned)ISD::PRE_INC;
         im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
      setIndexedLoadAction(im, VT, Legal);
      setIndexedStoreAction(im, VT, Legal);
    }
  }
}
 
bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
                                                          EVT OpVT) const {
  // Only SVE has a 1:1 mapping from intrinsic -> instruction (whilelo).
  if (!Subtarget->hasSVE())
    return true;
 
  // We can only support legal predicate result types. We can use the SVE
  // whilelo instruction for generating fixed-width predicates too.
  if (ResVT != MVT::nxv2i1 && ResVT != MVT::nxv4i1 && ResVT != MVT::nxv8i1 &&
      ResVT != MVT::nxv16i1 && ResVT != MVT::v2i1 && ResVT != MVT::v4i1 &&
      ResVT != MVT::v8i1 && ResVT != MVT::v16i1)
    return true;
 
  // The whilelo instruction only works with i32 or i64 scalar inputs.
  if (OpVT != MVT::i32 && OpVT != MVT::i64)
    return true;
 
  return false;
}
 
void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
 
  // By default everything must be expanded.
  for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
    setOperationAction(Op, VT, Expand);
 
  // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one.
  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
 
  if (VT.isFloatingPoint()) {
    setCondCodeAction(ISD::SETO, VT, Expand);
    setCondCodeAction(ISD::SETOLT, VT, Expand);
    setCondCodeAction(ISD::SETLT, VT, Expand);
    setCondCodeAction(ISD::SETOLE, VT, Expand);
    setCondCodeAction(ISD::SETLE, VT, Expand);
    setCondCodeAction(ISD::SETULT, VT, Expand);
    setCondCodeAction(ISD::SETULE, VT, Expand);
    setCondCodeAction(ISD::SETUGE, VT, Expand);
    setCondCodeAction(ISD::SETUGT, VT, Expand);
    setCondCodeAction(ISD::SETUEQ, VT, Expand);
    setCondCodeAction(ISD::SETONE, VT, Expand);
  }
 
  // Mark integer truncating stores/extending loads as having custom lowering
  if (VT.isInteger()) {
    MVT InnerVT = VT.changeVectorElementType(MVT::i8);
    while (InnerVT != VT) {
      setTruncStoreAction(VT, InnerVT, Custom);
      setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Custom);
      setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Custom);
      InnerVT = InnerVT.changeVectorElementType(
          MVT::getIntegerVT(2 * InnerVT.getScalarSizeInBits()));
    }
  }
 
  // Mark floating-point truncating stores/extending loads as having custom
  // lowering
  if (VT.isFloatingPoint()) {
    MVT InnerVT = VT.changeVectorElementType(MVT::f16);
    while (InnerVT != VT) {
      setTruncStoreAction(VT, InnerVT, Custom);
      setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Custom);
      InnerVT = InnerVT.changeVectorElementType(
          MVT::getFloatingPointVT(2 * InnerVT.getScalarSizeInBits()));
    }
  }
 
  // Lower fixed length vector operations to scalable equivalents.
  setOperationAction(ISD::ABS, VT, Custom);
  setOperationAction(ISD::ADD, VT, Custom);
  setOperationAction(ISD::AND, VT, Custom);
  setOperationAction(ISD::ANY_EXTEND, VT, Custom);
  setOperationAction(ISD::BITCAST, VT, Custom);
  setOperationAction(ISD::BITREVERSE, VT, Custom);
  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
  setOperationAction(ISD::BSWAP, VT, Custom);
  setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
  setOperationAction(ISD::CTLZ, VT, Custom);
  setOperationAction(ISD::CTPOP, VT, Custom);
  setOperationAction(ISD::CTTZ, VT, Custom);
  setOperationAction(ISD::FABS, VT, Custom);
  setOperationAction(ISD::FADD, VT, Custom);
  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
  setOperationAction(ISD::FCEIL, VT, Custom);
  setOperationAction(ISD::FCOPYSIGN, VT, Custom);
  setOperationAction(ISD::FDIV, VT, Custom);
  setOperationAction(ISD::FFLOOR, VT, Custom);
  setOperationAction(ISD::FMA, VT, Custom);
  setOperationAction(ISD::FMAXIMUM, VT, Custom);
  setOperationAction(ISD::FMAXNUM, VT, Custom);
  setOperationAction(ISD::FMINIMUM, VT, Custom);
  setOperationAction(ISD::FMINNUM, VT, Custom);
  setOperationAction(ISD::FMUL, VT, Custom);
  setOperationAction(ISD::FNEARBYINT, VT, Custom);
  setOperationAction(ISD::FNEG, VT, Custom);
  setOperationAction(ISD::FP_EXTEND, VT, Custom);
  setOperationAction(ISD::FP_ROUND, VT, Custom);
  setOperationAction(ISD::FP_TO_SINT, VT, Custom);
  setOperationAction(ISD::FP_TO_UINT, VT, Custom);
  setOperationAction(ISD::FRINT, VT, Custom);
  setOperationAction(ISD::FROUND, VT, Custom);
  setOperationAction(ISD::FROUNDEVEN, VT, Custom);
  setOperationAction(ISD::FSQRT, VT, Custom);
  setOperationAction(ISD::FSUB, VT, Custom);
  setOperationAction(ISD::FTRUNC, VT, Custom);
  setOperationAction(ISD::LOAD, VT, Custom);
  setOperationAction(ISD::MGATHER, VT, Custom);
  setOperationAction(ISD::MLOAD, VT, Custom);
  setOperationAction(ISD::MSCATTER, VT, Custom);
  setOperationAction(ISD::MSTORE, VT, Custom);
  setOperationAction(ISD::MUL, VT, Custom);
  setOperationAction(ISD::MULHS, VT, Custom);
  setOperationAction(ISD::MULHU, VT, Custom);
  setOperationAction(ISD::OR, VT, Custom);
  setOperationAction(ISD::SDIV, VT, Custom);
  setOperationAction(ISD::SELECT, VT, Custom);
  setOperationAction(ISD::SETCC, VT, Custom);
  setOperationAction(ISD::SHL, VT, Custom);
  setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
  setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
  setOperationAction(ISD::SINT_TO_FP, VT, Custom);
  setOperationAction(ISD::SMAX, VT, Custom);
  setOperationAction(ISD::SMIN, VT, Custom);
  setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
  setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
  setOperationAction(ISD::SRA, VT, Custom);
  setOperationAction(ISD::SRL, VT, Custom);
  setOperationAction(ISD::STORE, VT, Custom);
  setOperationAction(ISD::SUB, VT, Custom);
  setOperationAction(ISD::TRUNCATE, VT, Custom);
  setOperationAction(ISD::UDIV, VT, Custom);
  setOperationAction(ISD::UINT_TO_FP, VT, Custom);
  setOperationAction(ISD::UMAX, VT, Custom);
  setOperationAction(ISD::UMIN, VT, Custom);
  setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
  setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
  setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
  setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
  setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
  setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
  setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
  setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
  setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
  setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
  setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
  setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
  setOperationAction(ISD::VSELECT, VT, Custom);
  setOperationAction(ISD::XOR, VT, Custom);
  setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
}
 
void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
  addRegisterClass(VT, &AArch64::FPR64RegClass);
  addTypeForNEON(VT);
}
 
void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
  addRegisterClass(VT, &AArch64::FPR128RegClass);
  addTypeForNEON(VT);
}
 
EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &,
                                              LLVMContext &C, EVT VT) const {
  if (!VT.isVector())
    return MVT::i32;
  if (VT.isScalableVector())
    return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
  return VT.changeVectorElementTypeToInteger();
}
 
static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
                               const APInt &Demanded,
                               TargetLowering::TargetLoweringOpt &TLO,
                               unsigned NewOpc) {
  uint64_t OldImm = Imm, NewImm, Enc;
  uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
 
  // Return if the immediate is already all zeros, all ones, a bimm32 or a
  // bimm64.
  if (Imm == 0 || Imm == Mask ||
      AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
    return false;
 
  unsigned EltSize = Size;
  uint64_t DemandedBits = Demanded.getZExtValue();
 
  // Clear bits that are not demanded.
  Imm &= DemandedBits;
 
  while (true) {
    // The goal here is to set the non-demanded bits in a way that minimizes
    // the number of switching between 0 and 1. In order to achieve this goal,
    // we set the non-demanded bits to the value of the preceding demanded bits.
    // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
    // non-demanded bit), we copy bit0 (1) to the least significant 'x',
    // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
    // The final result is 0b11000011.
    uint64_t NonDemandedBits = ~DemandedBits;
    uint64_t InvertedImm = ~Imm & DemandedBits;
    uint64_t RotatedImm =
        ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
        NonDemandedBits;
    uint64_t Sum = RotatedImm + NonDemandedBits;
    bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
    uint64_t Ones = (Sum + Carry) & NonDemandedBits;
    NewImm = (Imm | Ones) & Mask;
 
    // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
    // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
    // we halve the element size and continue the search.
    if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
      break;
 
    // We cannot shrink the element size any further if it is 2-bits.
    if (EltSize == 2)
      return false;
 
    EltSize /= 2;
    Mask >>= EltSize;
    uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
 
    // Return if there is mismatch in any of the demanded bits of Imm and Hi.
    if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
      return false;
 
    // Merge the upper and lower halves of Imm and DemandedBits.
    Imm |= Hi;
    DemandedBits |= DemandedBitsHi;
  }
 
  ++NumOptimizedImms;
 
  // Replicate the element across the register width.
  while (EltSize < Size) {
    NewImm |= NewImm << EltSize;
    EltSize *= 2;
  }
 
  (void)OldImm;
  assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
         "demanded bits should never be altered");
  assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm");
 
  // Create the new constant immediate node.
  EVT VT = Op.getValueType();
  SDLoc DL(Op);
  SDValue New;
 
  // If the new constant immediate is all-zeros or all-ones, let the target
  // independent DAG combine optimize this node.
  if (NewImm == 0 || NewImm == OrigMask) {
    New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
                          TLO.DAG.getConstant(NewImm, DL, VT));
  // Otherwise, create a machine node so that target independent DAG combine
  // doesn't undo this optimization.
  } else {
    Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
    SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
    New = SDValue(
        TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
  }
 
  return TLO.CombineTo(Op, New);
}
 
bool AArch64TargetLowering::targetShrinkDemandedConstant(
    SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
    TargetLoweringOpt &TLO) const {
  // Delay this optimization to as late as possible.
  if (!TLO.LegalOps)
    return false;
 
  if (!EnableOptimizeLogicalImm)
    return false;
 
  EVT VT = Op.getValueType();
  if (VT.isVector())
    return false;
 
  unsigned Size = VT.getSizeInBits();
  assert((Size == 32 || Size == 64) &&
         "i32 or i64 is expected after legalization.");
 
  // Exit early if we demand all bits.
  if (DemandedBits.countPopulation() == Size)
    return false;
 
  unsigned NewOpc;
  switch (Op.getOpcode()) {
  default:
    return false;
  case ISD::AND:
    NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
    break;
  case ISD::OR:
    NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
    break;
  case ISD::XOR:
    NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
    break;
  }
  ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
  if (!C)
    return false;
  uint64_t Imm = C->getZExtValue();
  return optimizeLogicalImm(Op, Size, Imm, DemandedBits, TLO, NewOpc);
}
 
/// computeKnownBitsForTargetNode - Determine which of the bits specified in
/// Mask are known to be either zero or one and return them Known.
void AArch64TargetLowering::computeKnownBitsForTargetNode(
    const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
    const SelectionDAG &DAG, unsigned Depth) const {
  switch (Op.getOpcode()) {
  default:
    break;
  case AArch64ISD::DUP: {
    SDValue SrcOp = Op.getOperand(0);
    Known = DAG.computeKnownBits(SrcOp, Depth + 1);
    if (SrcOp.getValueSizeInBits() != Op.getScalarValueSizeInBits()) {
      assert(SrcOp.getValueSizeInBits() > Op.getScalarValueSizeInBits() &&
             "Expected DUP implicit truncation");
      Known = Known.trunc(Op.getScalarValueSizeInBits());
    }
    break;
  }
  case AArch64ISD::CSEL: {
    KnownBits Known2;
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
    Known = KnownBits::commonBits(Known, Known2);
    break;
  }
  case AArch64ISD::BICi: {
    // Compute the bit cleared value.
    uint64_t Mask =
        ~(Op->getConstantOperandVal(1) << Op->getConstantOperandVal(2));
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known &= KnownBits::makeConstant(APInt(Known.getBitWidth(), Mask));
    break;
  }
  case AArch64ISD::VLSHR: {
    KnownBits Known2;
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
    Known = KnownBits::lshr(Known, Known2);
    break;
  }
  case AArch64ISD::VASHR: {
    KnownBits Known2;
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
    Known = KnownBits::ashr(Known, Known2);
    break;
  }
  case AArch64ISD::LOADgot:
  case AArch64ISD::ADDlow: {
    if (!Subtarget->isTargetILP32())
      break;
    // In ILP32 mode all valid pointers are in the low 4GB of the address-space.
    Known.Zero = APInt::getHighBitsSet(64, 32);
    break;
  }
  case AArch64ISD::ASSERT_ZEXT_BOOL: {
    Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
    Known.Zero |= APInt(Known.getBitWidth(), 0xFE);
    break;
  }
  case ISD::INTRINSIC_W_CHAIN: {
    ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
    Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
    switch (IntID) {
    default: return;
    case Intrinsic::aarch64_ldaxr:
    case Intrinsic::aarch64_ldxr: {
      unsigned BitWidth = Known.getBitWidth();
      EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
      unsigned MemBits = VT.getScalarSizeInBits();
      Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
      return;
    }
    }
    break;
  }
  case ISD::INTRINSIC_WO_CHAIN:
  case ISD::INTRINSIC_VOID: {
    unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
    switch (IntNo) {
    default:
      break;
    case Intrinsic::aarch64_neon_umaxv:
    case Intrinsic::aarch64_neon_uminv: {
      // Figure out the datatype of the vector operand. The UMINV instruction
      // will zero extend the result, so we can mark as known zero all the
      // bits larger than the element datatype. 32-bit or larget doesn't need
      // this as those are legal types and will be handled by isel directly.
      MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
      unsigned BitWidth = Known.getBitWidth();
      if (VT == MVT::v8i8 || VT == MVT::v16i8) {
        assert(BitWidth >= 8 && "Unexpected width!");
        APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
        Known.Zero |= Mask;
      } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
        assert(BitWidth >= 16 && "Unexpected width!");
        APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
        Known.Zero |= Mask;
      }
      break;
    } break;
    }
  }
  }
}
 
MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
                                                  EVT) const {
  return MVT::i64;
}
 
bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
    EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
    bool *Fast) const {
  if (Subtarget->requiresStrictAlign())
    return false;
 
  if (Fast) {
    // Some CPUs are fine with unaligned stores except for 128-bit ones.
    *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
            // See comments in performSTORECombine() for more details about
            // these conditions.
 
            // Code that uses clang vector extensions can mark that it
            // wants unaligned accesses to be treated as fast by
            // underspecifying alignment to be 1 or 2.
            Alignment <= 2 ||
 
            // Disregard v2i64. Memcpy lowering produces those and splitting
            // them regresses performance on micro-benchmarks and olden/bh.
            VT == MVT::v2i64;
  }
  return true;
}
 
// Same as above but handling LLTs instead.
bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
    LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
    bool *Fast) const {
  if (Subtarget->requiresStrictAlign())
    return false;
 
  if (Fast) {
    // Some CPUs are fine with unaligned stores except for 128-bit ones.
    *Fast = !Subtarget->isMisaligned128StoreSlow() ||
            Ty.getSizeInBytes() != 16 ||
            // See comments in performSTORECombine() for more details about
            // these conditions.
 
            // Code that uses clang vector extensions can mark that it
            // wants unaligned accesses to be treated as fast by
            // underspecifying alignment to be 1 or 2.
            Alignment <= 2 ||
 
            // Disregard v2i64. Memcpy lowering produces those and splitting
            // them regresses performance on micro-benchmarks and olden/bh.
            Ty == LLT::fixed_vector(2, 64);
  }
  return true;
}
 
FastISel *
AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
                                      const TargetLibraryInfo *libInfo) const {
  return AArch64::createFastISel(funcInfo, libInfo);
}
 
const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
#define MAKE_CASE(V)                                                           \
  case V:                                                                      \
    return #V;
  switch ((AArch64ISD::NodeType)Opcode) {
  case AArch64ISD::FIRST_NUMBER:
    break;
    MAKE_CASE(AArch64ISD::SMSTART)
    MAKE_CASE(AArch64ISD::SMSTOP)
    MAKE_CASE(AArch64ISD::CALL)
    MAKE_CASE(AArch64ISD::ADRP)
    MAKE_CASE(AArch64ISD::ADR)
    MAKE_CASE(AArch64ISD::ADDlow)
    MAKE_CASE(AArch64ISD::LOADgot)
    MAKE_CASE(AArch64ISD::RET_FLAG)
    MAKE_CASE(AArch64ISD::BRCOND)
    MAKE_CASE(AArch64ISD::CSEL)
    MAKE_CASE(AArch64ISD::CSINV)
    MAKE_CASE(AArch64ISD::CSNEG)
    MAKE_CASE(AArch64ISD::CSINC)
    MAKE_CASE(AArch64ISD::THREAD_POINTER)
    MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
    MAKE_CASE(AArch64ISD::ABDS_PRED)
    MAKE_CASE(AArch64ISD::ABDU_PRED)
    MAKE_CASE(AArch64ISD::MUL_PRED)
    MAKE_CASE(AArch64ISD::MULHS_PRED)
    MAKE_CASE(AArch64ISD::MULHU_PRED)
    MAKE_CASE(AArch64ISD::SDIV_PRED)
    MAKE_CASE(AArch64ISD::SHL_PRED)
    MAKE_CASE(AArch64ISD::SMAX_PRED)
    MAKE_CASE(AArch64ISD::SMIN_PRED)
    MAKE_CASE(AArch64ISD::SRA_PRED)
    MAKE_CASE(AArch64ISD::SRL_PRED)
    MAKE_CASE(AArch64ISD::UDIV_PRED)
    MAKE_CASE(AArch64ISD::UMAX_PRED)
    MAKE_CASE(AArch64ISD::UMIN_PRED)
    MAKE_CASE(AArch64ISD::SRAD_MERGE_OP1)
    MAKE_CASE(AArch64ISD::FNEG_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FCEIL_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FFLOOR_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FRINT_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FROUND_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FTRUNC_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FP_ROUND_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FP_EXTEND_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FCVTZU_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FCVTZS_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FSQRT_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FRECPX_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::FABS_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::ABS_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::NEG_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::SETCC_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::ADC)
    MAKE_CASE(AArch64ISD::SBC)
    MAKE_CASE(AArch64ISD::ADDS)
    MAKE_CASE(AArch64ISD::SUBS)
    MAKE_CASE(AArch64ISD::ADCS)
    MAKE_CASE(AArch64ISD::SBCS)
    MAKE_CASE(AArch64ISD::ANDS)
    MAKE_CASE(AArch64ISD::CCMP)
    MAKE_CASE(AArch64ISD::CCMN)
    MAKE_CASE(AArch64ISD::FCCMP)
    MAKE_CASE(AArch64ISD::FCMP)
    MAKE_CASE(AArch64ISD::STRICT_FCMP)
    MAKE_CASE(AArch64ISD::STRICT_FCMPE)
    MAKE_CASE(AArch64ISD::DUP)
    MAKE_CASE(AArch64ISD::DUPLANE8)
    MAKE_CASE(AArch64ISD::DUPLANE16)
    MAKE_CASE(AArch64ISD::DUPLANE32)
    MAKE_CASE(AArch64ISD::DUPLANE64)
    MAKE_CASE(AArch64ISD::DUPLANE128)
    MAKE_CASE(AArch64ISD::MOVI)
    MAKE_CASE(AArch64ISD::MOVIshift)
    MAKE_CASE(AArch64ISD::MOVIedit)
    MAKE_CASE(AArch64ISD::MOVImsl)
    MAKE_CASE(AArch64ISD::FMOV)
    MAKE_CASE(AArch64ISD::MVNIshift)
    MAKE_CASE(AArch64ISD::MVNImsl)
    MAKE_CASE(AArch64ISD::BICi)
    MAKE_CASE(AArch64ISD::ORRi)
    MAKE_CASE(AArch64ISD::BSP)
    MAKE_CASE(AArch64ISD::EXTR)
    MAKE_CASE(AArch64ISD::ZIP1)
    MAKE_CASE(AArch64ISD::ZIP2)
    MAKE_CASE(AArch64ISD::UZP1)
    MAKE_CASE(AArch64ISD::UZP2)
    MAKE_CASE(AArch64ISD::TRN1)
    MAKE_CASE(AArch64ISD::TRN2)
    MAKE_CASE(AArch64ISD::REV16)
    MAKE_CASE(AArch64ISD::REV32)
    MAKE_CASE(AArch64ISD::REV64)
    MAKE_CASE(AArch64ISD::EXT)
    MAKE_CASE(AArch64ISD::SPLICE)
    MAKE_CASE(AArch64ISD::VSHL)
    MAKE_CASE(AArch64ISD::VLSHR)
    MAKE_CASE(AArch64ISD::VASHR)
    MAKE_CASE(AArch64ISD::VSLI)
    MAKE_CASE(AArch64ISD::VSRI)
    MAKE_CASE(AArch64ISD::CMEQ)
    MAKE_CASE(AArch64ISD::CMGE)
    MAKE_CASE(AArch64ISD::CMGT)
    MAKE_CASE(AArch64ISD::CMHI)
    MAKE_CASE(AArch64ISD::CMHS)
    MAKE_CASE(AArch64ISD::FCMEQ)
    MAKE_CASE(AArch64ISD::FCMGE)
    MAKE_CASE(AArch64ISD::FCMGT)
    MAKE_CASE(AArch64ISD::CMEQz)
    MAKE_CASE(AArch64ISD::CMGEz)
    MAKE_CASE(AArch64ISD::CMGTz)
    MAKE_CASE(AArch64ISD::CMLEz)
    MAKE_CASE(AArch64ISD::CMLTz)
    MAKE_CASE(AArch64ISD::FCMEQz)
    MAKE_CASE(AArch64ISD::FCMGEz)
    MAKE_CASE(AArch64ISD::FCMGTz)
    MAKE_CASE(AArch64ISD::FCMLEz)
    MAKE_CASE(AArch64ISD::FCMLTz)
    MAKE_CASE(AArch64ISD::SADDV)
    MAKE_CASE(AArch64ISD::UADDV)
    MAKE_CASE(AArch64ISD::SDOT)
    MAKE_CASE(AArch64ISD::UDOT)
    MAKE_CASE(AArch64ISD::SMINV)
    MAKE_CASE(AArch64ISD::UMINV)
    MAKE_CASE(AArch64ISD::SMAXV)
    MAKE_CASE(AArch64ISD::UMAXV)
    MAKE_CASE(AArch64ISD::SADDV_PRED)
    MAKE_CASE(AArch64ISD::UADDV_PRED)
    MAKE_CASE(AArch64ISD::SMAXV_PRED)
    MAKE_CASE(AArch64ISD::UMAXV_PRED)
    MAKE_CASE(AArch64ISD::SMINV_PRED)
    MAKE_CASE(AArch64ISD::UMINV_PRED)
    MAKE_CASE(AArch64ISD::ORV_PRED)
    MAKE_CASE(AArch64ISD::EORV_PRED)
    MAKE_CASE(AArch64ISD::ANDV_PRED)
    MAKE_CASE(AArch64ISD::CLASTA_N)
    MAKE_CASE(AArch64ISD::CLASTB_N)
    MAKE_CASE(AArch64ISD::LASTA)
    MAKE_CASE(AArch64ISD::LASTB)
    MAKE_CASE(AArch64ISD::REINTERPRET_CAST)
    MAKE_CASE(AArch64ISD::LS64_BUILD)
    MAKE_CASE(AArch64ISD::LS64_EXTRACT)
    MAKE_CASE(AArch64ISD::TBL)
    MAKE_CASE(AArch64ISD::FADD_PRED)
    MAKE_CASE(AArch64ISD::FADDA_PRED)
    MAKE_CASE(AArch64ISD::FADDV_PRED)
    MAKE_CASE(AArch64ISD::FDIV_PRED)
    MAKE_CASE(AArch64ISD::FMA_PRED)
    MAKE_CASE(AArch64ISD::FMAX_PRED)
    MAKE_CASE(AArch64ISD::FMAXV_PRED)
    MAKE_CASE(AArch64ISD::FMAXNM_PRED)
    MAKE_CASE(AArch64ISD::FMAXNMV_PRED)
    MAKE_CASE(AArch64ISD::FMIN_PRED)
    MAKE_CASE(AArch64ISD::FMINV_PRED)
    MAKE_CASE(AArch64ISD::FMINNM_PRED)
    MAKE_CASE(AArch64ISD::FMINNMV_PRED)
    MAKE_CASE(AArch64ISD::FMUL_PRED)
    MAKE_CASE(AArch64ISD::FSUB_PRED)
    MAKE_CASE(AArch64ISD::RDSVL)
    MAKE_CASE(AArch64ISD::BIC)
    MAKE_CASE(AArch64ISD::BIT)
    MAKE_CASE(AArch64ISD::CBZ)
    MAKE_CASE(AArch64ISD::CBNZ)
    MAKE_CASE(AArch64ISD::TBZ)
    MAKE_CASE(AArch64ISD::TBNZ)
    MAKE_CASE(AArch64ISD::TC_RETURN)
    MAKE_CASE(AArch64ISD::PREFETCH)
    MAKE_CASE(AArch64ISD::SITOF)
    MAKE_CASE(AArch64ISD::UITOF)
    MAKE_CASE(AArch64ISD::NVCAST)
    MAKE_CASE(AArch64ISD::MRS)
    MAKE_CASE(AArch64ISD::SQSHL_I)
    MAKE_CASE(AArch64ISD::UQSHL_I)
    MAKE_CASE(AArch64ISD::SRSHR_I)
    MAKE_CASE(AArch64ISD::URSHR_I)
    MAKE_CASE(AArch64ISD::SQSHLU_I)
    MAKE_CASE(AArch64ISD::WrapperLarge)
    MAKE_CASE(AArch64ISD::LD2post)
    MAKE_CASE(AArch64ISD::LD3post)
    MAKE_CASE(AArch64ISD::LD4post)
    MAKE_CASE(AArch64ISD::ST2post)
    MAKE_CASE(AArch64ISD::ST3post)
    MAKE_CASE(AArch64ISD::ST4post)
    MAKE_CASE(AArch64ISD::LD1x2post)
    MAKE_CASE(AArch64ISD::LD1x3post)
    MAKE_CASE(AArch64ISD::LD1x4post)
    MAKE_CASE(AArch64ISD::ST1x2post)
    MAKE_CASE(AArch64ISD::ST1x3post)
    MAKE_CASE(AArch64ISD::ST1x4post)
    MAKE_CASE(AArch64ISD::LD1DUPpost)
    MAKE_CASE(AArch64ISD::LD2DUPpost)
    MAKE_CASE(AArch64ISD::LD3DUPpost)
    MAKE_CASE(AArch64ISD::LD4DUPpost)
    MAKE_CASE(AArch64ISD::LD1LANEpost)
    MAKE_CASE(AArch64ISD::LD2LANEpost)
    MAKE_CASE(AArch64ISD::LD3LANEpost)
    MAKE_CASE(AArch64ISD::LD4LANEpost)
    MAKE_CASE(AArch64ISD::ST2LANEpost)
    MAKE_CASE(AArch64ISD::ST3LANEpost)
    MAKE_CASE(AArch64ISD::ST4LANEpost)
    MAKE_CASE(AArch64ISD::SMULL)
    MAKE_CASE(AArch64ISD::UMULL)
    MAKE_CASE(AArch64ISD::PMULL)
    MAKE_CASE(AArch64ISD::FRECPE)
    MAKE_CASE(AArch64ISD::FRECPS)
    MAKE_CASE(AArch64ISD::FRSQRTE)
    MAKE_CASE(AArch64ISD::FRSQRTS)
    MAKE_CASE(AArch64ISD::STG)
    MAKE_CASE(AArch64ISD::STZG)
    MAKE_CASE(AArch64ISD::ST2G)
    MAKE_CASE(AArch64ISD::STZ2G)
    MAKE_CASE(AArch64ISD::SUNPKHI)
    MAKE_CASE(AArch64ISD::SUNPKLO)
    MAKE_CASE(AArch64ISD::UUNPKHI)
    MAKE_CASE(AArch64ISD::UUNPKLO)
    MAKE_CASE(AArch64ISD::INSR)
    MAKE_CASE(AArch64ISD::PTEST)
    MAKE_CASE(AArch64ISD::PTRUE)
    MAKE_CASE(AArch64ISD::LD1_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::LD1S_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::LDNF1_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::LDNF1S_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::LDFF1_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::LDFF1S_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::LD1RQ_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::LD1RO_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::SVE_LD2_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::SVE_LD3_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::SVE_LD4_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1_SXTW_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1_UXTW_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1_IMM_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1S_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1S_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1S_SXTW_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1S_UXTW_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLD1S_IMM_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1_SXTW_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1_UXTW_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1_IMM_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1S_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDFF1S_IMM_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDNT1_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDNT1_INDEX_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::GLDNT1S_MERGE_ZERO)
    MAKE_CASE(AArch64ISD::ST1_PRED)
    MAKE_CASE(AArch64ISD::SST1_PRED)
    MAKE_CASE(AArch64ISD::SST1_SCALED_PRED)
    MAKE_CASE(AArch64ISD::SST1_SXTW_PRED)
    MAKE_CASE(AArch64ISD::SST1_UXTW_PRED)
    MAKE_CASE(AArch64ISD::SST1_SXTW_SCALED_PRED)
    MAKE_CASE(AArch64ISD::SST1_UXTW_SCALED_PRED)
    MAKE_CASE(AArch64ISD::SST1_IMM_PRED)
    MAKE_CASE(AArch64ISD::SSTNT1_PRED)
    MAKE_CASE(AArch64ISD::SSTNT1_INDEX_PRED)
    MAKE_CASE(AArch64ISD::LDP)
    MAKE_CASE(AArch64ISD::LDNP)
    MAKE_CASE(AArch64ISD::STP)
    MAKE_CASE(AArch64ISD::STNP)
    MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::REVH_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::REVW_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::REVD_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
    MAKE_CASE(AArch64ISD::INDEX_VECTOR)
    MAKE_CASE(AArch64ISD::ADDP)
    MAKE_CASE(AArch64ISD::SADDLP)
    MAKE_CASE(AArch64ISD::UADDLP)
    MAKE_CASE(AArch64ISD::CALL_RVMARKER)
    MAKE_CASE(AArch64ISD::ASSERT_ZEXT_BOOL)
    MAKE_CASE(AArch64ISD::MOPS_MEMSET)
    MAKE_CASE(AArch64ISD::MOPS_MEMSET_TAGGING)
    MAKE_CASE(AArch64ISD::MOPS_MEMCOPY)
    MAKE_CASE(AArch64ISD::MOPS_MEMMOVE)
    MAKE_CASE(AArch64ISD::CALL_BTI)
  }
#undef MAKE_CASE
  return nullptr;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
                                    MachineBasicBlock *MBB) const {
  // We materialise the F128CSEL pseudo-instruction as some control flow and a
  // phi node:
 
  // OrigBB:
  //     [... previous instrs leading to comparison ...]
  //     b.ne TrueBB
  //     b EndBB
  // TrueBB:
  //     ; Fallthrough
  // EndBB:
  //     Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
 
  MachineFunction *MF = MBB->getParent();
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
  DebugLoc DL = MI.getDebugLoc();
  MachineFunction::iterator It = ++MBB->getIterator();
 
  Register DestReg = MI.getOperand(0).getReg();
  Register IfTrueReg = MI.getOperand(1).getReg();
  Register IfFalseReg = MI.getOperand(2).getReg();
  unsigned CondCode = MI.getOperand(3).getImm();
  bool NZCVKilled = MI.getOperand(4).isKill();
 
  MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
  MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
  MF->insert(It, TrueBB);
  MF->insert(It, EndBB);
 
  // Transfer rest of current basic-block to EndBB
  EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
                MBB->end());
  EndBB->transferSuccessorsAndUpdatePHIs(MBB);
 
  BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
  BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
  MBB->addSuccessor(TrueBB);
  MBB->addSuccessor(EndBB);
 
  // TrueBB falls through to the end.
  TrueBB->addSuccessor(EndBB);
 
  if (!NZCVKilled) {
    TrueBB->addLiveIn(AArch64::NZCV);
    EndBB->addLiveIn(AArch64::NZCV);
  }
 
  BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
      .addReg(IfTrueReg)
      .addMBB(TrueBB)
      .addReg(IfFalseReg)
      .addMBB(MBB);
 
  MI.eraseFromParent();
  return EndBB;
}
 
MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchRet(
       MachineInstr &MI, MachineBasicBlock *BB) const {
  assert(!isAsynchronousEHPersonality(classifyEHPersonality(
             BB->getParent()->getFunction().getPersonalityFn())) &&
         "SEH does not use catchret!");
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitTileLoad(unsigned Opc, unsigned BaseReg,
                                    MachineInstr &MI,
                                    MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opc));
 
  MIB.addReg(BaseReg + MI.getOperand(0).getImm(), RegState::Define);
  MIB.add(MI.getOperand(1)); // slice index register
  MIB.add(MI.getOperand(2)); // slice index offset
  MIB.add(MI.getOperand(3)); // pg
  MIB.add(MI.getOperand(4)); // base
  MIB.add(MI.getOperand(5)); // offset
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitFill(MachineInstr &MI, MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB =
      BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::LDR_ZA));
 
  MIB.addReg(AArch64::ZA, RegState::Define);
  MIB.add(MI.getOperand(0)); // Vector select register
  MIB.add(MI.getOperand(1)); // Vector select offset
  MIB.add(MI.getOperand(2)); // Base
  MIB.add(MI.getOperand(1)); // Offset, same as vector select offset
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitMopa(unsigned Opc, unsigned BaseReg,
                                MachineInstr &MI, MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opc));
 
  MIB.addReg(BaseReg + MI.getOperand(0).getImm(), RegState::Define);
  MIB.addReg(BaseReg + MI.getOperand(0).getImm());
  MIB.add(MI.getOperand(1)); // pn
  MIB.add(MI.getOperand(2)); // pm
  MIB.add(MI.getOperand(3)); // zn
  MIB.add(MI.getOperand(4)); // zm
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitInsertVectorToTile(unsigned Opc, unsigned BaseReg,
                                              MachineInstr &MI,
                                              MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opc));
 
  MIB.addReg(BaseReg + MI.getOperand(0).getImm(), RegState::Define);
  MIB.addReg(BaseReg + MI.getOperand(0).getImm());
  MIB.add(MI.getOperand(1)); // Slice index register
  MIB.add(MI.getOperand(2)); // Slice index offset
  MIB.add(MI.getOperand(3)); // pg
  MIB.add(MI.getOperand(4)); // zn
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitZero(MachineInstr &MI, MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB =
      BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::ZERO_M));
  MIB.add(MI.getOperand(0)); // Mask
 
  unsigned Mask = MI.getOperand(0).getImm();
  for (unsigned I = 0; I < 8; I++) {
    if (Mask & (1 << I))
      MIB.addDef(AArch64::ZAD0 + I, RegState::ImplicitDefine);
  }
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *
AArch64TargetLowering::EmitAddVectorToTile(unsigned Opc, unsigned BaseReg,
                                           MachineInstr &MI,
                                           MachineBasicBlock *BB) const {
  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
  MachineInstrBuilder MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opc));
 
  MIB.addReg(BaseReg + MI.getOperand(0).getImm(), RegState::Define);
  MIB.addReg(BaseReg + MI.getOperand(0).getImm());
  MIB.add(MI.getOperand(1)); // pn
  MIB.add(MI.getOperand(2)); // pm
  MIB.add(MI.getOperand(3)); // zn
 
  MI.eraseFromParent(); // The pseudo is gone now.
  return BB;
}
 
MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
    MachineInstr &MI, MachineBasicBlock *BB) const {
  switch (MI.getOpcode()) {
  default:
#ifndef NDEBUG
    MI.dump();
#endif
    llvm_unreachable("Unexpected instruction for custom inserter!");
 
  case AArch64::F128CSEL:
    return EmitF128CSEL(MI, BB);
  case TargetOpcode::STATEPOINT:
    // STATEPOINT is a pseudo instruction which has no implicit defs/uses
    // while bl call instruction (where statepoint will be lowered at the end)
    // has implicit def. This def is early-clobber as it will be set at
    // the moment of the call and earlier than any use is read.
    // Add this implicit dead def here as a workaround.
    MI.addOperand(*MI.getMF(),
                  MachineOperand::CreateReg(
                      AArch64::LR, /*isDef*/ true,
                      /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
                      /*isUndef*/ false, /*isEarlyClobber*/ true));
    [[fallthrough]];
  case TargetOpcode::STACKMAP:
  case TargetOpcode::PATCHPOINT:
    return emitPatchPoint(MI, BB);
 
  case AArch64::CATCHRET:
    return EmitLoweredCatchRet(MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_B:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_B, AArch64::ZAB0, MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_H:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_H, AArch64::ZAH0, MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_S:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_S, AArch64::ZAS0, MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_D:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_D, AArch64::ZAD0, MI, BB);
  case AArch64::LD1_MXIPXX_H_PSEUDO_Q:
    return EmitTileLoad(AArch64::LD1_MXIPXX_H_Q, AArch64::ZAQ0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_B:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_B, AArch64::ZAB0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_H:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_H, AArch64::ZAH0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_S:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_S, AArch64::ZAS0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_D:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_D, AArch64::ZAD0, MI, BB);
  case AArch64::LD1_MXIPXX_V_PSEUDO_Q:
    return EmitTileLoad(AArch64::LD1_MXIPXX_V_Q, AArch64::ZAQ0, MI, BB);
  case AArch64::LDR_ZA_PSEUDO:
    return EmitFill(MI, BB);
  case AArch64::BFMOPA_MPPZZ_PSEUDO:
    return EmitMopa(AArch64::BFMOPA_MPPZZ, AArch64::ZAS0, MI, BB);
  case AArch64::BFMOPS_MPPZZ_PSEUDO:
    return EmitMopa(AArch64::BFMOPS_MPPZZ, AArch64::ZAS0, MI, BB);
  case AArch64::FMOPAL_MPPZZ_PSEUDO:
    return EmitMopa(AArch64::FMOPAL_MPPZZ, AArch64::ZAS0, MI, BB);
  case AArch64::FMOPSL_MPPZZ_PSEUDO:
    return EmitMopa(AArch64::FMOPSL_MPPZZ, AArch64::ZAS0, MI, BB);
  case AArch64::FMOPA_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::FMOPA_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::FMOPS_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::FMOPS_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::FMOPA_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::FMOPA_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::FMOPS_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::FMOPS_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::SMOPA_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::SMOPA_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::SMOPS_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::SMOPS_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::UMOPA_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::UMOPA_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::UMOPS_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::UMOPS_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::SUMOPA_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::SUMOPA_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::SUMOPS_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::SUMOPS_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::USMOPA_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::USMOPA_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::USMOPS_MPPZZ_S_PSEUDO:
    return EmitMopa(AArch64::USMOPS_MPPZZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::SMOPA_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::SMOPA_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::SMOPS_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::SMOPS_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::UMOPA_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::UMOPA_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::UMOPS_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::UMOPS_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::SUMOPA_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::SUMOPA_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::SUMOPS_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::SUMOPS_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::USMOPA_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::USMOPA_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::USMOPS_MPPZZ_D_PSEUDO:
    return EmitMopa(AArch64::USMOPS_MPPZZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::INSERT_MXIPZ_H_PSEUDO_B:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_H_B, AArch64::ZAB0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_H_PSEUDO_H:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_H_H, AArch64::ZAH0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_H_PSEUDO_S:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_H_S, AArch64::ZAS0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_H_PSEUDO_D:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_H_D, AArch64::ZAD0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_H_PSEUDO_Q:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_H_Q, AArch64::ZAQ0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_V_PSEUDO_B:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_V_B, AArch64::ZAB0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_V_PSEUDO_H:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_V_H, AArch64::ZAH0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_V_PSEUDO_S:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_V_S, AArch64::ZAS0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_V_PSEUDO_D:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_V_D, AArch64::ZAD0, MI,
                                  BB);
  case AArch64::INSERT_MXIPZ_V_PSEUDO_Q:
    return EmitInsertVectorToTile(AArch64::INSERT_MXIPZ_V_Q, AArch64::ZAQ0, MI,
                                  BB);
  case AArch64::ZERO_M_PSEUDO:
    return EmitZero(MI, BB);
  case AArch64::ADDHA_MPPZ_PSEUDO_S:
    return EmitAddVectorToTile(AArch64::ADDHA_MPPZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::ADDVA_MPPZ_PSEUDO_S:
    return EmitAddVectorToTile(AArch64::ADDVA_MPPZ_S, AArch64::ZAS0, MI, BB);
  case AArch64::ADDHA_MPPZ_PSEUDO_D:
    return EmitAddVectorToTile(AArch64::ADDHA_MPPZ_D, AArch64::ZAD0, MI, BB);
  case AArch64::ADDVA_MPPZ_PSEUDO_D:
    return EmitAddVectorToTile(AArch64::ADDVA_MPPZ_D, AArch64::ZAD0, MI, BB);
  }
}
 
//===----------------------------------------------------------------------===//
// AArch64 Lowering private implementation.
//===----------------------------------------------------------------------===//
 
//===----------------------------------------------------------------------===//
// Lowering Code
//===----------------------------------------------------------------------===//
 
// Forward declarations of SVE fixed length lowering helpers
static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT);
static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
static SDValue convertFixedMaskToScalableVector(SDValue Mask,
                                                SelectionDAG &DAG);
static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
                                             EVT VT);
 
/// isZerosVector - Check whether SDNode N is a zero-filled vector.
static bool isZerosVector(const SDNode *N) {
  // Look through a bit convert.
  while (N->getOpcode() == ISD::BITCAST)
    N = N->getOperand(0).getNode();
 
  if (ISD::isConstantSplatVectorAllZeros(N))
    return true;
 
  if (N->getOpcode() != AArch64ISD::DUP)
    return false;
 
  auto Opnd0 = N->getOperand(0);
  return isNullConstant(Opnd0) || isNullFPConstant(Opnd0);
}
 
/// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
/// CC
static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
  switch (CC) {
  default:
    llvm_unreachable("Unknown condition code!");
  case ISD::SETNE:
    return AArch64CC::NE;
  case ISD::SETEQ:
    return AArch64CC::EQ;
  case ISD::SETGT:
    return AArch64CC::GT;
  case ISD::SETGE:
    return AArch64CC::GE;
  case ISD::SETLT:
    return AArch64CC::LT;
  case ISD::SETLE:
    return AArch64CC::LE;
  case ISD::SETUGT:
    return AArch64CC::HI;
  case ISD::SETUGE:
    return AArch64CC::HS;
  case ISD::SETULT:
    return AArch64CC::LO;
  case ISD::SETULE:
    return AArch64CC::LS;
  }
}
 
/// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
static void changeFPCCToAArch64CC(ISD::CondCode CC,
                                  AArch64CC::CondCode &CondCode,
                                  AArch64CC::CondCode &CondCode2) {
  CondCode2 = AArch64CC::AL;
  switch (CC) {
  default:
    llvm_unreachable("Unknown FP condition!");
  case ISD::SETEQ:
  case ISD::SETOEQ:
    CondCode = AArch64CC::EQ;
    break;
  case ISD::SETGT:
  case ISD::SETOGT:
    CondCode = AArch64CC::GT;
    break;
  case ISD::SETGE:
  case ISD::SETOGE:
    CondCode = AArch64CC::GE;
    break;
  case ISD::SETOLT:
    CondCode = AArch64CC::MI;
    break;
  case ISD::SETOLE:
    CondCode = AArch64CC::LS;
    break;
  case ISD::SETONE:
    CondCode = AArch64CC::MI;
    CondCode2 = AArch64CC::GT;
    break;
  case ISD::SETO:
    CondCode = AArch64CC::VC;
    break;
  case ISD::SETUO:
    CondCode = AArch64CC::VS;
    break;
  case ISD::SETUEQ:
    CondCode = AArch64CC::EQ;
    CondCode2 = AArch64CC::VS;
    break;
  case ISD::SETUGT:
    CondCode = AArch64CC::HI;
    break;
  case ISD::SETUGE:
    CondCode = AArch64CC::PL;
    break;
  case ISD::SETLT:
  case ISD::SETULT:
    CondCode = AArch64CC::LT;
    break;
  case ISD::SETLE:
  case ISD::SETULE:
    CondCode = AArch64CC::LE;
    break;
  case ISD::SETNE:
  case ISD::SETUNE:
    CondCode = AArch64CC::NE;
    break;
  }
}
 
/// Convert a DAG fp condition code to an AArch64 CC.
/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
/// should be AND'ed instead of OR'ed.
static void changeFPCCToANDAArch64CC(ISD::CondCode CC,
                                     AArch64CC::CondCode &CondCode,
                                     AArch64CC::CondCode &CondCode2) {
  CondCode2 = AArch64CC::AL;
  switch (CC) {
  default:
    changeFPCCToAArch64CC(CC, CondCode, CondCode2);
    assert(CondCode2 == AArch64CC::AL);
    break;
  case ISD::SETONE:
    // (a one b)
    // == ((a olt b) || (a ogt b))
    // == ((a ord b) && (a une b))
    CondCode = AArch64CC::VC;
    CondCode2 = AArch64CC::NE;
    break;
  case ISD::SETUEQ:
    // (a ueq b)
    // == ((a uno b) || (a oeq b))
    // == ((a ule b) && (a uge b))
    CondCode = AArch64CC::PL;
    CondCode2 = AArch64CC::LE;
    break;
  }
}
 
/// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
/// CC usable with the vector instructions. Fewer operations are available
/// without a real NZCV register, so we have to use less efficient combinations
/// to get the same effect.
static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
                                        AArch64CC::CondCode &CondCode,
                                        AArch64CC::CondCode &CondCode2,
                                        bool &Invert) {
  Invert = false;
  switch (CC) {
  default:
    // Mostly the scalar mappings work fine.
    changeFPCCToAArch64CC(CC, CondCode, CondCode2);
    break;
  case ISD::SETUO:
    Invert = true;
    [[fallthrough]];
  case ISD::SETO:
    CondCode = AArch64CC::MI;
    CondCode2 = AArch64CC::GE;
    break;
  case ISD::SETUEQ:
  case ISD::SETULT:
  case ISD::SETULE:
  case ISD::SETUGT:
  case ISD::SETUGE:
    // All of the compare-mask comparisons are ordered, but we can switch
    // between the two by a double inversion. E.g. ULE == !OGT.
    Invert = true;
    changeFPCCToAArch64CC(getSetCCInverse(CC, /* FP inverse */ MVT::f32),
                          CondCode, CondCode2);
    break;
  }
}
 
static bool isLegalArithImmed(uint64_t C) {
  // Matches AArch64DAGToDAGISel::SelectArithImmed().
  bool IsLegal = (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0);
  LLVM_DEBUG(dbgs() << "Is imm " << C
                    << " legal: " << (IsLegal ? "yes\n" : "no\n"));
  return IsLegal;
}
 
// Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
// the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
// can be set differently by this operation. It comes down to whether
// "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
// everything is fine. If not then the optimization is wrong. Thus general
// comparisons are only valid if op2 != 0.
//
// So, finally, the only LLVM-native comparisons that don't mention C and V
// are SETEQ and SETNE. They're the only ones we can safely use CMN for in
// the absence of information about op2.
static bool isCMN(SDValue Op, ISD::CondCode CC) {
  return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
         (CC == ISD::SETEQ || CC == ISD::SETNE);
}
 
static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
                                      SelectionDAG &DAG, SDValue Chain,
                                      bool IsSignaling) {
  EVT VT = LHS.getValueType();
  assert(VT != MVT::f128);
 
  const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
 
  if (VT == MVT::f16 && !FullFP16) {
    LHS = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other},
                      {Chain, LHS});
    RHS = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {MVT::f32, MVT::Other},
                      {LHS.getValue(1), RHS});
    Chain = RHS.getValue(1);
    VT = MVT::f32;
  }
  unsigned Opcode =
      IsSignaling ? AArch64ISD::STRICT_FCMPE : AArch64ISD::STRICT_FCMP;
  return DAG.getNode(Opcode, dl, {VT, MVT::Other}, {Chain, LHS, RHS});
}
 
static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
                              const SDLoc &dl, SelectionDAG &DAG) {
  EVT VT = LHS.getValueType();
  const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
 
  if (VT.isFloatingPoint()) {
    assert(VT != MVT::f128);
    if (VT == MVT::f16 && !FullFP16) {
      LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
      RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
      VT = MVT::f32;
    }
    return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
  }
 
  // The CMP instruction is just an alias for SUBS, and representing it as
  // SUBS means that it's possible to get CSE with subtract operations.
  // A later phase can perform the optimization of setting the destination
  // register to WZR/XZR if it ends up being unused.
  unsigned Opcode = AArch64ISD::SUBS;
 
  if (isCMN(RHS, CC)) {
    // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
    Opcode = AArch64ISD::ADDS;
    RHS = RHS.getOperand(1);
  } else if (isCMN(LHS, CC)) {
    // As we are looking for EQ/NE compares, the operands can be commuted ; can
    // we combine a (CMP (sub 0, op1), op2) into a CMN instruction ?
    Opcode = AArch64ISD::ADDS;
    LHS = LHS.getOperand(1);
  } else if (isNullConstant(RHS) && !isUnsignedIntSetCC(CC)) {
    if (LHS.getOpcode() == ISD::AND) {
      // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
      // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
      // of the signed comparisons.
      const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl,
                                           DAG.getVTList(VT, MVT_CC),
                                           LHS.getOperand(0),
                                           LHS.getOperand(1));
      // Replace all users of (and X, Y) with newly generated (ands X, Y)
      DAG.ReplaceAllUsesWith(LHS, ANDSNode);
      return ANDSNode.getValue(1);
    } else if (LHS.getOpcode() == AArch64ISD::ANDS) {
      // Use result of ANDS
      return LHS.getValue(1);
    }
  }
 
  return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
      .getValue(1);
}
 
/// \defgroup AArch64CCMP CMP;CCMP matching
///
/// These functions deal with the formation of CMP;CCMP;... sequences.
/// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
/// a comparison. They set the NZCV flags to a predefined value if their
/// predicate is false. This allows to express arbitrary conjunctions, for
/// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B)))"
/// expressed as:
///   cmp A
///   ccmp B, inv(CB), CA
///   check for CB flags
///
/// This naturally lets us implement chains of AND operations with SETCC
/// operands. And we can even implement some other situations by transforming
/// them:
///   - We can implement (NEG SETCC) i.e. negating a single comparison by
///     negating the flags used in a CCMP/FCCMP operations.
///   - We can negate the result of a whole chain of CMP/CCMP/FCCMP operations
///     by negating the flags we test for afterwards. i.e.
///     NEG (CMP CCMP CCCMP ...) can be implemented.
///   - Note that we can only ever negate all previously processed results.
///     What we can not implement by flipping the flags to test is a negation
///     of two sub-trees (because the negation affects all sub-trees emitted so
///     far, so the 2nd sub-tree we emit would also affect the first).
/// With those tools we can implement some OR operations:
///   - (OR (SETCC A) (SETCC B)) can be implemented via:
///     NEG (AND (NEG (SETCC A)) (NEG (SETCC B)))
///   - After transforming OR to NEG/AND combinations we may be able to use NEG
///     elimination rules from earlier to implement the whole thing as a
///     CCMP/FCCMP chain.
///
/// As complete example:
///     or (or (setCA (cmp A)) (setCB (cmp B)))
///        (and (setCC (cmp C)) (setCD (cmp D)))"
/// can be reassociated to:
///     or (and (setCC (cmp C)) setCD (cmp D))
//         (or (setCA (cmp A)) (setCB (cmp B)))
/// can be transformed to:
///     not (and (not (and (setCC (cmp C)) (setCD (cmp D))))
///              (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
/// which can be implemented as:
///   cmp C
///   ccmp D, inv(CD), CC
///   ccmp A, CA, inv(CD)
///   ccmp B, CB, inv(CA)
///   check for CB flags
///
/// A counterexample is "or (and A B) (and C D)" which translates to
/// not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we
/// can only implement 1 of the inner (not) operations, but not both!
/// @{
 
/// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
                                         ISD::CondCode CC, SDValue CCOp,
                                         AArch64CC::CondCode Predicate,
                                         AArch64CC::CondCode OutCC,
                                         const SDLoc &DL, SelectionDAG &DAG) {
  unsigned Opcode = 0;
  const bool FullFP16 = DAG.getSubtarget<AArch64Subtarget>().hasFullFP16();
 
  if (LHS.getValueType().isFloatingPoint()) {
    assert(LHS.getValueType() != MVT::f128);
    if (LHS.getValueType() == MVT::f16 && !FullFP16) {
      LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
      RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
    }
    Opcode = AArch64ISD::FCCMP;
  } else if (RHS.getOpcode() == ISD::SUB) {
    SDValue SubOp0 = RHS.getOperand(0);
    if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
      // See emitComparison() on why we can only do this for SETEQ and SETNE.
      Opcode = AArch64ISD::CCMN;
      RHS = RHS.getOperand(1);
    }
  }
  if (Opcode == 0)
    Opcode = AArch64ISD::CCMP;
 
  SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
  AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
  unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
  SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
  return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
}
 
/// Returns true if @p Val is a tree of AND/OR/SETCC operations that can be
/// expressed as a conjunction. See \ref AArch64CCMP.
/// \param CanNegate    Set to true if we can negate the whole sub-tree just by
///                     changing the conditions on the SETCC tests.
///                     (this means we can call emitConjunctionRec() with
///                      Negate==true on this sub-tree)
/// \param MustBeFirst  Set to true if this subtree needs to be negated and we
///                     cannot do the negation naturally. We are required to
///                     emit the subtree first in this case.
/// \param WillNegate   Is true if are called when the result of this
///                     subexpression must be negated. This happens when the
///                     outer expression is an OR. We can use this fact to know
///                     that we have a double negation (or (or ...) ...) that
///                     can be implemented for free.
static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
                               bool &MustBeFirst, bool WillNegate,
                               unsigned Depth = 0) {
  if (!Val.hasOneUse())
    return false;
  unsigned Opcode = Val->getOpcode();
  if (Opcode == ISD::SETCC) {
    if (Val->getOperand(0).getValueType() == MVT::f128)
      return false;
    CanNegate = true;
    MustBeFirst = false;
    return true;
  }
  // Protect against exponential runtime and stack overflow.
  if (Depth > 6)
    return false;
  if (Opcode == ISD::AND || Opcode == ISD::OR) {
    bool IsOR = Opcode == ISD::OR;
    SDValue O0 = Val->getOperand(0);
    SDValue O1 = Val->getOperand(1);
    bool CanNegateL;
    bool MustBeFirstL;
    if (!canEmitConjunction(O0, CanNegateL, MustBeFirstL, IsOR, Depth+1))
      return false;
    bool CanNegateR;
    bool MustBeFirstR;
    if (!canEmitConjunction(O1, CanNegateR, MustBeFirstR, IsOR, Depth+1))
      return false;
 
    if (MustBeFirstL && MustBeFirstR)
      return false;
 
    if (IsOR) {
      // For an OR expression we need to be able to naturally negate at least
      // one side or we cannot do the transformation at all.
      if (!CanNegateL && !CanNegateR)
        return false;
      // If we the result of the OR will be negated and we can naturally negate
      // the leafs, then this sub-tree as a whole negates naturally.
      CanNegate = WillNegate && CanNegateL && CanNegateR;
      // If we cannot naturally negate the whole sub-tree, then this must be
      // emitted first.
      MustBeFirst = !CanNegate;
    } else {
      assert(Opcode == ISD::AND && "Must be OR or AND");
      // We cannot naturally negate an AND operation.
      CanNegate = false;
      MustBeFirst = MustBeFirstL || MustBeFirstR;
    }
    return true;
  }
  return false;
}
 
/// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
/// of CCMP/CFCMP ops. See @ref AArch64CCMP.
/// Tries to transform the given i1 producing node @p Val to a series compare
/// and conditional compare operations. @returns an NZCV flags producing node
/// and sets @p OutCC to the flags that should be tested or returns SDValue() if
/// transformation was not possible.
/// \p Negate is true if we want this sub-tree being negated just by changing
/// SETCC conditions.
static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
    AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
    AArch64CC::CondCode Predicate) {
  // We're at a tree leaf, produce a conditional comparison operation.
  unsigned Opcode = Val->getOpcode();
  if (Opcode == ISD::SETCC) {
    SDValue LHS = Val->getOperand(0);
    SDValue RHS = Val->getOperand(1);
    ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
    bool isInteger = LHS.getValueType().isInteger();
    if (Negate)
      CC = getSetCCInverse(CC, LHS.getValueType());
    SDLoc DL(Val);
    // Determine OutCC and handle FP special case.
    if (isInteger) {
      OutCC = changeIntCCToAArch64CC(CC);
    } else {
      assert(LHS.getValueType().isFloatingPoint());
      AArch64CC::CondCode ExtraCC;
      changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
      // Some floating point conditions can't be tested with a single condition
      // code. Construct an additional comparison in this case.
      if (ExtraCC != AArch64CC::AL) {
        SDValue ExtraCmp;
        if (!CCOp.getNode())
          ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
        else
          ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate,
                                               ExtraCC, DL, DAG);
        CCOp = ExtraCmp;
        Predicate = ExtraCC;
      }
    }
 
    // Produce a normal comparison if we are first in the chain
    if (!CCOp)
      return emitComparison(LHS, RHS, CC, DL, DAG);
    // Otherwise produce a ccmp.
    return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
                                     DAG);
  }
  assert(Val->hasOneUse() && "Valid conjunction/disjunction tree");
 
  bool IsOR = Opcode == ISD::OR;
 
  SDValue LHS = Val->getOperand(0);
  bool CanNegateL;
  bool MustBeFirstL;
  bool ValidL = canEmitConjunction(LHS, CanNegateL, MustBeFirstL, IsOR);
  assert(ValidL && "Valid conjunction/disjunction tree");
  (void)ValidL;
 
  SDValue RHS = Val->getOperand(1);
  bool CanNegateR;
  bool MustBeFirstR;
  bool ValidR = canEmitConjunction(RHS, CanNegateR, MustBeFirstR, IsOR);
  assert(ValidR && "Valid conjunction/disjunction tree");
  (void)ValidR;
 
  // Swap sub-tree that must come first to the right side.
  if (MustBeFirstL) {
    assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
    std::swap(LHS, RHS);
    std::swap(CanNegateL, CanNegateR);
    std::swap(MustBeFirstL, MustBeFirstR);
  }
 
  bool NegateR;
  bool NegateAfterR;
  bool NegateL;
  bool NegateAfterAll;
  if (Opcode == ISD::OR) {
    // Swap the sub-tree that we can negate naturally to the left.
    if (!CanNegateL) {
      assert(CanNegateR && "at least one side must be negatable");
      assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
      assert(!Negate);
      std::swap(LHS, RHS);
      NegateR = false;
      NegateAfterR = true;
    } else {
      // Negate the left sub-tree if possible, otherwise negate the result.
      NegateR = CanNegateR;
      NegateAfterR = !CanNegateR;
    }
    NegateL = true;
    NegateAfterAll = !Negate;
  } else {
    assert(Opcode == ISD::AND && "Valid conjunction/disjunction tree");
    assert(!Negate && "Valid conjunction/disjunction tree");
 
    NegateL = false;
    NegateR = false;
    NegateAfterR = false;
    NegateAfterAll = false;
  }
 
  // Emit sub-trees.
  AArch64CC::CondCode RHSCC;
  SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
  if (NegateAfterR)
    RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
  SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
  if (NegateAfterAll)
    OutCC = AArch64CC::getInvertedCondCode(OutCC);
  return CmpL;
}
 
/// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
/// In some cases this is even possible with OR operations in the expression.
/// See \ref AArch64CCMP.
/// \see emitConjunctionRec().
static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
                               AArch64CC::CondCode &OutCC) {
  bool DummyCanNegate;
  bool DummyMustBeFirst;
  if (!canEmitConjunction(Val, DummyCanNegate, DummyMustBeFirst, false))
    return SDValue();
 
  return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
}
 
/// @}
 
/// Returns how profitable it is to fold a comparison's operand's shift and/or
/// extension operations.
static unsigned getCmpOperandFoldingProfit(SDValue Op) {
  auto isSupportedExtend = [&](SDValue V) {
    if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
      return true;
 
    if (V.getOpcode() == ISD::AND)
      if (ConstantSDNode *MaskCst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
        uint64_t Mask = MaskCst->getZExtValue();
        return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
      }
 
    return false;
  };
 
  if (!Op.hasOneUse())
    return 0;
 
  if (isSupportedExtend(Op))
    return 1;
 
  unsigned Opc = Op.getOpcode();
  if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
    if (ConstantSDNode *ShiftCst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
      uint64_t Shift = ShiftCst->getZExtValue();
      if (isSupportedExtend(Op.getOperand(0)))
        return (Shift <= 4) ? 2 : 1;
      EVT VT = Op.getValueType();
      if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
        return 1;
    }
 
  return 0;
}
 
static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
                             SDValue &AArch64cc, SelectionDAG &DAG,
                             const SDLoc &dl) {
  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
    EVT VT = RHS.getValueType();
    uint64_t C = RHSC->getZExtValue();
    if (!isLegalArithImmed(C)) {
      // Constant does not fit, try adjusting it by one?
      switch (CC) {
      default:
        break;
      case ISD::SETLT:
      case ISD::SETGE:
        if ((VT == MVT::i32 && C != 0x80000000 &&
             isLegalArithImmed((uint32_t)(C - 1))) ||
            (VT == MVT::i64 && C != 0x80000000ULL &&
             isLegalArithImmed(C - 1ULL))) {
          CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
          C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
          RHS = DAG.getConstant(C, dl, VT);
        }
        break;
      case ISD::SETULT:
      case ISD::SETUGE:
        if ((VT == MVT::i32 && C != 0 &&
             isLegalArithImmed((uint32_t)(C - 1))) ||
            (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
          CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
          C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
          RHS = DAG.getConstant(C, dl, VT);
        }
        break;
      case ISD::SETLE:
      case ISD::SETGT:
        if ((VT == MVT::i32 && C != INT32_MAX &&
             isLegalArithImmed((uint32_t)(C + 1))) ||
            (VT == MVT::i64 && C != INT64_MAX &&
             isLegalArithImmed(C + 1ULL))) {
          CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
          C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
          RHS = DAG.getConstant(C, dl, VT);
        }
        break;
      case ISD::SETULE:
      case ISD::SETUGT:
        if ((VT == MVT::i32 && C != UINT32_MAX &&
             isLegalArithImmed((uint32_t)(C + 1))) ||
            (VT ==