RISCVInstructionSelector.cpp

Go to the documentation of this file.

//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/// \file
/// This file implements the targeting of the InstructionSelector class for
/// RISC-V.
/// \todo This should be generated by TableGen.
//===----------------------------------------------------------------------===//
 
#include "MCTargetDesc/RISCVMatInt.h"
#include "RISCVRegisterBankInfo.h"
#include "RISCVSubtarget.h"
#include "RISCVTargetMachine.h"
#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/Support/Debug.h"
 
#define DEBUG_TYPE "riscv-isel"
 
using namespace llvm;
using namespace MIPatternMatch;
 
#define GET_GLOBALISEL_PREDICATE_BITSET
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATE_BITSET
 
namespace {
 
class RISCVInstructionSelector : public InstructionSelector {
public:
  RISCVInstructionSelector(const RISCVTargetMachine &TM,
                           const RISCVSubtarget &STI,
                           const RISCVRegisterBankInfo &RBI);
 
  bool select(MachineInstr &MI) override;
 
  void setupMF(MachineFunction &MF, GISelValueTracking *VT,
               CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
               BlockFrequencyInfo *BFI) override {
    InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);
    MRI = &MF.getRegInfo();
  }
 
  static const char *getName() { return DEBUG_TYPE; }
 
private:
  static constexpr unsigned MaxRecursionDepth = 6;
 
  bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
                       const unsigned Depth = 0) const;
  bool hasAllHUsers(const MachineInstr &MI) const {
    return hasAllNBitUsers(MI, 16);
  }
  bool hasAllWUsers(const MachineInstr &MI) const {
    return hasAllNBitUsers(MI, 32);
  }
 
  bool isRegInGprb(Register Reg) const;
  bool isRegInFprb(Register Reg) const;
  bool isWorthFoldingAdd(Register AddResult) const;
 
  // tblgen-erated 'select' implementation, used as the initial selector for
  // the patterns that don't require complex C++.
  bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
 
  // A lowering phase that runs before any selection attempts.
  // Returns true if the instruction was modified.
  void preISelLower(MachineInstr &MI);
 
  bool replacePtrWithInt(MachineInstr &MI, unsigned OpIdx);
 
  // Custom selection methods
  bool selectCopy(MachineInstr &MI) const;
  bool selectImplicitDef(MachineInstr &MI) const;
  bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
  // Emit a constant-materialization instruction sequence.
  bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq,
                          MachineInstr &MI) const;
  bool selectAddr(MachineInstr &MI, bool IsLocal = true,
                  bool IsExternWeak = false) const;
  bool selectSelect(MachineInstr &MI) const;
  bool selectFPCompare(MachineInstr &MI) const;
  void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
                 MachineInstr &MI) const;
  bool selectUnmergeValues(MachineInstr &MI) const;
  bool selectMergeValues(MachineInstr &MI) const;
  void addVectorLoadStoreOperands(MachineInstr &I,
                                  SmallVectorImpl<Register> &SrcOps,
                                  unsigned &CurOp, bool IsMasked,
                                  bool IsStridedOrIndexed,
                                  LLT *IndexVT = nullptr) const;
  bool selectIntrinsicWithSideEffects(MachineInstr &I) const;
  bool selectIntrinsic(MachineInstr &I) const;
  bool selectExtractSubvector(MachineInstr &MI) const;
  bool selectInsertSubVector(MachineInstr &I) const;
  ComplexRendererFns selectShiftMask(MachineOperand &Root,
                                     unsigned ShiftWidth) const;
  ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
    return selectShiftMask(Root, STI.getXLen());
  }
  ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
    return selectShiftMask(Root, 32);
  }
  ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
  ComplexRendererFns selectBrindRegImm(MachineOperand &Root) const;
  ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
 
  // Plan for materializing a constant address as (Hi materialization, Lo12
  // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be
  // a multiple of 32.
  struct ConstAddrPlan {
    enum { X0, LUI, InstSeq } Kind = X0;
    int64_t Hi20 = 0;
    RISCVMatInt::InstSeq Seq;
    int64_t Lo12 = 0;
  };
  ComplexRendererFns computeConstAddr(int64_t CVal, bool IsPrefetch,
                                      Register OrigBase) const;
  // Materialize the high part of Plan into a register. If OrigBase is valid,
  // ADD it to the materialized high part (for G_PTR_ADD + large constant).
  Register materializeConstBase(MachineInstrBuilder &MIB,
                                const ConstAddrPlan &Plan,
                                Register OrigBase) const;
 
  ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
  template <unsigned Bits>
  ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
    return selectSExtBits(Root, Bits);
  }
 
  ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
  template <unsigned Bits>
  ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
    return selectZExtBits(Root, Bits);
  }
 
  ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
  template <unsigned ShAmt>
  ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
    return selectSHXADDOp(Root, ShAmt);
  }
 
  ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
                                       unsigned ShAmt) const;
  template <unsigned ShAmt>
  ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
    return selectSHXADD_UWOp(Root, ShAmt);
  }
 
  ComplexRendererFns renderVLOp(MachineOperand &Root) const;
  ComplexRendererFns renderAddiPair(Register BaseReg, int64_t AddiImm,
                                    int64_t OffsetImm) const;
  // Custom renderers for tablegen
  void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                    int OpIdx) const;
  void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
                            int OpIdx) const;
  void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
                          int OpIdx) const;
  void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
                      int OpIdx) const;
 
  void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
                           int OpIdx) const;
  void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
                                 const MachineInstr &MI, int OpIdx) const;
 
  void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
                              int OpIdx) const;
  void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
                              int OpIdx) const;
 
  const RISCVSubtarget &STI;
  const RISCVInstrInfo &TII;
  const RISCVRegisterInfo &TRI;
  const RISCVRegisterBankInfo &RBI;
  const RISCVTargetMachine &TM;
 
  MachineRegisterInfo *MRI = nullptr;
 
  // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
  // uses "STI." in the code generated by TableGen. We need to unify the name of
  // Subtarget variable.
  const RISCVSubtarget *Subtarget = &STI;
 
#define GET_GLOBALISEL_PREDICATES_DECL
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATES_DECL
 
#define GET_GLOBALISEL_TEMPORARIES_DECL
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_TEMPORARIES_DECL
};
 
} // end anonymous namespace
 
#define GET_GLOBALISEL_IMPL
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_IMPL
 
RISCVInstructionSelector::RISCVInstructionSelector(
    const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
    const RISCVRegisterBankInfo &RBI)
    : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
      TM(TM),
 
#define GET_GLOBALISEL_PREDICATES_INIT
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
#include "RISCVGenGlobalISel.inc"
#undef GET_GLOBALISEL_TEMPORARIES_INIT
{
}
 
// Mimics optimizations in ISel and RISCVOptWInst Pass
bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
                                               unsigned Bits,
                                               const unsigned Depth) const {
 
  assert((MI.getOpcode() == TargetOpcode::G_ADD ||
          MI.getOpcode() == TargetOpcode::G_SUB ||
          MI.getOpcode() == TargetOpcode::G_MUL ||
          MI.getOpcode() == TargetOpcode::G_SHL ||
          MI.getOpcode() == TargetOpcode::G_LSHR ||
          MI.getOpcode() == TargetOpcode::G_AND ||
          MI.getOpcode() == TargetOpcode::G_OR ||
          MI.getOpcode() == TargetOpcode::G_XOR ||
          MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
         "Unexpected opcode");
 
  if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
    return false;
 
  auto DestReg = MI.getOperand(0).getReg();
  for (auto &UserOp : MRI->use_nodbg_operands(DestReg)) {
    assert(UserOp.getParent() && "UserOp must have a parent");
    const MachineInstr &UserMI = *UserOp.getParent();
    unsigned OpIdx = UserOp.getOperandNo();
 
    switch (UserMI.getOpcode()) {
    default:
      return false;
    case RISCV::ADDW:
    case RISCV::ADDIW:
    case RISCV::SUBW:
    case RISCV::FCVT_D_W:
    case RISCV::FCVT_S_W:
      if (Bits >= 32)
        break;
      return false;
    case RISCV::SLL:
    case RISCV::SRA:
    case RISCV::SRL:
      // Shift amount operands only use log2(Xlen) bits.
      if (OpIdx == 2 && Bits >= Log2_32(Subtarget->getXLen()))
        break;
      return false;
    case RISCV::SLLI:
      // SLLI only uses the lower (XLen - ShAmt) bits.
      if (Bits >= Subtarget->getXLen() - UserMI.getOperand(2).getImm())
        break;
      return false;
    case RISCV::ANDI:
      if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
                      (uint64_t)UserMI.getOperand(2).getImm()))
        break;
      goto RecCheck;
    case RISCV::AND:
    case RISCV::OR:
    case RISCV::XOR:
    RecCheck:
      if (hasAllNBitUsers(UserMI, Bits, Depth + 1))
        break;
      return false;
    case RISCV::SRLI: {
      unsigned ShAmt = UserMI.getOperand(2).getImm();
      // If we are shifting right by less than Bits, and users don't demand any
      // bits that were shifted into [Bits-1:0], then we can consider this as an
      // N-Bit user.
      if (Bits > ShAmt && hasAllNBitUsers(UserMI, Bits - ShAmt, Depth + 1))
        break;
      return false;
    }
    }
  }
 
  return true;
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
                                          unsigned ShiftWidth) const {
  if (!Root.isReg())
    return std::nullopt;
 
  using namespace llvm::MIPatternMatch;
 
  Register ShAmtReg = Root.getReg();
  // Peek through zext.
  Register ZExtSrcReg;
  if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg))))
    ShAmtReg = ZExtSrcReg;
 
  APInt AndMask;
  Register AndSrcReg;
  // Try to combine the following pattern (applicable to other shift
  // instructions as well as 32-bit ones):
  //
  //   %4:gprb(s64) = G_AND %3, %2
  //   %5:gprb(s64) = G_LSHR %1, %4(s64)
  //
  // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
  // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
  // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
  // then it can be eliminated. Given register rs1 or rs2 holding a constant
  // (the and mask), there are two cases G_AND can be erased:
  //
  // 1. the lowest log2(XLEN) bits of the and mask are all set
  // 2. the bits of the register being masked are already unset (zero set)
  if (mi_match(ShAmtReg, *MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
    APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
    if (ShMask.isSubsetOf(AndMask)) {
      ShAmtReg = AndSrcReg;
    } else {
      // SimplifyDemandedBits may have optimized the mask so try restoring any
      // bits that are known zero.
      KnownBits Known = VT->getKnownBits(AndSrcReg);
      if (ShMask.isSubsetOf(AndMask | Known.Zero))
        ShAmtReg = AndSrcReg;
    }
  }
 
  APInt Imm;
  Register Reg;
  if (mi_match(ShAmtReg, *MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
    if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
      // If we are shifting by X+N where N == 0 mod Size, then just shift by X
      // to avoid the ADD.
      ShAmtReg = Reg;
  } else if (mi_match(ShAmtReg, *MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
    if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
      // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
      // to generate a NEG instead of a SUB of a constant.
      ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
      unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
      return {{[=](MachineInstrBuilder &MIB) {
        MachineIRBuilder(*MIB.getInstr())
            .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
        MIB.addReg(ShAmtReg);
      }}};
    }
    if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
      // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
      // to generate a NOT instead of a SUB of a constant.
      ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
      return {{[=](MachineInstrBuilder &MIB) {
        MachineIRBuilder(*MIB.getInstr())
            .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
            .addImm(-1);
        MIB.addReg(ShAmtReg);
      }}};
    }
  }
 
  return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
                                         unsigned Bits) const {
  if (!Root.isReg())
    return std::nullopt;
  Register RootReg = Root.getReg();
 
  Register SrcReg;
  if (mi_match(RootReg, *MRI,
               m_GSExtInReg(m_Reg(SrcReg), m_SpecificImm(Bits)))) {
    return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(SrcReg); }}};
  }
 
  unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
  if ((Size - VT->computeNumSignBits(RootReg)) < Bits)
    return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
 
  return std::nullopt;
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
                                         unsigned Bits) const {
  if (!Root.isReg())
    return std::nullopt;
  Register RootReg = Root.getReg();
 
  Register RegX;
  uint64_t Mask = maskTrailingOnes<uint64_t>(Bits);
  if (mi_match(RootReg, *MRI, m_GAnd(m_Reg(RegX), m_SpecificICst(Mask)))) {
    return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
  }
 
  if (mi_match(RootReg, *MRI, m_GZExt(m_Reg(RegX))) &&
      MRI->getType(RegX).getScalarSizeInBits() == Bits)
    return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
 
  unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
  if (VT->maskedValueIsZero(RootReg, APInt::getBitsSetFrom(Size, Bits)))
    return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
 
  return std::nullopt;
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
                                         unsigned ShAmt) const {
  using namespace llvm::MIPatternMatch;
 
  if (!Root.isReg())
    return std::nullopt;
  Register RootReg = Root.getReg();
 
  const unsigned XLen = STI.getXLen();
  APInt Mask;
  int64_t C2;
  Register RegY;
  std::optional<bool> LeftShift;
  // (and (shl y, c2), mask)
  if (mi_match(RootReg, *MRI,
               m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
    LeftShift = true;
  // (and (lshr y, c2), mask)
  else if (mi_match(RootReg, *MRI,
                    m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
    LeftShift = false;
 
  if (LeftShift.has_value()) {
    assert(Mask.getBitWidth() == XLen && "Unexpected mask size!");
    assert(C2 > 0 && C2 < XLen && "Unexpected shift amount!");
    if (*LeftShift)
      Mask &= maskTrailingZeros<uint64_t>(C2);
    else
      Mask &= maskTrailingOnes<uint64_t>(XLen - C2);
 
    if (Mask.isShiftedMask()) {
      unsigned Leading = XLen - Mask.getActiveBits();
      unsigned Trailing = Mask.countr_zero();
      // Given (and (shl y, c2), mask) in which mask has no leading zeros and
      // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
      if (*LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
        Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
        return {{[=](MachineInstrBuilder &MIB) {
          MachineIRBuilder(*MIB.getInstr())
              .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
              .addImm(Trailing - C2);
          MIB.addReg(DstReg);
        }}};
      }
 
      // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
      // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
      if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
        Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
        return {{[=](MachineInstrBuilder &MIB) {
          MachineIRBuilder(*MIB.getInstr())
              .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
              .addImm(Leading + Trailing);
          MIB.addReg(DstReg);
        }}};
      }
    }
  }
 
  LeftShift.reset();
 
  // (shl (and y, mask), c2)
  if (mi_match(RootReg, *MRI,
               m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
                      m_ICst(C2))))
    LeftShift = true;
  // (lshr (and y, mask), c2)
  else if (mi_match(RootReg, *MRI,
                    m_GLShr(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
                            m_ICst(C2))))
    LeftShift = false;
 
  if (LeftShift.has_value() && Mask.isShiftedMask()) {
    unsigned Leading = XLen - Mask.getActiveBits();
    unsigned Trailing = Mask.countr_zero();
 
    // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
    // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
    bool Cond =
        *LeftShift && Leading == 32 && Trailing > 0 && (Trailing + C2) == ShAmt;
    if (!Cond)
      // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
      // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
      Cond = !*LeftShift && Leading == 32 && C2 < Trailing &&
             (Trailing - C2) == ShAmt;
 
    if (Cond) {
      Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
      return {{[=](MachineInstrBuilder &MIB) {
        MachineIRBuilder(*MIB.getInstr())
            .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
            .addImm(Trailing);
        MIB.addReg(DstReg);
      }}};
    }
  }
 
  return std::nullopt;
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
                                            unsigned ShAmt) const {
  using namespace llvm::MIPatternMatch;
 
  if (!Root.isReg())
    return std::nullopt;
  Register RootReg = Root.getReg();
 
  // Given (and (shl x, c2), mask) in which mask is a shifted mask with
  // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
  // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
  APInt Mask;
  int64_t C2;
  Register RegX;
  if (mi_match(
          RootReg, *MRI,
          m_OneNonDBGUse(m_GAnd(m_OneNonDBGUse(m_GShl(m_Reg(RegX), m_ICst(C2))),
                                m_ICst(Mask))))) {
    assert(C2 > 0 && C2 < Mask.getBitWidth() && "Unexpected shift amount!");
    Mask &= maskTrailingZeros<uint64_t>(C2);
 
    if (Mask.isShiftedMask()) {
      unsigned Leading = Mask.countl_zero();
      unsigned Trailing = Mask.countr_zero();
      if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
        Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
        return {{[=](MachineInstrBuilder &MIB) {
          MachineIRBuilder(*MIB.getInstr())
              .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
              .addImm(C2 - ShAmt);
          MIB.addReg(DstReg);
        }}};
      }
    }
  }
 
  return std::nullopt;
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
  assert(Root.isReg() && "Expected operand to be a Register");
  std::optional<ValueAndVReg> C;
  if (mi_match(Root.getReg(), *MRI, m_GCst(C))) {
    if (C->Value.isAllOnes())
      // If the operand is a G_CONSTANT with value of all ones it is larger than
      // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
      // recognized specially by the vsetvli insertion pass.
      return {{[=](MachineInstrBuilder &MIB) {
        MIB.addImm(RISCV::VLMaxSentinel);
      }}};
 
    if (isUInt<5>(C->Value.getZExtValue())) {
      uint64_t ZExtC = C->Value.getZExtValue();
      return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(ZExtC); }}};
    }
  }
  return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }}};
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::renderAddiPair(Register BaseReg, int64_t AddiImm,
                                         int64_t OffsetImm) const {
  return {{[=](MachineInstrBuilder &MIB) {
             Register Tmp = MRI->createVirtualRegister(&RISCV::GPRRegClass);
             MachineInstr *Addi =
                 BuildMI(*MIB->getParent(), *MIB.getInstr(), MIB->getDebugLoc(),
                         TII.get(RISCV::ADDI), Tmp)
                     .addReg(BaseReg)
                     .addImm(AddiImm);
             constrainSelectedInstRegOperands(*Addi, TII, TRI, RBI);
             MIB.addReg(Tmp);
           },
           [=](MachineInstrBuilder &MIB) { MIB.addImm(OffsetImm); }}};
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
  if (!Root.isReg())
    return std::nullopt;
 
  Register RootReg = Root.getReg();
 
  // Frame index.
  int FI;
  if (mi_match(RootReg, *MRI, m_GFrameIndex(FI))) {
    return {{
        [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); },
        [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    }};
  }
 
  // base + constant offset (G_PTR_ADD).
  Register BaseReg;
  int64_t RHSC;
  if (mi_match(RootReg, *MRI, m_GPtrAdd(m_Reg(BaseReg), m_ICst(RHSC)))) {
    if (isInt<12>(RHSC)) {
      int BaseFI;
      if (mi_match(BaseReg, *MRI, m_GFrameIndex(BaseFI)))
        return {{
            [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(BaseFI); },
            [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
        }};
 
      return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(BaseReg); },
               [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
    }
 
    // Large constant offset. Fold a -2048/2047 adjustment so the whole
    // constant can be split across an ADDI and the load/store offset.
    if (RHSC >= -4096 && RHSC <= 4094) {
      int64_t Adj = RHSC < 0 ? -2048 : 2047;
      return renderAddiPair(BaseReg, Adj, RHSC - Adj);
    }
 
    if (isWorthFoldingAdd(RootReg))
      if (auto Fns = computeConstAddr(RHSC, /*IsPrefetch=*/false, BaseReg))
        return Fns;
  }
 
  // Bare constant address. IRTranslator lowers inttoptr(C) to
  // G_INTTOPTR(G_CONSTANT); look through it to reach the constant.
  int64_t CVal;
  if (mi_match(RootReg, *MRI, m_GIntToPtr(m_ICst(CVal))) ||
      mi_match(RootReg, *MRI, m_ICst(CVal))) {
    if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/false, Register()))
      return Fns;
  }
 
  return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RootReg); },
           [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectBrindRegImm(MachineOperand &Root) const {
  if (!Root.isReg())
    return std::nullopt;
 
  Register RootReg = Root.getReg();
 
  // base + constant offset (G_PTR_ADD). Doesn't match a FrameIndex, unlike
  // selectAddrRegImm, since the callers of this (e.g. indirect branches)
  // can't take a FrameIndex or global address operand.
  Register BaseReg;
  int64_t RHSC;
  if (mi_match(RootReg, *MRI, m_GPtrAdd(m_Reg(BaseReg), m_ICst(RHSC))) &&
      isInt<12>(RHSC)) {
    return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(BaseReg); },
             [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
  }
 
  return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RootReg); },
           [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
  if (!Root.isReg())
    return std::nullopt;
 
  Register RootReg = Root.getReg();
 
  // Frame index.
  int FI;
  if (mi_match(RootReg, *MRI, m_GFrameIndex(FI))) {
    return {{
        [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); },
        [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
    }};
  }
 
  // base + constant offset (G_PTR_ADD).
  Register BaseReg;
  int64_t RHSC;
  if (mi_match(RootReg, *MRI, m_GPtrAdd(m_Reg(BaseReg), m_ICst(RHSC)))) {
    if (isInt<12>(RHSC)) {
      // Not a multiple of 32: can't encode, use the address as-is.
      if ((RHSC & 0b11111) != 0) {
        return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RootReg); },
                 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
      }
      // Fold the offset.
      int BaseFI;
      if (mi_match(BaseReg, *MRI, m_GFrameIndex(BaseFI)))
        return {{
            [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(BaseFI); },
            [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
        }};
      return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(BaseReg); },
               [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
    }
 
    // Large constant: fold a -2048/2016 adjustment to save an instruction.
    if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) {
      int64_t Adj = RHSC < 0 ? -2048 : 2016;
      return renderAddiPair(BaseReg, RHSC - Adj, Adj);
    }
 
    // Otherwise split the constant into Hi (materialized + added to the base)
    // and Lo12 (folded offset).
    if (auto Fns = computeConstAddr(RHSC, /*IsPrefetch=*/true, BaseReg))
      return Fns;
  }
 
  // Bare constant address. IRTranslator emits inttoptr(C) as
  // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant.
  int64_t CVal;
  if (mi_match(RootReg, *MRI, m_GIntToPtr(m_ICst(CVal))) ||
      mi_match(RootReg, *MRI, m_ICst(CVal))) {
    if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/true, Register()))
      return Fns;
  }
 
  return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RootReg); },
           [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
}
 
/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
/// CC Must be an ICMP Predicate.
static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
  switch (CC) {
  default:
    llvm_unreachable("Expected ICMP CmpInst::Predicate.");
  case CmpInst::Predicate::ICMP_EQ:
    return RISCVCC::COND_EQ;
  case CmpInst::Predicate::ICMP_NE:
    return RISCVCC::COND_NE;
  case CmpInst::Predicate::ICMP_ULT:
    return RISCVCC::COND_LTU;
  case CmpInst::Predicate::ICMP_SLT:
    return RISCVCC::COND_LT;
  case CmpInst::Predicate::ICMP_UGE:
    return RISCVCC::COND_GEU;
  case CmpInst::Predicate::ICMP_SGE:
    return RISCVCC::COND_GE;
  }
}
 
static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC,
                                 Register &LHS, Register &RHS,
                                 MachineRegisterInfo &MRI) {
  // Try to fold an ICmp. If that fails, use a NE compare with X0.
  CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
  if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
    LHS = CondReg;
    RHS = RISCV::X0;
    CC = RISCVCC::COND_NE;
    return;
  }
 
  // We found an ICmp, do some canonicalization.
 
  // Adjust comparisons to use comparison with 0 if possible.
  if (auto Constant = getIConstantVRegSExtVal(RHS, MRI)) {
    switch (Pred) {
    case CmpInst::Predicate::ICMP_SGT:
      // Convert X > -1 to X >= 0
      if (*Constant == -1) {
        CC = RISCVCC::COND_GE;
        RHS = RISCV::X0;
        return;
      }
      break;
    case CmpInst::Predicate::ICMP_SLT:
      // Convert X < 1 to 0 >= X
      if (*Constant == 1) {
        CC = RISCVCC::COND_GE;
        RHS = LHS;
        LHS = RISCV::X0;
        return;
      }
      break;
    default:
      break;
    }
  }
 
  switch (Pred) {
  default:
    llvm_unreachable("Expected ICMP CmpInst::Predicate.");
  case CmpInst::Predicate::ICMP_EQ:
  case CmpInst::Predicate::ICMP_NE:
  case CmpInst::Predicate::ICMP_ULT:
  case CmpInst::Predicate::ICMP_SLT:
  case CmpInst::Predicate::ICMP_UGE:
  case CmpInst::Predicate::ICMP_SGE:
    // These CCs are supported directly by RISC-V branches.
    break;
  case CmpInst::Predicate::ICMP_SGT:
  case CmpInst::Predicate::ICMP_SLE:
  case CmpInst::Predicate::ICMP_UGT:
  case CmpInst::Predicate::ICMP_ULE:
    // These CCs are not supported directly by RISC-V branches, but changing the
    // direction of the CC and swapping LHS and RHS are.
    Pred = CmpInst::getSwappedPredicate(Pred);
    std::swap(LHS, RHS);
    break;
  }
 
  CC = getRISCVCCFromICmp(Pred);
}
 
/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
/// \p GenericOpc, appropriate for the GPR register bank and of memory access
/// size \p OpSize.
static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
  const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
  switch (OpSize) {
  default:
    llvm_unreachable("Unexpected memory size");
  case 8:
    return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
  case 16:
    return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
  case 32:
    return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
  case 64:
    return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
  }
}
 
/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
/// \p GenericOpc on \p STI, appropriate for the GPR register bank and of
/// memory access size \p OpSize. \returns \p GenericOpc if the combination
/// is unsupported.
static unsigned selectRegImmLoadStoreOp(const RISCVSubtarget &STI,
                                        unsigned GenericOpc, unsigned OpSize) {
  const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
  switch (OpSize) {
  case 8:
    // Prefer unsigned due to no c.lb in Zcb.
    return IsStore ? RISCV::SB : RISCV::LBU;
  case 16:
    return IsStore ? RISCV::SH : RISCV::LH;
  case 32:
    return IsStore ? RISCV::SW : RISCV::LW;
  case 64:
    // LD/SD are RV64-only. On RV32 a 64-bit access goes through an even/odd
    // GPR pair, which Zilsd provides via the LD_RV32/SD_RV32 pseudos.
    if (!STI.is64Bit() && STI.hasStdExtZilsd())
      return IsStore ? RISCV::SD_RV32 : RISCV::LD_RV32;
    return IsStore ? RISCV::SD : RISCV::LD;
  }
 
  return GenericOpc;
}
 
void RISCVInstructionSelector::addVectorLoadStoreOperands(
    MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp,
    bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
  // Base Pointer
  auto PtrReg = I.getOperand(CurOp++).getReg();
  SrcOps.push_back(PtrReg);
 
  // Stride or Index
  if (IsStridedOrIndexed) {
    auto StrideReg = I.getOperand(CurOp++).getReg();
    SrcOps.push_back(StrideReg);
    if (IndexVT)
      *IndexVT = MRI->getType(StrideReg);
  }
 
  // Mask
  if (IsMasked) {
    auto MaskReg = I.getOperand(CurOp++).getReg();
    SrcOps.push_back(MaskReg);
  }
}
 
bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
    MachineInstr &I) const {
  // Find the intrinsic ID.
  unsigned IntrinID = cast<GIntrinsic>(I).getIntrinsicID();
  // Select the instruction.
  switch (IntrinID) {
  default:
    return false;
  case Intrinsic::riscv_vlm:
  case Intrinsic::riscv_vle:
  case Intrinsic::riscv_vle_mask:
  case Intrinsic::riscv_vlse:
  case Intrinsic::riscv_vlse_mask: {
    bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
                    IntrinID == Intrinsic::riscv_vlse_mask;
    bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
                     IntrinID == Intrinsic::riscv_vlse_mask;
    LLT VT = MRI->getType(I.getOperand(0).getReg());
    unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
 
    // Result vector
    const Register DstReg = I.getOperand(0).getReg();
 
    // Sources
    bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
    unsigned CurOp = 2;
    SmallVector<Register, 4> SrcOps; // Source registers.
 
    // Passthru
    if (HasPassthruOperand) {
      auto PassthruReg = I.getOperand(CurOp++).getReg();
      SrcOps.push_back(PassthruReg);
    } else {
      SrcOps.push_back(Register(RISCV::NoRegister));
    }
 
    addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStrided);
 
    RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(getMVTForLLT(VT));
    const RISCV::VLEPseudo *P =
        RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, Log2SEW,
                            static_cast<unsigned>(LMUL));
 
    MachineInstrBuilder PseudoMI =
        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(P->Pseudo), DstReg);
    for (Register Reg : SrcOps)
      PseudoMI.addReg(Reg);
 
    // Select VL
    auto VLOpFn = renderVLOp(I.getOperand(CurOp++));
    for (auto &RenderFn : *VLOpFn)
      RenderFn(PseudoMI);
 
    // SEW
    PseudoMI.addImm(Log2SEW);
 
    // Policy
    uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
    if (IsMasked)
      Policy = I.getOperand(CurOp++).getImm();
    PseudoMI.addImm(Policy);
 
    // Memref
    PseudoMI.cloneMemRefs(I);
 
    I.eraseFromParent();
    constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
    return true;
  }
  case Intrinsic::riscv_vloxei:
  case Intrinsic::riscv_vloxei_mask:
  case Intrinsic::riscv_vluxei:
  case Intrinsic::riscv_vluxei_mask: {
    bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
                    IntrinID == Intrinsic::riscv_vluxei_mask;
    bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
                     IntrinID == Intrinsic::riscv_vloxei_mask;
    LLT VT = MRI->getType(I.getOperand(0).getReg());
    unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
 
    // Result vector
    const Register DstReg = I.getOperand(0).getReg();
 
    // Sources
    bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
    unsigned CurOp = 2;
    SmallVector<Register, 4> SrcOps; // Source registers.
 
    // Passthru
    if (HasPassthruOperand) {
      auto PassthruReg = I.getOperand(CurOp++).getReg();
      SrcOps.push_back(PassthruReg);
    } else {
      // Use NoRegister if there is no specified passthru.
      SrcOps.push_back(Register());
    }
    LLT IndexVT;
    addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, true, &IndexVT);
 
    RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(getMVTForLLT(VT));
    RISCVVType::VLMUL IndexLMUL =
        RISCVTargetLowering::getLMUL(getMVTForLLT(IndexVT));
    unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
    if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
      reportFatalUsageError("The V extension does not support EEW=64 for index "
                            "values when XLEN=32");
    }
    const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
        IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
        static_cast<unsigned>(IndexLMUL));
 
    MachineInstrBuilder PseudoMI =
        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(P->Pseudo), DstReg);
    for (Register Reg : SrcOps)
      PseudoMI.addReg(Reg);
 
    // Select VL
    auto VLOpFn = renderVLOp(I.getOperand(CurOp++));
    for (auto &RenderFn : *VLOpFn)
      RenderFn(PseudoMI);
 
    // SEW
    PseudoMI.addImm(Log2SEW);
 
    // Policy
    uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
    if (IsMasked)
      Policy = I.getOperand(CurOp++).getImm();
    PseudoMI.addImm(Policy);
 
    // Memref
    PseudoMI.cloneMemRefs(I);
 
    I.eraseFromParent();
    constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
    return true;
  }
  case Intrinsic::riscv_vsm:
  case Intrinsic::riscv_vse:
  case Intrinsic::riscv_vse_mask:
  case Intrinsic::riscv_vsse:
  case Intrinsic::riscv_vsse_mask: {
    bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
                    IntrinID == Intrinsic::riscv_vsse_mask;
    bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
                     IntrinID == Intrinsic::riscv_vsse_mask;
    LLT VT = MRI->getType(I.getOperand(1).getReg());
    unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
 
    // Sources
    unsigned CurOp = 1;
    SmallVector<Register, 4> SrcOps; // Source registers.
 
    // Store value
    auto PassthruReg = I.getOperand(CurOp++).getReg();
    SrcOps.push_back(PassthruReg);
 
    addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStrided);
 
    RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(getMVTForLLT(VT));
    const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
        IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
 
    MachineInstrBuilder PseudoMI =
        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(P->Pseudo));
    for (Register Reg : SrcOps)
      PseudoMI.addReg(Reg);
 
    // Select VL
    auto VLOpFn = renderVLOp(I.getOperand(CurOp++));
    for (auto &RenderFn : *VLOpFn)
      RenderFn(PseudoMI);
 
    // SEW
    PseudoMI.addImm(Log2SEW);
 
    // Memref
    PseudoMI.cloneMemRefs(I);
 
    I.eraseFromParent();
    constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
    return true;
  }
  case Intrinsic::riscv_vsoxei:
  case Intrinsic::riscv_vsoxei_mask:
  case Intrinsic::riscv_vsuxei:
  case Intrinsic::riscv_vsuxei_mask: {
    bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
                    IntrinID == Intrinsic::riscv_vsuxei_mask;
    bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
                     IntrinID == Intrinsic::riscv_vsoxei_mask;
    LLT VT = MRI->getType(I.getOperand(1).getReg());
    unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
 
    // Sources
    unsigned CurOp = 1;
    SmallVector<Register, 4> SrcOps; // Source registers.
 
    // Store value
    auto PassthruReg = I.getOperand(CurOp++).getReg();
    SrcOps.push_back(PassthruReg);
 
    LLT IndexVT;
    addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, true, &IndexVT);
 
    RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(getMVTForLLT(VT));
    RISCVVType::VLMUL IndexLMUL =
        RISCVTargetLowering::getLMUL(getMVTForLLT(IndexVT));
    unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
    if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
      reportFatalUsageError("The V extension does not support EEW=64 for index "
                            "values when XLEN=32");
    }
    const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
        IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
        static_cast<unsigned>(IndexLMUL));
 
    MachineInstrBuilder PseudoMI =
        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(P->Pseudo));
    for (Register Reg : SrcOps)
      PseudoMI.addReg(Reg);
 
    // Select VL
    auto VLOpFn = renderVLOp(I.getOperand(CurOp++));
    for (auto &RenderFn : *VLOpFn)
      RenderFn(PseudoMI);
 
    // SEW
    PseudoMI.addImm(Log2SEW);
 
    // Memref
    PseudoMI.cloneMemRefs(I);
 
    I.eraseFromParent();
    constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
    return true;
  }
  }
}
 
bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const {
  // Find the intrinsic ID.
  unsigned IntrinID = cast<GIntrinsic>(I).getIntrinsicID();
  // Select the instruction.
  switch (IntrinID) {
  default:
    return false;
  case Intrinsic::riscv_vsetvli:
  case Intrinsic::riscv_vsetvlimax: {
 
    bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
 
    unsigned Offset = VLMax ? 2 : 3;
    unsigned SEW = RISCVVType::decodeVSEW(I.getOperand(Offset).getImm() & 0x7);
    RISCVVType::VLMUL VLMul =
        static_cast<RISCVVType::VLMUL>(I.getOperand(Offset + 1).getImm() & 0x7);
 
    unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true,
                                              /*MaskAgnostic*/ true);
 
    Register DstReg = I.getOperand(0).getReg();
 
    Register VLOperand;
    unsigned Opcode = RISCV::PseudoVSETVLI;
 
    // Check if AVL is a constant that equals VLMAX.
    if (!VLMax) {
      Register AVLReg = I.getOperand(2).getReg();
      if (auto AVLConst = getIConstantVRegValWithLookThrough(AVLReg, *MRI)) {
        uint64_t AVL = AVLConst->Value.getZExtValue();
        if (auto VLEN = Subtarget->getRealVLen()) {
          if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
            VLMax = true;
        }
      }
 
      if (mi_match(AVLReg, *MRI, m_AllOnes()))
        VLMax = true;
    }
 
    if (VLMax) {
      VLOperand = Register(RISCV::X0);
      Opcode = RISCV::PseudoVSETVLIX0;
    } else {
      Register AVLReg = I.getOperand(2).getReg();
      VLOperand = AVLReg;
 
      // Check if AVL is a small constant that can use PseudoVSETIVLI.
      if (auto AVLConst = getIConstantVRegValWithLookThrough(AVLReg, *MRI)) {
        uint64_t AVL = AVLConst->Value.getZExtValue();
        if (isUInt<5>(AVL)) {
          MachineInstr *PseudoMI =
              BuildMI(*I.getParent(), I, I.getDebugLoc(),
                      TII.get(RISCV::PseudoVSETIVLI), DstReg)
                  .addImm(AVL)
                  .addImm(VTypeI);
          I.eraseFromParent();
          constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
          return true;
        }
      }
    }
 
    MachineInstr *PseudoMI =
        BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
            .addReg(VLOperand)
            .addImm(VTypeI);
    I.eraseFromParent();
    constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
    return true;
  }
  }
}
 
bool RISCVInstructionSelector::selectExtractSubvector(MachineInstr &MI) const {
  assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
 
  Register DstReg = MI.getOperand(0).getReg();
  Register SrcReg = MI.getOperand(1).getReg();
 
  LLT DstTy = MRI->getType(DstReg);
  LLT SrcTy = MRI->getType(SrcReg);
 
  unsigned Idx = static_cast<unsigned>(MI.getOperand(2).getImm());
 
  MVT DstMVT = getMVTForLLT(DstTy);
  MVT SrcMVT = getMVTForLLT(SrcTy);
 
  unsigned SubRegIdx;
  std::tie(SubRegIdx, Idx) =
      RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
          SrcMVT, DstMVT, Idx, &TRI);
 
  if (Idx != 0)
    return false;
 
  unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(DstMVT);
  const TargetRegisterClass *DstRC = TRI.getRegClass(DstRegClassID);
  if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
    return false;
 
  unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(SrcMVT);
  const TargetRegisterClass *SrcRC = TRI.getRegClass(SrcRegClassID);
  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
    return false;
 
  BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(TargetOpcode::COPY),
          DstReg)
      .addReg(SrcReg, {}, SubRegIdx);
 
  MI.eraseFromParent();
  return true;
}
 
bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const {
  assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR);
 
  Register DstReg = MI.getOperand(0).getReg();
  Register VecReg = MI.getOperand(1).getReg();
  Register SubVecReg = MI.getOperand(2).getReg();
 
  LLT VecTy = MRI->getType(VecReg);
  LLT SubVecTy = MRI->getType(SubVecReg);
 
  MVT VecMVT = getMVTForLLT(VecTy);
  MVT SubVecMVT = getMVTForLLT(SubVecTy);
 
  unsigned Idx = static_cast<unsigned>(MI.getOperand(3).getImm());
 
  unsigned SubRegIdx;
  std::tie(SubRegIdx, Idx) =
      RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
          VecMVT, SubVecMVT, Idx, &TRI);
 
  // If the Idx hasn't been completely eliminated then this is a subvector
  // insert which doesn't naturally align to a vector register. These must
  // be handled using instructions to manipulate the vector registers.
  if (Idx != 0)
    return false;
 
  // Constrain dst
  unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VecMVT);
  const TargetRegisterClass *DstRC = TRI.getRegClass(DstRegClassID);
  if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
    return false;
 
  // If we haven't set a SubRegIdx, then we must be going between
  // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
  if (SubRegIdx == RISCV::NoSubRegister) {
    assert(RISCVTargetLowering::getRegClassIDForVecVT(SubVecMVT) ==
               DstRegClassID &&
           "Unexpected subvector insert");
    BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(TargetOpcode::COPY),
            DstReg)
        .addReg(SubVecReg);
    MI.eraseFromParent();
    return true;
  }
 
  // Use INSERT_SUBREG to insert the subvector into the vector at the
  // appropriate subregister index.
  MachineInstr *Ins = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                              TII.get(TargetOpcode::INSERT_SUBREG), DstReg)
                          .addReg(VecReg)
                          .addReg(SubVecReg)
                          .addImm(SubRegIdx);
 
  MI.eraseFromParent();
  constrainSelectedInstRegOperands(*Ins, TII, TRI, RBI);
  return true;
}
 
bool RISCVInstructionSelector::select(MachineInstr &MI) {
  preISelLower(MI);
  const unsigned Opc = MI.getOpcode();
 
  if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
    if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
      const Register DefReg = MI.getOperand(0).getReg();
      const LLT DefTy = MRI->getType(DefReg);
 
      const RegClassOrRegBank &RegClassOrBank =
          MRI->getRegClassOrRegBank(DefReg);
 
      const TargetRegisterClass *DefRC =
          dyn_cast<const TargetRegisterClass *>(RegClassOrBank);
      if (!DefRC) {
        if (!DefTy.isValid()) {
          LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
          return false;
        }
 
        const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);
        DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB, STI.is64Bit());
        if (!DefRC) {
          LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
          return false;
        }
      }
 
      MI.setDesc(TII.get(TargetOpcode::PHI));
      return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
    }
 
    // Certain non-generic instructions also need some special handling.
    if (MI.isCopy())
      return selectCopy(MI);
 
    return true;
  }
 
  if (selectImpl(MI, *CoverageInfo))
    return true;
 
  switch (Opc) {
  case TargetOpcode::G_ANYEXT:
  case TargetOpcode::G_PTRTOINT:
  case TargetOpcode::G_INTTOPTR:
  case TargetOpcode::G_TRUNC:
  case TargetOpcode::G_FREEZE:
    return selectCopy(MI);
  case TargetOpcode::G_CONSTANT: {
    Register DstReg = MI.getOperand(0).getReg();
    int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
 
    if (!materializeImm(DstReg, Imm, MI))
      return false;
 
    MI.eraseFromParent();
    return true;
  }
  case TargetOpcode::G_ZEXT:
  case TargetOpcode::G_SEXT: {
    bool IsSigned = Opc != TargetOpcode::G_ZEXT;
    Register DstReg = MI.getOperand(0).getReg();
    Register SrcReg = MI.getOperand(1).getReg();
    LLT SrcTy = MRI->getType(SrcReg);
    unsigned SrcSize = SrcTy.getSizeInBits();
 
    if (SrcTy.isVector())
      return false; // Should be handled by imported patterns.
 
    assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
               RISCV::GPRBRegBankID &&
           "Unexpected ext regbank");
 
    // Use addiw SrcReg, 0 (sext.w) for i32.
    if (IsSigned && SrcSize == 32) {
      MI.setDesc(TII.get(RISCV::ADDIW));
      MI.addOperand(MachineOperand::CreateImm(0));
      constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
      return true;
    }
 
    // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
    if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
      MI.setDesc(TII.get(RISCV::ADD_UW));
      MI.addOperand(MachineOperand::CreateReg(RISCV::X0, /*isDef=*/false));
      constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
      return true;
    }
 
    // Use sext.h/zext.h for i16 with Zbb.
    if (SrcSize == 16 &&
        (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) {
      MI.setDesc(TII.get(IsSigned        ? RISCV::SEXT_H
                         : STI.is64Bit() ? RISCV::ZEXT_H_RV64
                                         : RISCV::ZEXT_H_RV32));
      constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
      return true;
    }
 
    // Fall back to shift pair.
    Register ShiftLeftReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    MachineInstr *ShiftLeft = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                      TII.get(RISCV::SLLI), ShiftLeftReg)
                                  .addReg(SrcReg)
                                  .addImm(STI.getXLen() - SrcSize);
    constrainSelectedInstRegOperands(*ShiftLeft, TII, TRI, RBI);
    MachineInstr *ShiftRight =
        BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                TII.get(IsSigned ? RISCV::SRAI : RISCV::SRLI), DstReg)
            .addReg(ShiftLeftReg)
            .addImm(STI.getXLen() - SrcSize);
    constrainSelectedInstRegOperands(*ShiftRight, TII, TRI, RBI);
    MI.eraseFromParent();
    return true;
  }
  case TargetOpcode::G_FCONSTANT: {
    // TODO: Use constant pool for complex constants.
    Register DstReg = MI.getOperand(0).getReg();
    const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
    unsigned Size = MRI->getType(DstReg).getSizeInBits();
    if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
      Register GPRReg;
      if (FPimm.isPosZero()) {
        GPRReg = RISCV::X0;
      } else {
        GPRReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
        APInt Imm = FPimm.bitcastToAPInt();
        if (!materializeImm(GPRReg, Imm.getSExtValue(), MI))
          return false;
      }
 
      unsigned Opcode = Size == 64   ? RISCV::FMV_D_X
                        : Size == 32 ? RISCV::FMV_W_X
                                     : RISCV::FMV_H_X;
      MachineInstr *FMV = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                  TII.get(Opcode), DstReg)
                              .addReg(GPRReg);
      constrainSelectedInstRegOperands(*FMV, TII, TRI, RBI);
    } else {
      // s64 on rv32
      assert(Size == 64 && !Subtarget->is64Bit() &&
             "Unexpected size or subtarget");
 
      if (FPimm.isPosZero()) {
        // Optimize +0.0 to use fcvt.d.w
        MachineInstr *FCVT = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                     TII.get(RISCV::FCVT_D_W), DstReg)
                                 .addReg(RISCV::X0)
                                 .addImm(RISCVFPRndMode::RNE);
        constrainSelectedInstRegOperands(*FCVT, TII, TRI, RBI);
 
        MI.eraseFromParent();
        return true;
      }
 
      // Split into two pieces and build through the stack.
      Register GPRRegHigh = MRI->createVirtualRegister(&RISCV::GPRRegClass);
      Register GPRRegLow = MRI->createVirtualRegister(&RISCV::GPRRegClass);
      APInt Imm = FPimm.bitcastToAPInt();
      if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
                          MI))
        return false;
      if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MI))
        return false;
      MachineInstr *PairF64 =
          BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                  TII.get(RISCV::BuildPairF64Pseudo), DstReg)
              .addReg(GPRRegLow)
              .addReg(GPRRegHigh);
      constrainSelectedInstRegOperands(*PairF64, TII, TRI, RBI);
    }
 
    MI.eraseFromParent();
    return true;
  }
  case TargetOpcode::G_GLOBAL_VALUE: {
    auto *GV = MI.getOperand(1).getGlobal();
    if (GV->isThreadLocal()) {
      // TODO: implement this case.
      return false;
    }
 
    return selectAddr(MI, GV->isDSOLocal(), GV->hasExternalWeakLinkage());
  }
  case TargetOpcode::G_JUMP_TABLE:
  case TargetOpcode::G_CONSTANT_POOL:
    return selectAddr(MI);
  case TargetOpcode::G_BRCOND: {
    Register LHS, RHS;
    RISCVCC::CondCode CC;
    getOperandsForBranch(MI.getOperand(0).getReg(), CC, LHS, RHS, *MRI);
 
    MachineInstr *Bcc = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                TII.get(RISCVCC::getBrCond(CC)))
                            .addReg(LHS)
                            .addReg(RHS)
                            .addMBB(MI.getOperand(1).getMBB());
    MI.eraseFromParent();
    constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
    return true;
  }
  case TargetOpcode::G_SELECT:
    return selectSelect(MI);
  case TargetOpcode::G_FCMP:
    return selectFPCompare(MI);
  case TargetOpcode::G_FENCE: {
    AtomicOrdering FenceOrdering =
        static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
    SyncScope::ID FenceSSID =
        static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
    emitFence(FenceOrdering, FenceSSID, MI);
    MI.eraseFromParent();
    return true;
  }
  case TargetOpcode::G_IMPLICIT_DEF:
    return selectImplicitDef(MI);
  case TargetOpcode::G_UNMERGE_VALUES:
    return selectUnmergeValues(MI);
  case TargetOpcode::G_MERGE_VALUES:
    return selectMergeValues(MI);
  case TargetOpcode::G_LOAD:
  case TargetOpcode::G_STORE: {
    GLoadStore &LdSt = cast<GLoadStore>(MI);
    const Register ValReg = LdSt.getReg(0);
    const Register PtrReg = LdSt.getPointerReg();
    LLT PtrTy = MRI->getType(PtrReg);
 
    const RegisterBank &RB = *RBI.getRegBank(ValReg, *MRI, TRI);
    if (RB.getID() != RISCV::GPRBRegBankID)
      return false;
 
#ifndef NDEBUG
    const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
    // Check that the pointer register is valid.
    assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
           "Load/Store pointer operand isn't a GPR");
    assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
#endif
 
    // Can only handle AddressSpace 0.
    if (PtrTy.getAddressSpace() != 0)
      return false;
 
    unsigned MemSize = LdSt.getMemSizeInBits().getValue();
    AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
 
    if (isStrongerThanMonotonic(Order)) {
      MI.setDesc(TII.get(selectZalasrLoadStoreOp(Opc, MemSize)));
      constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
      return true;
    }
 
    const unsigned NewOpc =
        selectRegImmLoadStoreOp(STI, MI.getOpcode(), MemSize);
    if (NewOpc == MI.getOpcode())
      return false;
 
    // Check if we can fold anything into the addressing mode.
    auto AddrModeFns = selectAddrRegImm(MI.getOperand(1));
    if (!AddrModeFns)
      return false;
 
    // Folded something. Create a new instruction and return it.
    MachineInstrBuilder NewInst =
        BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(NewOpc));
    NewInst.setMIFlags(MI.getFlags());
    if (isa<GStore>(MI))
      NewInst.addUse(ValReg);
    else
      NewInst.addDef(ValReg);
    NewInst.cloneMemRefs(MI);
    for (auto &Fn : *AddrModeFns)
      Fn(NewInst);
    MI.eraseFromParent();
 
    constrainSelectedInstRegOperands(*NewInst, TII, TRI, RBI);
    return true;
  }
  case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
    return selectIntrinsicWithSideEffects(MI);
  case TargetOpcode::G_INTRINSIC:
    return selectIntrinsic(MI);
  case TargetOpcode::G_EXTRACT_SUBVECTOR:
    return selectExtractSubvector(MI);
  case TargetOpcode::G_INSERT_SUBVECTOR:
    return selectInsertSubVector(MI);
  default:
    return false;
  }
}
 
bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const {
  assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
 
  // Split Src into two s32 parts.
  if (MI.getNumOperands() != 3)
    return false;
  Register Src = MI.getOperand(2).getReg();
  Register Lo = MI.getOperand(0).getReg();
  Register Hi = MI.getOperand(1).getReg();
 
  // RV32+Zilsd: split a GPRPair (i64) into its two s32 GPR halves.
  if (!Subtarget->is64Bit() && Subtarget->hasStdExtZilsd() &&
      MRI->getType(Src) == LLT::scalar(64) && isRegInGprb(Src) &&
      isRegInGprb(Lo) && isRegInGprb(Hi)) {
    // GPRPairNoX0 rather than GPRPair: a sub-register index is only valid for
    // a class when every register in it supports the index, and X0_Pair has no
    // sub_gpr_odd. SelectionDAG gets the same effect through
    // InstrEmitter::ConstrainForSubReg.
    if (!RBI.constrainGenericRegister(Src, RISCV::GPRPairNoX0RegClass, *MRI))
      return false;
    // COPY with a sub-register index rather than EXTRACT_SUBREG, which is what
    // SelectionDAG emits too: InstrEmitter lowers an EXTRACT_SUBREG SDNode to
    // %dst = COPY %src:sub and never builds the pseudo itself. A literal
    // EXTRACT_SUBREG would also survive to ExpandPostRAPseudos, which treats
    // one as unreachable.
    auto ExtractHalf = [&](Register Dst, unsigned SubIdx) {
      RBI.constrainGenericRegister(Dst, RISCV::GPRRegClass, *MRI);
      MachineInstr &C = *BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                 TII.get(TargetOpcode::COPY), Dst)
                             .addReg(Src, {}, SubIdx);
      constrainSelectedInstRegOperands(C, TII, TRI, RBI);
    };
    ExtractHalf(Lo, RISCV::sub_gpr_even);
    ExtractHalf(Hi, RISCV::sub_gpr_odd);
    MI.eraseFromParent();
    return true;
  }
 
  if (!Subtarget->hasStdExtZfa())
    return false;
 
  if (!isRegInFprb(Src) || !isRegInGprb(Lo) || !isRegInGprb(Hi))
    return false;
 
  MachineInstr *ExtractLo = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                    TII.get(RISCV::FMV_X_W_FPR64), Lo)
                                .addReg(Src);
  constrainSelectedInstRegOperands(*ExtractLo, TII, TRI, RBI);
 
  MachineInstr *ExtractHi = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                    TII.get(RISCV::FMVH_X_D), Hi)
                                .addReg(Src);
  constrainSelectedInstRegOperands(*ExtractHi, TII, TRI, RBI);
 
  MI.eraseFromParent();
  return true;
}
 
bool RISCVInstructionSelector::selectMergeValues(MachineInstr &MI) const {
  assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
 
  if (MI.getNumOperands() != 3)
    return false;
  Register Dst = MI.getOperand(0).getReg();
  Register Lo = MI.getOperand(1).getReg();
  Register Hi = MI.getOperand(2).getReg();
 
  // RV32+Zilsd: build a GPRPair (i64) from two s32 GPR halves.
  if (!Subtarget->is64Bit() && Subtarget->hasStdExtZilsd() &&
      MRI->getType(Dst) == LLT::scalar(64) && isRegInGprb(Dst) &&
      isRegInGprb(Lo) && isRegInGprb(Hi)) {
    // GPRPairNoX0 rather than GPRPair, because sub_gpr_odd is not valid for
    // X0_Pair. Constrain Dst up front: constrainSelectedInstRegOperands below
    // would otherwise infer GPRPair from the s64 type.
    if (!RBI.constrainGenericRegister(Dst, RISCV::GPRPairNoX0RegClass, *MRI))
      return false;
    MachineInstr &Seq = *BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                 TII.get(TargetOpcode::REG_SEQUENCE), Dst)
                             .addReg(Lo)
                             .addImm(RISCV::sub_gpr_even)
                             .addReg(Hi)
                             .addImm(RISCV::sub_gpr_odd);
    MI.eraseFromParent();
    constrainSelectedInstRegOperands(Seq, TII, TRI, RBI);
    return true;
  }
  return false;
}
 
bool RISCVInstructionSelector::replacePtrWithInt(MachineInstr &MI,
                                                 unsigned OpIdx) {
  MachineOperand &Op = MI.getOperand(OpIdx);
  Register PtrReg = Op.getReg();
  assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
 
  const LLT sXLen = LLT::scalar(STI.getXLen());
  Register IntReg = MRI->createGenericVirtualRegister(sXLen);
  MRI->setRegBank(IntReg, RBI.getRegBank(RISCV::GPRBRegBankID));
  MachineInstr *PtrToInt = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                   TII.get(TargetOpcode::G_PTRTOINT), IntReg)
                               .addReg(PtrReg);
  Op.setReg(IntReg);
  return select(*PtrToInt);
}
 
void RISCVInstructionSelector::preISelLower(MachineInstr &MI) {
  switch (MI.getOpcode()) {
  case TargetOpcode::G_PTR_ADD: {
    Register DstReg = MI.getOperand(0).getReg();
    const LLT sXLen = LLT::scalar(STI.getXLen());
 
    replacePtrWithInt(MI, 1);
    MI.setDesc(TII.get(TargetOpcode::G_ADD));
    MRI->setType(DstReg, sXLen);
    break;
  }
  case TargetOpcode::G_PTRMASK: {
    Register DstReg = MI.getOperand(0).getReg();
    const LLT sXLen = LLT::scalar(STI.getXLen());
    replacePtrWithInt(MI, 1);
    MI.setDesc(TII.get(TargetOpcode::G_AND));
    MRI->setType(DstReg, sXLen);
    break;
  }
  }
}
 
void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
                                            const MachineInstr &MI,
                                            int OpIdx) const {
  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
         "Expected G_CONSTANT");
  int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
  MIB.addImm(-CstVal);
}
 
void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
                                                    const MachineInstr &MI,
                                                    int OpIdx) const {
  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
         "Expected G_CONSTANT");
  uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
  MIB.addImm(STI.getXLen() - CstVal);
}
 
void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
                                                  const MachineInstr &MI,
                                                  int OpIdx) const {
  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
         "Expected G_CONSTANT");
  uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
  MIB.addImm(32 - CstVal);
}
 
void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
                                              const MachineInstr &MI,
                                              int OpIdx) const {
  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
         "Expected G_CONSTANT");
  int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
  MIB.addImm(CstVal + 1);
}
 
void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
                                                   const MachineInstr &MI,
                                                   int OpIdx) const {
  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
         "Expected G_CONSTANT");
  uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
  MIB.addImm(llvm::countr_zero(C));
}
 
void RISCVInstructionSelector::renderXLenSubTrailingOnes(
    MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
         "Expected G_CONSTANT");
  uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
  MIB.addImm(Subtarget->getXLen() - llvm::countr_one(C));
}
 
void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
                                                      const MachineInstr &MI,
                                                      int OpIdx) const {
  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
         "Expected G_CONSTANT");
  int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
  int64_t Adj = Imm < 0 ? -2048 : 2047;
  MIB.addImm(Imm - Adj);
}
 
void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
                                                      const MachineInstr &MI,
                                                      int OpIdx) const {
  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
         "Expected G_CONSTANT");
  int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
  MIB.addImm(Imm);
}
 
bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
  return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
}
 
bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
  return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
}
 
// A G_PTR_ADD result is worth splitting into Hi (materialized) +
// Lo12 (folded offset) only if every user is a plain scalar load/store
// using it as the address. Otherwise the ADD is selected on its own with
// the full materialized constant, making the Hi materialization here redundant.
bool RISCVInstructionSelector::isWorthFoldingAdd(Register AddResult) const {
  for (const MachineInstr &User : MRI->use_nodbg_instructions(AddResult)) {
    auto *LdSt = dyn_cast<GLoadStore>(&User);
    if (!LdSt)
      return false;
    // Must be used as the pointer, not the stored value.
    if (LdSt->getPointerReg() != AddResult)
      return false;
    if (isStrongerThanMonotonic(LdSt->getMMO().getSuccessOrdering()))
      return false;
    // Only scalar integer/f16/f32/f64 memory (exclude vectors, f128, ...).
    LLT Ty = MRI->getType(User.getOperand(0).getReg());
    if (!Ty.isScalar() || Ty.getSizeInBits() > 64)
      return false;
  }
  return true;
}
 
bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
  Register DstReg = MI.getOperand(0).getReg();
 
  if (DstReg.isPhysical())
    return true;
 
  const TargetRegisterClass *DstRC =
      TRI.getConstrainedRegClassForReg(DstReg, *MRI);
 
  assert(DstRC &&
         "Register class not available for LLT, register bank combination");
 
  // No need to constrain SrcReg. It will get constrained when
  // we hit another of its uses or its defs.
  // Copies do not have constraints.
  if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
                      << " operand\n");
    return false;
  }
 
  MI.setDesc(TII.get(RISCV::COPY));
  return true;
}
 
bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
  assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
 
  const Register DstReg = MI.getOperand(0).getReg();
  const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(
      MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI), STI.is64Bit());
 
  assert(DstRC &&
         "Register class not available for LLT, register bank combination");
 
  if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
    LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
                      << " operand\n");
  }
  MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
  return true;
}
 
bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
                                              MachineInstr &MI) const {
  if (Imm == 0) {
    MachineBasicBlock &MBB = *MI.getParent();
    DebugLoc DL = MI.getDebugLoc();
    BuildMI(MBB, MI, DL, TII.get(TargetOpcode::COPY), DstReg).addReg(RISCV::X0);
    RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, *MRI);
    return true;
  }
 
  RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, *Subtarget);
  return materializeInstSeq(DstReg, Seq, MI);
}
 
bool RISCVInstructionSelector::materializeInstSeq(
    Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const {
  assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence");
 
  MachineBasicBlock &MBB = *MI.getParent();
  DebugLoc DL = MI.getDebugLoc();
  unsigned NumInsts = Seq.size();
  Register SrcReg = RISCV::X0;
 
  for (unsigned i = 0; i < NumInsts; i++) {
    Register TmpReg = i < NumInsts - 1
                          ? MRI->createVirtualRegister(&RISCV::GPRRegClass)
                          : DstReg;
    const RISCVMatInt::Inst &I = Seq[i];
    MachineInstr *Result;
 
    switch (I.getOpndKind()) {
    case RISCVMatInt::Imm:
      Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
                   .addImm(I.getImm());
      break;
    case RISCVMatInt::RegX0:
      Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
                   .addReg(SrcReg)
                   .addReg(RISCV::X0);
      break;
    case RISCVMatInt::RegReg:
      Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
                   .addReg(SrcReg)
                   .addReg(SrcReg);
      break;
    case RISCVMatInt::RegImm:
      Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
                   .addReg(SrcReg)
                   .addImm(I.getImm());
      break;
    }
 
    constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
 
    SrcReg = TmpReg;
  }
 
  return true;
}
 
InstructionSelector::ComplexRendererFns
RISCVInstructionSelector::computeConstAddr(int64_t CVal, bool IsPrefetch,
                                           Register OrigBase) const {
  // Split the constant into a materialized high part (the base) and
  // a simm12 low part (the offset). For prefetch the low part
  // must additionally be a multiple of 32 (simm12_lsb00000).
  int64_t Lo12 = SignExtend64<12>(CVal);
  int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
  auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns {
    return {{[=](MachineInstrBuilder &MIB) {
               MIB.addReg(materializeConstBase(MIB, Plan, OrigBase));
             },
             [=](MachineInstrBuilder &MIB) { MIB.addImm(Plan.Lo12); }}};
  };
  if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
    if (IsPrefetch && (Lo12 & 0b11111) != 0)
      return std::nullopt;
    ConstAddrPlan Plan;
    Plan.Lo12 = Lo12;
    if (Hi) {
      Plan.Kind = ConstAddrPlan::LUI;
      Plan.Hi20 = (Hi >> 12) & 0xfffff;
    }
    return emit(std::move(Plan));
  }
 
  // Otherwise ask constant materialization how it would handle the constant
  // and fold the trailing ADDI into the offset.
  RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(CVal, *Subtarget);
  if (Seq.back().getOpcode() != RISCV::ADDI)
    return std::nullopt;
  Lo12 = Seq.back().getImm();
  if (IsPrefetch && (Lo12 & 0b11111) != 0)
    return std::nullopt;
  Seq.pop_back();
  if (Seq.empty())
    return std::nullopt;
  ConstAddrPlan Plan;
  Plan.Kind = ConstAddrPlan::InstSeq;
  Plan.Seq = std::move(Seq);
  Plan.Lo12 = Lo12;
  return emit(std::move(Plan));
}
 
Register
RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB,
                                               const ConstAddrPlan &Plan,
                                               Register OrigBase) const {
  MachineBasicBlock &MBB = *MIB->getParent();
  DebugLoc DL = MIB->getDebugLoc();
  MachineInstr &InsertPt = *MIB.getInstr();
 
  Register HiReg = RISCV::X0;
  switch (Plan.Kind) {
  case ConstAddrPlan::X0:
    break;
  case ConstAddrPlan::LUI: {
    HiReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    MachineInstr *LUI = BuildMI(MBB, InsertPt, DL, TII.get(RISCV::LUI), HiReg)
                            .addImm(Plan.Hi20);
    constrainSelectedInstRegOperands(*LUI, TII, TRI, RBI);
    break;
  }
  case ConstAddrPlan::InstSeq: {
    HiReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    materializeInstSeq(HiReg, Plan.Seq, InsertPt);
    break;
  }
  }
 
  // For G_PTR_ADD + large constant, add the original base to the materialized
  // high part.
  if (OrigBase.isValid() && HiReg != RISCV::X0) {
    Register BaseReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    MachineInstr *Add = BuildMI(MBB, InsertPt, DL, TII.get(RISCV::ADD), BaseReg)
                            .addReg(OrigBase)
                            .addReg(HiReg);
    constrainSelectedInstRegOperands(*Add, TII, TRI, RBI);
    return BaseReg;
  }
  return OrigBase.isValid() ? OrigBase : HiReg;
}
 
bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
                                          bool IsExternWeak) const {
  assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
          MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
          MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
         "Unexpected opcode");
 
  const MachineOperand &DispMO = MI.getOperand(1);
 
  Register DefReg = MI.getOperand(0).getReg();
  const LLT DefTy = MRI->getType(DefReg);
 
  // When HWASAN is used and tagging of global variables is enabled
  // they should be accessed via the GOT, since the tagged address of a global
  // is incompatible with existing code models. This also applies to non-pic
  // mode.
  if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
    if (IsLocal && !Subtarget->allowTaggedGlobals()) {
      // Use PC-relative addressing to access the symbol. This generates the
      // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
      // %pcrel_lo(auipc)).
      MI.setDesc(TII.get(RISCV::PseudoLLA));
      constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
      return true;
    }
 
    // Use PC-relative addressing to access the GOT for this symbol, then
    // load the address from the GOT. This generates the pattern (PseudoLGA
    // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
    // %pcrel_lo(auipc))).
    MachineFunction &MF = *MI.getParent()->getParent();
    MachineMemOperand *MemOp = MF.getMachineMemOperand(
        MachinePointerInfo::getGOT(MF),
        MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
            MachineMemOperand::MOInvariant,
        DefTy, Align(DefTy.getSizeInBits() / 8));
 
    MachineInstr *Result = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                   TII.get(RISCV::PseudoLGA), DefReg)
                               .addDisp(DispMO, 0)
                               .addMemOperand(MemOp);
 
    constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
 
    MI.eraseFromParent();
    return true;
  }
 
  switch (TM.getCodeModel()) {
  default: {
    reportGISelFailure(*MF, *MORE, getName(),
                       "Unsupported code model for lowering", MI);
    return false;
  }
  case CodeModel::Small: {
    // Must lie within a single 2 GiB address range and must lie between
    // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
    // (lui %hi(sym)) %lo(sym)).
    Register AddrHiDest = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    MachineInstr *AddrHi = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                   TII.get(RISCV::LUI), AddrHiDest)
                               .addDisp(DispMO, 0, RISCVII::MO_HI);
 
    constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI);
 
    MachineInstr *Result = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                   TII.get(RISCV::ADDI), DefReg)
                               .addReg(AddrHiDest)
                               .addDisp(DispMO, 0, RISCVII::MO_LO);
 
    constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
 
    MI.eraseFromParent();
    return true;
  }
  case CodeModel::Medium:
    // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
    // relocation needs to reference a label that points to the auipc
    // instruction itself, not the global. This cannot be done inside the
    // instruction selector.
    if (IsExternWeak) {
      // An extern weak symbol may be undefined, i.e. have value 0, which may
      // not be within 2GiB of PC, so use GOT-indirect addressing to access the
      // symbol. This generates the pattern (PseudoLGA sym), which expands to
      // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
      MachineFunction &MF = *MI.getParent()->getParent();
      MachineMemOperand *MemOp = MF.getMachineMemOperand(
          MachinePointerInfo::getGOT(MF),
          MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
              MachineMemOperand::MOInvariant,
          DefTy, Align(DefTy.getSizeInBits() / 8));
 
      MachineInstr *Result = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                     TII.get(RISCV::PseudoLGA), DefReg)
                                 .addDisp(DispMO, 0)
                                 .addMemOperand(MemOp);
 
      constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
 
      MI.eraseFromParent();
      return true;
    }
 
    // Generate a sequence for accessing addresses within any 2GiB range
    // within the address space. This generates the pattern (PseudoLLA sym),
    // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
    MI.setDesc(TII.get(RISCV::PseudoLLA));
    constrainSelectedInstRegOperands(MI, TII, TRI, RBI);
    return true;
  }
 
  return false;
}
 
bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const {
  auto &SelectMI = cast<GSelect>(MI);
 
  Register LHS, RHS;
  RISCVCC::CondCode CC;
  getOperandsForBranch(SelectMI.getCondReg(), CC, LHS, RHS, *MRI);
 
  Register DstReg = SelectMI.getReg(0);
 
  unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
  if (RBI.getRegBank(DstReg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
    unsigned Size = MRI->getType(DstReg).getSizeInBits();
    Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
                     : RISCV::Select_FPR64_Using_CC_GPR;
  }
 
  MachineInstr *Result =
      BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(Opc))
          .addDef(DstReg)
          .addReg(LHS)
          .addReg(RHS)
          .addImm(CC)
          .addReg(SelectMI.getTrueReg())
          .addReg(SelectMI.getFalseReg());
  MI.eraseFromParent();
  constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
  return true;
}
 
// Convert an FCMP predicate to one of the supported F or D instructions.
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
  assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
  switch (Pred) {
  default:
    llvm_unreachable("Unsupported predicate");
  case CmpInst::FCMP_OLT:
    return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
  case CmpInst::FCMP_OLE:
    return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
  case CmpInst::FCMP_OEQ:
    return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
  }
}
 
// Try legalizing an FCMP by swapping or inverting the predicate to one that
// is supported.
static bool legalizeFCmpPredicate(Register &LHS, Register &RHS,
                                  CmpInst::Predicate &Pred, bool &NeedInvert) {
  auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
    return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
           Pred == CmpInst::FCMP_OEQ;
  };
 
  assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
 
  CmpInst::Predicate InvPred = CmpInst::getSwappedPredicate(Pred);
  if (isLegalFCmpPredicate(InvPred)) {
    Pred = InvPred;
    std::swap(LHS, RHS);
    return true;
  }
 
  InvPred = CmpInst::getInversePredicate(Pred);
  NeedInvert = true;
  if (isLegalFCmpPredicate(InvPred)) {
    Pred = InvPred;
    return true;
  }
  InvPred = CmpInst::getSwappedPredicate(InvPred);
  if (isLegalFCmpPredicate(InvPred)) {
    Pred = InvPred;
    std::swap(LHS, RHS);
    return true;
  }
 
  return false;
}
 
// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
// the result in DstReg.
// FIXME: Maybe we should expand this earlier.
bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const {
  auto &CmpMI = cast<GFCmp>(MI);
  CmpInst::Predicate Pred = CmpMI.getCond();
 
  Register DstReg = CmpMI.getReg(0);
  Register LHS = CmpMI.getLHSReg();
  Register RHS = CmpMI.getRHSReg();
 
  unsigned Size = MRI->getType(LHS).getSizeInBits();
  assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
 
  Register TmpReg = DstReg;
 
  bool NeedInvert = false;
  // First try swapping operands or inverting.
  if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
    if (NeedInvert)
      TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    MachineInstr *Cmp = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                TII.get(getFCmpOpcode(Pred, Size)), TmpReg)
                            .setMIFlag(MachineInstr::NoFPExcept)
                            .addReg(LHS)
                            .addReg(RHS);
    constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
  } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
    // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
    NeedInvert = Pred == CmpInst::FCMP_UEQ;
    Register Cmp1Reg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    MachineInstr *Cmp1 =
        BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                TII.get(getFCmpOpcode(CmpInst::FCMP_OLT, Size)), Cmp1Reg)
            .setMIFlag(MachineInstr::NoFPExcept)
            .addReg(LHS)
            .addReg(RHS);
    constrainSelectedInstRegOperands(*Cmp1, TII, TRI, RBI);
    Register Cmp2Reg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    MachineInstr *Cmp2 =
        BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                TII.get(getFCmpOpcode(CmpInst::FCMP_OLT, Size)), Cmp2Reg)
            .setMIFlag(MachineInstr::NoFPExcept)
            .addReg(RHS)
            .addReg(LHS);
    constrainSelectedInstRegOperands(*Cmp2, TII, TRI, RBI);
    if (NeedInvert)
      TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    MachineInstr *Or = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                               TII.get(RISCV::OR), TmpReg)
                           .addReg(Cmp1Reg)
                           .addReg(Cmp2Reg);
    constrainSelectedInstRegOperands(*Or, TII, TRI, RBI);
  } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
    // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
    // If LHS and RHS are the same, a single FEQ suffices.
    NeedInvert = Pred == CmpInst::FCMP_UNO;
    if (NeedInvert)
      TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
    if (LHS == RHS) {
      MachineInstr *Cmp =
          BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                  TII.get(getFCmpOpcode(CmpInst::FCMP_OEQ, Size)), TmpReg)
              .setMIFlag(MachineInstr::NoFPExcept)
              .addReg(LHS)
              .addReg(LHS);
      constrainSelectedInstRegOperands(*Cmp, TII, TRI, RBI);
    } else {
      Register Cmp1Reg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
      MachineInstr *Cmp1 =
          BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                  TII.get(getFCmpOpcode(CmpInst::FCMP_OEQ, Size)), Cmp1Reg)
              .setMIFlag(MachineInstr::NoFPExcept)
              .addReg(LHS)
              .addReg(LHS);
      constrainSelectedInstRegOperands(*Cmp1, TII, TRI, RBI);
      Register Cmp2Reg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
      MachineInstr *Cmp2 =
          BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                  TII.get(getFCmpOpcode(CmpInst::FCMP_OEQ, Size)), Cmp2Reg)
              .setMIFlag(MachineInstr::NoFPExcept)
              .addReg(RHS)
              .addReg(RHS);
      constrainSelectedInstRegOperands(*Cmp2, TII, TRI, RBI);
      MachineInstr *And = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                  TII.get(RISCV::AND), TmpReg)
                              .addReg(Cmp1Reg)
                              .addReg(Cmp2Reg);
      constrainSelectedInstRegOperands(*And, TII, TRI, RBI);
    }
  } else
    llvm_unreachable("Unhandled predicate");
 
  // Emit an XORI to invert the result if needed.
  if (NeedInvert) {
    MachineInstr *Xor = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
                                TII.get(RISCV::XORI), DstReg)
                            .addReg(TmpReg)
                            .addImm(1);
    constrainSelectedInstRegOperands(*Xor, TII, TRI, RBI);
  }
 
  MI.eraseFromParent();
  return true;
}
 
void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
                                         SyncScope::ID FenceSSID,
                                         MachineInstr &MI) const {
  MachineBasicBlock &MBB = *MI.getParent();
  DebugLoc DL = MI.getDebugLoc();
 
  if (STI.hasStdExtZtso()) {
    // The only fence that needs an instruction is a sequentially-consistent
    // cross-thread fence.
    if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
        FenceSSID == SyncScope::System) {
      // fence rw, rw
      BuildMI(MBB, MI, DL, TII.get(RISCV::FENCE))
          .addImm(RISCVFenceField::R | RISCVFenceField::W)
          .addImm(RISCVFenceField::R | RISCVFenceField::W);
      return;
    }
 
    // MEMBARRIER is a compiler barrier; it codegens to a no-op.
    BuildMI(MBB, MI, DL, TII.get(TargetOpcode::MEMBARRIER));
    return;
  }
 
  // singlethread fences only synchronize with signal handlers on the same
  // thread and thus only need to preserve instruction order, not actually
  // enforce memory ordering.
  if (FenceSSID == SyncScope::SingleThread) {
    BuildMI(MBB, MI, DL, TII.get(TargetOpcode::MEMBARRIER));
    return;
  }
 
  // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
  // Manual: Volume I.
  unsigned Pred, Succ;
  switch (FenceOrdering) {
  default:
    llvm_unreachable("Unexpected ordering");
  case AtomicOrdering::AcquireRelease:
    // fence acq_rel -> fence.tso
    BuildMI(MBB, MI, DL, TII.get(RISCV::FENCE_TSO));
    return;
  case AtomicOrdering::Acquire:
    // fence acquire -> fence r, rw
    Pred = RISCVFenceField::R;
    Succ = RISCVFenceField::R | RISCVFenceField::W;
    break;
  case AtomicOrdering::Release:
    // fence release -> fence rw, w
    Pred = RISCVFenceField::R | RISCVFenceField::W;
    Succ = RISCVFenceField::W;
    break;
  case AtomicOrdering::SequentiallyConsistent:
    // fence seq_cst -> fence rw, rw
    Pred = RISCVFenceField::R | RISCVFenceField::W;
    Succ = RISCVFenceField::R | RISCVFenceField::W;
    break;
  }
  BuildMI(MBB, MI, DL, TII.get(RISCV::FENCE)).addImm(Pred).addImm(Succ);
}
 
namespace llvm {
InstructionSelector *
createRISCVInstructionSelector(const RISCVTargetMachine &TM,
                               const RISCVSubtarget &Subtarget,
                               const RISCVRegisterBankInfo &RBI) {
  return new RISCVInstructionSelector(TM, Subtarget, RBI);
}
} // end namespace llvm