AMDGPURegBankCombiner.cpp

Go to the documentation of this file.

//=== lib/CodeGen/GlobalISel/AMDGPURegBankCombiner.cpp ---------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass does combining of machine instructions at the generic MI level,
// after register banks are known.
//
//===----------------------------------------------------------------------===//
 
#include "AMDGPU.h"
#include "AMDGPULegalizerInfo.h"
#include "AMDGPURegisterBankInfo.h"
#include "GCNSubtarget.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/CodeGen/GlobalISel/Combiner.h"
#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFunctionAnalysisManager.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Target/TargetMachine.h"
 
#define GET_GICOMBINER_DEPS
#include "AMDGPUGenPreLegalizeGICombiner.inc"
#undef GET_GICOMBINER_DEPS
 
#define DEBUG_TYPE "amdgpu-regbank-combiner"
 
using namespace llvm;
using namespace MIPatternMatch;
 
namespace {
#define GET_GICOMBINER_TYPES
#include "AMDGPUGenRegBankGICombiner.inc"
#undef GET_GICOMBINER_TYPES
 
class AMDGPURegBankCombinerImpl : public Combiner {
protected:
  const AMDGPURegBankCombinerImplRuleConfig &RuleConfig;
  const GCNSubtarget &STI;
  const RegisterBankInfo &RBI;
  const TargetRegisterInfo &TRI;
  const SIInstrInfo &TII;
  const CombinerHelper Helper;
 
public:
  AMDGPURegBankCombinerImpl(
      MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
      GISelCSEInfo *CSEInfo,
      const AMDGPURegBankCombinerImplRuleConfig &RuleConfig,
      const GCNSubtarget &STI, MachineDominatorTree *MDT,
      const LegalizerInfo *LI);
 
  static const char *getName() { return "AMDGPURegBankCombinerImpl"; }
 
  bool tryCombineAll(MachineInstr &I) const override;
 
  bool isVgprRegBank(Register Reg) const;
  Register getAsVgpr(Register Reg) const;
 
  struct MinMaxMedOpc {
    unsigned Min, Max, Med;
  };
 
  struct Med3MatchInfo {
    unsigned Opc;
    Register Val0, Val1, Val2;
  };
 
  struct MinMaxToMinMax3MatchInfo {
    unsigned Opc;
    Register Val0, Val1, Val2;
  };
 
  MinMaxMedOpc getMinMaxPair(unsigned Opc) const;
 
  template <class m_Cst, typename CstTy>
  bool matchMed(MachineInstr &MI, MachineRegisterInfo &MRI, MinMaxMedOpc MMMOpc,
                Register &Val, CstTy &K0, CstTy &K1) const;
 
  bool matchIntMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
  bool matchFPMinMaxToMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
  bool matchFPMinMaxToClamp(MachineInstr &MI, Register &Reg) const;
  bool matchFPMed3ToClamp(MachineInstr &MI, Register &Reg) const;
  void applyMed3(MachineInstr &MI, Med3MatchInfo &MatchInfo) const;
  void applyClamp(MachineInstr &MI, Register &Reg) const;
 
  void applyCanonicalizeZextShiftAmt(MachineInstr &MI, MachineInstr &Ext) const;
 
  bool combineD16Load(MachineInstr &MI) const;
  void applyD16Load(unsigned D16Opc, MachineInstr &DstMI,
                    MachineInstr *SmallLoad, Register ToOverwriteD16,
                    MachineInstr &InsertPt) const;
 
  bool matchMinMaxToMinMax3(MachineInstr &MI,
                            MinMaxToMinMax3MatchInfo &MatchInfo) const;
  void applyMinMaxToMinMax3(MachineInstr &MI,
                            MinMaxToMinMax3MatchInfo &MatchInfo) const;
 
  bool matchCombine_mul24(MachineInstr &MI) const;
  void applyCombine_mul24(MachineInstr &MI) const;
 
private:
  SIModeRegisterDefaults getMode() const;
  bool getIEEE() const;
  bool getDX10Clamp() const;
  bool isFminnumIeee(const MachineInstr &MI) const;
  bool isFminnum(const MachineInstr &MI) const;
  bool isFminnumLike(const MachineInstr &MI) const;
  bool isFCst(MachineInstr *MI) const;
  bool isClampZeroToOne(MachineInstr *K0, MachineInstr *K1) const;
 
#define GET_GICOMBINER_CLASS_MEMBERS
#define AMDGPUSubtarget GCNSubtarget
#include "AMDGPUGenRegBankGICombiner.inc"
#undef GET_GICOMBINER_CLASS_MEMBERS
#undef AMDGPUSubtarget
};
 
#define GET_GICOMBINER_IMPL
#define AMDGPUSubtarget GCNSubtarget
#include "AMDGPUGenRegBankGICombiner.inc"
#undef AMDGPUSubtarget
#undef GET_GICOMBINER_IMPL
 
AMDGPURegBankCombinerImpl::AMDGPURegBankCombinerImpl(
    MachineFunction &MF, CombinerInfo &CInfo, GISelValueTracking &VT,
    GISelCSEInfo *CSEInfo,
    const AMDGPURegBankCombinerImplRuleConfig &RuleConfig,
    const GCNSubtarget &STI, MachineDominatorTree *MDT, const LegalizerInfo *LI)
    : Combiner(MF, CInfo, &VT, CSEInfo), RuleConfig(RuleConfig), STI(STI),
      RBI(*STI.getRegBankInfo()), TRI(*STI.getRegisterInfo()),
      TII(*STI.getInstrInfo()),
      Helper(Observer, B, /*IsPreLegalize*/ false, &VT, MDT, LI),
#define GET_GICOMBINER_CONSTRUCTOR_INITS
#include "AMDGPUGenRegBankGICombiner.inc"
#undef GET_GICOMBINER_CONSTRUCTOR_INITS
{
}
 
bool AMDGPURegBankCombinerImpl::isVgprRegBank(Register Reg) const {
  return RBI.getRegBank(Reg, MRI, TRI)->getID() == AMDGPU::VGPRRegBankID;
}
 
Register AMDGPURegBankCombinerImpl::getAsVgpr(Register Reg) const {
  if (isVgprRegBank(Reg))
    return Reg;
 
  // Search for existing copy of Reg to vgpr.
  for (MachineInstr &Use : MRI.use_instructions(Reg)) {
    Register Def = Use.getOperand(0).getReg();
    if (Use.getOpcode() == AMDGPU::COPY && isVgprRegBank(Def))
      return Def;
  }
 
  // Copy Reg to vgpr.
  Register VgprReg = B.buildCopy(MRI.getType(Reg), Reg).getReg(0);
  MRI.setRegBank(VgprReg, RBI.getRegBank(AMDGPU::VGPRRegBankID));
  return VgprReg;
}
 
AMDGPURegBankCombinerImpl::MinMaxMedOpc
AMDGPURegBankCombinerImpl::getMinMaxPair(unsigned Opc) const {
  switch (Opc) {
  default:
    llvm_unreachable("Unsupported opcode");
  case AMDGPU::G_SMAX:
  case AMDGPU::G_SMIN:
    return {AMDGPU::G_SMIN, AMDGPU::G_SMAX, AMDGPU::G_AMDGPU_SMED3};
  case AMDGPU::G_UMAX:
  case AMDGPU::G_UMIN:
    return {AMDGPU::G_UMIN, AMDGPU::G_UMAX, AMDGPU::G_AMDGPU_UMED3};
  case AMDGPU::G_FMAXNUM:
  case AMDGPU::G_FMINNUM:
    return {AMDGPU::G_FMINNUM, AMDGPU::G_FMAXNUM, AMDGPU::G_AMDGPU_FMED3};
  case AMDGPU::G_FMAXNUM_IEEE:
  case AMDGPU::G_FMINNUM_IEEE:
    return {AMDGPU::G_FMINNUM_IEEE, AMDGPU::G_FMAXNUM_IEEE,
            AMDGPU::G_AMDGPU_FMED3};
  }
}
 
template <class m_Cst, typename CstTy>
bool AMDGPURegBankCombinerImpl::matchMed(MachineInstr &MI,
                                         MachineRegisterInfo &MRI,
                                         MinMaxMedOpc MMMOpc, Register &Val,
                                         CstTy &K0, CstTy &K1) const {
  // 4 operand commutes of: min(max(Val, K0), K1).
  // Find K1 from outer instr: min(max(...), K1) or min(K1, max(...)).
  // Find K0 and Val from inner instr: max(K0, Val) or max(Val, K0).
  // 4 operand commutes of: max(min(Val, K1), K0).
  // Find K0 from outer instr: max(min(...), K0) or max(K0, min(...)).
  // Find K1 and Val from inner instr: min(K1, Val) or min(Val, K1).
  return mi_match(
      MI, MRI,
      m_any_of(
          m_CommutativeBinOp(
              MMMOpc.Min, m_CommutativeBinOp(MMMOpc.Max, m_Reg(Val), m_Cst(K0)),
              m_Cst(K1)),
          m_CommutativeBinOp(
              MMMOpc.Max, m_CommutativeBinOp(MMMOpc.Min, m_Reg(Val), m_Cst(K1)),
              m_Cst(K0))));
}
 
bool AMDGPURegBankCombinerImpl::matchIntMinMaxToMed3(
    MachineInstr &MI, Med3MatchInfo &MatchInfo) const {
  Register Dst = MI.getOperand(0).getReg();
  if (!isVgprRegBank(Dst))
    return false;
 
  // med3 for i16 is only available on gfx9+, and not available for v2i16.
  LLT Ty = MRI.getType(Dst);
  if ((Ty != LLT::scalar(16) || !STI.hasMed3_16()) && Ty != LLT::scalar(32))
    return false;
 
  MinMaxMedOpc OpcodeTriple = getMinMaxPair(MI.getOpcode());
  Register Val;
  std::optional<ValueAndVReg> K0, K1;
  // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). Then see if K0 <= K1.
  if (!matchMed<GCstAndRegMatch>(MI, MRI, OpcodeTriple, Val, K0, K1))
    return false;
 
  if (OpcodeTriple.Med == AMDGPU::G_AMDGPU_SMED3 && K0->Value.sgt(K1->Value))
    return false;
  if (OpcodeTriple.Med == AMDGPU::G_AMDGPU_UMED3 && K0->Value.ugt(K1->Value))
    return false;
 
  MatchInfo = {OpcodeTriple.Med, Val, K0->VReg, K1->VReg};
  return true;
}
 
// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1)
// ieee = true  : min/max(SNaN, K) = QNaN, min/max(QNaN, K) = K
// ieee = false : min/max(NaN, K) = K
// clamp(NaN) = dx10_clamp ? 0.0 : NaN
// Consider values of min(max(Val, K0), K1) and max(min(Val, K1), K0) as input.
// Other operand commutes (see matchMed) give same result since min and max are
// commutative.
 
// Try to replace fp min(max(Val, K0), K1) or max(min(Val, K1), K0), KO<=K1
// with fmed3(Val, K0, K1) or clamp(Val). Clamp requires K0 = 0.0 and K1 = 1.0.
// Val = SNaN only for ieee = true
// fmed3(SNaN, K0, K1) = min(min(SNaN, K0), K1) = min(QNaN, K1) = K1
// min(max(SNaN, K0), K1) = min(QNaN, K1) = K1
// max(min(SNaN, K1), K0) = max(K1, K0) = K1
// Val = NaN,ieee = false or Val = QNaN,ieee = true
// fmed3(NaN, K0, K1) = min(min(NaN, K0), K1) = min(K0, K1) = K0
// min(max(NaN, K0), K1) = min(K0, K1) = K0 (can clamp when dx10_clamp = true)
// max(min(NaN, K1), K0) = max(K1, K0) = K1 != K0
bool AMDGPURegBankCombinerImpl::matchFPMinMaxToMed3(
    MachineInstr &MI, Med3MatchInfo &MatchInfo) const {
  Register Dst = MI.getOperand(0).getReg();
  // Perform combine only when the destination is a VGPR.
  if (!isVgprRegBank(Dst))
    return false;
 
  LLT Ty = MRI.getType(Dst);
 
  // med3 for f16 is only available on gfx9+, and not available for v2f16.
  if ((Ty != LLT::scalar(16) || !STI.hasMed3_16()) && Ty != LLT::scalar(32))
    return false;
 
  auto OpcodeTriple = getMinMaxPair(MI.getOpcode());
 
  Register Val;
  std::optional<FPValueAndVReg> K0, K1;
  // Match min(max(Val, K0), K1) or max(min(Val, K1), K0). Then see if K0 <= K1.
  if (!matchMed<GFCstAndRegMatch>(MI, MRI, OpcodeTriple, Val, K0, K1))
    return false;
 
  if (K0->Value > K1->Value)
    return false;
 
  // For IEEE=false perform combine only when it's safe to assume that there are
  // no NaN inputs. Most often MI is marked with nnan fast math flag.
  // For IEEE=true consider NaN inputs. fmed3(NaN, K0, K1) is equivalent to
  // min(min(NaN, K0), K1). Safe to fold for min(max(Val, K0), K1) since inner
  // nodes(max/min) have same behavior when one input is NaN and other isn't.
  // Don't consider max(min(SNaN, K1), K0) since there is no isKnownNeverQNaN,
  // also post-legalizer inputs to min/max are fcanonicalized (never SNaN).
  if ((getIEEE() && isFminnumLike(MI)) || VT->isKnownNeverNaN(Dst)) {
    // Don't fold single use constant that can't be inlined.
    if ((!MRI.hasOneNonDBGUse(K0->VReg) || TII.isInlineConstant(K0->Value)) &&
        (!MRI.hasOneNonDBGUse(K1->VReg) || TII.isInlineConstant(K1->Value))) {
      MatchInfo = {OpcodeTriple.Med, Val, K0->VReg, K1->VReg};
      return true;
    }
  }
 
  return false;
}
 
bool AMDGPURegBankCombinerImpl::matchFPMinMaxToClamp(MachineInstr &MI,
                                                     Register &Reg) const {
  // Perform combine only when the destination is a VGPR.
  if (!isVgprRegBank(MI.getOperand(0).getReg()))
    return false;
 
  // Clamp is available on all types after regbankselect (f16, f32, f64, v2f16).
  auto OpcodeTriple = getMinMaxPair(MI.getOpcode());
  Register Val;
  std::optional<FPValueAndVReg> K0, K1;
  // Match min(max(Val, K0), K1) or max(min(Val, K1), K0).
  if (!matchMed<GFCstOrSplatGFCstMatch>(MI, MRI, OpcodeTriple, Val, K0, K1))
    return false;
 
  if (!K0->Value.isPosZero() || !K1->Value.isOne())
    return false;
 
  // For IEEE=false perform combine only when it's safe to assume that there are
  // no NaN inputs. Most often MI is marked with nnan fast math flag.
  // For IEEE=true consider NaN inputs. Only min(max(QNaN, 0.0), 1.0) evaluates
  // to 0.0 requires dx10_clamp = true.
  if ((getIEEE() && getDX10Clamp() && isFminnumLike(MI) &&
       VT->isKnownNeverSNaN(Val)) ||
      VT->isKnownNeverNaN(MI.getOperand(0).getReg())) {
    Reg = Val;
    return true;
  }
 
  return false;
}
 
// Replacing fmed3(NaN, 0.0, 1.0) with clamp. Requires dx10_clamp = true.
// Val = SNaN only for ieee = true. It is important which operand is NaN.
// min(min(SNaN, 0.0), 1.0) = min(QNaN, 1.0) = 1.0
// min(min(SNaN, 1.0), 0.0) = min(QNaN, 0.0) = 0.0
// min(min(0.0, 1.0), SNaN) = min(0.0, SNaN) = QNaN
// Val = NaN,ieee = false or Val = QNaN,ieee = true
// min(min(NaN, 0.0), 1.0) = min(0.0, 1.0) = 0.0
// min(min(NaN, 1.0), 0.0) = min(1.0, 0.0) = 0.0
// min(min(0.0, 1.0), NaN) = min(0.0, NaN) = 0.0
bool AMDGPURegBankCombinerImpl::matchFPMed3ToClamp(MachineInstr &MI,
                                                   Register &Reg) const {
  // Perform combine only when the destination is a VGPR.
  if (!isVgprRegBank(MI.getOperand(0).getReg()))
    return false;
 
  // In llvm-ir, clamp is often represented as an intrinsic call to
  // @llvm.amdgcn.fmed3.f32(%Val, 0.0, 1.0). Check for other operand orders.
  MachineInstr *Src0 = getDefIgnoringCopies(MI.getOperand(1).getReg(), MRI);
  MachineInstr *Src1 = getDefIgnoringCopies(MI.getOperand(2).getReg(), MRI);
  MachineInstr *Src2 = getDefIgnoringCopies(MI.getOperand(3).getReg(), MRI);
 
  if (isFCst(Src0) && !isFCst(Src1))
    std::swap(Src0, Src1);
  if (isFCst(Src1) && !isFCst(Src2))
    std::swap(Src1, Src2);
  if (isFCst(Src0) && !isFCst(Src1))
    std::swap(Src0, Src1);
  if (!isClampZeroToOne(Src1, Src2))
    return false;
 
  Register Val = Src0->getOperand(0).getReg();
 
  auto isOp3Zero = [&]() {
    MachineInstr *Op3 = getDefIgnoringCopies(MI.getOperand(3).getReg(), MRI);
    if (Op3->getOpcode() == TargetOpcode::G_FCONSTANT)
      return Op3->getOperand(1).getFPImm()->isPosZero();
    return false;
  };
  // For IEEE=false perform combine only when it's safe to assume that there are
  // no NaN inputs. Most often MI is marked with nnan fast math flag.
  // For IEEE=true consider NaN inputs. Requires dx10_clamp = true. Safe to fold
  // when Val could be QNaN. If Val can also be SNaN third input should be 0.0.
  if (VT->isKnownNeverNaN(MI.getOperand(0).getReg()) ||
      (getIEEE() && getDX10Clamp() &&
       (VT->isKnownNeverSNaN(Val) || isOp3Zero()))) {
    Reg = Val;
    return true;
  }
 
  return false;
}
 
void AMDGPURegBankCombinerImpl::applyClamp(MachineInstr &MI,
                                           Register &Reg) const {
  B.buildInstr(AMDGPU::G_AMDGPU_CLAMP, {MI.getOperand(0)}, {Reg},
               MI.getFlags());
  MI.eraseFromParent();
}
 
void AMDGPURegBankCombinerImpl::applyMed3(MachineInstr &MI,
                                          Med3MatchInfo &MatchInfo) const {
  B.buildInstr(MatchInfo.Opc, {MI.getOperand(0)},
               {getAsVgpr(MatchInfo.Val0), getAsVgpr(MatchInfo.Val1),
                getAsVgpr(MatchInfo.Val2)},
               MI.getFlags());
  MI.eraseFromParent();
}
 
void AMDGPURegBankCombinerImpl::applyCanonicalizeZextShiftAmt(
    MachineInstr &MI, MachineInstr &Ext) const {
  unsigned ShOpc = MI.getOpcode();
  assert(ShOpc == AMDGPU::G_SHL || ShOpc == AMDGPU::G_LSHR ||
         ShOpc == AMDGPU::G_ASHR);
  assert(Ext.getOpcode() == AMDGPU::G_ZEXT);
 
  Register AmtReg = Ext.getOperand(1).getReg();
  Register ShDst = MI.getOperand(0).getReg();
  Register ShSrc = MI.getOperand(1).getReg();
 
  LLT ExtAmtTy = MRI.getType(Ext.getOperand(0).getReg());
  LLT AmtTy = MRI.getType(AmtReg);
 
  auto &RB = *MRI.getRegBank(AmtReg);
 
  auto NewExt = B.buildAnyExt(ExtAmtTy, AmtReg);
  auto Mask = B.buildConstant(
      ExtAmtTy, maskTrailingOnes<uint64_t>(AmtTy.getScalarSizeInBits()));
  auto And = B.buildAnd(ExtAmtTy, NewExt, Mask);
  B.buildInstr(ShOpc, {ShDst}, {ShSrc, And});
 
  MRI.setRegBank(NewExt.getReg(0), RB);
  MRI.setRegBank(Mask.getReg(0), RB);
  MRI.setRegBank(And.getReg(0), RB);
  MI.eraseFromParent();
}
 
bool AMDGPURegBankCombinerImpl::combineD16Load(MachineInstr &MI) const {
  Register Dst;
  MachineInstr *Load;
  GSExtLoad *SextLoad;
  GZExtLoad *ZextLoad;
  const MachineMemOperand *MMO;
  const int64_t CleanLo16 = 0xFFFFFFFFFFFF0000;
  const int64_t CleanHi16 = 0x000000000000FFFF;
 
  auto TryApplyD16Load = [&](unsigned D16Opc, GAnyLoad *SmallLoad) {
    LLT DstTy = MRI.getType(Dst);
    if (DstTy != LLT::fixed_vector(2, LLT::integer(16)) &&
        DstTy != LLT::fixed_vector(2, LLT::float16()))
      return false;
 
    MachineInstr *DstDef = MRI.getVRegDef(Dst);
    MachineInstr *InsertPt = SmallLoad;
    if (SmallLoad->isSimple() && isObviouslySafeToFold(*SmallLoad, MI))
      InsertPt = &MI;
    else if (!Helper.dominates(*DstDef, *SmallLoad))
      return false;
 
    applyD16Load(D16Opc, MI, SmallLoad, Dst, *InsertPt);
    return true;
  };
 
  // Load lo
  if (mi_match(MI.getOperand(1).getReg(), MRI,
               m_OneUse(m_GOr(m_GAnd(m_GBitcast(m_Reg(Dst)),
                                     m_Copy(m_SpecificICst(CleanLo16))),
                              m_OneUse(m_MInstr(Load)))))) {
    if (mi_match(Load, MRI, m_GZExtLoad(ZextLoad, m_MMO(MMO)))) {
      unsigned LoadSize = MMO->getSizeInBits().getValue();
      if (LoadSize == 8)
        return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO_U8, ZextLoad);
      if (LoadSize == 16)
        return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO, ZextLoad);
 
      return false;
    }
 
    // s32 Load_lo16 holds SextLoad i8, Load_hi16 is zero.
    // fake16: and  (sextload i8 -> s32), 0xFFFF
    // true16: zext (sextload i8 -> s16) -> s32
    if (mi_match(Load, MRI,
                 m_any_of(m_GAnd(m_OneUse(m_GSExtLoad(SextLoad, m_MMO(MMO))),
                                 m_Copy(m_SpecificICst(CleanHi16))),
                          m_GZExt(m_all_of(
                              m_SpecificType(LLT::scalar(16)),
                              m_OneUse(m_GSExtLoad(SextLoad, m_MMO(MMO)))))))) {
      unsigned LoadSize = MMO->getSizeInBits().getValue();
      if (LoadSize == 8)
        return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_LO_I8, SextLoad);
    }
 
    return false;
  }
 
  // Load hi
  if (mi_match(MI.getOperand(1).getReg(), MRI,
               m_OneUse(m_GOr(m_GAnd(m_GBitcast(m_Reg(Dst)),
                                     m_Copy(m_SpecificICst(CleanHi16))),
                              m_OneUse(m_GShl(m_OneUse(m_MInstr(Load)),
                                              m_Copy(m_SpecificICst(16)))))))) {
    if (mi_match(Load, MRI, m_GZExtLoad(ZextLoad, m_MMO(MMO)))) {
      unsigned LoadSize = MMO->getSizeInBits().getValue();
      if (LoadSize == 8)
        return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI_U8, ZextLoad);
      if (LoadSize == 16)
        return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI, ZextLoad);
 
      return false;
    }
 
    // The shift discards the upper half of the sign-extending load.
    if (mi_match(Load, MRI,
                 m_any_of(m_GSExtLoad(SextLoad, m_MMO(MMO)),
                          m_GAnd(m_OneUse(m_GSExtLoad(SextLoad, m_MMO(MMO))),
                                 m_Copy(m_SpecificICst(CleanHi16))),
                          m_GZExt(m_all_of(
                              m_SpecificType(LLT::scalar(16)),
                              m_OneUse(m_GSExtLoad(SextLoad, m_MMO(MMO)))))))) {
      unsigned LoadSize = MMO->getSizeInBits().getValue();
      if (LoadSize == 8)
        return TryApplyD16Load(AMDGPU::G_AMDGPU_LOAD_D16_HI_I8, SextLoad);
    }
 
    return false;
  }
 
  return false;
}
 
void AMDGPURegBankCombinerImpl::applyMinMaxToMinMax3(
    MachineInstr &MI, MinMaxToMinMax3MatchInfo &MatchInfo) const {
  B.buildInstr(MatchInfo.Opc, {MI.getOperand(0)},
               {MatchInfo.Val0, MatchInfo.Val1, MatchInfo.Val2}, MI.getFlags());
  MI.eraseFromParent();
  return;
}
 
bool AMDGPURegBankCombinerImpl::matchCombine_mul24(MachineInstr &MI) const {
  Register Dst = MI.getOperand(0).getReg();
  if (MRI.getType(Dst) != LLT::scalar(32) || !isVgprRegBank(Dst))
    return false;
  Register Src0 = MI.getOperand(1).getReg();
  Register Src1 = MI.getOperand(2).getReg();
  if (STI.hasMulU24() && VT->getKnownBits(Src0).countMinLeadingZeros() >= 8 &&
      VT->getKnownBits(Src1).countMinLeadingZeros() >= 8)
    return true;
  return STI.hasMulI24() && VT->computeNumSignBits(Src0) >= 9 &&
         VT->computeNumSignBits(Src1) >= 9;
}
 
void AMDGPURegBankCombinerImpl::applyCombine_mul24(MachineInstr &MI) const {
  Register Dst = MI.getOperand(0).getReg();
  Register Src0 = MI.getOperand(1).getReg();
  Register Src1 = MI.getOperand(2).getReg();
  Intrinsic::ID IntrID =
      (STI.hasMulU24() && VT->getKnownBits(Src0).countMinLeadingZeros() >= 8 &&
       VT->getKnownBits(Src1).countMinLeadingZeros() >= 8)
          ? Intrinsic::amdgcn_mul_u24
          : Intrinsic::amdgcn_mul_i24;
  B.buildIntrinsic(IntrID, ArrayRef<Register>{Dst}).addUse(Src0).addUse(Src1);
  MI.eraseFromParent();
}
 
// min(min(a, b), c) == min(a, min(b, c)) == min3(a, b, c)
// supported scalar type: S32 S16 U32 U16 F32 F16
bool AMDGPURegBankCombinerImpl::matchMinMaxToMinMax3(
    MachineInstr &MI, MinMaxToMinMax3MatchInfo &MatchInfo) const {
  Register Dst = MI.getOperand(0).getReg();
  Register Src1 = MI.getOperand(1).getReg();
  Register Src2 = MI.getOperand(2).getReg();
  // If the register is SGPR, don't optimize it.
  if (!(isVgprRegBank(Dst) && isVgprRegBank(Src1) && isVgprRegBank(Src2))) {
    return false;
  }
 
  LLT Ty = MRI.getType(Dst);
  unsigned Opc = MI.getOpcode();
  if (!(Ty == LLT::scalar(32) ||
        (Ty == LLT::scalar(16) && STI.hasMin3Max3_16())))
    return false;
 
  Register R0, R1, R2;
  if (!mi_match(MI, MRI,
                m_CommutativeBinOp(Opc,
                                   m_OneUse(m_BinOp(Opc, m_Reg(R0), m_Reg(R1))),
                                   m_Reg(R2)))) {
    return false;
  }
 
  unsigned AMDGPUOpc = 0;
  switch (Opc) {
  case AMDGPU::G_SMAX:
    AMDGPUOpc = AMDGPU::G_AMDGPU_SMAX3;
    break;
  case AMDGPU::G_SMIN:
    AMDGPUOpc = AMDGPU::G_AMDGPU_SMIN3;
    break;
  case AMDGPU::G_UMAX:
    AMDGPUOpc = AMDGPU::G_AMDGPU_UMAX3;
    break;
  case AMDGPU::G_UMIN:
    AMDGPUOpc = AMDGPU::G_AMDGPU_UMIN3;
    break;
  case AMDGPU::G_FMAXNUM:
  case AMDGPU::G_FMAXNUM_IEEE:
  case AMDGPU::G_FMAXIMUMNUM:
    AMDGPUOpc = AMDGPU::G_AMDGPU_FMAX3;
    break;
  case AMDGPU::G_FMINNUM:
  case AMDGPU::G_FMINNUM_IEEE:
  case AMDGPU::G_FMINIMUMNUM:
    AMDGPUOpc = AMDGPU::G_AMDGPU_FMIN3;
    break;
  case AMDGPU::G_FMAXIMUM:
    AMDGPUOpc = AMDGPU::G_AMDGPU_FMAXIMUM3;
    break;
  case AMDGPU::G_FMINIMUM:
    AMDGPUOpc = AMDGPU::G_AMDGPU_FMINIMUM3;
    break;
  default:
    return false;
  }
 
  MatchInfo = {AMDGPUOpc, R0, R1, R2};
  return true;
}
 
void AMDGPURegBankCombinerImpl::applyD16Load(unsigned D16Opc,
                                             MachineInstr &DstMI,
                                             MachineInstr *SmallLoad,
                                             Register SrcReg32ToOverwriteD16,
                                             MachineInstr &InsertPt) const {
  Register DstReg = DstMI.getOperand(0).getReg();
  LLT SrcTy = MRI.getType(SrcReg32ToOverwriteD16);
 
  // Dst and Src for D16 load need to have same type.
  Register D16Dst =
      SrcTy == MRI.getType(DstReg)
          ? DstReg
          : MRI.createVirtualRegister({MRI.getRegBank(DstReg), SrcTy});
 
  B.setInstrAndDebugLoc(InsertPt);
  B.setDebugLoc(SmallLoad->getDebugLoc());
  B.buildInstr(D16Opc, {D16Dst},
               {SmallLoad->getOperand(1).getReg(), SrcReg32ToOverwriteD16})
      .setMemRefs(SmallLoad->memoperands());
 
  if (D16Dst != DstReg)
    B.buildBitcast(DstReg, D16Dst);
 
  // Recursive cleanup may erase SmallLoad.
  Register LoadDst = SmallLoad->getOperand(0).getReg();
  eraseInstr(DstMI, MRI);
  // Volatile and ordered atomic loads are not trivially dead.
  if (MachineInstr *OldLoad = MRI.getVRegDef(LoadDst))
    eraseInstr(*OldLoad, MRI);
}
 
SIModeRegisterDefaults AMDGPURegBankCombinerImpl::getMode() const {
  return MF.getInfo<SIMachineFunctionInfo>()->getMode();
}
 
bool AMDGPURegBankCombinerImpl::getIEEE() const { return getMode().IEEE; }
 
bool AMDGPURegBankCombinerImpl::getDX10Clamp() const {
  return getMode().DX10Clamp;
}
 
bool AMDGPURegBankCombinerImpl::isFminnumIeee(const MachineInstr &MI) const {
  return MI.getOpcode() == AMDGPU::G_FMINNUM_IEEE;
}
 
bool AMDGPURegBankCombinerImpl::isFminnum(const MachineInstr &MI) const {
  return MI.getOpcode() == AMDGPU::G_FMINNUM;
}
 
bool AMDGPURegBankCombinerImpl::isFminnumLike(const MachineInstr &MI) const {
  return isFminnumIeee(MI) || isFminnum(MI);
}
 
bool AMDGPURegBankCombinerImpl::isFCst(MachineInstr *MI) const {
  return MI->getOpcode() == AMDGPU::G_FCONSTANT;
}
 
bool AMDGPURegBankCombinerImpl::isClampZeroToOne(MachineInstr *K0,
                                                 MachineInstr *K1) const {
  if (isFCst(K0) && isFCst(K1)) {
    const ConstantFP *KO_FPImm = K0->getOperand(1).getFPImm();
    const ConstantFP *K1_FPImm = K1->getOperand(1).getFPImm();
    return (KO_FPImm->isPosZero() && K1_FPImm->isOne()) ||
           (KO_FPImm->isOne() && K1_FPImm->isPosZero());
  }
  return false;
}
 
static bool runCombiner(MachineFunction &MF,
                        function_ref<GISelValueTracking *()> GetVT,
                        function_ref<MachineDominatorTree *()> GetMDT,
                        bool EnableOpt) {
  AMDGPURegBankCombinerImplRuleConfig RuleConfig;
  if (!RuleConfig.parseCommandLineOption())
    reportFatalUsageError("Invalid rule identifier");
 
  if (MF.getProperties().hasFailedISel())
    return false;
 
  const Function &F = MF.getFunction();
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
  const auto *LI = ST.getLegalizerInfo();
 
  CombinerInfo CInfo(/*AllowIllegalOps=*/false, /*ShouldLegalizeIllegal=*/true,
                     LI, EnableOpt, F.hasOptSize(), F.hasMinSize());
  // Disable fixed-point iteration to reduce compile-time
  CInfo.MaxIterations = 1;
  CInfo.ObserverLvl = CombinerInfo::ObserverLevel::SinglePass;
  // RegBankSelect seems not to leave dead instructions, so a full DCE pass is
  // unnecessary.
  CInfo.EnableFullDCE = false;
 
  GISelValueTracking *VT = GetVT();
  MachineDominatorTree *MDT = GetMDT();
  AMDGPURegBankCombinerImpl Impl(MF, CInfo, *VT, /*CSEInfo=*/nullptr,
                                 RuleConfig, ST, MDT, LI);
  return Impl.combineMachineInstrs();
}
 
// Pass boilerplate
// ================
 
class AMDGPURegBankCombinerLegacy : public MachineFunctionPass {
public:
  static char ID;
 
  AMDGPURegBankCombinerLegacy(bool IsOptLevelNone = false)
      : MachineFunctionPass(ID), IsOptLevelNone(IsOptLevelNone) {}
 
  StringRef getPassName() const override { return "AMDGPURegBankCombiner"; }
 
  bool runOnMachineFunction(MachineFunction &MF) override;
 
  void getAnalysisUsage(AnalysisUsage &AU) const override;
 
private:
  bool IsOptLevelNone;
};
} // end anonymous namespace
 
void AMDGPURegBankCombinerLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
  AU.setPreservesCFG();
  getSelectionDAGFallbackAnalysisUsage(AU);
  AU.addRequired<GISelValueTrackingAnalysisLegacy>();
  AU.addPreserved<GISelValueTrackingAnalysisLegacy>();
  if (!IsOptLevelNone) {
    AU.addRequired<MachineDominatorTreeWrapperPass>();
  }
  MachineFunctionPass::getAnalysisUsage(AU);
}
 
bool AMDGPURegBankCombinerLegacy::runOnMachineFunction(MachineFunction &MF) {
  const Function &F = MF.getFunction();
  bool EnableOpt =
      MF.getTarget().getOptLevel() != CodeGenOptLevel::None && !skipFunction(F);
 
  return runCombiner(
      MF,
      [&]() {
        return &getAnalysis<GISelValueTrackingAnalysisLegacy>().get(MF);
      },
      [&]() -> MachineDominatorTree * {
        return IsOptLevelNone ? nullptr
                              : &getAnalysis<MachineDominatorTreeWrapperPass>()
                                     .getDomTree();
      },
      EnableOpt);
}
 
char AMDGPURegBankCombinerLegacy::ID = 0;
INITIALIZE_PASS_BEGIN(AMDGPURegBankCombinerLegacy, DEBUG_TYPE,
                      "Combine AMDGPU machine instrs after regbankselect",
                      false, false)
INITIALIZE_PASS_DEPENDENCY(GISelValueTrackingAnalysisLegacy)
INITIALIZE_PASS_END(AMDGPURegBankCombinerLegacy, DEBUG_TYPE,
                    "Combine AMDGPU machine instrs after regbankselect", false,
                    false)
 
FunctionPass *llvm::createAMDGPURegBankCombinerLegacy(bool IsOptLevelNone) {
  return new AMDGPURegBankCombinerLegacy(IsOptLevelNone);
}
 
AMDGPURegBankCombinerPass::AMDGPURegBankCombinerPass(bool IsOptLevelNone)
    : IsOptLevelNone(IsOptLevelNone) {}
 
PreservedAnalyses
AMDGPURegBankCombinerPass::run(MachineFunction &MF,
                               MachineFunctionAnalysisManager &MFAM) {
  const Function &F = MF.getFunction();
  bool EnableOpt = MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
                   !F.hasOptNone() && !shouldSkipOptimizationForOptBisect(F);
 
  if (!runCombiner(
          MF, [&]() { return &MFAM.getResult<GISelValueTrackingAnalysis>(MF); },
          [&]() -> MachineDominatorTree * {
            return IsOptLevelNone
                       ? nullptr
                       : &MFAM.getResult<MachineDominatorTreeAnalysis>(MF);
          },
          EnableOpt))
    return PreservedAnalyses::all();
 
  PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
  PA.preserveSet<CFGAnalyses>();
  PA.preserve<GISelValueTrackingAnalysis>();
  return PA;
}