AArch64MacroFusion.cpp

Go to the documentation of this file.

//===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file This file contains the AArch64 implementation of the DAG scheduling
///  mutation to pair instructions back to back.
//
//===----------------------------------------------------------------------===//
 
#include "AArch64MacroFusion.h"
#include "AArch64Subtarget.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MacroFusion.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
 
#define DEBUG_TYPE "aarch64-macro-fusion"
 
using namespace llvm;
 
STATISTIC(NumFusedArithmeticBcc, "Number of arithmetic-Bcc fusions");
STATISTIC(NumFusedArithmeticCbz, "Number of arithmetic-Cbz fusions");
STATISTIC(NumFusedAES, "Number of AES fusions");
STATISTIC(NumFusedCryptoEOR, "Number of crypto-EOR fusions");
STATISTIC(NumFusedAdrpAdd, "Number of ADRP-ADD fusions");
STATISTIC(NumFusedLiterals, "Number of literal-generation fusions");
STATISTIC(NumFusedAddress, "Number of address-generation load/store fusions");
STATISTIC(NumFusedCmpCSel, "Number of compare-CSEL fusions");
STATISTIC(NumFusedFCmpFCSel, "Number of FP-compare-FCSEL fusions");
STATISTIC(NumFusedCmpCSet, "Number of compare-CSET fusions");
STATISTIC(NumFusedArithmeticLogic, "Number of arithmetic-logic fusions");
STATISTIC(NumFusedAddSub2RegAndConstOne,
          "Number of add/sub-two-register-and-constant-one fusions");
STATISTIC(NumFusedAppleSMECompute, "Number of Apple SME compute fusions");
STATISTIC(NumFusedFMinFMax, "Number of FMIN-FMAX fusions");
 
/// CMN, CMP, TST followed by Bcc
static bool isArithmeticBccPair(const MachineInstr *FirstMI,
                                const MachineInstr &SecondMI, bool CmpOnly) {
  if (SecondMI.getOpcode() != AArch64::Bcc)
    return false;
 
  // Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;
 
  // If we're in CmpOnly mode, we only fuse arithmetic instructions that
  // discard their result.
  if (CmpOnly && FirstMI->getOperand(0).isReg() &&
      !(FirstMI->getOperand(0).getReg() == AArch64::XZR ||
        FirstMI->getOperand(0).getReg() == AArch64::WZR)) {
    return false;
  }
 
  switch (FirstMI->getOpcode()) {
  case AArch64::ADDSWri:
  case AArch64::ADDSWrr:
  case AArch64::ADDSXri:
  case AArch64::ADDSXrr:
  case AArch64::ANDSWri:
  case AArch64::ANDSWrr:
  case AArch64::ANDSXri:
  case AArch64::ANDSXrr:
  case AArch64::SUBSWri:
  case AArch64::SUBSWrr:
  case AArch64::SUBSXri:
  case AArch64::SUBSXrr:
  case AArch64::BICSWrr:
  case AArch64::BICSXrr:
    return true;
  case AArch64::ADDSWrs:
  case AArch64::ADDSXrs:
  case AArch64::ANDSWrs:
  case AArch64::ANDSXrs:
  case AArch64::SUBSWrs:
  case AArch64::SUBSXrs:
  case AArch64::BICSWrs:
  case AArch64::BICSXrs:
    // Shift value can be 0 making these behave like the "rr" variant...
    return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
  }
 
  return false;
}
 
/// ALU operations followed by CBZ/CBNZ.
static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
                                const MachineInstr &SecondMI) {
  if (SecondMI.getOpcode() != AArch64::CBZW &&
      SecondMI.getOpcode() != AArch64::CBZX &&
      SecondMI.getOpcode() != AArch64::CBNZW &&
      SecondMI.getOpcode() != AArch64::CBNZX &&
      SecondMI.getOpcode() != AArch64::TBZW &&
      SecondMI.getOpcode() != AArch64::TBZX &&
      SecondMI.getOpcode() != AArch64::TBNZW &&
      SecondMI.getOpcode() != AArch64::TBNZX)
    return false;
 
  // Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;
 
  switch (FirstMI->getOpcode()) {
  case AArch64::ADDWri:
  case AArch64::ADDWrr:
  case AArch64::ADDXri:
  case AArch64::ADDXrr:
  case AArch64::ANDWri:
  case AArch64::ANDWrr:
  case AArch64::ANDXri:
  case AArch64::ANDXrr:
  case AArch64::EORWri:
  case AArch64::EORWrr:
  case AArch64::EORXri:
  case AArch64::EORXrr:
  case AArch64::ORRWri:
  case AArch64::ORRWrr:
  case AArch64::ORRXri:
  case AArch64::ORRXrr:
  case AArch64::ORNWrr:
  case AArch64::ORNXrr:
  case AArch64::SUBWri:
  case AArch64::SUBWrr:
  case AArch64::SUBXri:
  case AArch64::SUBXrr:
  case AArch64::BICWrr:
  case AArch64::BICXrr:
    return true;
  case AArch64::ADDWrs:
  case AArch64::ADDXrs:
  case AArch64::ANDWrs:
  case AArch64::ANDXrs:
  case AArch64::EORWrs:
  case AArch64::EORXrs:
  case AArch64::ORNWrs:
  case AArch64::ORNXrs:
  case AArch64::ORRWrs:
  case AArch64::ORRXrs:
  case AArch64::SUBWrs:
  case AArch64::SUBXrs:
  case AArch64::BICWrs:
  case AArch64::BICXrs:
    // Shift value can be 0 making these behave like the "rr" variant...
    return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
  }
 
  return false;
}
 
// True unless the pair provably writes non overlapping physical registers.
// Pre-RA the dests are still virtual, and post-RA it requires a genuine WAW,
// that is overlapping dest regs. Overlapping includes sub and super register
// relations, e.g. W0 and X0, which matches the register unit based dependency
// model of the scheduling DAG.
static bool mayHaveWAWDependency(const MachineInstr &FirstMI,
                                 const MachineInstr &SecondMI,
                                 const TargetRegisterInfo *TRI) {
  Register DestFirst = FirstMI.getOperand(0).getReg();
  Register DestSecond = SecondMI.getOperand(0).getReg();
  if (!DestFirst.isPhysical() || !DestSecond.isPhysical())
    return true;
  return TRI->regsOverlap(DestFirst, DestSecond);
}
 
/// AES crypto encoding or decoding.
static bool isAESPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI,
                      const TargetRegisterInfo *TRI) {
  // Assume the 1st instr to be a wildcard if it is unspecified.
  unsigned SecondOpcode = SecondMI.getOpcode();
  switch (SecondOpcode) {
  // AES encode.
  case AArch64::AESMCrr:
  case AArch64::AESMCrrTied:
    if (FirstMI == nullptr)
      return true;
    if (FirstMI->getOpcode() != AArch64::AESErr)
      return false;
    return SecondOpcode == AArch64::AESMCrrTied ||
           mayHaveWAWDependency(*FirstMI, SecondMI, TRI);
  // AES decode.
  case AArch64::AESIMCrr:
  case AArch64::AESIMCrrTied:
    if (FirstMI == nullptr)
      return true;
    if (FirstMI->getOpcode() != AArch64::AESDrr)
      return false;
    return SecondOpcode == AArch64::AESIMCrrTied ||
           mayHaveWAWDependency(*FirstMI, SecondMI, TRI);
  }
 
  return false;
}
 
/// AESE/AESD/PMULL + EOR.
static bool isCryptoEORPair(const MachineInstr *FirstMI,
                            const MachineInstr &SecondMI) {
  if (SecondMI.getOpcode() != AArch64::EORv16i8)
    return false;
 
  // Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;
 
  switch (FirstMI->getOpcode()) {
  case AArch64::AESErr:
  case AArch64::AESDrr:
  case AArch64::PMULLv16i8:
  case AArch64::PMULLv8i8:
  case AArch64::PMULLv1i64:
  case AArch64::PMULLv2i64:
    return true;
  }
 
  return false;
}
 
static bool isAdrpAddPair(const MachineInstr *FirstMI,
                          const MachineInstr &SecondMI) {
  // Assume the 1st instr to be a wildcard if it is unspecified.
  if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
      SecondMI.getOpcode() == AArch64::ADDXri)
    return true;
  return false;
}
 
/// Fuse address generation and loads or stores.
static bool isAddressLdStPair(const MachineInstr *FirstMI,
                              const MachineInstr &SecondMI) {
  switch (SecondMI.getOpcode()) {
  case AArch64::STRBBui:
  case AArch64::STRBui:
  case AArch64::STRDui:
  case AArch64::STRHHui:
  case AArch64::STRHui:
  case AArch64::STRQui:
  case AArch64::STRSui:
  case AArch64::STRWui:
  case AArch64::STRXui:
  case AArch64::LDRBBui:
  case AArch64::LDRBui:
  case AArch64::LDRDui:
  case AArch64::LDRHHui:
  case AArch64::LDRHui:
  case AArch64::LDRQui:
  case AArch64::LDRSui:
  case AArch64::LDRWui:
  case AArch64::LDRXui:
  case AArch64::LDRSBWui:
  case AArch64::LDRSBXui:
  case AArch64::LDRSHWui:
  case AArch64::LDRSHXui:
  case AArch64::LDRSWui:
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;
 
   switch (FirstMI->getOpcode()) {
    case AArch64::ADR:
      return SecondMI.getOperand(2).getImm() == 0;
    case AArch64::ADRP:
      return true;
    }
  }
 
  return false;
}
 
/// Compare and conditional select.
static bool isCmpCSelPair(const MachineInstr *FirstMI,
                          const MachineInstr &SecondMI) {
  // 32 bits
  if (SecondMI.getOpcode() == AArch64::CSELWr) {
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;
 
    if (FirstMI->definesRegister(AArch64::WZR, /*TRI=*/nullptr))
      switch (FirstMI->getOpcode()) {
      case AArch64::SUBSWrs:
        return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
      case AArch64::SUBSWrx:
        return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
      case AArch64::SUBSWrr:
      case AArch64::SUBSWri:
        return true;
      }
  }
 
  // 64 bits
  if (SecondMI.getOpcode() == AArch64::CSELXr) {
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;
 
    if (FirstMI->definesRegister(AArch64::XZR, /*TRI=*/nullptr))
      switch (FirstMI->getOpcode()) {
      case AArch64::SUBSXrs:
        return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
      case AArch64::SUBSXrx:
      case AArch64::SUBSXrx64:
        return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
      case AArch64::SUBSXrr:
      case AArch64::SUBSXri:
        return true;
      }
  }
 
  return false;
}
 
/// Floating-point compare and floating-point conditional select.
static bool isFCmpFCSelPair(const MachineInstr *FirstMI,
                            const MachineInstr &SecondMI) {
  switch (SecondMI.getOpcode()) {
  case AArch64::FCSELSrrr:
  case AArch64::FCSELDrrr:
  case AArch64::FCSELHrrr:
    break;
  default:
    return false;
  }
 
  // Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;
 
  switch (FirstMI->getOpcode()) {
  case AArch64::FCMPSrr:
  case AArch64::FCMPDrr:
  case AArch64::FCMPESrr:
  case AArch64::FCMPEDrr:
  case AArch64::FCMPHrr:
  case AArch64::FCMPEHrr:
    return true;
  default:
    return false;
  }
}
 
/// Compare and cset.
static bool isCmpCSetPair(const MachineInstr *FirstMI,
                          const MachineInstr &SecondMI) {
  if ((SecondMI.getOpcode() == AArch64::CSINCWr &&
       SecondMI.getOperand(1).getReg() == AArch64::WZR &&
       SecondMI.getOperand(2).getReg() == AArch64::WZR) ||
      (SecondMI.getOpcode() == AArch64::CSINCXr &&
       SecondMI.getOperand(1).getReg() == AArch64::XZR &&
       SecondMI.getOperand(2).getReg() == AArch64::XZR)) {
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;
 
    if (FirstMI->definesRegister(AArch64::WZR, /*TRI=*/nullptr) ||
        FirstMI->definesRegister(AArch64::XZR, /*TRI=*/nullptr))
      switch (FirstMI->getOpcode()) {
      case AArch64::SUBSWrs:
      case AArch64::SUBSXrs:
        return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
      case AArch64::SUBSWrx:
      case AArch64::SUBSXrx:
      case AArch64::SUBSXrx64:
        return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
      case AArch64::SUBSWri:
      case AArch64::SUBSWrr:
      case AArch64::SUBSXri:
      case AArch64::SUBSXrr:
        return true;
      }
  }
 
  return false;
}
 
// Arithmetic and logic.
static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
                                  const MachineInstr &SecondMI) {
  if (AArch64InstrInfo::hasShiftedReg(SecondMI))
    return false;
 
  switch (SecondMI.getOpcode()) {
  // Arithmetic
  case AArch64::ADDWrr:
  case AArch64::ADDXrr:
  case AArch64::SUBWrr:
  case AArch64::SUBXrr:
  case AArch64::ADDWrs:
  case AArch64::ADDXrs:
  case AArch64::SUBWrs:
  case AArch64::SUBXrs:
  // Logic
  case AArch64::ANDWrr:
  case AArch64::ANDXrr:
  case AArch64::BICWrr:
  case AArch64::BICXrr:
  case AArch64::EONWrr:
  case AArch64::EONXrr:
  case AArch64::EORWrr:
  case AArch64::EORXrr:
  case AArch64::ORNWrr:
  case AArch64::ORNXrr:
  case AArch64::ORRWrr:
  case AArch64::ORRXrr:
  case AArch64::ANDWrs:
  case AArch64::ANDXrs:
  case AArch64::BICWrs:
  case AArch64::BICXrs:
  case AArch64::EONWrs:
  case AArch64::EONXrs:
  case AArch64::EORWrs:
  case AArch64::EORXrs:
  case AArch64::ORNWrs:
  case AArch64::ORNXrs:
  case AArch64::ORRWrs:
  case AArch64::ORRXrs:
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;
 
    // Arithmetic
    switch (FirstMI->getOpcode()) {
    case AArch64::ADDWrr:
    case AArch64::ADDXrr:
    case AArch64::ADDSWrr:
    case AArch64::ADDSXrr:
    case AArch64::SUBWrr:
    case AArch64::SUBXrr:
    case AArch64::SUBSWrr:
    case AArch64::SUBSXrr:
      return true;
    case AArch64::ADDWrs:
    case AArch64::ADDXrs:
    case AArch64::ADDSWrs:
    case AArch64::ADDSXrs:
    case AArch64::SUBWrs:
    case AArch64::SUBXrs:
    case AArch64::SUBSWrs:
    case AArch64::SUBSXrs:
      return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
    }
    break;
 
  // Arithmetic, setting flags.
  case AArch64::ADDSWrr:
  case AArch64::ADDSXrr:
  case AArch64::SUBSWrr:
  case AArch64::SUBSXrr:
  case AArch64::ADDSWrs:
  case AArch64::ADDSXrs:
  case AArch64::SUBSWrs:
  case AArch64::SUBSXrs:
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;
 
    // Arithmetic, not setting flags.
    switch (FirstMI->getOpcode()) {
    case AArch64::ADDWrr:
    case AArch64::ADDXrr:
    case AArch64::SUBWrr:
    case AArch64::SUBXrr:
      return true;
    case AArch64::ADDWrs:
    case AArch64::ADDXrs:
    case AArch64::SUBWrs:
    case AArch64::SUBXrs:
      return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
    }
    break;
  }
 
  return false;
}
 
// "(A + B) + 1" or "(A - B) - 1"
static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI,
                                        const MachineInstr &SecondMI) {
  bool NeedsSubtract = false;
 
  // The 2nd instr must be an add-immediate or subtract-immediate.
  switch (SecondMI.getOpcode()) {
  case AArch64::SUBWri:
  case AArch64::SUBXri:
    NeedsSubtract = true;
    [[fallthrough]];
  case AArch64::ADDWri:
  case AArch64::ADDXri:
    break;
 
  default:
    return false;
  }
 
  // The immediate in the 2nd instr must be "1".
  if (!SecondMI.getOperand(2).isImm() || SecondMI.getOperand(2).getImm() != 1) {
    return false;
  }
 
  // Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr) {
    return true;
  }
 
  switch (FirstMI->getOpcode()) {
  case AArch64::SUBWrs:
  case AArch64::SUBXrs:
    if (AArch64InstrInfo::hasShiftedReg(*FirstMI))
      return false;
    [[fallthrough]];
  case AArch64::SUBWrr:
  case AArch64::SUBXrr:
    if (NeedsSubtract) {
      return true;
    }
    break;
 
  case AArch64::ADDWrs:
  case AArch64::ADDXrs:
    if (AArch64InstrInfo::hasShiftedReg(*FirstMI))
      return false;
    [[fallthrough]];
  case AArch64::ADDWrr:
  case AArch64::ADDXrr:
    if (!NeedsSubtract) {
      return true;
    }
    break;
  }
 
  return false;
}
 
static bool definesRegInClass(const MachineInstr &MI,
                              const TargetRegisterInfo *TRI,
                              const TargetRegisterClass &Class) {
  return llvm::any_of(Class, [&MI, TRI](MCPhysReg Reg) {
    return MI.definesRegister(Reg, TRI);
  });
}
 
static bool readsRegInClass(const MachineInstr &MI,
                            const TargetRegisterInfo *TRI,
                            const TargetRegisterClass &Class) {
  return llvm::any_of(
      Class, [&MI, TRI](MCPhysReg Reg) { return MI.readsRegister(Reg, TRI); });
}
 
static bool isFusableAppleSMEComputeOp(const MachineInstr &MI,
                                       const TargetInstrInfo &TII,
                                       const TargetRegisterInfo *TRI) {
  const bool ReadOrWriteZA = MI.readsRegister(AArch64::ZA, TRI) ||
                             MI.definesRegister(AArch64::ZA, TRI);
 
  // (read/write ZA or read/write Z)
  if (!ReadOrWriteZA && !definesRegInClass(MI, TRI, AArch64::ZPRRegClass))
    return false;
 
  // (NOT load/store)
  if (MI.mayLoad() || MI.mayStore())
    return false;
 
  // (NOT write P)
  if (definesRegInClass(MI, TRI, AArch64::PPRRegClass))
    return false;
 
  // (NOT write GPR)
  const bool WriteGPR = definesRegInClass(MI, TRI, AArch64::GPR32RegClass) ||
                        definesRegInClass(MI, TRI, AArch64::GPR64RegClass);
  if (WriteGPR)
    return false;
 
  // (NOT read/write NZCV)
  if (MI.readsRegister(AArch64::NZCV, TRI) ||
      MI.definesRegister(AArch64::NZCV, TRI))
    return false;
 
  const bool ReadGPR = readsRegInClass(MI, TRI, AArch64::GPR32RegClass) &&
                       readsRegInClass(MI, TRI, AArch64::GPR64RegClass);
 
  // ( (NOT read GPR) or read/write ZA )
  if (ReadGPR && !ReadOrWriteZA)
    return false;
 
  return true;
}
 
static bool isAppleSMEComputePair(const MachineInstr *FirstMI,
                                  const MachineInstr &SecondMI,
                                  const TargetInstrInfo &TII,
                                  const TargetRegisterInfo *TRI) {
  if (!isFusableAppleSMEComputeOp(SecondMI, TII, TRI))
    return false;
  // Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;
  if (isFusableAppleSMEComputeOp(*FirstMI, TII, TRI))
    return true;
  return false;
}
 
// Floating-point minimum or maximum, scalar (H/S/D) or vector (Vd).
static bool isFMinFMax(unsigned Opcode) {
  switch (Opcode) {
  // Scalar.
  case AArch64::FMAXHrr:
  case AArch64::FMAXSrr:
  case AArch64::FMAXDrr:
  case AArch64::FMINHrr:
  case AArch64::FMINSrr:
  case AArch64::FMINDrr:
  // Vector.
  case AArch64::FMAXv4f16:
  case AArch64::FMAXv8f16:
  case AArch64::FMAXv2f32:
  case AArch64::FMAXv4f32:
  case AArch64::FMAXv2f64:
  case AArch64::FMINv4f16:
  case AArch64::FMINv8f16:
  case AArch64::FMINv2f32:
  case AArch64::FMINv4f32:
  case AArch64::FMINv2f64:
    return true;
  }
  return false;
}
 
// FMIN + FMAX.
static bool isFMinFMaxPair(const MachineInstr *FirstMI,
                           const MachineInstr &SecondMI,
                           const TargetRegisterInfo *TRI) {
  if (!isFMinFMax(SecondMI.getOpcode()))
    return false;
 
  // Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;
 
  if (!isFMinFMax(FirstMI->getOpcode()))
    return false;
 
  return mayHaveWAWDependency(*FirstMI, SecondMI, TRI);
}
 
/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
/// together. Given SecondMI, when FirstMI is unspecified, then check if
/// SecondMI may be part of a fused pair at all.
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
                                   const TargetSubtargetInfo &TSI,
                                   const MachineInstr *FirstMI,
                                   const MachineInstr &SecondMI,
                                   const SDep *Dep) {
  const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
  const TargetRegisterInfo *TRI = TSI.getRegisterInfo();
 
  // All checking functions assume that the 1st instr is a wildcard if it is
  // unspecified.
 
  // FuseAppleSMECompute does not require a specific dependency kind
  if (ST.hasFuseAppleSMECompute() &&
      isAppleSMEComputePair(FirstMI, SecondMI, TII, TRI)) {
    ++NumFusedAppleSMECompute;
    return true;
  }
 
  // All the other fusions require RAW dependency
  if (isNonDataDep(Dep))
    return false;
 
  if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
    bool CmpOnly = !ST.hasArithmeticBccFusion();
    if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly)) {
      ++NumFusedArithmeticBcc;
      return true;
    }
  }
  if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI)) {
    ++NumFusedArithmeticCbz;
    return true;
  }
  if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI, TRI)) {
    ++NumFusedAES;
    return true;
  }
  if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI)) {
    ++NumFusedCryptoEOR;
    return true;
  }
  if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI)) {
    ++NumFusedAdrpAdd;
    return true;
  }
  if (ST.hasFuseLiterals() && ST.fusesMOVImmPair(FirstMI, SecondMI)) {
    ++NumFusedLiterals;
    return true;
  }
  if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI)) {
    ++NumFusedAddress;
    return true;
  }
  if (ST.hasFuseCmpCSel() && isCmpCSelPair(FirstMI, SecondMI)) {
    ++NumFusedCmpCSel;
    return true;
  }
  if (ST.hasFuseFCmpFCSel() && isFCmpFCSelPair(FirstMI, SecondMI)) {
    ++NumFusedFCmpFCSel;
    return true;
  }
  if (ST.hasFuseCmpCSet() && isCmpCSetPair(FirstMI, SecondMI)) {
    ++NumFusedCmpCSet;
    return true;
  }
  if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI)) {
    ++NumFusedArithmeticLogic;
    return true;
  }
  if (ST.hasFuseAddSub2RegAndConstOne() &&
      isAddSub2RegAndConstOnePair(FirstMI, SecondMI)) {
    ++NumFusedAddSub2RegAndConstOne;
    return true;
  }
  if (ST.hasFuseFMinFMax() && isFMinFMaxPair(FirstMI, SecondMI, TRI)) {
    ++NumFusedFMinFMax;
    return true;
  }
 
  return false;
}
 
std::unique_ptr<ScheduleDAGMutation>
llvm::createAArch64MacroFusionDAGMutation() {
  return createMacroFusionDAGMutation(shouldScheduleAdjacent);
}