HexagonXQFloatGenerator.cpp

Go to the documentation of this file.

//===-------------------- HexagonXQFloatGenerator.cpp --------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass enables generation of XQFloat instructions. XQF instructions
// are more efficient, but can be less precise in comparison to IEEE ones.
// Based on the accuracy preservation of the generated code, we enabled four
// modes - Strict IEEE-754 compliant, IEEE-754 compliant, Lossy subnormals and
// legacy mode.
//
// Strict IEEE mode adheres to similar accuracy and precision as of IEEE-754.
//
// IEEE-754 compliant mode excludes IEEE-754 overflows and lower precision
// subnormals due to larger dynamic range than IEEE-754.
// All subnormals have extra precision.
//
// Lossy subnormals mode without normalization result in a loss of accuracy.
// This provides greater precision than a clamp of subnormals to 0.
// If dataset excludes subnormals, it behavas as IEEE-754 compliant mode.
//
// The direct mode has a loss of 1 bit of accuracy compared to IEEE-754.
//
// V79 replaces the prior internal HVX floating point format for floating-point
// arithmetic. The new internal HVX floating-point format yields results
// identical to IEEE-754 round-to-even mode. The new format contains more bits
// than IEEE-754, which optionally produces results with greater range and
// accuracy. Only the HVX vector registers use the HVX floating-point format.
// Memory maintains all floating-point data in IEEE-754 format,
// and all loads/stores use the IEEE-754 format. A subset of HVX floating-point
// operations transform IEEE-754 floating-point data to HVX floating-point data.
// Subsequent HVX floating-point instructions may consume operands in the HVX
// floating-point without conversion to IEEE-754, which allows for performant
// & energy efficient code. The program does not need to switch between formats
// continuously. The program must convert the HVX floating-point results to
// IEEE-754 prior to storing to memory.
 
// HVX floating-point achieves IEEE-754 compliance through normalization.
// The program may skip normalization when faster calculation is desired, and
// IEEE-754 compliance isn’t required. HVX floating-point contains two input
// types: qf32, single precision floating point, and qf16, half precision
// floating point. In Hexagon, IEEE-754 contains two input types: sf, single
// precision floating point, and hf, half precision floating point.
//
// Only HVX floating-point source and destination instructions use HVX
// floating-point values. Instructions specify the HVX floating-point format
// with the qf16 and qf32 identifier. A source vector register will drop the
// extended state of a HVX floating-point value when an instruction reads the
// source vector register without the qf16 or qf32 identifier. A destination
// vector register will reset its extended state when an instruction writes to
// a vector register without the qf16 or qf32 identifier. When dropping the
// extended state, the floating-point value loses accuracy. The program may
// preserve the floating-point value by converting HVX floating-point values
// to IEEE-754 values. Compiler must convert HVX floating-point values to
// IEEE-754 values before using as an input to stores, permutes, shifts, and
// any other operations that do not source the HVX floating-point format.
//
// Depending on the desired results, HVX floating-point operations may have
// some requirements on the input sources. The HVX floating-point values
// require normalization to achieve IEEE-754 compliance, while faster operations
// may skip normalization. The program normalizes HVX floating-point values
// before subsequent HVX floating-point operations, so the floating-point value
// does not lose precision. The program also obtains results identical to
// IEEE-754 by converting all HVX floating-point results to IEEE-754 format
// before consumed in any subsequent operation. There are however cases where
// this conversion is redundant, or the differences between IEEE-754 and HVX
// floating-point may not be a concern.
//
// The conversion logic can be understood by the table below:
//
// ================================================================================================================================================
//            |                              | |                               |
//            |    Inputs to add/subtarct    |                  Inputs to
//            multiplication instuctions              |     Non-HVX floating
//            point    | |    instructions              | |          instruction
//            | |                              | | |
// ===============================================================================================================================================|
// Sources    | IEEE-  | HVX      | HVX      | sf        | qf32      | qf32 | hf
// | qf16     | qf16     | IEE-754 | HVX      | HVX      |
//            |  754   | floating | floating |           | from      | from | |
//            from     | from     |         | floating | floating | |        |
//            point    | point    |           | mult      | adder     | | mult
//            | adder    |         | point    | point    | |        | from     |
//            from     |           |           |           |          | | | |
//            from     | from     | |        | multi    | adder    |           |
//            |           |          |          |          |         | mult |
//            adder    | |        |          |          |           | | | | | |
//            |          |          |
// ===============================================================================================================================================|
// Strict     | Direct | Convert  | Convert  | Normalize | Convert   | Convert
// | widening | Convert  | Convert  | Direct  | Convert  | Convert  | IEEE-754
// | Use    | to       | to       |           | to IEEE   | to IEEE   | multiply
// | to IEEE, | to IEEE, | use     | to       | to       | compliance |        |
// IEEE     | IEEE     |           | then      | then      | then     | widening
// | widening |         | IEEE     | IEEE     |
//            |        |          |          |           | normalize | normalize
//            | convert  | multiply,| multiply,|         |          |          |
//            |        |          |          |           |           | | to IEEE
//            | convert  | convert  |         |          |          | |        |
//            |          |           |           |           |          | to
//            IEEE  | to IEEE  |         |          |          |
// -----------------------------------------------------------------------------------------------------------------------------------------------|
// IEEE-754   | Direct | Direct   | Direct   | Normalize | Direct    | Normalize
// | Widening | Direct   | Widening | Direct  | Convert  | Convert  | compliance
// | Use    | Use      | Use      |           | use       |           | multiply
// | use      | multiply | use     | to IEEE  | to IEEE  |
// -----------------------------------------------------------------------------------------------------------------------------------------------|
// Lossy      | Direct | Direct   | Direct   | Direct    | Direct    | Normalize
// | Direct   | Direct   | Widening | Direct  | Convert  | Convert  | Subnormals
// | Use    | Use      | Use      | Use       | use       |           | use |
// use      | multiply | use     | to IEEE  | to IEEE  |
// -----------------------------------------------------------------------------------------------------------------------------------------------|
// Direct     | Direct | Direct   | Direct   | Direct    | Direct    | Direct |
// Direct   | Direct   | Direct   | Direct  | Direct   | Direct   | Lossy      |
// Use    | Use      | Use      | Use       | use       | use       | use      |
// use      | use      | use     | use      | use      |
// -----------------------------------------------------------------------------------------------------------------------------------------------|
//
// For v81, the normalization sequence changes. Instead of multiplying 0
// and -0, a simple copy operation normalizes the unnormal value. Both
// qf and IEEE-754 value can be unnormal.
// Additionally for v81, we have two new vsub instructions which are handled.
 
#define HEXAGON_XQFLOAT_GENERATOR "XQFloat Generator pass"
 
#include "Hexagon.h"
#include "HexagonInstrInfo.h"
#include "HexagonSubtarget.h"
#include "HexagonTargetMachine.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/Passes.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include <vector>
 
#define DEBUG_TYPE "hexagon-xqf-gen"
 
using namespace llvm;
 
extern cl::opt<QFloatMode> QFloatModeValue;
 
// Master flag to enable XQF generations
cl::opt<bool> EnableHVXXQFloat("enable-xqf-gen", cl::init(false),
                               cl::desc("Enable XQFloat generations"));
// This vector contains the opcodes which generate qf32 from add/subtract
static constexpr unsigned XQFPAdd32[] = {
    // vector add instructions
    Hexagon::V6_vadd_sf, Hexagon::V6_vadd_qf32, Hexagon::V6_vadd_qf32_mix,
 
    // vector subtract instructions
    Hexagon::V6_vsub_qf32, Hexagon::V6_vsub_qf32_mix, Hexagon::V6_vsub_sf,
    Hexagon::V6_vsub_sf_mix};
 
// This vector contains the opcodes which generate qf16 from add/subtract
static constexpr unsigned XQFPAdd16[] = {
    // vector add instructions
    Hexagon::V6_vadd_hf, Hexagon::V6_vadd_qf16, Hexagon::V6_vadd_qf16_mix,
 
    // vector subtract intrutions
    Hexagon::V6_vsub_hf, Hexagon::V6_vsub_qf16, Hexagon::V6_vsub_qf16_mix,
    Hexagon::V6_vsub_hf_mix};
 
// This vector contains the opcodes which generate qf32 from multiplication
static constexpr unsigned XQFPMult32[] = {
    Hexagon::V6_vmpy_qf32, Hexagon::V6_vmpy_qf32_qf16, Hexagon::V6_vmpy_qf32_hf,
    Hexagon::V6_vmpy_qf32_sf, Hexagon::V6_vmpy_qf32_mix_hf};
// This vector contains the opcodes which generate qf16 from multiplication
static constexpr unsigned XQFPMult16[] = {Hexagon::V6_vmpy_qf16,
                                          Hexagon::V6_vmpy_qf16_hf,
                                          Hexagon::V6_vmpy_qf16_mix_hf};
 
namespace llvm {
FunctionPass *createHexagonXQFloatGenerator();
void initializeHexagonXQFloatGeneratorPass(PassRegistry &);
} // namespace llvm
 
namespace {
 
struct HexagonXQFloatGenerator : public MachineFunctionPass {
public:
  static char ID;
  HexagonXQFloatGenerator() : MachineFunctionPass(ID) {}
 
  bool runOnMachineFunction(MachineFunction &MF) override;
 
  StringRef getPassName() const override { return HEXAGON_XQFLOAT_GENERATOR; }
 
  void getAnalysisUsage(AnalysisUsage &AU) const override {
    MachineFunctionPass::getAnalysisUsage(AU);
  }
 
private:
  // Handle each XQF optimization level
  bool HandleStrictIEEE(MachineFunction &);
  bool HandleCompliantIEEE(MachineFunction &);
  bool HandleLossySubnormals(MachineFunction &);
  bool HandleLossyLegacy(MachineFunction &);
 
  // Checkers functions for input operands
  bool checkIfInputFromAdder32(Register Reg);
  bool checkIfInputFromAdder16(Register Reg);
  bool checkIfInputFromMult32(Register Reg);
  bool checkIfInputFromMult16(Register Reg);
  bool deleteList();
 
  // Helper functions for conversion/normalization/widening
  bool widenMultiplicationInputF16(MachineInstr &, Register &, Register &,
                                   Register &, bool);
  bool widenMultiplicationInputF16Rt(MachineInstr &, Register &, Register &,
                                     Register &);
  void widenMultiplyInputHF(MachineInstr &, Register &, Register &, Register &);
  bool normalizeMultiplicationInputF32(MachineInstr &, Register &, Register &,
                                       Register &, Register &, bool &);
  void normalizeMultiplicationInputSF(MachineInstr &, Register &, Register &,
                                      Register &, Register &, bool &);
  bool convertNormalizeMultOp32(MachineInstr &, Register &, Register &,
                                Register &, Register &, bool &);
  bool convertWidenMultOp16(MachineInstr &, Register &, Register &, Register &,
                            bool);
  bool convertWidenMultOp32(MachineInstr &, Register &, Register &, Register &,
                            bool);
  void createPrologInstructions(MachineInstr &, Register &);
  bool convertAddOpToIEEE16(MachineInstr &, Register &, Register &, Register &,
                            bool, bool, bool);
  bool convertAddOpToIEEE32(MachineInstr &, Register &, Register &, Register &,
                            bool, bool, bool);
  void generateQF16FromQF32(MachineInstr &, Register &, Register &);
  bool convertIfInputToNonHVX(MachineInstr &, bool);
  void createConvertInstr(MachineInstr *, Register &, Register &, bool);
 
  // V81 specific normalization function
  bool V81normalizeMultF32(MachineInstr &, Register &, Register &, Register &,
                           bool, bool, bool);
 
  const HexagonSubtarget *HST = nullptr;
  const HexagonInstrInfo *HII = nullptr;
  MachineRegisterInfo *MRI = nullptr;
 
  SmallVector<MachineInstr *, 16>
      OriginalMI; // Hold the instructions to be deleted
};
 
// This class removes redundant vector convert instructions from qf to hf/sf.
// Additionally, it relaces use of sf/hf registers with qf types.
// The resulting code is complete without dangling instructions.
// FIXME: Liveness is not preserved.
char HexagonXQFloatGenerator::ID = 0;
 
} // namespace
 
INITIALIZE_PASS(HexagonXQFloatGenerator, "hexagon-xqfloat-generator",
                HEXAGON_XQFLOAT_GENERATOR, false, false)
 
FunctionPass *llvm::createHexagonXQFloatGenerator() {
  return new HexagonXQFloatGenerator();
}
 
// Returns true if qf32 input is from an adder/subtract unit
bool HexagonXQFloatGenerator::checkIfInputFromAdder32(Register Reg) {
  MachineInstr *Def = MRI->getVRegDef(Reg);
  if (!Def)
    return false;
 
  // If the definition is a copy, we need to analyze its def again
  if (Def->getOpcode() == TargetOpcode::COPY) {
    Register SrcReg = Def->getOperand(1).getReg();
    if (SrcReg.isValid())
      return checkIfInputFromAdder32(SrcReg);
    return false;
  } else if (Def->getOpcode() == TargetOpcode::REG_SEQUENCE) {
    Register SrcReg1 = Def->getOperand(1).getReg();
    Register SrcReg2 = Def->getOperand(2).getReg();
    bool isTrue = false;
    if (SrcReg1.isValid())
      isTrue = checkIfInputFromAdder32(SrcReg1);
    if (SrcReg2.isValid())
      isTrue |= checkIfInputFromAdder32(SrcReg2);
    return isTrue;
  } else
    return llvm::is_contained(XQFPAdd32, Def->getOpcode());
}
 
// Returns true if qf16 input is from an adder/subtract unit
bool HexagonXQFloatGenerator::checkIfInputFromAdder16(Register Reg) {
  MachineInstr *Def = MRI->getVRegDef(Reg);
  if (!Def)
    return false;
 
  // if the definition is a copy, we need to analyze its def again
  if (Def->getOpcode() == TargetOpcode::COPY) {
    Register SrcReg = Def->getOperand(1).getReg();
    if (SrcReg.isValid())
      return checkIfInputFromAdder16(SrcReg);
    return false;
  } else
    return llvm::is_contained(XQFPAdd16, Def->getOpcode());
}
 
// Returns true if qf32 input is from a multiplier unit
bool HexagonXQFloatGenerator::checkIfInputFromMult32(Register Reg) {
  MachineInstr *Def = MRI->getVRegDef(Reg);
  if (!Def)
    return false;
 
  // if the definition is a copy, we need to analyze its def again
  if (Def->getOpcode() == TargetOpcode::COPY) {
    Register SrcReg = Def->getOperand(1).getReg();
    if (SrcReg.isValid())
      return checkIfInputFromMult32(SrcReg);
    return false;
  } else if (Def->getOpcode() == TargetOpcode::REG_SEQUENCE) {
    Register SrcReg1 = Def->getOperand(1).getReg();
    Register SrcReg2 = Def->getOperand(2).getReg();
    bool isTrue = false;
    if (SrcReg1.isValid())
      isTrue |= checkIfInputFromMult32(SrcReg1);
    if (SrcReg2.isValid())
      isTrue |= checkIfInputFromMult32(SrcReg2);
    return isTrue;
  } else
    return llvm::is_contained(XQFPMult32, Def->getOpcode());
}
 
// Returns true if qf16 input is from a multiplier unit
bool HexagonXQFloatGenerator::checkIfInputFromMult16(Register Reg) {
  MachineInstr *Def = MRI->getVRegDef(Reg);
  if (!Def)
    return false;
 
  // if the definition is a copy, we need to analyze its def again
  if (Def->getOpcode() == TargetOpcode::COPY) {
    Register SrcReg = Def->getOperand(1).getReg();
    if (SrcReg.isValid())
      return checkIfInputFromMult16(SrcReg);
    return false;
  } else
    return llvm::is_contained(XQFPMult16, Def->getOpcode());
}
 
// Generates sf = qf32 instruction or hf = qf16 intruction
void HexagonXQFloatGenerator::createConvertInstr(MachineInstr *UseMI,
                                                 Register &NewR, Register &OldR,
                                                 bool is32bit) {
  const DebugLoc &DL = UseMI->getDebugLoc();
  MachineBasicBlock *MBB = UseMI->getParent();
  NewR = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  if (is32bit)
    BuildMI(*MBB, *UseMI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), NewR)
        .addReg(OldR);
  else
    BuildMI(*MBB, *UseMI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), NewR)
        .addReg(OldR);
}
 
// Generate HVX to IEEE conversion instruction for all non-HVX uses
bool HexagonXQFloatGenerator::convertIfInputToNonHVX(MachineInstr &MI,
                                                     bool is32bit) {
  Register NewR;
  bool Changed = false;
  ;
  Register Dest = MI.getOperand(0).getReg();
 
  // Iterate over all uses of the Def we are analyzing
  for (auto &MO : make_range(MRI->use_begin(Dest), MRI->use_end())) {
    MachineInstr *UseMI = MO.getParent();
    // Omit if the use is a REG_SEQUENCE instruction, since the only
    // use of REG_SEQUENCE in qf context is transforming to IEEE.
    // Omit for use in DBG instructions.
    // Omit for use in PHI instructions since PHI result can be used as a qf
    // operand.
    if (UseMI->getOpcode() == TargetOpcode::REG_SEQUENCE ||
        UseMI->getOpcode() == TargetOpcode::DBG_VALUE ||
        UseMI->getOpcode() == TargetOpcode::DBG_LABEL ||
        UseMI->getOpcode() == TargetOpcode::PHI)
      continue;
 
    // If 32-bit operand
    if (is32bit) {
      // If it is a copy instruction, we need to analyze it uses
      if (UseMI->getOpcode() == TargetOpcode::COPY)
        return convertIfInputToNonHVX(*UseMI, /* 32 bit */ true);
      if (!HII->usesQFOperand(UseMI)) {
        createConvertInstr(UseMI, NewR, Dest, /*32 bit*/ true);
        MO.setReg(NewR);
        Changed = true;
      }
      // If 16-bit operand
    } else {
      // If it is a copy instruction, we need to analyze it uses
      if (UseMI->getOpcode() == TargetOpcode::COPY)
        return convertIfInputToNonHVX(*UseMI, /* 16 bit */ false);
      if (!HII->usesQFOperand(UseMI)) {
        createConvertInstr(UseMI, NewR, Dest, /*16 bit*/ false);
        MO.setReg(NewR);
        Changed = true;
      }
    }
  }
  return Changed;
}
 
// generate qf16 = qf32 via:
// hf = qf32
// V0 = #0
// qf16 = vsub(hf,V0)
void HexagonXQFloatGenerator::generateQF16FromQF32(MachineInstr &MI,
                                                   Register &Dest,
                                                   Register &SrcReg) {
 
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  Register convertReg = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf32), convertReg)
      .addReg(SrcReg);
  Register VR0 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vd0), VR0);
 
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_hf), Dest)
      .addReg(convertReg)
      .addReg(VR0);
}
 
// Widen qf16 = vmpy(hf, hf) result unconditionally
void HexagonXQFloatGenerator::widenMultiplyInputHF(MachineInstr &MI,
                                                   Register &Reg1,
                                                   Register &Reg2,
                                                   Register &Dest) {
  Register output_mpy = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass);
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_hf), output_mpy)
      .addReg(Reg1)
      .addReg(Reg2);
  generateQF16FromQF32(MI, Dest, output_mpy);
}
 
// Widen vmpy(qf16, qf16/hf) result conditionally
bool HexagonXQFloatGenerator::widenMultiplicationInputF16(MachineInstr &MI,
                                                          Register &Reg1,
                                                          Register &Reg2,
                                                          Register &Dest,
                                                          bool twoOps) {
  bool firstconvert = false, secondconvert = false;
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  // We widen only that operand which comes from add/subtract unit.
  if (checkIfInputFromAdder16(Reg1))
    firstconvert = true;
  // twoOps == true suggest 2nd operand is qf16, else it is hf
  if (twoOps && checkIfInputFromAdder16(Reg2))
    secondconvert = true;
 
  Register widenReg;
  // if either operands from add/subtract unit, we widen
  if (twoOps) {
    if (firstconvert || secondconvert) {
      widenReg = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_qf16), widenReg)
          .addReg(Reg1)
          .addReg(Reg2);
    } else {
      return false;
    }
  } else {
    if (firstconvert) {
      widenReg = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_mix_hf), widenReg)
          .addReg(Reg1)
          .addReg(Reg2);
    } else {
      return false;
    }
  }
 
  // generate qf16 = qf32
  generateQF16FromQF32(MI, Dest, widenReg);
 
  return true;
}
 
// Handle qf16 = vmpy(qf16, Rt)
// For strict IEEE mode, convert the qf16 to IEEE before widening
bool HexagonXQFloatGenerator::widenMultiplicationInputF16Rt(MachineInstr &MI,
                                                            Register &Reg1,
                                                            Register &Reg2,
                                                            Register &Dest) {
  // If the first input is not from an adder, for strict-ieee check if
  // input from mult, else return false.
  if (!checkIfInputFromAdder16(Reg1)) {
    if (QFloatModeValue == QFloatMode::StrictIEEE) {
      if (!checkIfInputFromMult16(Reg1))
        return false;
    } else
      return false;
  }
 
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  Register VSplatReg = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_lvsplatw), VSplatReg).addReg(Reg2);
 
  Register widenReg = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass);
  if (QFloatModeValue == QFloatMode::StrictIEEE) {
    Register VHf = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VHf).addReg(Reg1);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_hf), widenReg)
        .addReg(VHf)
        .addReg(VSplatReg);
  } else {
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_mix_hf), widenReg)
        .addReg(Reg1)
        .addReg(VSplatReg);
  }
 
  // generate qf16 = qf32
  generateQF16FromQF32(MI, Dest, widenReg);
  return true;
}
 
// Handle qf32 = vadd/vsub(qf32/sf, qf32/sf)
// Handle vadd/vsub instructions with qf32 operands conditionally
// isAdd:  true if an add instruction is analyzed, false for subtract
// isFirstOpQf: true if 1st operand is qf32 type, false if sf type
// isSecOpQf: true if 2nd operand is qf32 type, false if sf type
bool HexagonXQFloatGenerator::convertAddOpToIEEE32(
    MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
    bool isAdd, bool isFirstOpQf, bool isSecOpQf) {
 
  Register VR1;
  Register VR2;
  bool firstconvert = false, secondconvert = false;
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  // If the first operand is qf32 type
  if (isFirstOpQf) {
    // If the first operand is from add/sub/mul unit,
    // generate IEEE conversion instruction sf = qf32
    if (checkIfInputFromAdder32(Reg1) || checkIfInputFromMult32(Reg1)) {
      VR1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR1)
          .addReg(Reg1);
      firstconvert = true;
    }
  }
 
  // If 2nd operand is of qf32 type
  if (isSecOpQf) {
    // If the second operand is from add/sub/mul unit,
    // generate IEEE conversion instruction
    if (checkIfInputFromAdder32(Reg2) || checkIfInputFromMult32(Reg2)) {
      VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR2)
          .addReg(Reg2);
      secondconvert = true;
    }
  }
 
  // If both operands are qf32 type, use V6_v[add/sub]_sf instruction
  // If one of them is of sf type, use V6_v[add/sub]_qf32_mix instruction
  // Output is qf32
  if (isFirstOpQf && isSecOpQf) {
    if (firstconvert && secondconvert) {
      BuildMI(MBB, MI, DL,
              HII->get(isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), Dest)
          .addReg(VR1)
          .addReg(VR2);
    } else if (firstconvert) {
      if (isAdd)
        BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), Dest)
            .addReg(Reg2)
            .addReg(VR1);
      // For vsub type, for v81 we use a different opcode,
      // for v79, we convert the 2nd op to IEEE too.
      else {
        if (HST->useHVXV81Ops())
          BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_sf_mix), Dest)
              .addReg(VR1)
              .addReg(Reg2);
        else {
          VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
          BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR2)
              .addReg(Reg2);
          BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_sf), Dest)
              .addReg(VR1)
              .addReg(VR2);
        }
      }
    } else if (secondconvert) {
      BuildMI(MBB, MI, DL,
              HII->get(isAdd ? Hexagon::V6_vadd_qf32_mix
                             : Hexagon::V6_vsub_qf32_mix),
              Dest)
          .addReg(Reg1)
          .addReg(VR2);
    } else { // none of the inputs is from an add/sub/mul unit
      return false;
    }
    // handle vadd/vsub when the 1st op of original instruction is qf type
  } else if (isFirstOpQf) {
    if (firstconvert)
      BuildMI(MBB, MI, DL,
              HII->get(isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), Dest)
          .addReg(VR1)
          .addReg(Reg2);
    else
      return false;
    // handle vadd/vsub when the 2nd op of original instruction is qf type
  } else if (isSecOpQf) {
    if (secondconvert)
      BuildMI(MBB, MI, DL,
              HII->get(isAdd ? Hexagon::V6_vadd_sf : Hexagon::V6_vsub_sf), Dest)
          .addReg(Reg1)
          .addReg(VR2);
    else
      return false;
  } else
    return false;
  return true;
}
 
// Handle qf16 = vadd/vsub(qf16, qf16/hf)
// Handle vadd/vsub instructions with qf16 operands conditionally
// isAdd:  true if an add instruction is analyzed, false for subtract
// isFirstOpQf: true if 1st operand is qf16 type, false if hf type
// isSecOpQf: true if 2nd operand is qf16 type, false if hf type
bool HexagonXQFloatGenerator::convertAddOpToIEEE16(
    MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
    bool isAdd, bool isFirstOpQf, bool isSecOpQf) {
 
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
  Register VR1;
  Register VR2;
  bool firstconvert = false, secondconvert = false;
 
  // If the first qf16 operand is from add/sub/mul unit,
  // generate IEEE conversion instruction
  if (isFirstOpQf) {
    if (checkIfInputFromAdder16(Reg1) || checkIfInputFromMult16(Reg1)) {
      VR1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR1)
          .addReg(Reg1);
      firstconvert = true;
    }
  }
  if (isSecOpQf) {
    // If the second operand is from add/sub/mul unit,
    // generate IEEE conversion instruction
    if (checkIfInputFromAdder16(Reg2) || checkIfInputFromMult16(Reg2)) {
      VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR2)
          .addReg(Reg2);
      secondconvert = true;
    }
  }
 
  // If both operands are qf16 type, use V6_v[add/sub]_hf instruction
  // If one of them is of hf type, use V6_v[add/sub]_qf16_mix instruction
  // Output is qf16
  if (isFirstOpQf && isSecOpQf) {
    if (firstconvert && secondconvert) {
      BuildMI(MBB, MI, DL,
              HII->get(isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), Dest)
          .addReg(VR1)
          .addReg(VR2);
    } else if (firstconvert) {
      if (isAdd)
        BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf16_mix), Dest)
            .addReg(Reg2)
            .addReg(VR1);
      // For vsub type, for v81 we use a different opcode,
      // for v79, we convert the 2nd op to IEEE too.
      else {
        if (HST->useHVXV81Ops())
          BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_hf_mix), Dest)
              .addReg(VR1)
              .addReg(Reg2);
        else {
          VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
          BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR2)
              .addReg(Reg2);
          BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vsub_hf), Dest)
              .addReg(VR1)
              .addReg(VR2);
        }
      }
    } else if (secondconvert) {
      BuildMI(MBB, MI, DL,
              HII->get(isAdd ? Hexagon::V6_vadd_qf16_mix
                             : Hexagon::V6_vsub_qf16_mix),
              Dest)
          .addReg(Reg1)
          .addReg(VR2);
    } else { // none of the inputs is from an add/sub/mul unit
      return false;
    }
    // handle vadd/vsub when the 1st op of original instruction is qf type
  } else if (isFirstOpQf) {
    if (firstconvert)
      BuildMI(MBB, MI, DL,
              HII->get(isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), Dest)
          .addReg(VR1)
          .addReg(Reg2);
    else
      return false;
    // handle vadd/vsub when the 2nd op of original instruction is qf type
  } else if (isSecOpQf) {
    if (secondconvert)
      BuildMI(MBB, MI, DL,
              HII->get(isAdd ? Hexagon::V6_vadd_hf : Hexagon::V6_vsub_hf), Dest)
          .addReg(Reg1)
          .addReg(VR2);
    else
      return false;
  } else
    return false;
  return true;
}
 
// Create the prolog
// v0 = #0
// R1 = #0x80000000
// v1.sf = vsplat(R1)
// v2.sf = vmpy(v0.sf, v1.sf)
void HexagonXQFloatGenerator::createPrologInstructions(MachineInstr &MI,
                                                       Register &R_mpy) {
 
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  Register VR0 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vd0), VR0);
 
  Register R_0 = MRI->createVirtualRegister(&Hexagon::IntRegsRegClass);
  BuildMI(MBB, MI, DL, HII->get(Hexagon::A2_tfrsi), R_0).addImm(0x80000000);
 
  Register VR_0 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_lvsplatw), VR_0).addReg(R_0);
 
  R_mpy = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_sf), R_mpy)
      .addReg(VR0)
      .addReg(VR_0);
}
 
bool HexagonXQFloatGenerator::V81normalizeMultF32(
    MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
    bool firstconvert, bool secondconvert, bool strictieee) {
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
  Register input_mpy1, input_mpy2;
 
  auto Op =
      strictieee ? Hexagon::V6_vconv_qf32_sf : Hexagon::V6_vconv_qf32_qf32;
 
  // Normalize both input operands
  if (firstconvert && secondconvert) {
    input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
 
    BuildMI(MBB, MI, DL, HII->get(Op), input_mpy1).addReg(Reg1);
    BuildMI(MBB, MI, DL, HII->get(Op), input_mpy2).addReg(Reg2);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy1)
        .addReg(input_mpy2);
  }
  // Normalize only first operand
  else if (firstconvert) {
    input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    BuildMI(MBB, MI, DL, HII->get(Op), input_mpy1).addReg(Reg1);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy1)
        .addReg(Reg2);
  }
  // Normalize only second operand
  else if (secondconvert) {
    input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    BuildMI(MBB, MI, DL, HII->get(Op), input_mpy2).addReg(Reg2);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(Reg1)
        .addReg(input_mpy2);
  } else
    // we do nothing if the inputs are not from adder/sub/mult unit
    return false;
 
  return true;
}
 
// Normalize qf32 = vmpy(sf, sf) instruction unconditionally
void HexagonXQFloatGenerator::normalizeMultiplicationInputSF(
    MachineInstr &MI, Register &Src1, Register &Src2, Register &Dest,
    Register &R_mpy, bool &PrologCreated) {
 
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  if (HST->useHVXV81Ops()) {
    Register input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    Register input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
 
    // Normalize both inputs
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_qf32_sf), input_mpy1)
        .addReg(Src1);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_qf32_sf), input_mpy2)
        .addReg(Src2);
    // Add the new vmpy
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy1)
        .addReg(input_mpy2);
    return;
  }
 
  if (!PrologCreated) {
    createPrologInstructions(MI, R_mpy);
    PrologCreated = true;
  }
 
  Register input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  Register input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
  // Normalize both inputs
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy1)
      .addReg(R_mpy)
      .addReg(Src1);
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy2)
      .addReg(R_mpy)
      .addReg(Src2);
  // Add the new vmpy
  BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
      .addReg(input_mpy1)
      .addReg(input_mpy2);
}
 
// Convert and normalize qf32 = vmpy(qf32, qf32) instructions conditionally
bool HexagonXQFloatGenerator::convertNormalizeMultOp32(
    MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
    Register &R_mpy, bool &PrologCreated) {
 
  Register VR1, VR2;
  bool firstconvert = false, secondconvert = false;
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  // If the first operand is from add/subtract/multiply unit, generate IEEE
  // conversion instruction
  if (checkIfInputFromAdder32(Reg1) || checkIfInputFromMult32(Reg1)) {
    VR1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR1).addReg(Reg1);
    firstconvert = true;
  }
 
  if (checkIfInputFromAdder32(Reg2) || checkIfInputFromMult32(Reg2)) {
    VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_sf_qf32), VR2).addReg(Reg2);
    secondconvert = true;
  }
 
  if (HST->useHVXV81Ops()) {
    if (firstconvert && secondconvert)
      return V81normalizeMultF32(MI, VR1, VR2, Dest, true, true, true);
    else if (firstconvert)
      return V81normalizeMultF32(MI, VR1, Reg2, Dest, true, false, true);
    else if (secondconvert)
      return V81normalizeMultF32(MI, Reg1, VR2, Dest, false, true, true);
    else
      return false;
  }
 
  // create prolog if not already created
  if (!PrologCreated && (firstconvert || secondconvert)) {
    createPrologInstructions(MI, R_mpy);
    PrologCreated = true;
  }
 
  Register input_mpy1, input_mpy2;
 
  // Normalize both IEEE converts
  if (firstconvert && secondconvert) {
    input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
 
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy1)
        .addReg(R_mpy)
        .addReg(VR1);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy2)
        .addReg(R_mpy)
        .addReg(VR2);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy1)
        .addReg(input_mpy2);
    // Normalize only first operand
  } else if (firstconvert) {
    input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
 
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy1)
        .addReg(R_mpy)
        .addReg(VR1);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy1)
        .addReg(Reg2);
    // Normalize only second operand
  } else if (secondconvert) {
    input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
 
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32_mix), input_mpy2)
        .addReg(R_mpy)
        .addReg(VR2);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy2)
        .addReg(Reg2);
  } else {
    // we do nothing if the inputs are not fromadder/subtracter/multiplier unit
    return false;
  }
  return true;
}
 
// Convert to IEEE and widen qf16 = vmpy(qf16/hf, qf16) conditionally
// Then convert qf32 to qf16
// twoOps: true if the first operand is qf type, false if hf type
bool HexagonXQFloatGenerator::convertWidenMultOp16(MachineInstr &MI,
                                                   Register &Reg1,
                                                   Register &Reg2,
                                                   Register &Dest,
                                                   bool twoOps) {
 
  Register VR1, VR2, output_mpy;
  bool firstconvert = false,
       secondconvert = false; // normalize with hf or qf16 operands
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  // If the first operand is from add/sub/mul unit,
  // generate IEEE conversion instruction
  if (checkIfInputFromAdder16(Reg1) || checkIfInputFromMult16(Reg1)) {
    VR1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR1).addReg(Reg1);
    firstconvert = true;
  }
 
  if (twoOps) {
    if (checkIfInputFromAdder16(Reg2) || checkIfInputFromMult16(Reg2)) {
      VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR2)
          .addReg(Reg2);
      secondconvert = true;
    }
  }
 
  if (twoOps) {
    // Both operands have been converted to IEEE
    if (firstconvert && secondconvert) {
      output_mpy = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_hf), output_mpy)
          .addReg(VR1)
          .addReg(VR2);
      // Only one operand has been converted to IEEE
    } else if (firstconvert) {
      output_mpy = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_mix_hf), output_mpy)
          .addReg(Reg2)
          .addReg(VR1);
    } else if (secondconvert) {
      output_mpy = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_mix_hf), output_mpy)
          .addReg(Reg1)
          .addReg(VR2);
    } else {
      // Neither have to be transformed
      return false;
    }
  } else {
    if (firstconvert) {
      output_mpy = MRI->createVirtualRegister(&Hexagon::HvxWRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_hf), output_mpy)
          .addReg(VR1)
          .addReg(Reg2);
    } else
      return false;
  }
 
  // convert qf32 to qf16
  generateQF16FromQF32(MI, Dest, output_mpy);
 
  return true;
}
 
// Convert to IEEE and perform qf32 = vmpy(qf16/hf, qf16) conditionally
// Final output is qf32 type
bool HexagonXQFloatGenerator::convertWidenMultOp32(MachineInstr &MI,
                                                   Register &Reg1,
                                                   Register &Reg2,
                                                   Register &Dest,
                                                   bool twoOps) {
  Register VR1, VR2;
  bool firstconvert = false,
       secondconvert = false; // normalize with hf or qf16 operands
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  // If the first operand is from add/subtract/multiply unit, generate IEEE
  // conversion instruction
  if (checkIfInputFromAdder16(Reg1) || checkIfInputFromMult16(Reg1)) {
    VR1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR1).addReg(Reg1);
    firstconvert = true;
  }
 
  if (twoOps) {
    if (checkIfInputFromAdder16(Reg2) || checkIfInputFromMult16(Reg2)) {
      VR2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vconv_hf_qf16), VR2)
          .addReg(Reg2);
      secondconvert = true;
    }
  }
 
  if (twoOps) {
    // Both operands have been converted to IEEE
    if (firstconvert && secondconvert) {
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_hf), Dest)
          .addReg(VR1)
          .addReg(VR2);
      // Only one operand has been converted to IEEE
    } else if (firstconvert) {
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_mix_hf), Dest)
          .addReg(Reg2)
          .addReg(VR1);
    } else if (secondconvert) {
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_mix_hf), Dest)
          .addReg(Reg1)
          .addReg(VR2);
    } else
      // Neither have to be transformed
      return false;
  } else {
    if (firstconvert)
      BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32_hf), Dest)
          .addReg(VR1)
          .addReg(Reg2);
    else
      return false;
  }
 
  return true;
}
 
// Normalize instructions of type qf32 = vmpy(qf32, qf32)
bool HexagonXQFloatGenerator::normalizeMultiplicationInputF32(
    MachineInstr &MI, Register &Reg1, Register &Reg2, Register &Dest,
    Register &R_mpy, bool &PrologCreated) {
  bool firstconvert = false, secondconvert = false;
  MachineBasicBlock &MBB = *MI.getParent();
  const DebugLoc &DL = MI.getDebugLoc();
 
  // We normalize only that operand which comes from add/subtract unit.
  if (checkIfInputFromAdder32(Reg1))
    firstconvert = true;
  if (checkIfInputFromAdder32(Reg2))
    secondconvert = true;
 
  // v81 normalization
  if (HST->useHVXV81Ops())
    return V81normalizeMultF32(MI, Reg1, Reg2, Dest, firstconvert,
                               secondconvert, false);
 
  // create normalization operand conditionally for v79
  if ((!PrologCreated && (firstconvert || secondconvert))) {
    createPrologInstructions(MI, R_mpy);
    PrologCreated = true;
  }
 
  Register input_mpy1, input_mpy2;
 
  // Normalize both input operands
  if (firstconvert && secondconvert) {
    input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
    input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
 
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32), input_mpy1)
        .addReg(R_mpy)
        .addReg(Reg1);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32), input_mpy2)
        .addReg(R_mpy)
        .addReg(Reg2);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy1)
        .addReg(input_mpy2);
    // Normalize only first operand
  } else if (firstconvert) {
    input_mpy1 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
 
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32), input_mpy1)
        .addReg(R_mpy)
        .addReg(Reg1);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy1)
        .addReg(Reg2);
    // Normalize only second operand
  } else if (secondconvert) {
    input_mpy2 = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
 
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vadd_qf32), input_mpy2)
        .addReg(R_mpy)
        .addReg(Reg2);
    BuildMI(MBB, MI, DL, HII->get(Hexagon::V6_vmpy_qf32), Dest)
        .addReg(input_mpy2)
        .addReg(Reg1);
  } else {
    // we do nothing if the inputs are not from adder/subtracter/multiplier unit
    return false;
  }
 
  return true;
}
 
bool HexagonXQFloatGenerator::deleteList() {
  if (OriginalMI.empty())
    return false;
  bool Changed = false;
  for (MachineInstr *origMI : OriginalMI) {
    LLVM_DEBUG(dbgs() << "deleting redundant instruction");
    LLVM_DEBUG(origMI->dump());
    origMI->eraseFromParent();
    Changed = true;
  }
  OriginalMI.clear();
  return Changed;
}
 
// Parent function to handle Loosy subnormal transformations
bool HexagonXQFloatGenerator::HandleLossySubnormals(MachineFunction &MF) {
  bool Changed = false;
  Register R_mpy;
  for (auto &MBB : MF) {
    bool PrologCreated = false;
    for (auto &MI : MBB) {
      Changed |= deleteList();
      // Skip if the instruction does not have two operands,
      // or is a bundle instruction
      // or is a debug instruction
      if (MI.getNumOperands() != 3 || MI.isDebugInstr())
        continue;
      auto Op1 = MI.getOperand(1);
      if (!Op1.isReg())
        continue;
      auto Op2 = MI.getOperand(2);
      if (!Op2.isReg())
        continue;
      auto Op0 = MI.getOperand(0);
      if (!Op0.isReg())
        continue;
      Register Reg1 = Op1.getReg();
      Register Reg2 = Op2.getReg();
      Register Dest = Op0.getReg();
 
      // FIXME Do not process physical registers as operands
      if (!Reg1.isVirtual() || !Reg2.isVirtual() || !Dest.isVirtual())
        continue;
 
      switch (MI.getOpcode()) {
      // qf32 = vmpy(qf32, qf32)
      // Normalize one or both input operands
      // if from add/sub unit
      case Hexagon::V6_vmpy_qf32:
        if (normalizeMultiplicationInputF32(MI, Reg1, Reg2, Dest, R_mpy,
                                            PrologCreated))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
 
      // qf16 = vmpy(qf16, qf16)
      // Widening multiply to qf32 and convert back to qf16
      // if any of the operands are from add/sub unit
      case Hexagon::V6_vmpy_qf16:
        if (widenMultiplicationInputF16(MI, Reg1, Reg2, Dest, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // qf16 = vmpy(qf16, Rt.hf)
      // Splat Rt to vector and then widening multiply
      // and then convert back to qf16
      // if first operand is from add/sub unit
      case Hexagon::V6_vmpy_rt_qf16:
        if (widenMultiplicationInputF16Rt(MI, Reg1, Reg2, Dest))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // qf16 = vmpy(qf16, hf)
      // Widening multiply to qf32 and convert back to qf16
      // if first operand is from add/sub unit
      case Hexagon::V6_vmpy_qf16_mix_hf:
        if (widenMultiplicationInputF16(MI, Reg1, Reg2, Dest, false))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
      // Check if use of qf32 generating add/sub/mul instructions
      // are used as non-HVX operands.
      // If yes, convert the use to IEEE
      case Hexagon::V6_vadd_sf:
      case Hexagon::V6_vadd_qf32:
      case Hexagon::V6_vadd_qf32_mix:
      case Hexagon::V6_vsub_sf:
      case Hexagon::V6_vsub_qf32:
      case Hexagon::V6_vsub_qf32_mix:
      case Hexagon::V6_vsub_sf_mix:
      case Hexagon::V6_vmpy_qf32_qf16:
      case Hexagon::V6_vmpy_qf32_hf:
      case Hexagon::V6_vmpy_qf32_mix_hf:
      case Hexagon::V6_vmpy_rt_sf:
      case Hexagon::V6_vmpy_qf32_sf:
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
      // Check if use of qf16 generating add/sub/mul instructions
      // are used as non-HVX operands.
      // If yes, convert the use to IEEE
      case Hexagon::V6_vadd_hf:
      case Hexagon::V6_vsub_hf:
      case Hexagon::V6_vadd_qf16:
      case Hexagon::V6_vsub_qf16:
      case Hexagon::V6_vadd_qf16_mix:
      case Hexagon::V6_vsub_qf16_mix:
      case Hexagon::V6_vsub_hf_mix:
      case Hexagon::V6_vmpy_qf16_hf:
      case Hexagon::V6_vmpy_rt_hf:
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
      default:
        break;
      }
    }
  }
  if (OriginalMI.empty() || !Changed)
    return false;
  return true;
}
 
// Parent function to handle all IEEE-754 compliant transformations
bool HexagonXQFloatGenerator::HandleCompliantIEEE(MachineFunction &MF) {
  bool Changed = false;
  Register R_mpy;
  for (auto &MBB : MF) {
    bool PrologCreated = false;
    for (auto &MI : MBB) {
      Changed |= deleteList();
      // Skip if the instruction does not have two operands,
      // or is a bundle instruction
      // or is a debug instruction
      if (MI.getNumOperands() != 3 || MI.isDebugInstr())
        continue;
 
      auto Op1 = MI.getOperand(1);
      if (!Op1.isReg())
        continue;
      auto Op2 = MI.getOperand(2);
      if (!Op2.isReg())
        continue;
      auto Op0 = MI.getOperand(0);
      if (!Op0.isReg())
        continue;
      Register Reg1 = Op1.getReg();
      Register Reg2 = Op2.getReg();
      Register Dest = Op0.getReg();
      Register VRtSplat;
 
      // FIXME Do not process physical registers as operands
      if (!Reg1.isVirtual() || !Reg2.isVirtual() || !Dest.isVirtual())
        continue;
 
      switch (MI.getOpcode()) {
 
      // ==== Handle multiplication instructions ====
 
      // qf32 = vmpy(sf, Rt.sf)
      // Splat Rt to a vector
      // Normalize both input operands unconditionally
      case Hexagon::V6_vmpy_rt_sf:
        VRtSplat = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
        BuildMI(MBB, MI, MI.getDebugLoc(), HII->get(Hexagon::V6_lvsplatw),
                VRtSplat)
            .addReg(Reg2);
        normalizeMultiplicationInputSF(MI, Reg1, VRtSplat, Dest, R_mpy,
                                       PrologCreated);
        OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
 
      // qf32 = vmpy(sf, sf)
      // Normalize both operands unconditionally
      case Hexagon::V6_vmpy_qf32_sf:
        normalizeMultiplicationInputSF(MI, Reg1, Reg2, Dest, R_mpy,
                                       PrologCreated);
        OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
 
      // qf32 = vmpy(qf32, qf32)
      // Normalize one or both input operands
      // if from add/sub unit
      case Hexagon::V6_vmpy_qf32:
        if (normalizeMultiplicationInputF32(MI, Reg1, Reg2, Dest, R_mpy,
                                            PrologCreated))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
 
      // qf16 = vmpy(hf, rt)
      // Splat Rt to vector and then widening multiply
      case Hexagon::V6_vmpy_rt_hf:
        VRtSplat = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
        BuildMI(MBB, MI, MI.getDebugLoc(), HII->get(Hexagon::V6_lvsplatw),
                VRtSplat)
            .addReg(Reg2);
        widenMultiplyInputHF(MI, Reg1, VRtSplat, Dest);
        OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // Widening multiply
      // qf16 = vmpy(hf, hf)
      case Hexagon::V6_vmpy_qf16_hf:
        widenMultiplyInputHF(MI, Reg1, Reg2, Dest);
        OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // qf16 = vmpy(qf16, qf16)
      // Widening multiply to qf32 and convert back to qf16
      // if any of the operands are from add/sub unit
      case Hexagon::V6_vmpy_qf16:
        if (widenMultiplicationInputF16(MI, Reg1, Reg2, Dest, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // qf16 = vmpy(qf16, Rt.hf)
      // Splat Rt to vector and then widening multiply
      // and then convert back to qf16
      // if first operand is from add/sub unit
      case Hexagon::V6_vmpy_rt_qf16:
        if (widenMultiplicationInputF16Rt(MI, Reg1, Reg2, Dest))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // qf16 = vmpy(qf16, hf)
      // Widening multiply to qf32 and convert back to qf16
      // if first operand is from add/sub unit
      case Hexagon::V6_vmpy_qf16_mix_hf:
        if (widenMultiplicationInputF16(MI, Reg1, Reg2, Dest, false))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // Check if use of qf32/qf16 generating add/sub/mul
      // instructions are used as non-HVX operands.
      // If yes, convert the use to IEEE
      case Hexagon::V6_vadd_sf:
      case Hexagon::V6_vadd_qf32:
      case Hexagon::V6_vadd_qf32_mix:
      case Hexagon::V6_vsub_sf:
      case Hexagon::V6_vsub_qf32:
      case Hexagon::V6_vsub_qf32_mix:
      case Hexagon::V6_vsub_sf_mix:
      case Hexagon::V6_vmpy_qf32_qf16:
      case Hexagon::V6_vmpy_qf32_hf:
      case Hexagon::V6_vmpy_qf32_mix_hf:
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
      case Hexagon::V6_vadd_hf:
      case Hexagon::V6_vsub_hf:
      case Hexagon::V6_vadd_qf16:
      case Hexagon::V6_vsub_qf16:
      case Hexagon::V6_vadd_qf16_mix:
      case Hexagon::V6_vsub_qf16_mix:
      case Hexagon::V6_vsub_hf_mix:
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
      default:
        break;
      }
    }
  }
  if (OriginalMI.empty() || !Changed)
    return false;
  return true;
}
 
// Parent function to do strict IEEE transformations
bool HexagonXQFloatGenerator::HandleStrictIEEE(MachineFunction &MF) {
 
  bool Changed = false;
  Register R_mpy;
  for (auto &MBB : MF) {
    bool PrologCreated = false;
    for (auto &MI : MBB) {
      Changed |= deleteList();
      // Skip if the instruction does not have two operands,
      // or is a bundle instruction
      // or is a debug instruction
      if (MI.getNumOperands() != 3 || MI.isDebugInstr())
        continue;
 
      auto Op1 = MI.getOperand(1);
      if (!Op1.isReg())
        continue;
      auto Op2 = MI.getOperand(2);
      if (!Op2.isReg())
        continue;
      auto Op0 = MI.getOperand(0);
      if (!Op0.isReg())
        continue;
      Register Reg1 = Op1.getReg();
      Register Reg2 = Op2.getReg();
      Register Dest = Op0.getReg();
      Register VRtSplat;
 
      // FIXME Do not process physical registers as operands
      if (!Reg1.isVirtual() || !Reg2.isVirtual() || !Dest.isVirtual())
        continue;
 
      switch (MI.getOpcode()) {
      // ==== Handle add/subtract instructions ====
      // Convert one or both the input operands to IEEE 32-bit
      // if from add/sub/mult unit(s)
      // qf32 = vadd(qf32, qf32)
      case Hexagon::V6_vadd_qf32:
        if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, true, true, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
      // qf32 = vsub(qf32, qf32)
      case Hexagon::V6_vsub_qf32:
        if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, false, true, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
      // Convert only the first input operand to IEEE 32-bit
      // if it is from add/sub/mult unit
      // qf32 = vadd(qf32, sf)
      case Hexagon::V6_vadd_qf32_mix:
        if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, true, true, false))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
      // qf32 = vsub(qf32, sf)
      case Hexagon::V6_vsub_qf32_mix:
        if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, false, true, false))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
      // qf32 = vsub(sf, qf32)
      case Hexagon::V6_vsub_sf_mix:
        if (convertAddOpToIEEE32(MI, Reg1, Reg2, Dest, false, false, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
        break;
 
      // Convert one or both the input operands to IEEE 16-bit
      // if from add/sub/mult unit(s)
      // qf16 = vadd(qf16, qf16)
      case Hexagon::V6_vadd_qf16:
        if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, true, true, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
      // qf16 = vsub(qf16, qf16)
      case Hexagon::V6_vsub_qf16:
        if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, false, true, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
      // Convert only the first input operand IEEE 16-bit
      // if it is from add/sub/mul unit
      // qf16 = vadd(qf16, hf)
      case Hexagon::V6_vadd_qf16_mix:
        if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, true, true, false))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
      // qf16 = vsub(qf16, hf)
      case Hexagon::V6_vsub_qf16_mix:
        if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, false, true, false))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
      // qf16 = vsub(hf, qf16)
      case Hexagon::V6_vsub_hf_mix:
        if (convertAddOpToIEEE16(MI, Reg1, Reg2, Dest, false, false, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // ==== Handle multiplication instructions ====
 
      // qf32 = vmpy(sf, Rt.sf)
      // Splat Rt to a vector
      // Normalize both input operands unconditionally
      case Hexagon::V6_vmpy_rt_sf:
        VRtSplat = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
        BuildMI(MBB, MI, MI.getDebugLoc(), HII->get(Hexagon::V6_lvsplatw),
                VRtSplat)
            .addReg(Reg2);
        normalizeMultiplicationInputSF(MI, Reg1, VRtSplat, Dest, R_mpy,
                                       PrologCreated);
        OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
 
      // Normalize both operands unconditionally
      // qf32 = vmpy(sf, sf)
      case Hexagon::V6_vmpy_qf32_sf:
        normalizeMultiplicationInputSF(MI, Reg1, Reg2, Dest, R_mpy,
                                       PrologCreated);
        Changed |= convertIfInputToNonHVX(MI, true);
        OriginalMI.push_back(&MI);
        break;
 
      // Convert one or both input operands to IEEE 32-bit
      // if from add/sub/mult unit and normalize
      // qf32 = vmpy(qf32, qf32)
      case Hexagon::V6_vmpy_qf32:
        if (convertNormalizeMultOp32(MI, Reg1, Reg2, Dest, R_mpy,
                                     PrologCreated))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
 
      // Convert one or both input operands to IEEE 16-bit
      // if from mul/add/sub unit;
      // then widening multiply to generate qf32
      // then convert to qf16
      // qf16 = vmpy(qf16, qf16)
      case Hexagon::V6_vmpy_qf16:
        if (convertWidenMultOp16(MI, Reg1, Reg2, Dest, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // Convert one or both input operands to IEEE 16-bit
      // if from mul/add/sub unit;
      // then widening multiply to generate qf32
      // qf32 = vmpy(qf16, qf16)
      case Hexagon::V6_vmpy_qf32_qf16:
        if (convertWidenMultOp32(MI, Reg1, Reg2, Dest, true))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
 
      // qf16 = vmpy(hf, rt)
      // Splat Rt to vector and then widening multiply
      case Hexagon::V6_vmpy_rt_hf:
        VRtSplat = MRI->createVirtualRegister(&Hexagon::HvxVRRegClass);
        BuildMI(MBB, MI, MI.getDebugLoc(), HII->get(Hexagon::V6_lvsplatw),
                VRtSplat)
            .addReg(Reg2);
        widenMultiplyInputHF(MI, Reg1, VRtSplat, Dest);
        OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // Widening multiply, then convert to IEEE
      // qf16 = vmpy(hf, hf)
      case Hexagon::V6_vmpy_qf16_hf:
        widenMultiplyInputHF(MI, Reg1, Reg2, Dest);
        OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // qf16 = vmpy(qf16, Rt.hf)
      // Splat Rt to vector and then widening multiply
      // and then convert back to qf16
      // if first operand is from add/sub unit
      case Hexagon::V6_vmpy_rt_qf16:
        if (widenMultiplicationInputF16Rt(MI, Reg1, Reg2, Dest))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // qf16 = vmpy(qf16, hf)
      // Convert only the first input operans to IEEE 16-bit
      // if from mul/add/sub unit;
      // then widening multiply to generate qf32
      // then convert back to qf16
      case Hexagon::V6_vmpy_qf16_mix_hf:
        if (convertWidenMultOp16(MI, Reg1, Reg2, Dest, false))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
 
      // qf32 = vmpy(qf16, hf)
      // Convert only the first input operans to IEEE 16-bit
      // if from mul/add/sub unit;
      // then widening multiply to generate qf32
      case Hexagon::V6_vmpy_qf32_mix_hf:
        if (convertWidenMultOp32(MI, Reg1, Reg2, Dest, false))
          OriginalMI.push_back(&MI);
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
      // Check if use of qf32/qf16 generating add/sub/mul
      // instructions are used as non-HVX operands.
      // If yes, convert the use to IEEE
      case Hexagon::V6_vadd_sf:
      case Hexagon::V6_vsub_sf:
        Changed |= convertIfInputToNonHVX(MI, true);
        break;
      case Hexagon::V6_vadd_hf:
      case Hexagon::V6_vsub_hf:
        Changed |= convertIfInputToNonHVX(MI, false);
        break;
      default:
        break;
      }
    }
  }
  if (OriginalMI.empty() || !Changed)
    return false;
  return true;
}
 
// There is no conversions in lossy mode
bool HexagonXQFloatGenerator::HandleLossyLegacy(MachineFunction &MF) {
  return false;
}
 
bool HexagonXQFloatGenerator::runOnMachineFunction(MachineFunction &MF) {
  if (!EnableHVXXQFloat || (QFloatModeValue == QFloatMode::Legacy))
    return false;
 
  bool Changed = false;
  HST = &MF.getSubtarget<HexagonSubtarget>();
  HII = HST->getInstrInfo();
  MRI = &MF.getRegInfo();
 
  switch (QFloatModeValue) {
  case QFloatMode::StrictIEEE:
    LLVM_DEBUG(dbgs() << "\nGenerating code for STRICT-IEEE mode.\n");
    Changed = HandleStrictIEEE(MF);
    break;
  case QFloatMode::IEEE:
    LLVM_DEBUG(dbgs() << "\nGenerating code for IEEE mode.\n");
    Changed = HandleCompliantIEEE(MF);
    break;
  case QFloatMode::Lossy:
    LLVM_DEBUG(dbgs() << "\nGenerating code for LOSSY mode.\n");
    Changed = HandleLossySubnormals(MF);
    break;
  case QFloatMode::Legacy:
    LLVM_DEBUG(dbgs() << "\nGenerating code for LEGACY mode.\n");
    Changed = HandleLossyLegacy(MF);
    break;
  }
  LLVM_DEBUG(dbgs() << "...fine");
 
  // Delete the original instructions
  for (MachineInstr *origMI : OriginalMI) {
    LLVM_DEBUG(origMI->dump());
    origMI->eraseFromParent();
  }
  OriginalMI.clear();
 
  return Changed;
}