X86InstrInfo.cpp

Go to the documentation of this file.

//===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file contains the X86 implementation of the TargetInstrInfo class.
//
//===----------------------------------------------------------------------===//
 
#include "X86InstrInfo.h"
#include "X86.h"
#include "X86InstrBuilder.h"
#include "X86InstrFoldTables.h"
#include "X86MachineFunctionInfo.h"
#include "X86Subtarget.h"
#include "X86TargetMachine.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/LiveVariables.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetOptions.h"
#include <optional>
 
using namespace llvm;
 
#define DEBUG_TYPE "x86-instr-info"
 
#define GET_INSTRINFO_CTOR_DTOR
#include "X86GenInstrInfo.inc"
 
static cl::opt<bool>
    NoFusing("disable-spill-fusing",
             cl::desc("Disable fusing of spill code into instructions"),
             cl::Hidden);
static cl::opt<bool>
PrintFailedFusing("print-failed-fuse-candidates",
                  cl::desc("Print instructions that the allocator wants to"
                           " fuse, but the X86 backend currently can't"),
                  cl::Hidden);
static cl::opt<bool>
ReMatPICStubLoad("remat-pic-stub-load",
                 cl::desc("Re-materialize load from stub in PIC mode"),
                 cl::init(false), cl::Hidden);
static cl::opt<unsigned>
PartialRegUpdateClearance("partial-reg-update-clearance",
                          cl::desc("Clearance between two register writes "
                                   "for inserting XOR to avoid partial "
                                   "register update"),
                          cl::init(64), cl::Hidden);
static cl::opt<unsigned>
UndefRegClearance("undef-reg-clearance",
                  cl::desc("How many idle instructions we would like before "
                           "certain undef register reads"),
                  cl::init(128), cl::Hidden);
 
 
// Pin the vtable to this file.
void X86InstrInfo::anchor() {}
 
X86InstrInfo::X86InstrInfo(X86Subtarget &STI)
    : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64
                                               : X86::ADJCALLSTACKDOWN32),
                      (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64
                                               : X86::ADJCALLSTACKUP32),
                      X86::CATCHRET,
                      (STI.is64Bit() ? X86::RET64 : X86::RET32)),
      Subtarget(STI), RI(STI.getTargetTriple()) {
}
 
bool
X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
                                    Register &SrcReg, Register &DstReg,
                                    unsigned &SubIdx) const {
  switch (MI.getOpcode()) {
  default: break;
  case X86::MOVSX16rr8:
  case X86::MOVZX16rr8:
  case X86::MOVSX32rr8:
  case X86::MOVZX32rr8:
  case X86::MOVSX64rr8:
    if (!Subtarget.is64Bit())
      // It's not always legal to reference the low 8-bit of the larger
      // register in 32-bit mode.
      return false;
    [[fallthrough]];
  case X86::MOVSX32rr16:
  case X86::MOVZX32rr16:
  case X86::MOVSX64rr16:
  case X86::MOVSX64rr32: {
    if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
      // Be conservative.
      return false;
    SrcReg = MI.getOperand(1).getReg();
    DstReg = MI.getOperand(0).getReg();
    switch (MI.getOpcode()) {
    default: llvm_unreachable("Unreachable!");
    case X86::MOVSX16rr8:
    case X86::MOVZX16rr8:
    case X86::MOVSX32rr8:
    case X86::MOVZX32rr8:
    case X86::MOVSX64rr8:
      SubIdx = X86::sub_8bit;
      break;
    case X86::MOVSX32rr16:
    case X86::MOVZX32rr16:
    case X86::MOVSX64rr16:
      SubIdx = X86::sub_16bit;
      break;
    case X86::MOVSX64rr32:
      SubIdx = X86::sub_32bit;
      break;
    }
    return true;
  }
  }
  return false;
}
 
bool X86InstrInfo::isDataInvariant(MachineInstr &MI) {
  if (MI.mayLoad() || MI.mayStore())
    return false;
 
  // Some target-independent operations that trivially lower to data-invariant
  // instructions.
  if (MI.isCopyLike() || MI.isInsertSubreg())
    return true;
 
  unsigned Opcode = MI.getOpcode();
  using namespace X86;
  // On x86 it is believed that imul is constant time w.r.t. the loaded data.
  // However, they set flags and are perhaps the most surprisingly constant
  // time operations so we call them out here separately.
  if (isIMUL(Opcode))
    return true;
  // Bit scanning and counting instructions that are somewhat surprisingly
  // constant time as they scan across bits and do other fairly complex
  // operations like popcnt, but are believed to be constant time on x86.
  // However, these set flags.
  if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) ||
      isTZCNT(Opcode))
    return true;
  // Bit manipulation instructions are effectively combinations of basic
  // arithmetic ops, and should still execute in constant time. These also
  // set flags.
  if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) ||
      isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) ||
      isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) ||
      isTZMSK(Opcode))
    return true;
  // Bit extracting and clearing instructions should execute in constant time,
  // and set flags.
  if (isBEXTR(Opcode) || isBZHI(Opcode))
    return true;
  // Shift and rotate.
  if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) ||
      isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode))
    return true;
  // Basic arithmetic is constant time on the input but does set flags.
  if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) ||
      isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode))
    return true;
  // Arithmetic with just 32-bit and 64-bit variants and no immediates.
  if (isADCX(Opcode) || isADOX(Opcode) || isANDN(Opcode))
    return true;
  // Unary arithmetic operations.
  if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode))
    return true;
  // Unlike other arithmetic, NOT doesn't set EFLAGS.
  if (isNOT(Opcode))
    return true;
  // Various move instructions used to zero or sign extend things. Note that we
  // intentionally don't support the _NOREX variants as we can't handle that
  // register constraint anyways.
  if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode))
    return true;
  // Arithmetic instructions that are both constant time and don't set flags.
  if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode))
    return true;
  // LEA doesn't actually access memory, and its arithmetic is constant time.
  if (isLEA(Opcode))
    return true;
  // By default, assume that the instruction is not data invariant.
  return false;
}
 
bool X86InstrInfo::isDataInvariantLoad(MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default:
    // By default, assume that the load will immediately leak.
    return false;
 
  // On x86 it is believed that imul is constant time w.r.t. the loaded data.
  // However, they set flags and are perhaps the most surprisingly constant
  // time operations so we call them out here separately.
  case X86::IMUL16rm:
  case X86::IMUL16rmi8:
  case X86::IMUL16rmi:
  case X86::IMUL32rm:
  case X86::IMUL32rmi8:
  case X86::IMUL32rmi:
  case X86::IMUL64rm:
  case X86::IMUL64rmi32:
  case X86::IMUL64rmi8:
 
  // Bit scanning and counting instructions that are somewhat surprisingly
  // constant time as they scan across bits and do other fairly complex
  // operations like popcnt, but are believed to be constant time on x86.
  // However, these set flags.
  case X86::BSF16rm:
  case X86::BSF32rm:
  case X86::BSF64rm:
  case X86::BSR16rm:
  case X86::BSR32rm:
  case X86::BSR64rm:
  case X86::LZCNT16rm:
  case X86::LZCNT32rm:
  case X86::LZCNT64rm:
  case X86::POPCNT16rm:
  case X86::POPCNT32rm:
  case X86::POPCNT64rm:
  case X86::TZCNT16rm:
  case X86::TZCNT32rm:
  case X86::TZCNT64rm:
 
  // Bit manipulation instructions are effectively combinations of basic
  // arithmetic ops, and should still execute in constant time. These also
  // set flags.
  case X86::BLCFILL32rm:
  case X86::BLCFILL64rm:
  case X86::BLCI32rm:
  case X86::BLCI64rm:
  case X86::BLCIC32rm:
  case X86::BLCIC64rm:
  case X86::BLCMSK32rm:
  case X86::BLCMSK64rm:
  case X86::BLCS32rm:
  case X86::BLCS64rm:
  case X86::BLSFILL32rm:
  case X86::BLSFILL64rm:
  case X86::BLSI32rm:
  case X86::BLSI64rm:
  case X86::BLSIC32rm:
  case X86::BLSIC64rm:
  case X86::BLSMSK32rm:
  case X86::BLSMSK64rm:
  case X86::BLSR32rm:
  case X86::BLSR64rm:
  case X86::TZMSK32rm:
  case X86::TZMSK64rm:
 
  // Bit extracting and clearing instructions should execute in constant time,
  // and set flags.
  case X86::BEXTR32rm:
  case X86::BEXTR64rm:
  case X86::BEXTRI32mi:
  case X86::BEXTRI64mi:
  case X86::BZHI32rm:
  case X86::BZHI64rm:
 
  // Basic arithmetic is constant time on the input but does set flags.
  case X86::ADC8rm:
  case X86::ADC16rm:
  case X86::ADC32rm:
  case X86::ADC64rm:
  case X86::ADCX32rm:
  case X86::ADCX64rm:
  case X86::ADD8rm:
  case X86::ADD16rm:
  case X86::ADD32rm:
  case X86::ADD64rm:
  case X86::ADOX32rm:
  case X86::ADOX64rm:
  case X86::AND8rm:
  case X86::AND16rm:
  case X86::AND32rm:
  case X86::AND64rm:
  case X86::ANDN32rm:
  case X86::ANDN64rm:
  case X86::OR8rm:
  case X86::OR16rm:
  case X86::OR32rm:
  case X86::OR64rm:
  case X86::SBB8rm:
  case X86::SBB16rm:
  case X86::SBB32rm:
  case X86::SBB64rm:
  case X86::SUB8rm:
  case X86::SUB16rm:
  case X86::SUB32rm:
  case X86::SUB64rm:
  case X86::XOR8rm:
  case X86::XOR16rm:
  case X86::XOR32rm:
  case X86::XOR64rm:
 
  // Integer multiply w/o affecting flags is still believed to be constant
  // time on x86. Called out separately as this is among the most surprising
  // instructions to exhibit that behavior.
  case X86::MULX32rm:
  case X86::MULX64rm:
 
  // Arithmetic instructions that are both constant time and don't set flags.
  case X86::RORX32mi:
  case X86::RORX64mi:
  case X86::SARX32rm:
  case X86::SARX64rm:
  case X86::SHLX32rm:
  case X86::SHLX64rm:
  case X86::SHRX32rm:
  case X86::SHRX64rm:
 
  // Conversions are believed to be constant time and don't set flags.
  case X86::CVTTSD2SI64rm:
  case X86::VCVTTSD2SI64rm:
  case X86::VCVTTSD2SI64Zrm:
  case X86::CVTTSD2SIrm:
  case X86::VCVTTSD2SIrm:
  case X86::VCVTTSD2SIZrm:
  case X86::CVTTSS2SI64rm:
  case X86::VCVTTSS2SI64rm:
  case X86::VCVTTSS2SI64Zrm:
  case X86::CVTTSS2SIrm:
  case X86::VCVTTSS2SIrm:
  case X86::VCVTTSS2SIZrm:
  case X86::CVTSI2SDrm:
  case X86::VCVTSI2SDrm:
  case X86::VCVTSI2SDZrm:
  case X86::CVTSI2SSrm:
  case X86::VCVTSI2SSrm:
  case X86::VCVTSI2SSZrm:
  case X86::CVTSI642SDrm:
  case X86::VCVTSI642SDrm:
  case X86::VCVTSI642SDZrm:
  case X86::CVTSI642SSrm:
  case X86::VCVTSI642SSrm:
  case X86::VCVTSI642SSZrm:
  case X86::CVTSS2SDrm:
  case X86::VCVTSS2SDrm:
  case X86::VCVTSS2SDZrm:
  case X86::CVTSD2SSrm:
  case X86::VCVTSD2SSrm:
  case X86::VCVTSD2SSZrm:
  // AVX512 added unsigned integer conversions.
  case X86::VCVTTSD2USI64Zrm:
  case X86::VCVTTSD2USIZrm:
  case X86::VCVTTSS2USI64Zrm:
  case X86::VCVTTSS2USIZrm:
  case X86::VCVTUSI2SDZrm:
  case X86::VCVTUSI642SDZrm:
  case X86::VCVTUSI2SSZrm:
  case X86::VCVTUSI642SSZrm:
 
  // Loads to register don't set flags.
  case X86::MOV8rm:
  case X86::MOV8rm_NOREX:
  case X86::MOV16rm:
  case X86::MOV32rm:
  case X86::MOV64rm:
  case X86::MOVSX16rm8:
  case X86::MOVSX32rm16:
  case X86::MOVSX32rm8:
  case X86::MOVSX32rm8_NOREX:
  case X86::MOVSX64rm16:
  case X86::MOVSX64rm32:
  case X86::MOVSX64rm8:
  case X86::MOVZX16rm8:
  case X86::MOVZX32rm16:
  case X86::MOVZX32rm8:
  case X86::MOVZX32rm8_NOREX:
  case X86::MOVZX64rm16:
  case X86::MOVZX64rm8:
    return true;
  }
}
 
int X86InstrInfo::getSPAdjust(const MachineInstr &MI) const {
  const MachineFunction *MF = MI.getParent()->getParent();
  const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
 
  if (isFrameInstr(MI)) {
    int SPAdj = alignTo(getFrameSize(MI), TFI->getStackAlign());
    SPAdj -= getFrameAdjustment(MI);
    if (!isFrameSetup(MI))
      SPAdj = -SPAdj;
    return SPAdj;
  }
 
  // To know whether a call adjusts the stack, we need information
  // that is bound to the following ADJCALLSTACKUP pseudo.
  // Look for the next ADJCALLSTACKUP that follows the call.
  if (MI.isCall()) {
    const MachineBasicBlock *MBB = MI.getParent();
    auto I = ++MachineBasicBlock::const_iterator(MI);
    for (auto E = MBB->end(); I != E; ++I) {
      if (I->getOpcode() == getCallFrameDestroyOpcode() ||
          I->isCall())
        break;
    }
 
    // If we could not find a frame destroy opcode, then it has already
    // been simplified, so we don't care.
    if (I->getOpcode() != getCallFrameDestroyOpcode())
      return 0;
 
    return -(I->getOperand(1).getImm());
  }
 
  // Currently handle only PUSHes we can reasonably expect to see
  // in call sequences
  switch (MI.getOpcode()) {
  default:
    return 0;
  case X86::PUSH32i8:
  case X86::PUSH32r:
  case X86::PUSH32rmm:
  case X86::PUSH32rmr:
  case X86::PUSHi32:
    return 4;
  case X86::PUSH64i8:
  case X86::PUSH64r:
  case X86::PUSH64rmm:
  case X86::PUSH64rmr:
  case X86::PUSH64i32:
    return 8;
  }
}
 
/// Return true and the FrameIndex if the specified
/// operand and follow operands form a reference to the stack frame.
bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op,
                                  int &FrameIndex) const {
  if (MI.getOperand(Op + X86::AddrBaseReg).isFI() &&
      MI.getOperand(Op + X86::AddrScaleAmt).isImm() &&
      MI.getOperand(Op + X86::AddrIndexReg).isReg() &&
      MI.getOperand(Op + X86::AddrDisp).isImm() &&
      MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 &&
      MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 &&
      MI.getOperand(Op + X86::AddrDisp).getImm() == 0) {
    FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex();
    return true;
  }
  return false;
}
 
static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) {
  switch (Opcode) {
  default:
    return false;
  case X86::MOV8rm:
  case X86::KMOVBkm:
    MemBytes = 1;
    return true;
  case X86::MOV16rm:
  case X86::KMOVWkm:
  case X86::VMOVSHZrm:
  case X86::VMOVSHZrm_alt:
    MemBytes = 2;
    return true;
  case X86::MOV32rm:
  case X86::MOVSSrm:
  case X86::MOVSSrm_alt:
  case X86::VMOVSSrm:
  case X86::VMOVSSrm_alt:
  case X86::VMOVSSZrm:
  case X86::VMOVSSZrm_alt:
  case X86::KMOVDkm:
    MemBytes = 4;
    return true;
  case X86::MOV64rm:
  case X86::LD_Fp64m:
  case X86::MOVSDrm:
  case X86::MOVSDrm_alt:
  case X86::VMOVSDrm:
  case X86::VMOVSDrm_alt:
  case X86::VMOVSDZrm:
  case X86::VMOVSDZrm_alt:
  case X86::MMX_MOVD64rm:
  case X86::MMX_MOVQ64rm:
  case X86::KMOVQkm:
    MemBytes = 8;
    return true;
  case X86::MOVAPSrm:
  case X86::MOVUPSrm:
  case X86::MOVAPDrm:
  case X86::MOVUPDrm:
  case X86::MOVDQArm:
  case X86::MOVDQUrm:
  case X86::VMOVAPSrm:
  case X86::VMOVUPSrm:
  case X86::VMOVAPDrm:
  case X86::VMOVUPDrm:
  case X86::VMOVDQArm:
  case X86::VMOVDQUrm:
  case X86::VMOVAPSZ128rm:
  case X86::VMOVUPSZ128rm:
  case X86::VMOVAPSZ128rm_NOVLX:
  case X86::VMOVUPSZ128rm_NOVLX:
  case X86::VMOVAPDZ128rm:
  case X86::VMOVUPDZ128rm:
  case X86::VMOVDQU8Z128rm:
  case X86::VMOVDQU16Z128rm:
  case X86::VMOVDQA32Z128rm:
  case X86::VMOVDQU32Z128rm:
  case X86::VMOVDQA64Z128rm:
  case X86::VMOVDQU64Z128rm:
    MemBytes = 16;
    return true;
  case X86::VMOVAPSYrm:
  case X86::VMOVUPSYrm:
  case X86::VMOVAPDYrm:
  case X86::VMOVUPDYrm:
  case X86::VMOVDQAYrm:
  case X86::VMOVDQUYrm:
  case X86::VMOVAPSZ256rm:
  case X86::VMOVUPSZ256rm:
  case X86::VMOVAPSZ256rm_NOVLX:
  case X86::VMOVUPSZ256rm_NOVLX:
  case X86::VMOVAPDZ256rm:
  case X86::VMOVUPDZ256rm:
  case X86::VMOVDQU8Z256rm:
  case X86::VMOVDQU16Z256rm:
  case X86::VMOVDQA32Z256rm:
  case X86::VMOVDQU32Z256rm:
  case X86::VMOVDQA64Z256rm:
  case X86::VMOVDQU64Z256rm:
    MemBytes = 32;
    return true;
  case X86::VMOVAPSZrm:
  case X86::VMOVUPSZrm:
  case X86::VMOVAPDZrm:
  case X86::VMOVUPDZrm:
  case X86::VMOVDQU8Zrm:
  case X86::VMOVDQU16Zrm:
  case X86::VMOVDQA32Zrm:
  case X86::VMOVDQU32Zrm:
  case X86::VMOVDQA64Zrm:
  case X86::VMOVDQU64Zrm:
    MemBytes = 64;
    return true;
  }
}
 
static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) {
  switch (Opcode) {
  default:
    return false;
  case X86::MOV8mr:
  case X86::KMOVBmk:
    MemBytes = 1;
    return true;
  case X86::MOV16mr:
  case X86::KMOVWmk:
  case X86::VMOVSHZmr:
    MemBytes = 2;
    return true;
  case X86::MOV32mr:
  case X86::MOVSSmr:
  case X86::VMOVSSmr:
  case X86::VMOVSSZmr:
  case X86::KMOVDmk:
    MemBytes = 4;
    return true;
  case X86::MOV64mr:
  case X86::ST_FpP64m:
  case X86::MOVSDmr:
  case X86::VMOVSDmr:
  case X86::VMOVSDZmr:
  case X86::MMX_MOVD64mr:
  case X86::MMX_MOVQ64mr:
  case X86::MMX_MOVNTQmr:
  case X86::KMOVQmk:
    MemBytes = 8;
    return true;
  case X86::MOVAPSmr:
  case X86::MOVUPSmr:
  case X86::MOVAPDmr:
  case X86::MOVUPDmr:
  case X86::MOVDQAmr:
  case X86::MOVDQUmr:
  case X86::VMOVAPSmr:
  case X86::VMOVUPSmr:
  case X86::VMOVAPDmr:
  case X86::VMOVUPDmr:
  case X86::VMOVDQAmr:
  case X86::VMOVDQUmr:
  case X86::VMOVUPSZ128mr:
  case X86::VMOVAPSZ128mr:
  case X86::VMOVUPSZ128mr_NOVLX:
  case X86::VMOVAPSZ128mr_NOVLX:
  case X86::VMOVUPDZ128mr:
  case X86::VMOVAPDZ128mr:
  case X86::VMOVDQA32Z128mr:
  case X86::VMOVDQU32Z128mr:
  case X86::VMOVDQA64Z128mr:
  case X86::VMOVDQU64Z128mr:
  case X86::VMOVDQU8Z128mr:
  case X86::VMOVDQU16Z128mr:
    MemBytes = 16;
    return true;
  case X86::VMOVUPSYmr:
  case X86::VMOVAPSYmr:
  case X86::VMOVUPDYmr:
  case X86::VMOVAPDYmr:
  case X86::VMOVDQUYmr:
  case X86::VMOVDQAYmr:
  case X86::VMOVUPSZ256mr:
  case X86::VMOVAPSZ256mr:
  case X86::VMOVUPSZ256mr_NOVLX:
  case X86::VMOVAPSZ256mr_NOVLX:
  case X86::VMOVUPDZ256mr:
  case X86::VMOVAPDZ256mr:
  case X86::VMOVDQU8Z256mr:
  case X86::VMOVDQU16Z256mr:
  case X86::VMOVDQA32Z256mr:
  case X86::VMOVDQU32Z256mr:
  case X86::VMOVDQA64Z256mr:
  case X86::VMOVDQU64Z256mr:
    MemBytes = 32;
    return true;
  case X86::VMOVUPSZmr:
  case X86::VMOVAPSZmr:
  case X86::VMOVUPDZmr:
  case X86::VMOVAPDZmr:
  case X86::VMOVDQU8Zmr:
  case X86::VMOVDQU16Zmr:
  case X86::VMOVDQA32Zmr:
  case X86::VMOVDQU32Zmr:
  case X86::VMOVDQA64Zmr:
  case X86::VMOVDQU64Zmr:
    MemBytes = 64;
    return true;
  }
  return false;
}
 
unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
                                           int &FrameIndex) const {
  unsigned Dummy;
  return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy);
}
 
unsigned X86InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
                                           int &FrameIndex,
                                           unsigned &MemBytes) const {
  if (isFrameLoadOpcode(MI.getOpcode(), MemBytes))
    if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex))
      return MI.getOperand(0).getReg();
  return 0;
}
 
unsigned X86InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
                                                 int &FrameIndex) const {
  unsigned Dummy;
  if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) {
    unsigned Reg;
    if ((Reg = isLoadFromStackSlot(MI, FrameIndex)))
      return Reg;
    // Check for post-frame index elimination operations
    SmallVector<const MachineMemOperand *, 1> Accesses;
    if (hasLoadFromStackSlot(MI, Accesses)) {
      FrameIndex =
          cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
              ->getFrameIndex();
      return MI.getOperand(0).getReg();
    }
  }
  return 0;
}
 
unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
                                          int &FrameIndex) const {
  unsigned Dummy;
  return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy);
}
 
unsigned X86InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
                                          int &FrameIndex,
                                          unsigned &MemBytes) const {
  if (isFrameStoreOpcode(MI.getOpcode(), MemBytes))
    if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 &&
        isFrameOperand(MI, 0, FrameIndex))
      return MI.getOperand(X86::AddrNumOperands).getReg();
  return 0;
}
 
unsigned X86InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
                                                int &FrameIndex) const {
  unsigned Dummy;
  if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) {
    unsigned Reg;
    if ((Reg = isStoreToStackSlot(MI, FrameIndex)))
      return Reg;
    // Check for post-frame index elimination operations
    SmallVector<const MachineMemOperand *, 1> Accesses;
    if (hasStoreToStackSlot(MI, Accesses)) {
      FrameIndex =
          cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
              ->getFrameIndex();
      return MI.getOperand(X86::AddrNumOperands).getReg();
    }
  }
  return 0;
}
 
/// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r.
static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) {
  // Don't waste compile time scanning use-def chains of physregs.
  if (!BaseReg.isVirtual())
    return false;
  bool isPICBase = false;
  for (MachineRegisterInfo::def_instr_iterator I = MRI.def_instr_begin(BaseReg),
         E = MRI.def_instr_end(); I != E; ++I) {
    MachineInstr *DefMI = &*I;
    if (DefMI->getOpcode() != X86::MOVPC32r)
      return false;
    assert(!isPICBase && "More than one PIC base?");
    isPICBase = true;
  }
  return isPICBase;
}
 
bool X86InstrInfo::isReallyTriviallyReMaterializable(
    const MachineInstr &MI) const {
  switch (MI.getOpcode()) {
  default:
    // This function should only be called for opcodes with the ReMaterializable
    // flag set.
    llvm_unreachable("Unknown rematerializable operation!");
    break;
 
  case X86::LOAD_STACK_GUARD:
  case X86::AVX1_SETALLONES:
  case X86::AVX2_SETALLONES:
  case X86::AVX512_128_SET0:
  case X86::AVX512_256_SET0:
  case X86::AVX512_512_SET0:
  case X86::AVX512_512_SETALLONES:
  case X86::AVX512_FsFLD0SD:
  case X86::AVX512_FsFLD0SH:
  case X86::AVX512_FsFLD0SS:
  case X86::AVX512_FsFLD0F128:
  case X86::AVX_SET0:
  case X86::FsFLD0SD:
  case X86::FsFLD0SS:
  case X86::FsFLD0SH:
  case X86::FsFLD0F128:
  case X86::KSET0D:
  case X86::KSET0Q:
  case X86::KSET0W:
  case X86::KSET1D:
  case X86::KSET1Q:
  case X86::KSET1W:
  case X86::MMX_SET0:
  case X86::MOV32ImmSExti8:
  case X86::MOV32r0:
  case X86::MOV32r1:
  case X86::MOV32r_1:
  case X86::MOV32ri64:
  case X86::MOV64ImmSExti8:
  case X86::V_SET0:
  case X86::V_SETALLONES:
  case X86::MOV16ri:
  case X86::MOV32ri:
  case X86::MOV64ri:
  case X86::MOV64ri32:
  case X86::MOV8ri:
  case X86::PTILEZEROV:
    return true;
 
  case X86::MOV8rm:
  case X86::MOV8rm_NOREX:
  case X86::MOV16rm:
  case X86::MOV32rm:
  case X86::MOV64rm:
  case X86::MOVSSrm:
  case X86::MOVSSrm_alt:
  case X86::MOVSDrm:
  case X86::MOVSDrm_alt:
  case X86::MOVAPSrm:
  case X86::MOVUPSrm:
  case X86::MOVAPDrm:
  case X86::MOVUPDrm:
  case X86::MOVDQArm:
  case X86::MOVDQUrm:
  case X86::VMOVSSrm:
  case X86::VMOVSSrm_alt:
  case X86::VMOVSDrm:
  case X86::VMOVSDrm_alt:
  case X86::VMOVAPSrm:
  case X86::VMOVUPSrm:
  case X86::VMOVAPDrm:
  case X86::VMOVUPDrm:
  case X86::VMOVDQArm:
  case X86::VMOVDQUrm:
  case X86::VMOVAPSYrm:
  case X86::VMOVUPSYrm:
  case X86::VMOVAPDYrm:
  case X86::VMOVUPDYrm:
  case X86::VMOVDQAYrm:
  case X86::VMOVDQUYrm:
  case X86::MMX_MOVD64rm:
  case X86::MMX_MOVQ64rm:
  // AVX-512
  case X86::VMOVSSZrm:
  case X86::VMOVSSZrm_alt:
  case X86::VMOVSDZrm:
  case X86::VMOVSDZrm_alt:
  case X86::VMOVSHZrm:
  case X86::VMOVSHZrm_alt:
  case X86::VMOVAPDZ128rm:
  case X86::VMOVAPDZ256rm:
  case X86::VMOVAPDZrm:
  case X86::VMOVAPSZ128rm:
  case X86::VMOVAPSZ256rm:
  case X86::VMOVAPSZ128rm_NOVLX:
  case X86::VMOVAPSZ256rm_NOVLX:
  case X86::VMOVAPSZrm:
  case X86::VMOVDQA32Z128rm:
  case X86::VMOVDQA32Z256rm:
  case X86::VMOVDQA32Zrm:
  case X86::VMOVDQA64Z128rm:
  case X86::VMOVDQA64Z256rm:
  case X86::VMOVDQA64Zrm:
  case X86::VMOVDQU16Z128rm:
  case X86::VMOVDQU16Z256rm:
  case X86::VMOVDQU16Zrm:
  case X86::VMOVDQU32Z128rm:
  case X86::VMOVDQU32Z256rm:
  case X86::VMOVDQU32Zrm:
  case X86::VMOVDQU64Z128rm:
  case X86::VMOVDQU64Z256rm:
  case X86::VMOVDQU64Zrm:
  case X86::VMOVDQU8Z128rm:
  case X86::VMOVDQU8Z256rm:
  case X86::VMOVDQU8Zrm:
  case X86::VMOVUPDZ128rm:
  case X86::VMOVUPDZ256rm:
  case X86::VMOVUPDZrm:
  case X86::VMOVUPSZ128rm:
  case X86::VMOVUPSZ256rm:
  case X86::VMOVUPSZ128rm_NOVLX:
  case X86::VMOVUPSZ256rm_NOVLX:
  case X86::VMOVUPSZrm: {
    // Loads from constant pools are trivially rematerializable.
    if (MI.getOperand(1 + X86::AddrBaseReg).isReg() &&
        MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
        MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
        MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
        MI.isDereferenceableInvariantLoad()) {
      Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
      if (BaseReg == 0 || BaseReg == X86::RIP)
        return true;
      // Allow re-materialization of PIC load.
      if (!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal())
        return false;
      const MachineFunction &MF = *MI.getParent()->getParent();
      const MachineRegisterInfo &MRI = MF.getRegInfo();
      return regIsPICBase(BaseReg, MRI);
    }
    return false;
  }
 
  case X86::LEA32r:
  case X86::LEA64r: {
    if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
        MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
        MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
        !MI.getOperand(1 + X86::AddrDisp).isReg()) {
      // lea fi#, lea GV, etc. are all rematerializable.
      if (!MI.getOperand(1 + X86::AddrBaseReg).isReg())
        return true;
      Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
      if (BaseReg == 0)
        return true;
      // Allow re-materialization of lea PICBase + x.
      const MachineFunction &MF = *MI.getParent()->getParent();
      const MachineRegisterInfo &MRI = MF.getRegInfo();
      return regIsPICBase(BaseReg, MRI);
    }
    return false;
  }
  }
}
 
void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB,
                                 MachineBasicBlock::iterator I,
                                 Register DestReg, unsigned SubIdx,
                                 const MachineInstr &Orig,
                                 const TargetRegisterInfo &TRI) const {
  bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI);
  if (ClobbersEFLAGS && MBB.computeRegisterLiveness(&TRI, X86::EFLAGS, I) !=
                            MachineBasicBlock::LQR_Dead) {
    // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side
    // effects.
    int Value;
    switch (Orig.getOpcode()) {
    case X86::MOV32r0:  Value = 0; break;
    case X86::MOV32r1:  Value = 1; break;
    case X86::MOV32r_1: Value = -1; break;
    default:
      llvm_unreachable("Unexpected instruction!");
    }
 
    const DebugLoc &DL = Orig.getDebugLoc();
    BuildMI(MBB, I, DL, get(X86::MOV32ri))
        .add(Orig.getOperand(0))
        .addImm(Value);
  } else {
    MachineInstr *MI = MBB.getParent()->CloneMachineInstr(&Orig);
    MBB.insert(I, MI);
  }
 
  MachineInstr &NewMI = *std::prev(I);
  NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
}
 
/// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead.
bool X86InstrInfo::hasLiveCondCodeDef(MachineInstr &MI) const {
  for (const MachineOperand &MO : MI.operands()) {
    if (MO.isReg() && MO.isDef() &&
        MO.getReg() == X86::EFLAGS && !MO.isDead()) {
      return true;
    }
  }
  return false;
}
 
/// Check whether the shift count for a machine operand is non-zero.
inline static unsigned getTruncatedShiftCount(const MachineInstr &MI,
                                              unsigned ShiftAmtOperandIdx) {
  // The shift count is six bits with the REX.W prefix and five bits without.
  unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31;
  unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm();
  return Imm & ShiftCountMask;
}
 
/// Check whether the given shift count is appropriate
/// can be represented by a LEA instruction.
inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) {
  // Left shift instructions can be transformed into load-effective-address
  // instructions if we can encode them appropriately.
  // A LEA instruction utilizes a SIB byte to encode its scale factor.
  // The SIB.scale field is two bits wide which means that we can encode any
  // shift amount less than 4.
  return ShAmt < 4 && ShAmt > 0;
}
 
static bool findRedundantFlagInstr(MachineInstr &CmpInstr,
                                   MachineInstr &CmpValDefInstr,
                                   const MachineRegisterInfo *MRI,
                                   MachineInstr **AndInstr,
                                   const TargetRegisterInfo *TRI,
                                   bool &NoSignFlag, bool &ClearsOverflowFlag) {
  if (CmpValDefInstr.getOpcode() != X86::SUBREG_TO_REG)
    return false;
 
  if (CmpInstr.getOpcode() != X86::TEST64rr)
    return false;
 
  // CmpInstr is a TEST64rr instruction, and `X86InstrInfo::analyzeCompare`
  // guarantees that it's analyzable only if two registers are identical.
  assert(
      (CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) &&
      "CmpInstr is an analyzable TEST64rr, and `X86InstrInfo::analyzeCompare` "
      "requires two reg operands are the same.");
 
  // Caller (`X86InstrInfo::optimizeCompareInstr`) guarantees that
  // `CmpValDefInstr` defines the value that's used by `CmpInstr`; in this case
  // if `CmpValDefInstr` sets the EFLAGS, it is likely that `CmpInstr` is
  // redundant.
  assert(
      (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) &&
      "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG.");
 
  // As seen in X86 td files, CmpValDefInstr.getOperand(1).getImm() is typically
  // 0.
  if (CmpValDefInstr.getOperand(1).getImm() != 0)
    return false;
 
  // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically
  // sub_32bit or sub_xmm.
  if (CmpValDefInstr.getOperand(3).getImm() != X86::sub_32bit)
    return false;
 
  MachineInstr *VregDefInstr =
      MRI->getVRegDef(CmpValDefInstr.getOperand(2).getReg());
 
  assert(VregDefInstr && "Must have a definition (SSA)");
 
  // Requires `CmpValDefInstr` and `VregDefInstr` are from the same MBB
  // to simplify the subsequent analysis.
  //
  // FIXME: If `VregDefInstr->getParent()` is the only predecessor of
  // `CmpValDefInstr.getParent()`, this could be handled.
  if (VregDefInstr->getParent() != CmpValDefInstr.getParent())
    return false;
 
  if (X86::isAND(VregDefInstr->getOpcode())) {
    // Get a sequence of instructions like
    //   %reg = and* ...                    // Set EFLAGS
    //   ...                                // EFLAGS not changed
    //   %extended_reg = subreg_to_reg 0, %reg, %subreg.sub_32bit
    //   test64rr %extended_reg, %extended_reg, implicit-def $eflags
    //
    // If subsequent readers use a subset of bits that don't change
    // after `and*` instructions, it's likely that the test64rr could
    // be optimized away.
    for (const MachineInstr &Instr :
         make_range(std::next(MachineBasicBlock::iterator(VregDefInstr)),
                    MachineBasicBlock::iterator(CmpValDefInstr))) {
      // There are instructions between 'VregDefInstr' and
      // 'CmpValDefInstr' that modifies EFLAGS.
      if (Instr.modifiesRegister(X86::EFLAGS, TRI))
        return false;
    }
 
    *AndInstr = VregDefInstr;
 
    // AND instruction will essentially update SF and clear OF, so
    // NoSignFlag should be false in the sense that SF is modified by `AND`.
    //
    // However, the implementation artifically sets `NoSignFlag` to true
    // to poison the SF bit; that is to say, if SF is looked at later, the
    // optimization (to erase TEST64rr) will be disabled.
    //
    // The reason to poison SF bit is that SF bit value could be different
    // in the `AND` and `TEST` operation; signed bit is not known for `AND`,
    // and is known to be 0 as a result of `TEST64rr`.
    //
    // FIXME: As opposed to poisoning the SF bit directly, consider peeking into
    // the AND instruction and using the static information to guide peephole
    // optimization if possible. For example, it's possible to fold a
    // conditional move into a copy if the relevant EFLAG bits could be deduced
    // from an immediate operand of and operation.
    //
    NoSignFlag = true;
    // ClearsOverflowFlag is true for AND operation (no surprise).
    ClearsOverflowFlag = true;
    return true;
  }
  return false;
}
 
bool X86InstrInfo::classifyLEAReg(MachineInstr &MI, const MachineOperand &Src,
                                  unsigned Opc, bool AllowSP, Register &NewSrc,
                                  bool &isKill, MachineOperand &ImplicitOp,
                                  LiveVariables *LV, LiveIntervals *LIS) const {
  MachineFunction &MF = *MI.getParent()->getParent();
  const TargetRegisterClass *RC;
  if (AllowSP) {
    RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
  } else {
    RC = Opc != X86::LEA32r ?
      &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
  }
  Register SrcReg = Src.getReg();
  isKill = MI.killsRegister(SrcReg);
 
  // For both LEA64 and LEA32 the register already has essentially the right
  // type (32-bit or 64-bit) we may just need to forbid SP.
  if (Opc != X86::LEA64_32r) {
    NewSrc = SrcReg;
    assert(!Src.isUndef() && "Undef op doesn't need optimization");
 
    if (NewSrc.isVirtual() && !MF.getRegInfo().constrainRegClass(NewSrc, RC))
      return false;
 
    return true;
  }
 
  // This is for an LEA64_32r and incoming registers are 32-bit. One way or
  // another we need to add 64-bit registers to the final MI.
  if (SrcReg.isPhysical()) {
    ImplicitOp = Src;
    ImplicitOp.setImplicit();
 
    NewSrc = getX86SubSuperRegister(SrcReg, 64);
    assert(NewSrc.isValid() && "Invalid Operand");
    assert(!Src.isUndef() && "Undef op doesn't need optimization");
  } else {
    // Virtual register of the wrong class, we have to create a temporary 64-bit
    // vreg to feed into the LEA.
    NewSrc = MF.getRegInfo().createVirtualRegister(RC);
    MachineInstr *Copy =
        BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY))
            .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit)
            .addReg(SrcReg, getKillRegState(isKill));
 
    // Which is obviously going to be dead after we're done with it.
    isKill = true;
 
    if (LV)
      LV->replaceKillInstruction(SrcReg, MI, *Copy);
 
    if (LIS) {
      SlotIndex CopyIdx = LIS->InsertMachineInstrInMaps(*Copy);
      SlotIndex Idx = LIS->getInstructionIndex(MI);
      LiveInterval &LI = LIS->getInterval(SrcReg);
      LiveRange::Segment *S = LI.getSegmentContaining(Idx);
      if (S->end.getBaseIndex() == Idx)
        S->end = CopyIdx.getRegSlot();
    }
  }
 
  // We've set all the parameters without issue.
  return true;
}
 
MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
                                                         MachineInstr &MI,
                                                         LiveVariables *LV,
                                                         LiveIntervals *LIS,
                                                         bool Is8BitOp) const {
  // We handle 8-bit adds and various 16-bit opcodes in the switch below.
  MachineBasicBlock &MBB = *MI.getParent();
  MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
  assert((Is8BitOp || RegInfo.getTargetRegisterInfo()->getRegSizeInBits(
              *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) &&
         "Unexpected type for LEA transform");
 
  // TODO: For a 32-bit target, we need to adjust the LEA variables with
  // something like this:
  //   Opcode = X86::LEA32r;
  //   InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
  //   OutRegLEA =
  //       Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass)
  //                : RegInfo.createVirtualRegister(&X86::GR32RegClass);
  if (!Subtarget.is64Bit())
    return nullptr;
 
  unsigned Opcode = X86::LEA64_32r;
  Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
  Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass);
  Register InRegLEA2;
 
  // Build and insert into an implicit UNDEF value. This is OK because
  // we will be shifting and then extracting the lower 8/16-bits.
  // This has the potential to cause partial register stall. e.g.
  //   movw    (%rbp,%rcx,2), %dx
  //   leal    -65(%rdx), %esi
  // But testing has shown this *does* help performance in 64-bit mode (at
  // least on modern x86 machines).
  MachineBasicBlock::iterator MBBI = MI.getIterator();
  Register Dest = MI.getOperand(0).getReg();
  Register Src = MI.getOperand(1).getReg();
  Register Src2;
  bool IsDead = MI.getOperand(0).isDead();
  bool IsKill = MI.getOperand(1).isKill();
  unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
  assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization");
  MachineInstr *ImpDef =
      BuildMI(MBB, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA);
  MachineInstr *InsMI =
      BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY))
          .addReg(InRegLEA, RegState::Define, SubReg)
          .addReg(Src, getKillRegState(IsKill));
  MachineInstr *ImpDef2 = nullptr;
  MachineInstr *InsMI2 = nullptr;
 
  MachineInstrBuilder MIB =
      BuildMI(MBB, MBBI, MI.getDebugLoc(), get(Opcode), OutRegLEA);
  switch (MIOpc) {
  default: llvm_unreachable("Unreachable!");
  case X86::SHL8ri:
  case X86::SHL16ri: {
    unsigned ShAmt = MI.getOperand(2).getImm();
    MIB.addReg(0)
        .addImm(1LL << ShAmt)
        .addReg(InRegLEA, RegState::Kill)
        .addImm(0)
        .addReg(0);
    break;
  }
  case X86::INC8r:
  case X86::INC16r:
    addRegOffset(MIB, InRegLEA, true, 1);
    break;
  case X86::DEC8r:
  case X86::DEC16r:
    addRegOffset(MIB, InRegLEA, true, -1);
    break;
  case X86::ADD8ri:
  case X86::ADD8ri_DB:
  case X86::ADD16ri:
  case X86::ADD16ri8:
  case X86::ADD16ri_DB:
  case X86::ADD16ri8_DB:
    addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm());
    break;
  case X86::ADD8rr:
  case X86::ADD8rr_DB:
  case X86::ADD16rr:
  case X86::ADD16rr_DB: {
    Src2 = MI.getOperand(2).getReg();
    bool IsKill2 = MI.getOperand(2).isKill();
    assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization");
    if (Src == Src2) {
      // ADD8rr/ADD16rr killed %reg1028, %reg1028
      // just a single insert_subreg.
      addRegReg(MIB, InRegLEA, true, InRegLEA, false);
    } else {
      if (Subtarget.is64Bit())
        InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
      else
        InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
      // Build and insert into an implicit UNDEF value. This is OK because
      // we will be shifting and then extracting the lower 8/16-bits.
      ImpDef2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF),
                        InRegLEA2);
      InsMI2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY))
                   .addReg(InRegLEA2, RegState::Define, SubReg)
                   .addReg(Src2, getKillRegState(IsKill2));
      addRegReg(MIB, InRegLEA, true, InRegLEA2, true);
    }
    if (LV && IsKill2 && InsMI2)
      LV->replaceKillInstruction(Src2, MI, *InsMI2);
    break;
  }
  }
 
  MachineInstr *NewMI = MIB;
  MachineInstr *ExtMI =
      BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY))
          .addReg(Dest, RegState::Define | getDeadRegState(IsDead))
          .addReg(OutRegLEA, RegState::Kill, SubReg);
 
  if (LV) {
    // Update live variables.
    LV->getVarInfo(InRegLEA).Kills.push_back(NewMI);
    if (InRegLEA2)
      LV->getVarInfo(InRegLEA2).Kills.push_back(NewMI);
    LV->getVarInfo(OutRegLEA).Kills.push_back(ExtMI);
    if (IsKill)
      LV->replaceKillInstruction(Src, MI, *InsMI);
    if (IsDead)
      LV->replaceKillInstruction(Dest, MI, *ExtMI);
  }
 
  if (LIS) {
    LIS->InsertMachineInstrInMaps(*ImpDef);
    SlotIndex InsIdx = LIS->InsertMachineInstrInMaps(*InsMI);
    if (ImpDef2)
      LIS->InsertMachineInstrInMaps(*ImpDef2);
    SlotIndex Ins2Idx;
    if (InsMI2)
      Ins2Idx = LIS->InsertMachineInstrInMaps(*InsMI2);
    SlotIndex NewIdx = LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
    SlotIndex ExtIdx = LIS->InsertMachineInstrInMaps(*ExtMI);
    LIS->getInterval(InRegLEA);
    LIS->getInterval(OutRegLEA);
    if (InRegLEA2)
      LIS->getInterval(InRegLEA2);
 
    // Move the use of Src up to InsMI.
    LiveInterval &SrcLI = LIS->getInterval(Src);
    LiveRange::Segment *SrcSeg = SrcLI.getSegmentContaining(NewIdx);
    if (SrcSeg->end == NewIdx.getRegSlot())
      SrcSeg->end = InsIdx.getRegSlot();
 
    if (InsMI2) {
      // Move the use of Src2 up to InsMI2.
      LiveInterval &Src2LI = LIS->getInterval(Src2);
      LiveRange::Segment *Src2Seg = Src2LI.getSegmentContaining(NewIdx);
      if (Src2Seg->end == NewIdx.getRegSlot())
        Src2Seg->end = Ins2Idx.getRegSlot();
    }
 
    // Move the definition of Dest down to ExtMI.
    LiveInterval &DestLI = LIS->getInterval(Dest);
    LiveRange::Segment *DestSeg =
        DestLI.getSegmentContaining(NewIdx.getRegSlot());
    assert(DestSeg->start == NewIdx.getRegSlot() &&
           DestSeg->valno->def == NewIdx.getRegSlot());
    DestSeg->start = ExtIdx.getRegSlot();
    DestSeg->valno->def = ExtIdx.getRegSlot();
  }
 
  return ExtMI;
}
 
/// This method must be implemented by targets that
/// set the M_CONVERTIBLE_TO_3_ADDR flag.  When this flag is set, the target
/// may be able to convert a two-address instruction into a true
/// three-address instruction on demand.  This allows the X86 target (for
/// example) to convert ADD and SHL instructions into LEA instructions if they
/// would require register copies due to two-addressness.
///
/// This method returns a null pointer if the transformation cannot be
/// performed, otherwise it returns the new instruction.
///
MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI,
                                                  LiveVariables *LV,
                                                  LiveIntervals *LIS) const {
  // The following opcodes also sets the condition code register(s). Only
  // convert them to equivalent lea if the condition code register def's
  // are dead!
  if (hasLiveCondCodeDef(MI))
    return nullptr;
 
  MachineFunction &MF = *MI.getParent()->getParent();
  // All instructions input are two-addr instructions.  Get the known operands.
  const MachineOperand &Dest = MI.getOperand(0);
  const MachineOperand &Src = MI.getOperand(1);
 
  // Ideally, operations with undef should be folded before we get here, but we
  // can't guarantee it. Bail out because optimizing undefs is a waste of time.
  // Without this, we have to forward undef state to new register operands to
  // avoid machine verifier errors.
  if (Src.isUndef())
    return nullptr;
  if (MI.getNumOperands() > 2)
    if (MI.getOperand(2).isReg() && MI.getOperand(2).isUndef())
      return nullptr;
 
  MachineInstr *NewMI = nullptr;
  Register SrcReg, SrcReg2;
  bool Is64Bit = Subtarget.is64Bit();
 
  bool Is8BitOp = false;
  unsigned NumRegOperands = 2;
  unsigned MIOpc = MI.getOpcode();
  switch (MIOpc) {
  default: llvm_unreachable("Unreachable!");
  case X86::SHL64ri: {
    assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
    unsigned ShAmt = getTruncatedShiftCount(MI, 2);
    if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr;
 
    // LEA can't handle RSP.
    if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass(
                                        Src.getReg(), &X86::GR64_NOSPRegClass))
      return nullptr;
 
    NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
                .add(Dest)
                .addReg(0)
                .addImm(1LL << ShAmt)
                .add(Src)
                .addImm(0)
                .addReg(0);
    break;
  }
  case X86::SHL32ri: {
    assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
    unsigned ShAmt = getTruncatedShiftCount(MI, 2);
    if (!isTruncatedShiftCountForLEA(ShAmt)) return nullptr;
 
    unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
 
    // LEA can't handle ESP.
    bool isKill;
    MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
    if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
                        ImplicitOp, LV, LIS))
      return nullptr;
 
    MachineInstrBuilder MIB =
        BuildMI(MF, MI.getDebugLoc(), get(Opc))
            .add(Dest)
            .addReg(0)
            .addImm(1LL << ShAmt)
            .addReg(SrcReg, getKillRegState(isKill))
            .addImm(0)
            .addReg(0);
    if (ImplicitOp.getReg() != 0)
      MIB.add(ImplicitOp);
    NewMI = MIB;
 
    // Add kills if classifyLEAReg created a new register.
    if (LV && SrcReg != Src.getReg())
      LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
    break;
  }
  case X86::SHL8ri:
    Is8BitOp = true;
    [[fallthrough]];
  case X86::SHL16ri: {
    assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
    unsigned ShAmt = getTruncatedShiftCount(MI, 2);
    if (!isTruncatedShiftCountForLEA(ShAmt))
      return nullptr;
    return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
  }
  case X86::INC64r:
  case X86::INC32r: {
    assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
    unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r :
        (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
    bool isKill;
    MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
    if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
                        ImplicitOp, LV, LIS))
      return nullptr;
 
    MachineInstrBuilder MIB =
        BuildMI(MF, MI.getDebugLoc(), get(Opc))
            .add(Dest)
            .addReg(SrcReg, getKillRegState(isKill));
    if (ImplicitOp.getReg() != 0)
      MIB.add(ImplicitOp);
 
    NewMI = addOffset(MIB, 1);
 
    // Add kills if classifyLEAReg created a new register.
    if (LV && SrcReg != Src.getReg())
      LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
    break;
  }
  case X86::DEC64r:
  case X86::DEC32r: {
    assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
    unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r
        : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
 
    bool isKill;
    MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
    if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
                        ImplicitOp, LV, LIS))
      return nullptr;
 
    MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
                                  .add(Dest)
                                  .addReg(SrcReg, getKillRegState(isKill));
    if (ImplicitOp.getReg() != 0)
      MIB.add(ImplicitOp);
 
    NewMI = addOffset(MIB, -1);
 
    // Add kills if classifyLEAReg created a new register.
    if (LV && SrcReg != Src.getReg())
      LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
    break;
  }
  case X86::DEC8r:
  case X86::INC8r:
    Is8BitOp = true;
    [[fallthrough]];
  case X86::DEC16r:
  case X86::INC16r:
    return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
  case X86::ADD64rr:
  case X86::ADD64rr_DB:
  case X86::ADD32rr:
  case X86::ADD32rr_DB: {
    assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
    unsigned Opc;
    if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB)
      Opc = X86::LEA64r;
    else
      Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
 
    const MachineOperand &Src2 = MI.getOperand(2);
    bool isKill2;
    MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false);
    if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/false, SrcReg2, isKill2,
                        ImplicitOp2, LV, LIS))
      return nullptr;
 
    bool isKill;
    MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
    if (Src.getReg() == Src2.getReg()) {
      // Don't call classify LEAReg a second time on the same register, in case
      // the first call inserted a COPY from Src2 and marked it as killed.
      isKill = isKill2;
      SrcReg = SrcReg2;
    } else {
      if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
                          ImplicitOp, LV, LIS))
        return nullptr;
    }
 
    MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest);
    if (ImplicitOp.getReg() != 0)
      MIB.add(ImplicitOp);
    if (ImplicitOp2.getReg() != 0)
      MIB.add(ImplicitOp2);
 
    NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2);
 
    // Add kills if classifyLEAReg created a new register.
    if (LV) {
      if (SrcReg2 != Src2.getReg())
        LV->getVarInfo(SrcReg2).Kills.push_back(NewMI);
      if (SrcReg != SrcReg2 && SrcReg != Src.getReg())
        LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
    }
    NumRegOperands = 3;
    break;
  }
  case X86::ADD8rr:
  case X86::ADD8rr_DB:
    Is8BitOp = true;
    [[fallthrough]];
  case X86::ADD16rr:
  case X86::ADD16rr_DB:
    return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
  case X86::ADD64ri32:
  case X86::ADD64ri8:
  case X86::ADD64ri32_DB:
  case X86::ADD64ri8_DB:
    assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
    NewMI = addOffset(
        BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src),
        MI.getOperand(2));
    break;
  case X86::ADD32ri:
  case X86::ADD32ri8:
  case X86::ADD32ri_DB:
  case X86::ADD32ri8_DB: {
    assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
    unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
 
    bool isKill;
    MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
    if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
                        ImplicitOp, LV, LIS))
      return nullptr;
 
    MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
                                  .add(Dest)
                                  .addReg(SrcReg, getKillRegState(isKill));
    if (ImplicitOp.getReg() != 0)
      MIB.add(ImplicitOp);
 
    NewMI = addOffset(MIB, MI.getOperand(2));
 
    // Add kills if classifyLEAReg created a new register.
    if (LV && SrcReg != Src.getReg())
      LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
    break;
  }
  case X86::ADD8ri:
  case X86::ADD8ri_DB:
    Is8BitOp = true;
    [[fallthrough]];
  case X86::ADD16ri:
  case X86::ADD16ri8:
  case X86::ADD16ri_DB:
  case X86::ADD16ri8_DB:
    return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
  case X86::SUB8ri:
  case X86::SUB16ri8:
  case X86::SUB16ri:
    /// FIXME: Support these similar to ADD8ri/ADD16ri*.
    return nullptr;
  case X86::SUB32ri8:
  case X86::SUB32ri: {
    if (!MI.getOperand(2).isImm())
      return nullptr;
    int64_t Imm = MI.getOperand(2).getImm();
    if (!isInt<32>(-Imm))
      return nullptr;
 
    assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
    unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
 
    bool isKill;
    MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
    if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
                        ImplicitOp, LV, LIS))
      return nullptr;
 
    MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
                                  .add(Dest)
                                  .addReg(SrcReg, getKillRegState(isKill));
    if (ImplicitOp.getReg() != 0)
      MIB.add(ImplicitOp);
 
    NewMI = addOffset(MIB, -Imm);
 
    // Add kills if classifyLEAReg created a new register.
    if (LV && SrcReg != Src.getReg())
      LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
    break;
  }
 
  case X86::SUB64ri8:
  case X86::SUB64ri32: {
    if (!MI.getOperand(2).isImm())
      return nullptr;
    int64_t Imm = MI.getOperand(2).getImm();
    if (!isInt<32>(-Imm))
      return nullptr;
 
    assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
 
    MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(),
                                      get(X86::LEA64r)).add(Dest).add(Src);
    NewMI = addOffset(MIB, -Imm);
    break;
  }
 
  case X86::VMOVDQU8Z128rmk:
  case X86::VMOVDQU8Z256rmk:
  case X86::VMOVDQU8Zrmk:
  case X86::VMOVDQU16Z128rmk:
  case X86::VMOVDQU16Z256rmk:
  case X86::VMOVDQU16Zrmk:
  case X86::VMOVDQU32Z128rmk: case X86::VMOVDQA32Z128rmk:
  case X86::VMOVDQU32Z256rmk: case X86::VMOVDQA32Z256rmk:
  case X86::VMOVDQU32Zrmk:    case X86::VMOVDQA32Zrmk:
  case X86::VMOVDQU64Z128rmk: case X86::VMOVDQA64Z128rmk:
  case X86::VMOVDQU64Z256rmk: case X86::VMOVDQA64Z256rmk:
  case X86::VMOVDQU64Zrmk:    case X86::VMOVDQA64Zrmk:
  case X86::VMOVUPDZ128rmk:   case X86::VMOVAPDZ128rmk:
  case X86::VMOVUPDZ256rmk:   case X86::VMOVAPDZ256rmk:
  case X86::VMOVUPDZrmk:      case X86::VMOVAPDZrmk:
  case X86::VMOVUPSZ128rmk:   case X86::VMOVAPSZ128rmk:
  case X86::VMOVUPSZ256rmk:   case X86::VMOVAPSZ256rmk:
  case X86::VMOVUPSZrmk:      case X86::VMOVAPSZrmk:
  case X86::VBROADCASTSDZ256rmk:
  case X86::VBROADCASTSDZrmk:
  case X86::VBROADCASTSSZ128rmk:
  case X86::VBROADCASTSSZ256rmk:
  case X86::VBROADCASTSSZrmk:
  case X86::VPBROADCASTDZ128rmk:
  case X86::VPBROADCASTDZ256rmk:
  case X86::VPBROADCASTDZrmk:
  case X86::VPBROADCASTQZ128rmk:
  case X86::VPBROADCASTQZ256rmk:
  case X86::VPBROADCASTQZrmk: {
    unsigned Opc;
    switch (MIOpc) {
    default: llvm_unreachable("Unreachable!");
    case X86::VMOVDQU8Z128rmk:     Opc = X86::VPBLENDMBZ128rmk; break;
    case X86::VMOVDQU8Z256rmk:     Opc = X86::VPBLENDMBZ256rmk; break;
    case X86::VMOVDQU8Zrmk:        Opc = X86::VPBLENDMBZrmk;    break;
    case X86::VMOVDQU16Z128rmk:    Opc = X86::VPBLENDMWZ128rmk; break;
    case X86::VMOVDQU16Z256rmk:    Opc = X86::VPBLENDMWZ256rmk; break;
    case X86::VMOVDQU16Zrmk:       Opc = X86::VPBLENDMWZrmk;    break;
    case X86::VMOVDQU32Z128rmk:    Opc = X86::VPBLENDMDZ128rmk; break;
    case X86::VMOVDQU32Z256rmk:    Opc = X86::VPBLENDMDZ256rmk; break;
    case X86::VMOVDQU32Zrmk:       Opc = X86::VPBLENDMDZrmk;    break;
    case X86::VMOVDQU64Z128rmk:    Opc = X86::VPBLENDMQZ128rmk; break;
    case X86::VMOVDQU64Z256rmk:    Opc = X86::VPBLENDMQZ256rmk; break;
    case X86::VMOVDQU64Zrmk:       Opc = X86::VPBLENDMQZrmk;    break;
    case X86::VMOVUPDZ128rmk:      Opc = X86::VBLENDMPDZ128rmk; break;
    case X86::VMOVUPDZ256rmk:      Opc = X86::VBLENDMPDZ256rmk; break;
    case X86::VMOVUPDZrmk:         Opc = X86::VBLENDMPDZrmk;    break;
    case X86::VMOVUPSZ128rmk:      Opc = X86::VBLENDMPSZ128rmk; break;
    case X86::VMOVUPSZ256rmk:      Opc = X86::VBLENDMPSZ256rmk; break;
    case X86::VMOVUPSZrmk:         Opc = X86::VBLENDMPSZrmk;    break;
    case X86::VMOVDQA32Z128rmk:    Opc = X86::VPBLENDMDZ128rmk; break;
    case X86::VMOVDQA32Z256rmk:    Opc = X86::VPBLENDMDZ256rmk; break;
    case X86::VMOVDQA32Zrmk:       Opc = X86::VPBLENDMDZrmk;    break;
    case X86::VMOVDQA64Z128rmk:    Opc = X86::VPBLENDMQZ128rmk; break;
    case X86::VMOVDQA64Z256rmk:    Opc = X86::VPBLENDMQZ256rmk; break;
    case X86::VMOVDQA64Zrmk:       Opc = X86::VPBLENDMQZrmk;    break;
    case X86::VMOVAPDZ128rmk:      Opc = X86::VBLENDMPDZ128rmk; break;
    case X86::VMOVAPDZ256rmk:      Opc = X86::VBLENDMPDZ256rmk; break;
    case X86::VMOVAPDZrmk:         Opc = X86::VBLENDMPDZrmk;    break;
    case X86::VMOVAPSZ128rmk:      Opc = X86::VBLENDMPSZ128rmk; break;
    case X86::VMOVAPSZ256rmk:      Opc = X86::VBLENDMPSZ256rmk; break;
    case X86::VMOVAPSZrmk:         Opc = X86::VBLENDMPSZrmk;    break;
    case X86::VBROADCASTSDZ256rmk: Opc = X86::VBLENDMPDZ256rmbk; break;
    case X86::VBROADCASTSDZrmk:    Opc = X86::VBLENDMPDZrmbk;    break;
    case X86::VBROADCASTSSZ128rmk: Opc = X86::VBLENDMPSZ128rmbk; break;
    case X86::VBROADCASTSSZ256rmk: Opc = X86::VBLENDMPSZ256rmbk; break;
    case X86::VBROADCASTSSZrmk:    Opc = X86::VBLENDMPSZrmbk;    break;
    case X86::VPBROADCASTDZ128rmk: Opc = X86::VPBLENDMDZ128rmbk; break;
    case X86::VPBROADCASTDZ256rmk: Opc = X86::VPBLENDMDZ256rmbk; break;
    case X86::VPBROADCASTDZrmk:    Opc = X86::VPBLENDMDZrmbk;    break;
    case X86::VPBROADCASTQZ128rmk: Opc = X86::VPBLENDMQZ128rmbk; break;
    case X86::VPBROADCASTQZ256rmk: Opc = X86::VPBLENDMQZ256rmbk; break;
    case X86::VPBROADCASTQZrmk:    Opc = X86::VPBLENDMQZrmbk;    break;
    }
 
    NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
              .add(Dest)
              .add(MI.getOperand(2))
              .add(Src)
              .add(MI.getOperand(3))
              .add(MI.getOperand(4))
              .add(MI.getOperand(5))
              .add(MI.getOperand(6))
              .add(MI.getOperand(7));
    NumRegOperands = 4;
    break;
  }
 
  case X86::VMOVDQU8Z128rrk:
  case X86::VMOVDQU8Z256rrk:
  case X86::VMOVDQU8Zrrk:
  case X86::VMOVDQU16Z128rrk:
  case X86::VMOVDQU16Z256rrk:
  case X86::VMOVDQU16Zrrk:
  case X86::VMOVDQU32Z128rrk: case X86::VMOVDQA32Z128rrk:
  case X86::VMOVDQU32Z256rrk: case X86::VMOVDQA32Z256rrk:
  case X86::VMOVDQU32Zrrk:    case X86::VMOVDQA32Zrrk:
  case X86::VMOVDQU64Z128rrk: case X86::VMOVDQA64Z128rrk:
  case X86::VMOVDQU64Z256rrk: case X86::VMOVDQA64Z256rrk:
  case X86::VMOVDQU64Zrrk:    case X86::VMOVDQA64Zrrk:
  case X86::VMOVUPDZ128rrk:   case X86::VMOVAPDZ128rrk:
  case X86::VMOVUPDZ256rrk:   case X86::VMOVAPDZ256rrk:
  case X86::VMOVUPDZrrk:      case X86::VMOVAPDZrrk:
  case X86::VMOVUPSZ128rrk:   case X86::VMOVAPSZ128rrk:
  case X86::VMOVUPSZ256rrk:   case X86::VMOVAPSZ256rrk:
  case X86::VMOVUPSZrrk:      case X86::VMOVAPSZrrk: {
    unsigned Opc;
    switch (MIOpc) {
    default: llvm_unreachable("Unreachable!");
    case X86::VMOVDQU8Z128rrk:  Opc = X86::VPBLENDMBZ128rrk; break;
    case X86::VMOVDQU8Z256rrk:  Opc = X86::VPBLENDMBZ256rrk; break;
    case X86::VMOVDQU8Zrrk:     Opc = X86::VPBLENDMBZrrk;    break;
    case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk; break;
    case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk; break;
    case X86::VMOVDQU16Zrrk:    Opc = X86::VPBLENDMWZrrk;    break;
    case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break;
    case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break;
    case X86::VMOVDQU32Zrrk:    Opc = X86::VPBLENDMDZrrk;    break;
    case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break;
    case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break;
    case X86::VMOVDQU64Zrrk:    Opc = X86::VPBLENDMQZrrk;    break;
    case X86::VMOVUPDZ128rrk:   Opc = X86::VBLENDMPDZ128rrk; break;
    case X86::VMOVUPDZ256rrk:   Opc = X86::VBLENDMPDZ256rrk; break;
    case X86::VMOVUPDZrrk:      Opc = X86::VBLENDMPDZrrk;    break;
    case X86::VMOVUPSZ128rrk:   Opc = X86::VBLENDMPSZ128rrk; break;
    case X86::VMOVUPSZ256rrk:   Opc = X86::VBLENDMPSZ256rrk; break;
    case X86::VMOVUPSZrrk:      Opc = X86::VBLENDMPSZrrk;    break;
    case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk; break;
    case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk; break;
    case X86::VMOVDQA32Zrrk:    Opc = X86::VPBLENDMDZrrk;    break;
    case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk; break;
    case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk; break;
    case X86::VMOVDQA64Zrrk:    Opc = X86::VPBLENDMQZrrk;    break;
    case X86::VMOVAPDZ128rrk:   Opc = X86::VBLENDMPDZ128rrk; break;
    case X86::VMOVAPDZ256rrk:   Opc = X86::VBLENDMPDZ256rrk; break;
    case X86::VMOVAPDZrrk:      Opc = X86::VBLENDMPDZrrk;    break;
    case X86::VMOVAPSZ128rrk:   Opc = X86::VBLENDMPSZ128rrk; break;
    case X86::VMOVAPSZ256rrk:   Opc = X86::VBLENDMPSZ256rrk; break;
    case X86::VMOVAPSZrrk:      Opc = X86::VBLENDMPSZrrk;    break;
    }
 
    NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
              .add(Dest)
              .add(MI.getOperand(2))
              .add(Src)
              .add(MI.getOperand(3));
    NumRegOperands = 4;
    break;
  }
  }
 
  if (!NewMI) return nullptr;
 
  if (LV) {  // Update live variables
    for (unsigned I = 0; I < NumRegOperands; ++I) {
      MachineOperand &Op = MI.getOperand(I);
      if (Op.isReg() && (Op.isDead() || Op.isKill()))
        LV->replaceKillInstruction(Op.getReg(), MI, *NewMI);
    }
  }
 
  MachineBasicBlock &MBB = *MI.getParent();
  MBB.insert(MI.getIterator(), NewMI); // Insert the new inst
 
  if (LIS) {
    LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
    if (SrcReg)
      LIS->getInterval(SrcReg);
    if (SrcReg2)
      LIS->getInterval(SrcReg2);
  }
 
  return NewMI;
}
 
/// This determines which of three possible cases of a three source commute
/// the source indexes correspond to taking into account any mask operands.
/// All prevents commuting a passthru operand. Returns -1 if the commute isn't
/// possible.
/// Case 0 - Possible to commute the first and second operands.
/// Case 1 - Possible to commute the first and third operands.
/// Case 2 - Possible to commute the second and third operands.
static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1,
                                       unsigned SrcOpIdx2) {
  // Put the lowest index to SrcOpIdx1 to simplify the checks below.
  if (SrcOpIdx1 > SrcOpIdx2)
    std::swap(SrcOpIdx1, SrcOpIdx2);
 
  unsigned Op1 = 1, Op2 = 2, Op3 = 3;
  if (X86II::isKMasked(TSFlags)) {
    Op2++;
    Op3++;
  }
 
  if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2)
    return 0;
  if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3)
    return 1;
  if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3)
    return 2;
  llvm_unreachable("Unknown three src commute case.");
}
 
unsigned X86InstrInfo::getFMA3OpcodeToCommuteOperands(
    const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2,
    const X86InstrFMA3Group &FMA3Group) const {
 
  unsigned Opc = MI.getOpcode();
 
  // TODO: Commuting the 1st operand of FMA*_Int requires some additional
  // analysis. The commute optimization is legal only if all users of FMA*_Int
  // use only the lowest element of the FMA*_Int instruction. Such analysis are
  // not implemented yet. So, just return 0 in that case.
  // When such analysis are available this place will be the right place for
  // calling it.
  assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) &&
         "Intrinsic instructions can't commute operand 1");
 
  // Determine which case this commute is or if it can't be done.
  unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1,
                                         SrcOpIdx2);
  assert(Case < 3 && "Unexpected case number!");
 
  // Define the FMA forms mapping array that helps to map input FMA form
  // to output FMA form to preserve the operation semantics after
  // commuting the operands.
  const unsigned Form132Index = 0;
  const unsigned Form213Index = 1;
  const unsigned Form231Index = 2;
  static const unsigned FormMapping[][3] = {
    // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2;
    // FMA132 A, C, b; ==> FMA231 C, A, b;
    // FMA213 B, A, c; ==> FMA213 A, B, c;
    // FMA231 C, A, b; ==> FMA132 A, C, b;
    { Form231Index, Form213Index, Form132Index },
    // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3;
    // FMA132 A, c, B; ==> FMA132 B, c, A;
    // FMA213 B, a, C; ==> FMA231 C, a, B;
    // FMA231 C, a, B; ==> FMA213 B, a, C;
    { Form132Index, Form231Index, Form213Index },
    // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3;
    // FMA132 a, C, B; ==> FMA213 a, B, C;
    // FMA213 b, A, C; ==> FMA132 b, C, A;
    // FMA231 c, A, B; ==> FMA231 c, B, A;
    { Form213Index, Form132Index, Form231Index }
  };
 
  unsigned FMAForms[3];
  FMAForms[0] = FMA3Group.get132Opcode();
  FMAForms[1] = FMA3Group.get213Opcode();
  FMAForms[2] = FMA3Group.get231Opcode();
 
  // Everything is ready, just adjust the FMA opcode and return it.
  for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
    if (Opc == FMAForms[FormIndex])
      return FMAForms[FormMapping[Case][FormIndex]];
 
  llvm_unreachable("Illegal FMA3 format");
}
 
static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1,
                             unsigned SrcOpIdx2) {
  // Determine which case this commute is or if it can't be done.
  unsigned Case = getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1,
                                         SrcOpIdx2);
  assert(Case < 3 && "Unexpected case value!");
 
  // For each case we need to swap two pairs of bits in the final immediate.
  static const uint8_t SwapMasks[3][4] = {
    { 0x04, 0x10, 0x08, 0x20 }, // Swap bits 2/4 and 3/5.
    { 0x02, 0x10, 0x08, 0x40 }, // Swap bits 1/4 and 3/6.
    { 0x02, 0x04, 0x20, 0x40 }, // Swap bits 1/2 and 5/6.
  };
 
  uint8_t Imm = MI.getOperand(MI.getNumOperands()-1).getImm();
  // Clear out the bits we are swapping.
  uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
                           SwapMasks[Case][2] | SwapMasks[Case][3]);
  // If the immediate had a bit of the pair set, then set the opposite bit.
  if (Imm & SwapMasks[Case][0]) NewImm |= SwapMasks[Case][1];
  if (Imm & SwapMasks[Case][1]) NewImm |= SwapMasks[Case][0];
  if (Imm & SwapMasks[Case][2]) NewImm |= SwapMasks[Case][3];
  if (Imm & SwapMasks[Case][3]) NewImm |= SwapMasks[Case][2];
  MI.getOperand(MI.getNumOperands()-1).setImm(NewImm);
}
 
// Returns true if this is a VPERMI2 or VPERMT2 instruction that can be
// commuted.
static bool isCommutableVPERMV3Instruction(unsigned Opcode) {
#define VPERM_CASES(Suffix) \
  case X86::VPERMI2##Suffix##128rr:    case X86::VPERMT2##Suffix##128rr:    \
  case X86::VPERMI2##Suffix##256rr:    case X86::VPERMT2##Suffix##256rr:    \
  case X86::VPERMI2##Suffix##rr:       case X86::VPERMT2##Suffix##rr:       \
  case X86::VPERMI2##Suffix##128rm:    case X86::VPERMT2##Suffix##128rm:    \
  case X86::VPERMI2##Suffix##256rm:    case X86::VPERMT2##Suffix##256rm:    \
  case X86::VPERMI2##Suffix##rm:       case X86::VPERMT2##Suffix##rm:       \
  case X86::VPERMI2##Suffix##128rrkz:  case X86::VPERMT2##Suffix##128rrkz:  \
  case X86::VPERMI2##Suffix##256rrkz:  case X86::VPERMT2##Suffix##256rrkz:  \
  case X86::VPERMI2##Suffix##rrkz:     case X86::VPERMT2##Suffix##rrkz:     \
  case X86::VPERMI2##Suffix##128rmkz:  case X86::VPERMT2##Suffix##128rmkz:  \
  case X86::VPERMI2##Suffix##256rmkz:  case X86::VPERMT2##Suffix##256rmkz:  \
  case X86::VPERMI2##Suffix##rmkz:     case X86::VPERMT2##Suffix##rmkz:
 
#define VPERM_CASES_BROADCAST(Suffix) \
  VPERM_CASES(Suffix) \
  case X86::VPERMI2##Suffix##128rmb:   case X86::VPERMT2##Suffix##128rmb:   \
  case X86::VPERMI2##Suffix##256rmb:   case X86::VPERMT2##Suffix##256rmb:   \
  case X86::VPERMI2##Suffix##rmb:      case X86::VPERMT2##Suffix##rmb:      \
  case X86::VPERMI2##Suffix##128rmbkz: case X86::VPERMT2##Suffix##128rmbkz: \
  case X86::VPERMI2##Suffix##256rmbkz: case X86::VPERMT2##Suffix##256rmbkz: \
  case X86::VPERMI2##Suffix##rmbkz:    case X86::VPERMT2##Suffix##rmbkz:
 
  switch (Opcode) {
  default: return false;
  VPERM_CASES(B)
  VPERM_CASES_BROADCAST(D)
  VPERM_CASES_BROADCAST(PD)
  VPERM_CASES_BROADCAST(PS)
  VPERM_CASES_BROADCAST(Q)
  VPERM_CASES(W)
    return true;
  }
#undef VPERM_CASES_BROADCAST
#undef VPERM_CASES
}
 
// Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching
// from the I opcode to the T opcode and vice versa.
static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) {
#define VPERM_CASES(Orig, New) \
  case X86::Orig##128rr:    return X86::New##128rr;   \
  case X86::Orig##128rrkz:  return X86::New##128rrkz; \
  case X86::Orig##128rm:    return X86::New##128rm;   \
  case X86::Orig##128rmkz:  return X86::New##128rmkz; \
  case X86::Orig##256rr:    return X86::New##256rr;   \
  case X86::Orig##256rrkz:  return X86::New##256rrkz; \
  case X86::Orig##256rm:    return X86::New##256rm;   \
  case X86::Orig##256rmkz:  return X86::New##256rmkz; \
  case X86::Orig##rr:       return X86::New##rr;      \
  case X86::Orig##rrkz:     return X86::New##rrkz;    \
  case X86::Orig##rm:       return X86::New##rm;      \
  case X86::Orig##rmkz:     return X86::New##rmkz;
 
#define VPERM_CASES_BROADCAST(Orig, New) \
  VPERM_CASES(Orig, New) \
  case X86::Orig##128rmb:   return X86::New##128rmb;   \
  case X86::Orig##128rmbkz: return X86::New##128rmbkz; \
  case X86::Orig##256rmb:   return X86::New##256rmb;   \
  case X86::Orig##256rmbkz: return X86::New##256rmbkz; \
  case X86::Orig##rmb:      return X86::New##rmb;      \
  case X86::Orig##rmbkz:    return X86::New##rmbkz;
 
  switch (Opcode) {
  VPERM_CASES(VPERMI2B, VPERMT2B)
  VPERM_CASES_BROADCAST(VPERMI2D,  VPERMT2D)
  VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD)
  VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS)
  VPERM_CASES_BROADCAST(VPERMI2Q,  VPERMT2Q)
  VPERM_CASES(VPERMI2W, VPERMT2W)
  VPERM_CASES(VPERMT2B, VPERMI2B)
  VPERM_CASES_BROADCAST(VPERMT2D,  VPERMI2D)
  VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD)
  VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS)
  VPERM_CASES_BROADCAST(VPERMT2Q,  VPERMI2Q)
  VPERM_CASES(VPERMT2W, VPERMI2W)
  }
 
  llvm_unreachable("Unreachable!");
#undef VPERM_CASES_BROADCAST
#undef VPERM_CASES
}
 
MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
                                                   unsigned OpIdx1,
                                                   unsigned OpIdx2) const {
  auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
    if (NewMI)
      return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
    return MI;
  };
 
  switch (MI.getOpcode()) {
  case X86::SHRD16rri8: // A = SHRD16rri8 B, C, I -> A = SHLD16rri8 C, B, (16-I)
  case X86::SHLD16rri8: // A = SHLD16rri8 B, C, I -> A = SHRD16rri8 C, B, (16-I)
  case X86::SHRD32rri8: // A = SHRD32rri8 B, C, I -> A = SHLD32rri8 C, B, (32-I)
  case X86::SHLD32rri8: // A = SHLD32rri8 B, C, I -> A = SHRD32rri8 C, B, (32-I)
  case X86::SHRD64rri8: // A = SHRD64rri8 B, C, I -> A = SHLD64rri8 C, B, (64-I)
  case X86::SHLD64rri8:{// A = SHLD64rri8 B, C, I -> A = SHRD64rri8 C, B, (64-I)
    unsigned Opc;
    unsigned Size;
    switch (MI.getOpcode()) {
    default: llvm_unreachable("Unreachable!");
    case X86::SHRD16rri8: Size = 16; Opc = X86::SHLD16rri8; break;
    case X86::SHLD16rri8: Size = 16; Opc = X86::SHRD16rri8; break;
    case X86::SHRD32rri8: Size = 32; Opc = X86::SHLD32rri8; break;
    case X86::SHLD32rri8: Size = 32; Opc = X86::SHRD32rri8; break;
    case X86::SHRD64rri8: Size = 64; Opc = X86::SHLD64rri8; break;
    case X86::SHLD64rri8: Size = 64; Opc = X86::SHRD64rri8; break;
    }
    unsigned Amt = MI.getOperand(3).getImm();
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.setDesc(get(Opc));
    WorkingMI.getOperand(3).setImm(Size - Amt);
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::PFSUBrr:
  case X86::PFSUBRrr: {
    // PFSUB  x, y: x = x - y
    // PFSUBR x, y: x = y - x
    unsigned Opc =
        (X86::PFSUBRrr == MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr);
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.setDesc(get(Opc));
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::BLENDPDrri:
  case X86::BLENDPSrri:
  case X86::VBLENDPDrri:
  case X86::VBLENDPSrri:
    // If we're optimizing for size, try to use MOVSD/MOVSS.
    if (MI.getParent()->getParent()->getFunction().hasOptSize()) {
      unsigned Mask, Opc;
      switch (MI.getOpcode()) {
      default: llvm_unreachable("Unreachable!");
      case X86::BLENDPDrri:  Opc = X86::MOVSDrr;  Mask = 0x03; break;
      case X86::BLENDPSrri:  Opc = X86::MOVSSrr;  Mask = 0x0F; break;
      case X86::VBLENDPDrri: Opc = X86::VMOVSDrr; Mask = 0x03; break;
      case X86::VBLENDPSrri: Opc = X86::VMOVSSrr; Mask = 0x0F; break;
      }
      if ((MI.getOperand(3).getImm() ^ Mask) == 1) {
        auto &WorkingMI = cloneIfNew(MI);
        WorkingMI.setDesc(get(Opc));
        WorkingMI.removeOperand(3);
        return TargetInstrInfo::commuteInstructionImpl(WorkingMI,
                                                       /*NewMI=*/false,
                                                       OpIdx1, OpIdx2);
      }
    }
    [[fallthrough]];
  case X86::PBLENDWrri:
  case X86::VBLENDPDYrri:
  case X86::VBLENDPSYrri:
  case X86::VPBLENDDrri:
  case X86::VPBLENDWrri:
  case X86::VPBLENDDYrri:
  case X86::VPBLENDWYrri:{
    int8_t Mask;
    switch (MI.getOpcode()) {
    default: llvm_unreachable("Unreachable!");
    case X86::BLENDPDrri:    Mask = (int8_t)0x03; break;
    case X86::BLENDPSrri:    Mask = (int8_t)0x0F; break;
    case X86::PBLENDWrri:    Mask = (int8_t)0xFF; break;
    case X86::VBLENDPDrri:   Mask = (int8_t)0x03; break;
    case X86::VBLENDPSrri:   Mask = (int8_t)0x0F; break;
    case X86::VBLENDPDYrri:  Mask = (int8_t)0x0F; break;
    case X86::VBLENDPSYrri:  Mask = (int8_t)0xFF; break;
    case X86::VPBLENDDrri:   Mask = (int8_t)0x0F; break;
    case X86::VPBLENDWrri:   Mask = (int8_t)0xFF; break;
    case X86::VPBLENDDYrri:  Mask = (int8_t)0xFF; break;
    case X86::VPBLENDWYrri:  Mask = (int8_t)0xFF; break;
    }
    // Only the least significant bits of Imm are used.
    // Using int8_t to ensure it will be sign extended to the int64_t that
    // setImm takes in order to match isel behavior.
    int8_t Imm = MI.getOperand(3).getImm() & Mask;
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.getOperand(3).setImm(Mask ^ Imm);
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::INSERTPSrr:
  case X86::VINSERTPSrr:
  case X86::VINSERTPSZrr: {
    unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
    unsigned ZMask = Imm & 15;
    unsigned DstIdx = (Imm >> 4) & 3;
    unsigned SrcIdx = (Imm >> 6) & 3;
 
    // We can commute insertps if we zero 2 of the elements, the insertion is
    // "inline" and we don't override the insertion with a zero.
    if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
        llvm::popcount(ZMask) == 2) {
      unsigned AltIdx = llvm::countr_zero((ZMask | (1 << DstIdx)) ^ 15);
      assert(AltIdx < 4 && "Illegal insertion index");
      unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
      auto &WorkingMI = cloneIfNew(MI);
      WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(AltImm);
      return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                     OpIdx1, OpIdx2);
    }
    return nullptr;
  }
  case X86::MOVSDrr:
  case X86::MOVSSrr:
  case X86::VMOVSDrr:
  case X86::VMOVSSrr:{
    // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD.
    if (Subtarget.hasSSE41()) {
      unsigned Mask, Opc;
      switch (MI.getOpcode()) {
      default: llvm_unreachable("Unreachable!");
      case X86::MOVSDrr:  Opc = X86::BLENDPDrri;  Mask = 0x02; break;
      case X86::MOVSSrr:  Opc = X86::BLENDPSrri;  Mask = 0x0E; break;
      case X86::VMOVSDrr: Opc = X86::VBLENDPDrri; Mask = 0x02; break;
      case X86::VMOVSSrr: Opc = X86::VBLENDPSrri; Mask = 0x0E; break;
      }
 
      auto &WorkingMI = cloneIfNew(MI);
      WorkingMI.setDesc(get(Opc));
      WorkingMI.addOperand(MachineOperand::CreateImm(Mask));
      return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                     OpIdx1, OpIdx2);
    }
 
    // Convert to SHUFPD.
    assert(MI.getOpcode() == X86::MOVSDrr &&
           "Can only commute MOVSDrr without SSE4.1");
 
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.setDesc(get(X86::SHUFPDrri));
    WorkingMI.addOperand(MachineOperand::CreateImm(0x02));
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::SHUFPDrri: {
    // Commute to MOVSD.
    assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!");
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.setDesc(get(X86::MOVSDrr));
    WorkingMI.removeOperand(3);
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::PCLMULQDQrr:
  case X86::VPCLMULQDQrr:
  case X86::VPCLMULQDQYrr:
  case X86::VPCLMULQDQZrr:
  case X86::VPCLMULQDQZ128rr:
  case X86::VPCLMULQDQZ256rr: {
    // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0]
    // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0]
    unsigned Imm = MI.getOperand(3).getImm();
    unsigned Src1Hi = Imm & 0x01;
    unsigned Src2Hi = Imm & 0x10;
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4));
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::VPCMPBZ128rri:  case X86::VPCMPUBZ128rri:
  case X86::VPCMPBZ256rri:  case X86::VPCMPUBZ256rri:
  case X86::VPCMPBZrri:     case X86::VPCMPUBZrri:
  case X86::VPCMPDZ128rri:  case X86::VPCMPUDZ128rri:
  case X86::VPCMPDZ256rri:  case X86::VPCMPUDZ256rri:
  case X86::VPCMPDZrri:     case X86::VPCMPUDZrri:
  case X86::VPCMPQZ128rri:  case X86::VPCMPUQZ128rri:
  case X86::VPCMPQZ256rri:  case X86::VPCMPUQZ256rri:
  case X86::VPCMPQZrri:     case X86::VPCMPUQZrri:
  case X86::VPCMPWZ128rri:  case X86::VPCMPUWZ128rri:
  case X86::VPCMPWZ256rri:  case X86::VPCMPUWZ256rri:
  case X86::VPCMPWZrri:     case X86::VPCMPUWZrri:
  case X86::VPCMPBZ128rrik: case X86::VPCMPUBZ128rrik:
  case X86::VPCMPBZ256rrik: case X86::VPCMPUBZ256rrik:
  case X86::VPCMPBZrrik:    case X86::VPCMPUBZrrik:
  case X86::VPCMPDZ128rrik: case X86::VPCMPUDZ128rrik:
  case X86::VPCMPDZ256rrik: case X86::VPCMPUDZ256rrik:
  case X86::VPCMPDZrrik:    case X86::VPCMPUDZrrik:
  case X86::VPCMPQZ128rrik: case X86::VPCMPUQZ128rrik:
  case X86::VPCMPQZ256rrik: case X86::VPCMPUQZ256rrik:
  case X86::VPCMPQZrrik:    case X86::VPCMPUQZrrik:
  case X86::VPCMPWZ128rrik: case X86::VPCMPUWZ128rrik:
  case X86::VPCMPWZ256rrik: case X86::VPCMPUWZ256rrik:
  case X86::VPCMPWZrrik:    case X86::VPCMPUWZrrik: {
    // Flip comparison mode immediate (if necessary).
    unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7;
    Imm = X86::getSwappedVPCMPImm(Imm);
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(Imm);
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::VPCOMBri: case X86::VPCOMUBri:
  case X86::VPCOMDri: case X86::VPCOMUDri:
  case X86::VPCOMQri: case X86::VPCOMUQri:
  case X86::VPCOMWri: case X86::VPCOMUWri: {
    // Flip comparison mode immediate (if necessary).
    unsigned Imm = MI.getOperand(3).getImm() & 0x7;
    Imm = X86::getSwappedVPCOMImm(Imm);
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.getOperand(3).setImm(Imm);
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::VCMPSDZrr:
  case X86::VCMPSSZrr:
  case X86::VCMPPDZrri:
  case X86::VCMPPSZrri:
  case X86::VCMPSHZrr:
  case X86::VCMPPHZrri:
  case X86::VCMPPHZ128rri:
  case X86::VCMPPHZ256rri:
  case X86::VCMPPDZ128rri:
  case X86::VCMPPSZ128rri:
  case X86::VCMPPDZ256rri:
  case X86::VCMPPSZ256rri:
  case X86::VCMPPDZrrik:
  case X86::VCMPPSZrrik:
  case X86::VCMPPDZ128rrik:
  case X86::VCMPPSZ128rrik:
  case X86::VCMPPDZ256rrik:
  case X86::VCMPPSZ256rrik: {
    unsigned Imm =
                MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 0x1f;
    Imm = X86::getSwappedVCMPImm(Imm);
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.getOperand(MI.getNumExplicitOperands() - 1).setImm(Imm);
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::VPERM2F128rr:
  case X86::VPERM2I128rr: {
    // Flip permute source immediate.
    // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi.
    // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi.
    int8_t Imm = MI.getOperand(3).getImm() & 0xFF;
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.getOperand(3).setImm(Imm ^ 0x22);
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::MOVHLPSrr:
  case X86::UNPCKHPDrr:
  case X86::VMOVHLPSrr:
  case X86::VUNPCKHPDrr:
  case X86::VMOVHLPSZrr:
  case X86::VUNPCKHPDZ128rr: {
    assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!");
 
    unsigned Opc = MI.getOpcode();
    switch (Opc) {
    default: llvm_unreachable("Unreachable!");
    case X86::MOVHLPSrr:       Opc = X86::UNPCKHPDrr;      break;
    case X86::UNPCKHPDrr:      Opc = X86::MOVHLPSrr;       break;
    case X86::VMOVHLPSrr:      Opc = X86::VUNPCKHPDrr;     break;
    case X86::VUNPCKHPDrr:     Opc = X86::VMOVHLPSrr;      break;
    case X86::VMOVHLPSZrr:     Opc = X86::VUNPCKHPDZ128rr; break;
    case X86::VUNPCKHPDZ128rr: Opc = X86::VMOVHLPSZrr;     break;
    }
    auto &WorkingMI = cloneIfNew(MI);
    WorkingMI.setDesc(get(Opc));
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::CMOV16rr:  case X86::CMOV32rr:  case X86::CMOV64rr: {
    auto &WorkingMI = cloneIfNew(MI);
    unsigned OpNo = MI.getDesc().getNumOperands() - 1;
    X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
    WorkingMI.getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC));
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  case X86::VPTERNLOGDZrri:      case X86::VPTERNLOGDZrmi:
  case X86::VPTERNLOGDZ128rri:   case X86::VPTERNLOGDZ128rmi:
  case X86::VPTERNLOGDZ256rri:   case X86::VPTERNLOGDZ256rmi:
  case X86::VPTERNLOGQZrri:      case X86::VPTERNLOGQZrmi:
  case X86::VPTERNLOGQZ128rri:   case X86::VPTERNLOGQZ128rmi:
  case X86::VPTERNLOGQZ256rri:   case X86::VPTERNLOGQZ256rmi:
  case X86::VPTERNLOGDZrrik:
  case X86::VPTERNLOGDZ128rrik:
  case X86::VPTERNLOGDZ256rrik:
  case X86::VPTERNLOGQZrrik:
  case X86::VPTERNLOGQZ128rrik:
  case X86::VPTERNLOGQZ256rrik:
  case X86::VPTERNLOGDZrrikz:    case X86::VPTERNLOGDZrmikz:
  case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz:
  case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz:
  case X86::VPTERNLOGQZrrikz:    case X86::VPTERNLOGQZrmikz:
  case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz:
  case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz:
  case X86::VPTERNLOGDZ128rmbi:
  case X86::VPTERNLOGDZ256rmbi:
  case X86::VPTERNLOGDZrmbi:
  case X86::VPTERNLOGQZ128rmbi:
  case X86::VPTERNLOGQZ256rmbi:
  case X86::VPTERNLOGQZrmbi:
  case X86::VPTERNLOGDZ128rmbikz:
  case X86::VPTERNLOGDZ256rmbikz:
  case X86::VPTERNLOGDZrmbikz:
  case X86::VPTERNLOGQZ128rmbikz:
  case X86::VPTERNLOGQZ256rmbikz:
  case X86::VPTERNLOGQZrmbikz: {
    auto &WorkingMI = cloneIfNew(MI);
    commuteVPTERNLOG(WorkingMI, OpIdx1, OpIdx2);
    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                   OpIdx1, OpIdx2);
  }
  default: {
    if (isCommutableVPERMV3Instruction(MI.getOpcode())) {
      unsigned Opc = getCommutedVPERMV3Opcode(MI.getOpcode());
      auto &WorkingMI = cloneIfNew(MI);
      WorkingMI.setDesc(get(Opc));
      return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                     OpIdx1, OpIdx2);
    }
 
    const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(),
                                                      MI.getDesc().TSFlags);
    if (FMA3Group) {
      unsigned Opc =
        getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group);
      auto &WorkingMI = cloneIfNew(MI);
      WorkingMI.setDesc(get(Opc));
      return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                     OpIdx1, OpIdx2);
    }
 
    return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
  }
  }
}
 
bool
X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI,
                                            unsigned &SrcOpIdx1,
                                            unsigned &SrcOpIdx2,
                                            bool IsIntrinsic) const {
  uint64_t TSFlags = MI.getDesc().TSFlags;
 
  unsigned FirstCommutableVecOp = 1;
  unsigned LastCommutableVecOp = 3;
  unsigned KMaskOp = -1U;
  if (X86II::isKMasked(TSFlags)) {
    // For k-zero-masked operations it is Ok to commute the first vector
    // operand. Unless this is an intrinsic instruction.
    // For regular k-masked operations a conservative choice is done as the
    // elements of the first vector operand, for which the corresponding bit
    // in the k-mask operand is set to 0, are copied to the result of the
    // instruction.
    // TODO/FIXME: The commute still may be legal if it is known that the
    // k-mask operand is set to either all ones or all zeroes.
    // It is also Ok to commute the 1st operand if all users of MI use only
    // the elements enabled by the k-mask operand. For example,
    //   v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i]
    //                                                     : v1[i];
    //   VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 ->
    //                                  // Ok, to commute v1 in FMADD213PSZrk.
 
    // The k-mask operand has index = 2 for masked and zero-masked operations.
    KMaskOp = 2;
 
    // The operand with index = 1 is used as a source for those elements for
    // which the corresponding bit in the k-mask is set to 0.
    if (X86II::isKMergeMasked(TSFlags) || IsIntrinsic)
      FirstCommutableVecOp = 3;
 
    LastCommutableVecOp++;
  } else if (IsIntrinsic) {
    // Commuting the first operand of an intrinsic instruction isn't possible
    // unless we can prove that only the lowest element of the result is used.
    FirstCommutableVecOp = 2;
  }
 
  if (isMem(MI, LastCommutableVecOp))
    LastCommutableVecOp--;
 
  // Only the first RegOpsNum operands are commutable.
  // Also, the value 'CommuteAnyOperandIndex' is valid here as it means
  // that the operand is not specified/fixed.
  if (SrcOpIdx1 != CommuteAnyOperandIndex &&
      (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp ||
       SrcOpIdx1 == KMaskOp))
    return false;
  if (SrcOpIdx2 != CommuteAnyOperandIndex &&
      (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp ||
       SrcOpIdx2 == KMaskOp))
    return false;
 
  // Look for two different register operands assumed to be commutable
  // regardless of the FMA opcode. The FMA opcode is adjusted later.
  if (SrcOpIdx1 == CommuteAnyOperandIndex ||
      SrcOpIdx2 == CommuteAnyOperandIndex) {
    unsigned CommutableOpIdx2 = SrcOpIdx2;
 
    // At least one of operands to be commuted is not specified and
    // this method is free to choose appropriate commutable operands.
    if (SrcOpIdx1 == SrcOpIdx2)
      // Both of operands are not fixed. By default set one of commutable
      // operands to the last register operand of the instruction.
      CommutableOpIdx2 = LastCommutableVecOp;
    else if (SrcOpIdx2 == CommuteAnyOperandIndex)
      // Only one of operands is not fixed.
      CommutableOpIdx2 = SrcOpIdx1;
 
    // CommutableOpIdx2 is well defined now. Let's choose another commutable
    // operand and assign its index to CommutableOpIdx1.
    Register Op2Reg = MI.getOperand(CommutableOpIdx2).getReg();
 
    unsigned CommutableOpIdx1;
    for (CommutableOpIdx1 = LastCommutableVecOp;
         CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) {
      // Just ignore and skip the k-mask operand.
      if (CommutableOpIdx1 == KMaskOp)
        continue;
 
      // The commuted operands must have different registers.
      // Otherwise, the commute transformation does not change anything and
      // is useless then.
      if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg())
        break;
    }
 
    // No appropriate commutable operands were found.
    if (CommutableOpIdx1 < FirstCommutableVecOp)
      return false;
 
    // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2
    // to return those values.
    if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
                              CommutableOpIdx1, CommutableOpIdx2))
      return false;
  }
 
  return true;
}
 
bool X86InstrInfo::findCommutedOpIndices(const MachineInstr &MI,
                                         unsigned &SrcOpIdx1,
                                         unsigned &SrcOpIdx2) const {
  const MCInstrDesc &Desc = MI.getDesc();
  if (!Desc.isCommutable())
    return false;
 
  switch (MI.getOpcode()) {
  case X86::CMPSDrr:
  case X86::CMPSSrr:
  case X86::CMPPDrri:
  case X86::CMPPSrri:
  case X86::VCMPSDrr:
  case X86::VCMPSSrr:
  case X86::VCMPPDrri:
  case X86::VCMPPSrri:
  case X86::VCMPPDYrri:
  case X86::VCMPPSYrri:
  case X86::VCMPSDZrr:
  case X86::VCMPSSZrr:
  case X86::VCMPPDZrri:
  case X86::VCMPPSZrri:
  case X86::VCMPSHZrr:
  case X86::VCMPPHZrri:
  case X86::VCMPPHZ128rri:
  case X86::VCMPPHZ256rri:
  case X86::VCMPPDZ128rri:
  case X86::VCMPPSZ128rri:
  case X86::VCMPPDZ256rri:
  case X86::VCMPPSZ256rri:
  case X86::VCMPPDZrrik:
  case X86::VCMPPSZrrik:
  case X86::VCMPPDZ128rrik:
  case X86::VCMPPSZ128rrik:
  case X86::VCMPPDZ256rrik:
  case X86::VCMPPSZ256rrik: {
    unsigned OpOffset = X86II::isKMasked(Desc.TSFlags) ? 1 : 0;
 
    // Float comparison can be safely commuted for
    // Ordered/Unordered/Equal/NotEqual tests
    unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7;
    switch (Imm) {
    default:
      // EVEX versions can be commuted.
      if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX)
        break;
      return false;
    case 0x00: // EQUAL
    case 0x03: // UNORDERED
    case 0x04: // NOT EQUAL
    case 0x07: // ORDERED
      break;
    }
 
    // The indices of the commutable operands are 1 and 2 (or 2 and 3
    // when masked).
    // Assign them to the returned operand indices here.
    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
                                2 + OpOffset);
  }
  case X86::MOVSSrr:
    // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can
    // form sse4.1 blend. We assume VMOVSSrr/VMOVSDrr is always commutable since
    // AVX implies sse4.1.
    if (Subtarget.hasSSE41())
      return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
    return false;
  case X86::SHUFPDrri:
    // We can commute this to MOVSD.
    if (MI.getOperand(3).getImm() == 0x02)
      return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
    return false;
  case X86::MOVHLPSrr:
  case X86::UNPCKHPDrr:
  case X86::VMOVHLPSrr:
  case X86::VUNPCKHPDrr:
  case X86::VMOVHLPSZrr:
  case X86::VUNPCKHPDZ128rr:
    if (Subtarget.hasSSE2())
      return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
    return false;
  case X86::VPTERNLOGDZrri:      case X86::VPTERNLOGDZrmi:
  case X86::VPTERNLOGDZ128rri:   case X86::VPTERNLOGDZ128rmi:
  case X86::VPTERNLOGDZ256rri:   case X86::VPTERNLOGDZ256rmi:
  case X86::VPTERNLOGQZrri:      case X86::VPTERNLOGQZrmi:
  case X86::VPTERNLOGQZ128rri:   case X86::VPTERNLOGQZ128rmi:
  case X86::VPTERNLOGQZ256rri:   case X86::VPTERNLOGQZ256rmi:
  case X86::VPTERNLOGDZrrik:
  case X86::VPTERNLOGDZ128rrik:
  case X86::VPTERNLOGDZ256rrik:
  case X86::VPTERNLOGQZrrik:
  case X86::VPTERNLOGQZ128rrik:
  case X86::VPTERNLOGQZ256rrik:
  case X86::VPTERNLOGDZrrikz:    case X86::VPTERNLOGDZrmikz:
  case X86::VPTERNLOGDZ128rrikz: case X86::VPTERNLOGDZ128rmikz:
  case X86::VPTERNLOGDZ256rrikz: case X86::VPTERNLOGDZ256rmikz:
  case X86::VPTERNLOGQZrrikz:    case X86::VPTERNLOGQZrmikz:
  case X86::VPTERNLOGQZ128rrikz: case X86::VPTERNLOGQZ128rmikz:
  case X86::VPTERNLOGQZ256rrikz: case X86::VPTERNLOGQZ256rmikz:
  case X86::VPTERNLOGDZ128rmbi:
  case X86::VPTERNLOGDZ256rmbi:
  case X86::VPTERNLOGDZrmbi:
  case X86::VPTERNLOGQZ128rmbi:
  case X86::VPTERNLOGQZ256rmbi:
  case X86::VPTERNLOGQZrmbi:
  case X86::VPTERNLOGDZ128rmbikz:
  case X86::VPTERNLOGDZ256rmbikz:
  case X86::VPTERNLOGDZrmbikz:
  case X86::VPTERNLOGQZ128rmbikz:
  case X86::VPTERNLOGQZ256rmbikz:
  case X86::VPTERNLOGQZrmbikz:
    return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
  case X86::VPDPWSSDYrr:
  case X86::VPDPWSSDrr:
  case X86::VPDPWSSDSYrr:
  case X86::VPDPWSSDSrr:
  case X86::VPDPBSSDSrr:
  case X86::VPDPBSSDSYrr:
  case X86::VPDPBSSDrr:
  case X86::VPDPBSSDYrr:
  case X86::VPDPBUUDSrr:
  case X86::VPDPBUUDSYrr:
  case X86::VPDPBUUDrr:
  case X86::VPDPBUUDYrr:
  case X86::VPDPWSSDZ128r:
  case X86::VPDPWSSDZ128rk:
  case X86::VPDPWSSDZ128rkz:
  case X86::VPDPWSSDZ256r:
  case X86::VPDPWSSDZ256rk:
  case X86::VPDPWSSDZ256rkz:
  case X86::VPDPWSSDZr:
  case X86::VPDPWSSDZrk:
  case X86::VPDPWSSDZrkz:
  case X86::VPDPWSSDSZ128r:
  case X86::VPDPWSSDSZ128rk:
  case X86::VPDPWSSDSZ128rkz:
  case X86::VPDPWSSDSZ256r:
  case X86::VPDPWSSDSZ256rk:
  case X86::VPDPWSSDSZ256rkz:
  case X86::VPDPWSSDSZr:
  case X86::VPDPWSSDSZrk:
  case X86::VPDPWSSDSZrkz:
  case X86::VPMADD52HUQrr:
  case X86::VPMADD52HUQYrr:
  case X86::VPMADD52HUQZ128r:
  case X86::VPMADD52HUQZ128rk:
  case X86::VPMADD52HUQZ128rkz:
  case X86::VPMADD52HUQZ256r:
  case X86::VPMADD52HUQZ256rk:
  case X86::VPMADD52HUQZ256rkz:
  case X86::VPMADD52HUQZr:
  case X86::VPMADD52HUQZrk:
  case X86::VPMADD52HUQZrkz:
  case X86::VPMADD52LUQrr:
  case X86::VPMADD52LUQYrr:
  case X86::VPMADD52LUQZ128r:
  case X86::VPMADD52LUQZ128rk:
  case X86::VPMADD52LUQZ128rkz:
  case X86::VPMADD52LUQZ256r:
  case X86::VPMADD52LUQZ256rk:
  case X86::VPMADD52LUQZ256rkz:
  case X86::VPMADD52LUQZr:
  case X86::VPMADD52LUQZrk:
  case X86::VPMADD52LUQZrkz:
  case X86::VFMADDCPHZr:
  case X86::VFMADDCPHZrk:
  case X86::VFMADDCPHZrkz:
  case X86::VFMADDCPHZ128r:
  case X86::VFMADDCPHZ128rk:
  case X86::VFMADDCPHZ128rkz:
  case X86::VFMADDCPHZ256r:
  case X86::VFMADDCPHZ256rk:
  case X86::VFMADDCPHZ256rkz:
  case X86::VFMADDCSHZr:
  case X86::VFMADDCSHZrk:
  case X86::VFMADDCSHZrkz: {
    unsigned CommutableOpIdx1 = 2;
    unsigned CommutableOpIdx2 = 3;
    if (X86II::isKMasked(Desc.TSFlags)) {
      // Skip the mask register.
      ++CommutableOpIdx1;
      ++CommutableOpIdx2;
    }
    if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
                              CommutableOpIdx1, CommutableOpIdx2))
      return false;
    if (!MI.getOperand(SrcOpIdx1).isReg() ||
        !MI.getOperand(SrcOpIdx2).isReg())
      // No idea.
      return false;
    return true;
  }
 
  default:
    const X86InstrFMA3Group *FMA3Group = getFMA3Group(MI.getOpcode(),
                                                      MI.getDesc().TSFlags);
    if (FMA3Group)
      return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2,
                                           FMA3Group->isIntrinsic());
 
    // Handled masked instructions since we need to skip over the mask input
    // and the preserved input.
    if (X86II::isKMasked(Desc.TSFlags)) {
      // First assume that the first input is the mask operand and skip past it.
      unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1;
      unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2;
      // Check if the first input is tied. If there isn't one then we only
      // need to skip the mask operand which we did above.
      if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(),
                                             MCOI::TIED_TO) != -1)) {
        // If this is zero masking instruction with a tied operand, we need to
        // move the first index back to the first input since this must
        // be a 3 input instruction and we want the first two non-mask inputs.
        // Otherwise this is a 2 input instruction with a preserved input and
        // mask, so we need to move the indices to skip one more input.
        if (X86II::isKMergeMasked(Desc.TSFlags)) {
          ++CommutableOpIdx1;
          ++CommutableOpIdx2;
        } else {
          --CommutableOpIdx1;
        }
      }
 
      if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
                                CommutableOpIdx1, CommutableOpIdx2))
        return false;
 
      if (!MI.getOperand(SrcOpIdx1).isReg() ||
          !MI.getOperand(SrcOpIdx2).isReg())
        // No idea.
        return false;
      return true;
    }
 
    return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
  }
  return false;
}
 
static bool isConvertibleLEA(MachineInstr *MI) {
  unsigned Opcode = MI->getOpcode();
  if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
      Opcode != X86::LEA64_32r)
    return false;
 
  const MachineOperand &Scale = MI->getOperand(1 + X86::AddrScaleAmt);
  const MachineOperand &Disp = MI->getOperand(1 + X86::AddrDisp);
  const MachineOperand &Segment = MI->getOperand(1 + X86::AddrSegmentReg);
 
  if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 ||
      Scale.getImm() > 1)
    return false;
 
  return true;
}
 
bool X86InstrInfo::hasCommutePreference(MachineInstr &MI, bool &Commute) const {
  // Currently we're interested in following sequence only.
  //   r3 = lea r1, r2
  //   r5 = add r3, r4
  // Both r3 and r4 are killed in add, we hope the add instruction has the
  // operand order
  //   r5 = add r4, r3
  // So later in X86FixupLEAs the lea instruction can be rewritten as add.
  unsigned Opcode = MI.getOpcode();
  if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
    return false;
 
  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
  Register Reg1 = MI.getOperand(1).getReg();
  Register Reg2 = MI.getOperand(2).getReg();
 
  // Check if Reg1 comes from LEA in the same MBB.
  if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg1)) {
    if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) {
      Commute = true;
      return true;
    }
  }
 
  // Check if Reg2 comes from LEA in the same MBB.
  if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg2)) {
    if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) {
      Commute = false;
      return true;
    }
  }
 
  return false;
}
 
int X86::getCondSrcNoFromDesc(const MCInstrDesc &MCID) {
  unsigned Opcode = MCID.getOpcode();
  if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode)))
    return -1;
  // Assume that condition code is always the last use operand.
  unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs();
  return NumUses - 1;
}
 
X86::CondCode X86::getCondFromMI(const MachineInstr &MI) {
  const MCInstrDesc &MCID = MI.getDesc();
  int CondNo = getCondSrcNoFromDesc(MCID);
  if (CondNo < 0)
    return X86::COND_INVALID;
  CondNo += MCID.getNumDefs();
  return static_cast<X86::CondCode>(MI.getOperand(CondNo).getImm());
}
 
X86::CondCode X86::getCondFromBranch(const MachineInstr &MI) {
  return X86::isJCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
                                    : X86::COND_INVALID;
}
 
X86::CondCode X86::getCondFromSETCC(const MachineInstr &MI) {
  return X86::isSETCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
                                      : X86::COND_INVALID;
}
 
X86::CondCode X86::getCondFromCMov(const MachineInstr &MI) {
  return X86::isCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
                                       : X86::COND_INVALID;
}
 
/// Return the inverse of the specified condition,
/// e.g. turning COND_E to COND_NE.
X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) {
  switch (CC) {
  default: llvm_unreachable("Illegal condition code!");
  case X86::COND_E:  return X86::COND_NE;
  case X86::COND_NE: return X86::COND_E;
  case X86::COND_L:  return X86::COND_GE;
  case X86::COND_LE: return X86::COND_G;
  case X86::COND_G:  return X86::COND_LE;
  case X86::COND_GE: return X86::COND_L;
  case X86::COND_B:  return X86::COND_AE;
  case X86::COND_BE: return X86::COND_A;
  case X86::COND_A:  return X86::COND_BE;
  case X86::COND_AE: return X86::COND_B;
  case X86::COND_S:  return X86::COND_NS;
  case X86::COND_NS: return X86::COND_S;
  case X86::COND_P:  return X86::COND_NP;
  case X86::COND_NP: return X86::COND_P;
  case X86::COND_O:  return X86::COND_NO;
  case X86::COND_NO: return X86::COND_O;
  case X86::COND_NE_OR_P:  return X86::COND_E_AND_NP;
  case X86::COND_E_AND_NP: return X86::COND_NE_OR_P;
  }
}
 
/// Assuming the flags are set by MI(a,b), return the condition code if we
/// modify the instructions such that flags are set by MI(b,a).
static X86::CondCode getSwappedCondition(X86::CondCode CC) {
  switch (CC) {
  default: return X86::COND_INVALID;
  case X86::COND_E:  return X86::COND_E;
  case X86::COND_NE: return X86::COND_NE;
  case X86::COND_L:  return X86::COND_G;
  case X86::COND_LE: return X86::COND_GE;
  case X86::COND_G:  return X86::COND_L;
  case X86::COND_GE: return X86::COND_LE;
  case X86::COND_B:  return X86::COND_A;
  case X86::COND_BE: return X86::COND_AE;
  case X86::COND_A:  return X86::COND_B;
  case X86::COND_AE: return X86::COND_BE;
  }
}
 
std::pair<X86::CondCode, bool>
X86::getX86ConditionCode(CmpInst::Predicate Predicate) {
  X86::CondCode CC = X86::COND_INVALID;
  bool NeedSwap = false;
  switch (Predicate) {
  default: break;
  // Floating-point Predicates
  case CmpInst::FCMP_UEQ: CC = X86::COND_E;       break;
  case CmpInst::FCMP_OLT: NeedSwap = true;        [[fallthrough]];
  case CmpInst::FCMP_OGT: CC = X86::COND_A;       break;
  case CmpInst::FCMP_OLE: NeedSwap = true;        [[fallthrough]];
  case CmpInst::FCMP_OGE: CC = X86::COND_AE;      break;
  case CmpInst::FCMP_UGT: NeedSwap = true;        [[fallthrough]];
  case CmpInst::FCMP_ULT: CC = X86::COND_B;       break;
  case CmpInst::FCMP_UGE: NeedSwap = true;        [[fallthrough]];
  case CmpInst::FCMP_ULE: CC = X86::COND_BE;      break;
  case CmpInst::FCMP_ONE: CC = X86::COND_NE;      break;
  case CmpInst::FCMP_UNO: CC = X86::COND_P;       break;
  case CmpInst::FCMP_ORD: CC = X86::COND_NP;      break;
  case CmpInst::FCMP_OEQ:                         [[fallthrough]];
  case CmpInst::FCMP_UNE: CC = X86::COND_INVALID; break;
 
  // Integer Predicates
  case CmpInst::ICMP_EQ:  CC = X86::COND_E;       break;
  case CmpInst::ICMP_NE:  CC = X86::COND_NE;      break;
  case CmpInst::ICMP_UGT: CC = X86::COND_A;       break;
  case CmpInst::ICMP_UGE: CC = X86::COND_AE;      break;
  case CmpInst::ICMP_ULT: CC = X86::COND_B;       break;
  case CmpInst::ICMP_ULE: CC = X86::COND_BE;      break;
  case CmpInst::ICMP_SGT: CC = X86::COND_G;       break;
  case CmpInst::ICMP_SGE: CC = X86::COND_GE;      break;
  case CmpInst::ICMP_SLT: CC = X86::COND_L;       break;
  case CmpInst::ICMP_SLE: CC = X86::COND_LE;      break;
  }
 
  return std::make_pair(CC, NeedSwap);
}
 
/// Return a cmov opcode for the given register size in bytes, and operand type.
unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand) {
  switch(RegBytes) {
  default: llvm_unreachable("Illegal register size!");
  case 2: return HasMemoryOperand ? X86::CMOV16rm : X86::CMOV16rr;
  case 4: return HasMemoryOperand ? X86::CMOV32rm : X86::CMOV32rr;
  case 8: return HasMemoryOperand ? X86::CMOV64rm : X86::CMOV64rr;
  }
}
 
/// Get the VPCMP immediate for the given condition.
unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) {
  switch (CC) {
  default: llvm_unreachable("Unexpected SETCC condition");
  case ISD::SETNE:  return 4;
  case ISD::SETEQ:  return 0;
  case ISD::SETULT:
  case ISD::SETLT: return 1;
  case ISD::SETUGT:
  case ISD::SETGT: return 6;
  case ISD::SETUGE:
  case ISD::SETGE: return 5;
  case ISD::SETULE:
  case ISD::SETLE: return 2;
  }
}
 
/// Get the VPCMP immediate if the operands are swapped.
unsigned X86::getSwappedVPCMPImm(unsigned Imm) {
  switch (Imm) {
  default: llvm_unreachable("Unreachable!");
  case 0x01: Imm = 0x06; break; // LT  -> NLE
  case 0x02: Imm = 0x05; break; // LE  -> NLT
  case 0x05: Imm = 0x02; break; // NLT -> LE
  case 0x06: Imm = 0x01; break; // NLE -> LT
  case 0x00: // EQ
  case 0x03: // FALSE
  case 0x04: // NE
  case 0x07: // TRUE
    break;
  }
 
  return Imm;
}
 
/// Get the VPCOM immediate if the operands are swapped.
unsigned X86::getSwappedVPCOMImm(unsigned Imm) {
  switch (Imm) {
  default: llvm_unreachable("Unreachable!");
  case 0x00: Imm = 0x02; break; // LT -> GT
  case 0x01: Imm = 0x03; break; // LE -> GE
  case 0x02: Imm = 0x00; break; // GT -> LT
  case 0x03: Imm = 0x01; break; // GE -> LE
  case 0x04: // EQ
  case 0x05: // NE
  case 0x06: // FALSE
  case 0x07: // TRUE
    break;
  }
 
  return Imm;
}
 
/// Get the VCMP immediate if the operands are swapped.
unsigned X86::getSwappedVCMPImm(unsigned Imm) {
  // Only need the lower 2 bits to distinquish.
  switch (Imm & 0x3) {
  default: llvm_unreachable("Unreachable!");
  case 0x00: case 0x03:
    // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted.
    break;
  case 0x01: case 0x02:
    // Need to toggle bits 3:0. Bit 4 stays the same.
    Imm ^= 0xf;
    break;
  }
 
  return Imm;
}
 
/// Return true if the Reg is X87 register.
static bool isX87Reg(unsigned Reg) {
  return (Reg == X86::FPCW || Reg == X86::FPSW ||
          (Reg >= X86::ST0 && Reg <= X86::ST7));
}
 
/// check if the instruction is X87 instruction
bool X86::isX87Instruction(MachineInstr &MI) {
  for (const MachineOperand &MO : MI.operands()) {
    if (!MO.isReg())
      continue;
    if (isX87Reg(MO.getReg()))
      return true;
  }
  return false;
}
 
bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const {
  switch (MI.getOpcode()) {
  case X86::TCRETURNdi:
  case X86::TCRETURNri:
  case X86::TCRETURNmi:
  case X86::TCRETURNdi64:
  case X86::TCRETURNri64:
  case X86::TCRETURNmi64:
    return true;
  default:
    return false;
  }
}
 
bool X86InstrInfo::canMakeTailCallConditional(
    SmallVectorImpl<MachineOperand> &BranchCond,
    const MachineInstr &TailCall) const {
 
  const MachineFunction *MF = TailCall.getMF();
 
  if (MF->getTarget().getCodeModel() == CodeModel::Kernel) {
    // Kernel patches thunk calls in runtime, these should never be conditional.
    const MachineOperand &Target = TailCall.getOperand(0);
    if (Target.isSymbol()) {
      StringRef Symbol(Target.getSymbolName());
      // this is currently only relevant to r11/kernel indirect thunk.
      if (Symbol.equals("__x86_indirect_thunk_r11"))
        return false;
    }
  }
 
  if (TailCall.getOpcode() != X86::TCRETURNdi &&
      TailCall.getOpcode() != X86::TCRETURNdi64) {
    // Only direct calls can be done with a conditional branch.
    return false;
  }
 
  if (Subtarget.isTargetWin64() && MF->hasWinCFI()) {
    // Conditional tail calls confuse the Win64 unwinder.
    return false;
  }
 
  assert(BranchCond.size() == 1);
  if (BranchCond[0].getImm() > X86::LAST_VALID_COND) {
    // Can't make a conditional tail call with this condition.
    return false;
  }
 
  const X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
  if (X86FI->getTCReturnAddrDelta() != 0 ||
      TailCall.getOperand(1).getImm() != 0) {
    // A conditional tail call cannot do any stack adjustment.
    return false;
  }
 
  return true;
}
 
void X86InstrInfo::replaceBranchWithTailCall(
    MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &BranchCond,
    const MachineInstr &TailCall) const {
  assert(canMakeTailCallConditional(BranchCond, TailCall));
 
  MachineBasicBlock::iterator I = MBB.end();
  while (I != MBB.begin()) {
    --I;
    if (I->isDebugInstr())
      continue;
    if (!I->isBranch())
      assert(0 && "Can't find the branch to replace!");
 
    X86::CondCode CC = X86::getCondFromBranch(*I);
    assert(BranchCond.size() == 1);
    if (CC != BranchCond[0].getImm())
      continue;
 
    break;
  }
 
  unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
                                                         : X86::TCRETURNdi64cc;
 
  auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc));
  MIB->addOperand(TailCall.getOperand(0)); // Destination.
  MIB.addImm(0); // Stack offset (not used).
  MIB->addOperand(BranchCond[0]); // Condition.
  MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters.
 
  // Add implicit uses and defs of all live regs potentially clobbered by the
  // call. This way they still appear live across the call.
  LivePhysRegs LiveRegs(getRegisterInfo());
  LiveRegs.addLiveOuts(MBB);
  SmallVector<std::pair<MCPhysReg, const MachineOperand *>, 8> Clobbers;
  LiveRegs.stepForward(*MIB, Clobbers);
  for (const auto &C : Clobbers) {
    MIB.addReg(C.first, RegState::Implicit);
    MIB.addReg(C.first, RegState::Implicit | RegState::Define);
  }
 
  I->eraseFromParent();
}
 
// Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may
// not be a fallthrough MBB now due to layout changes). Return nullptr if the
// fallthrough MBB cannot be identified.
static MachineBasicBlock *getFallThroughMBB(MachineBasicBlock *MBB,
                                            MachineBasicBlock *TBB) {
  // Look for non-EHPad successors other than TBB. If we find exactly one, it
  // is the fallthrough MBB. If we find zero, then TBB is both the target MBB
  // and fallthrough MBB. If we find more than one, we cannot identify the
  // fallthrough MBB and should return nullptr.
  MachineBasicBlock *FallthroughBB = nullptr;
  for (MachineBasicBlock *Succ : MBB->successors()) {
    if (Succ->isEHPad() || (Succ == TBB && FallthroughBB))
      continue;
    // Return a nullptr if we found more than one fallthrough successor.
    if (FallthroughBB && FallthroughBB != TBB)
      return nullptr;
    FallthroughBB = Succ;
  }
  return FallthroughBB;
}
 
bool X86InstrInfo::AnalyzeBranchImpl(
    MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB,
    SmallVectorImpl<MachineOperand> &Cond,
    SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const {
 
  // Start from the bottom of the block and work up, examining the
  // terminator instructions.
  MachineBasicBlock::iterator I = MBB.end();
  MachineBasicBlock::iterator UnCondBrIter = MBB.end();
  while (I != MBB.begin()) {
    --I;
    if (I->isDebugInstr())
      continue;
 
    // Working from the bottom, when we see a non-terminator instruction, we're
    // done.
    if (!isUnpredicatedTerminator(*I))
      break;
 
    // A terminator that isn't a branch can't easily be handled by this
    // analysis.
    if (!I->isBranch())
      return true;
 
    // Handle unconditional branches.
    if (I->getOpcode() == X86::JMP_1) {
      UnCondBrIter = I;
 
      if (!AllowModify) {
        TBB = I->getOperand(0).getMBB();
        continue;
      }
 
      // If the block has any instructions after a JMP, delete them.
      MBB.erase(std::next(I), MBB.end());
 
      Cond.clear();
      FBB = nullptr;
 
      // Delete the JMP if it's equivalent to a fall-through.
      if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
        TBB = nullptr;
        I->eraseFromParent();
        I = MBB.end();
        UnCondBrIter = MBB.end();
        continue;
      }
 
      // TBB is used to indicate the unconditional destination.
      TBB = I->getOperand(0).getMBB();
      continue;
    }
 
    // Handle conditional branches.
    X86::CondCode BranchCode = X86::getCondFromBranch(*I);
    if (BranchCode == X86::COND_INVALID)
      return true;  // Can't handle indirect branch.
 
    // In practice we should never have an undef eflags operand, if we do
    // abort here as we are not prepared to preserve the flag.
    if (I->findRegisterUseOperand(X86::EFLAGS)->isUndef())
      return true;
 
    // Working from the bottom, handle the first conditional branch.
    if (Cond.empty()) {
      FBB = TBB;
      TBB = I->getOperand(0).getMBB();
      Cond.push_back(MachineOperand::CreateImm(BranchCode));
      CondBranches.push_back(&*I);
      continue;
    }
 
    // Handle subsequent conditional branches. Only handle the case where all
    // conditional branches branch to the same destination and their condition
    // opcodes fit one of the special multi-branch idioms.
    assert(Cond.size() == 1);
    assert(TBB);
 
    // If the conditions are the same, we can leave them alone.
    X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm();
    auto NewTBB = I->getOperand(0).getMBB();
    if (OldBranchCode == BranchCode && TBB == NewTBB)
      continue;
 
    // If they differ, see if they fit one of the known patterns. Theoretically,
    // we could handle more patterns here, but we shouldn't expect to see them
    // if instruction selection has done a reasonable job.
    if (TBB == NewTBB &&
               ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) ||
                (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) {
      BranchCode = X86::COND_NE_OR_P;
    } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) ||
               (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) {
      if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB)))
        return true;
 
      // X86::COND_E_AND_NP usually has two different branch destinations.
      //
      // JP B1
      // JE B2
      // JMP B1
      // B1:
      // B2:
      //
      // Here this condition branches to B2 only if NP && E. It has another
      // equivalent form:
      //
      // JNE B1
      // JNP B2
      // JMP B1
      // B1:
      // B2:
      //
      // Similarly it branches to B2 only if E && NP. That is why this condition
      // is named with COND_E_AND_NP.
      BranchCode = X86::COND_E_AND_NP;
    } else
      return true;
 
    // Update the MachineOperand.
    Cond[0].setImm(BranchCode);
    CondBranches.push_back(&*I);
  }
 
  return false;
}
 
bool X86InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
                                 MachineBasicBlock *&TBB,
                                 MachineBasicBlock *&FBB,
                                 SmallVectorImpl<MachineOperand> &Cond,
                                 bool AllowModify) const {
  SmallVector<MachineInstr *, 4> CondBranches;
  return AnalyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify);
}
 
bool X86InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
                                          MachineBranchPredicate &MBP,
                                          bool AllowModify) const {
  using namespace std::placeholders;
 
  SmallVector<MachineOperand, 4> Cond;
  SmallVector<MachineInstr *, 4> CondBranches;
  if (AnalyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches,
                        AllowModify))
    return true;
 
  if (Cond.size() != 1)
    return true;
 
  assert(MBP.TrueDest && "expected!");
 
  if (!MBP.FalseDest)
    MBP.FalseDest = MBB.getNextNode();
 
  const TargetRegisterInfo *TRI = &getRegisterInfo();
 
  MachineInstr *ConditionDef = nullptr;
  bool SingleUseCondition = true;
 
  for (MachineInstr &MI : llvm::drop_begin(llvm::reverse(MBB))) {
    if (MI.modifiesRegister(X86::EFLAGS, TRI)) {
      ConditionDef = &MI;
      break;
    }
 
    if (MI.readsRegister(X86::EFLAGS, TRI))
      SingleUseCondition = false;
  }
 
  if (!ConditionDef)
    return true;
 
  if (SingleUseCondition) {
    for (auto *Succ : MBB.successors())
      if (Succ->isLiveIn(X86::EFLAGS))
        SingleUseCondition = false;
  }
 
  MBP.ConditionDef = ConditionDef;
  MBP.SingleUseCondition = SingleUseCondition;
 
  // Currently we only recognize the simple pattern:
  //
  //   test %reg, %reg
  //   je %label
  //
  const unsigned TestOpcode =
      Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
 
  if (ConditionDef->getOpcode() == TestOpcode &&
      ConditionDef->getNumOperands() == 3 &&
      ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) &&
      (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) {
    MBP.LHS = ConditionDef->getOperand(0);
    MBP.RHS = MachineOperand::CreateImm(0);
    MBP.Predicate = Cond[0].getImm() == X86::COND_NE
                        ? MachineBranchPredicate::PRED_NE
                        : MachineBranchPredicate::PRED_EQ;
    return false;
  }
 
  return true;
}
 
unsigned X86InstrInfo::removeBranch(MachineBasicBlock &MBB,
                                    int *BytesRemoved) const {
  assert(!BytesRemoved && "code size not handled");
 
  MachineBasicBlock::iterator I = MBB.end();
  unsigned Count = 0;
 
  while (I != MBB.begin()) {
    --I;
    if (I->isDebugInstr())
      continue;
    if (I->getOpcode() != X86::JMP_1 &&
        X86::getCondFromBranch(*I) == X86::COND_INVALID)
      break;
    // Remove the branch.
    I->eraseFromParent();
    I = MBB.end();
    ++Count;
  }
 
  return Count;
}
 
unsigned X86InstrInfo::insertBranch(MachineBasicBlock &MBB,
                                    MachineBasicBlock *TBB,
                                    MachineBasicBlock *FBB,
                                    ArrayRef<MachineOperand> Cond,
                                    const DebugLoc &DL,
                                    int *BytesAdded) const {
  // Shouldn't be a fall through.
  assert(TBB && "insertBranch must not be told to insert a fallthrough");
  assert((Cond.size() == 1 || Cond.size() == 0) &&
         "X86 branch conditions have one component!");
  assert(!BytesAdded && "code size not handled");
 
  if (Cond.empty()) {
    // Unconditional branch?
    assert(!FBB && "Unconditional branch with multiple successors!");
    BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB);
    return 1;
  }
 
  // If FBB is null, it is implied to be a fall-through block.
  bool FallThru = FBB == nullptr;
 
  // Conditional branch.
  unsigned Count = 0;
  X86::CondCode CC = (X86::CondCode)Cond[0].getImm();
  switch (CC) {
  case X86::COND_NE_OR_P:
    // Synthesize NE_OR_P with two branches.
    BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NE);
    ++Count;
    BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_P);
    ++Count;
    break;
  case X86::COND_E_AND_NP:
    // Use the next block of MBB as FBB if it is null.
    if (FBB == nullptr) {
      FBB = getFallThroughMBB(&MBB, TBB);
      assert(FBB && "MBB cannot be the last block in function when the false "
                    "body is a fall-through.");
    }
    // Synthesize COND_E_AND_NP with two branches.
    BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(FBB).addImm(X86::COND_NE);
    ++Count;
    BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NP);
    ++Count;
    break;
  default: {
    BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(CC);
    ++Count;
  }
  }
  if (!FallThru) {
    // Two-way Conditional branch. Insert the second branch.
    BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB);
    ++Count;
  }
  return Count;
}
 
bool X86InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
                                   ArrayRef<MachineOperand> Cond,
                                   Register DstReg, Register TrueReg,
                                   Register FalseReg, int &CondCycles,
                                   int &TrueCycles, int &FalseCycles) const {
  // Not all subtargets have cmov instructions.
  if (!Subtarget.canUseCMOV())
    return false;
  if (Cond.size() != 1)
    return false;
  // We cannot do the composite conditions, at least not in SSA form.
  if ((X86::CondCode)Cond[0].getImm() > X86::LAST_VALID_COND)
    return false;
 
  // Check register classes.
  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
  const TargetRegisterClass *RC =
    RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
  if (!RC)
    return false;
 
  // We have cmov instructions for 16, 32, and 64 bit general purpose registers.
  if (X86::GR16RegClass.hasSubClassEq(RC) ||
      X86::GR32RegClass.hasSubClassEq(RC) ||
      X86::GR64RegClass.hasSubClassEq(RC)) {
    // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy
    // Bridge. Probably Ivy Bridge as well.
    CondCycles = 2;
    TrueCycles = 2;
    FalseCycles = 2;
    return true;
  }
 
  // Can't do vectors.
  return false;
}
 
void X86InstrInfo::insertSelect(MachineBasicBlock &MBB,
                                MachineBasicBlock::iterator I,
                                const DebugLoc &DL, Register DstReg,
                                ArrayRef<MachineOperand> Cond, Register TrueReg,
                                Register FalseReg) const {
  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
  const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
  const TargetRegisterClass &RC = *MRI.getRegClass(DstReg);
  assert(Cond.size() == 1 && "Invalid Cond array");
  unsigned Opc = X86::getCMovOpcode(TRI.getRegSizeInBits(RC) / 8,
                                    false /*HasMemoryOperand*/);
  BuildMI(MBB, I, DL, get(Opc), DstReg)
      .addReg(FalseReg)
      .addReg(TrueReg)
      .addImm(Cond[0].getImm());
}
 
/// Test if the given register is a physical h register.
static bool isHReg(unsigned Reg) {
  return X86::GR8_ABCD_HRegClass.contains(Reg);
}
 
// Try and copy between VR128/VR64 and GR64 registers.
static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
                                        const X86Subtarget &Subtarget) {
  bool HasAVX = Subtarget.hasAVX();
  bool HasAVX512 = Subtarget.hasAVX512();
 
  // SrcReg(MaskReg) -> DestReg(GR64)
  // SrcReg(MaskReg) -> DestReg(GR32)
 
  // All KMASK RegClasses hold the same k registers, can be tested against anyone.
  if (X86::VK16RegClass.contains(SrcReg)) {
    if (X86::GR64RegClass.contains(DestReg)) {
      assert(Subtarget.hasBWI());
      return X86::KMOVQrk;
    }
    if (X86::GR32RegClass.contains(DestReg))
      return Subtarget.hasBWI() ? X86::KMOVDrk : X86::KMOVWrk;
  }
 
  // SrcReg(GR64) -> DestReg(MaskReg)
  // SrcReg(GR32) -> DestReg(MaskReg)
 
  // All KMASK RegClasses hold the same k registers, can be tested against anyone.
  if (X86::VK16RegClass.contains(DestReg)) {
    if (X86::GR64RegClass.contains(SrcReg)) {
      assert(Subtarget.hasBWI());
      return X86::KMOVQkr;
    }
    if (X86::GR32RegClass.contains(SrcReg))
      return Subtarget.hasBWI() ? X86::KMOVDkr : X86::KMOVWkr;
  }
 
 
  // SrcReg(VR128) -> DestReg(GR64)
  // SrcReg(VR64)  -> DestReg(GR64)
  // SrcReg(GR64)  -> DestReg(VR128)
  // SrcReg(GR64)  -> DestReg(VR64)
 
  if (X86::GR64RegClass.contains(DestReg)) {
    if (X86::VR128XRegClass.contains(SrcReg))
      // Copy from a VR128 register to a GR64 register.
      return HasAVX512 ? X86::VMOVPQIto64Zrr :
             HasAVX    ? X86::VMOVPQIto64rr  :
                         X86::MOVPQIto64rr;
    if (X86::VR64RegClass.contains(SrcReg))
      // Copy from a VR64 register to a GR64 register.
      return X86::MMX_MOVD64from64rr;
  } else if (X86::GR64RegClass.contains(SrcReg)) {
    // Copy from a GR64 register to a VR128 register.
    if (X86::VR128XRegClass.contains(DestReg))
      return HasAVX512 ? X86::VMOV64toPQIZrr :
             HasAVX    ? X86::VMOV64toPQIrr  :
                         X86::MOV64toPQIrr;
    // Copy from a GR64 register to a VR64 register.
    if (X86::VR64RegClass.contains(DestReg))
      return X86::MMX_MOVD64to64rr;
  }
 
  // SrcReg(VR128) -> DestReg(GR32)
  // SrcReg(GR32)  -> DestReg(VR128)
 
  if (X86::GR32RegClass.contains(DestReg) &&
      X86::VR128XRegClass.contains(SrcReg))
    // Copy from a VR128 register to a GR32 register.
    return HasAVX512 ? X86::VMOVPDI2DIZrr :
           HasAVX    ? X86::VMOVPDI2DIrr  :
                       X86::MOVPDI2DIrr;
 
  if (X86::VR128XRegClass.contains(DestReg) &&
      X86::GR32RegClass.contains(SrcReg))
    // Copy from a VR128 register to a VR128 register.
    return HasAVX512 ? X86::VMOVDI2PDIZrr :
           HasAVX    ? X86::VMOVDI2PDIrr  :
                       X86::MOVDI2PDIrr;
  return 0;
}
 
void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator MI,
                               const DebugLoc &DL, MCRegister DestReg,
                               MCRegister SrcReg, bool KillSrc) const {
  // First deal with the normal symmetric copies.
  bool HasAVX = Subtarget.hasAVX();
  bool HasVLX = Subtarget.hasVLX();
  unsigned Opc = 0;
  if (X86::GR64RegClass.contains(DestReg, SrcReg))
    Opc = X86::MOV64rr;
  else if (X86::GR32RegClass.contains(DestReg, SrcReg))
    Opc = X86::MOV32rr;
  else if (X86::GR16RegClass.contains(DestReg, SrcReg))
    Opc = X86::MOV16rr;
  else if (X86::GR8RegClass.contains(DestReg, SrcReg)) {
    // Copying to or from a physical H register on x86-64 requires a NOREX
    // move.  Otherwise use a normal move.
    if ((isHReg(DestReg) || isHReg(SrcReg)) &&
        Subtarget.is64Bit()) {
      Opc = X86::MOV8rr_NOREX;
      // Both operands must be encodable without an REX prefix.
      assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&
             "8-bit H register can not be copied outside GR8_NOREX");
    } else
      Opc = X86::MOV8rr;
  }
  else if (X86::VR64RegClass.contains(DestReg, SrcReg))
    Opc = X86::MMX_MOVQ64rr;
  else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) {
    if (HasVLX)
      Opc = X86::VMOVAPSZ128rr;
    else if (X86::VR128RegClass.contains(DestReg, SrcReg))
      Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
    else {
      // If this an extended register and we don't have VLX we need to use a
      // 512-bit move.
      Opc = X86::VMOVAPSZrr;
      const TargetRegisterInfo *TRI = &getRegisterInfo();
      DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_xmm,
                                         &X86::VR512RegClass);
      SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm,
                                        &X86::VR512RegClass);
    }
  } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) {
    if (HasVLX)
      Opc = X86::VMOVAPSZ256rr;
    else if (X86::VR256RegClass.contains(DestReg, SrcReg))
      Opc = X86::VMOVAPSYrr;
    else {
      // If this an extended register and we don't have VLX we need to use a
      // 512-bit move.
      Opc = X86::VMOVAPSZrr;
      const TargetRegisterInfo *TRI = &getRegisterInfo();
      DestReg = TRI->getMatchingSuperReg(DestReg, X86::sub_ymm,
                                         &X86::VR512RegClass);
      SrcReg = TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm,
                                        &X86::VR512RegClass);
    }
  } else if (X86::VR512RegClass.contains(DestReg, SrcReg))
    Opc = X86::VMOVAPSZrr;
  // All KMASK RegClasses hold the same k registers, can be tested against anyone.
  else if (X86::VK16RegClass.contains(DestReg, SrcReg))
    Opc = Subtarget.hasBWI() ? X86::KMOVQkk : X86::KMOVWkk;
  if (!Opc)
    Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget);
 
  if (Opc) {
    BuildMI(MBB, MI, DL, get(Opc), DestReg)
      .addReg(SrcReg, getKillRegState(KillSrc));
    return;
  }
 
  if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
    // FIXME: We use a fatal error here because historically LLVM has tried
    // lower some of these physreg copies and we want to ensure we get
    // reasonable bug reports if someone encounters a case no other testing
    // found. This path should be removed after the LLVM 7 release.
    report_fatal_error("Unable to copy EFLAGS physical register!");
  }
 
  LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to "
                    << RI.getName(DestReg) << '\n');
  report_fatal_error("Cannot emit physreg copy instruction");
}
 
std::optional<DestSourcePair>
X86InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
  if (MI.isMoveReg())
    return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
  return std::nullopt;
}
 
static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI) {
  if (STI.hasFP16())
    return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
  if (Load)
    return STI.hasAVX512() ? X86::VMOVSSZrm
           : STI.hasAVX()  ? X86::VMOVSSrm
                           : X86::MOVSSrm;
  else
    return STI.hasAVX512() ? X86::VMOVSSZmr
           : STI.hasAVX()  ? X86::VMOVSSmr
                           : X86::MOVSSmr;
}
 
static unsigned getLoadStoreRegOpcode(Register Reg,
                                      const TargetRegisterClass *RC,
                                      bool IsStackAligned,
                                      const X86Subtarget &STI, bool Load) {
  bool HasAVX = STI.hasAVX();
  bool HasAVX512 = STI.hasAVX512();
  bool HasVLX = STI.hasVLX();
 
  switch (STI.getRegisterInfo()->getSpillSize(*RC)) {
  default:
    llvm_unreachable("Unknown spill size");
  case 1:
    assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass");
    if (STI.is64Bit())
      // Copying to or from a physical H register on x86-64 requires a NOREX
      // move.  Otherwise use a normal move.
      if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
        return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
    return Load ? X86::MOV8rm : X86::MOV8mr;
  case 2:
    if (X86::VK16RegClass.hasSubClassEq(RC))
      return Load ? X86::KMOVWkm : X86::KMOVWmk;
    assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass");
    return Load ? X86::MOV16rm : X86::MOV16mr;
  case 4:
    if (X86::GR32RegClass.hasSubClassEq(RC))
      return Load ? X86::MOV32rm : X86::MOV32mr;
    if (X86::FR32XRegClass.hasSubClassEq(RC))
      return Load ?
        (HasAVX512 ? X86::VMOVSSZrm_alt :
         HasAVX    ? X86::VMOVSSrm_alt :
                     X86::MOVSSrm_alt) :
        (HasAVX512 ? X86::VMOVSSZmr :
         HasAVX    ? X86::VMOVSSmr :
                     X86::MOVSSmr);
    if (X86::RFP32RegClass.hasSubClassEq(RC))
      return Load ? X86::LD_Fp32m : X86::ST_Fp32m;
    if (X86::VK32RegClass.hasSubClassEq(RC)) {
      assert(STI.hasBWI() && "KMOVD requires BWI");
      return Load ? X86::KMOVDkm : X86::KMOVDmk;
    }
    // All of these mask pair classes have the same spill size, the same kind
    // of kmov instructions can be used with all of them.
    if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
        X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
        X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
        X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
        X86::VK16PAIRRegClass.hasSubClassEq(RC))
      return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
    if (X86::FR16RegClass.hasSubClassEq(RC) ||
        X86::FR16XRegClass.hasSubClassEq(RC))
      return getLoadStoreOpcodeForFP16(Load, STI);
    llvm_unreachable("Unknown 4-byte regclass");
  case 8:
    if (X86::GR64RegClass.hasSubClassEq(RC))
      return Load ? X86::MOV64rm : X86::MOV64mr;
    if (X86::FR64XRegClass.hasSubClassEq(RC))
      return Load ?
        (HasAVX512 ? X86::VMOVSDZrm_alt :
         HasAVX    ? X86::VMOVSDrm_alt :
                     X86::MOVSDrm_alt) :
        (HasAVX512 ? X86::VMOVSDZmr :
         HasAVX    ? X86::VMOVSDmr :
                     X86::MOVSDmr);
    if (X86::VR64RegClass.hasSubClassEq(RC))
      return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
    if (X86::RFP64RegClass.hasSubClassEq(RC))
      return Load ? X86::LD_Fp64m : X86::ST_Fp64m;
    if (X86::VK64RegClass.hasSubClassEq(RC)) {
      assert(STI.hasBWI() && "KMOVQ requires BWI");
      return Load ? X86::KMOVQkm : X86::KMOVQmk;
    }
    llvm_unreachable("Unknown 8-byte regclass");
  case 10:
    assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass");
    return Load ? X86::LD_Fp80m : X86::ST_FpP80m;
  case 16: {
    if (X86::VR128XRegClass.hasSubClassEq(RC)) {
      // If stack is realigned we can use aligned stores.
      if (IsStackAligned)
        return Load ?
          (HasVLX    ? X86::VMOVAPSZ128rm :
           HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX :
           HasAVX    ? X86::VMOVAPSrm :
                       X86::MOVAPSrm):
          (HasVLX    ? X86::VMOVAPSZ128mr :
           HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX :
           HasAVX    ? X86::VMOVAPSmr :
                       X86::MOVAPSmr);
      else
        return Load ?
          (HasVLX    ? X86::VMOVUPSZ128rm :
           HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX :
           HasAVX    ? X86::VMOVUPSrm :
                       X86::MOVUPSrm):
          (HasVLX    ? X86::VMOVUPSZ128mr :
           HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX :
           HasAVX    ? X86::VMOVUPSmr :
                       X86::MOVUPSmr);
    }
    llvm_unreachable("Unknown 16-byte regclass");
  }
  case 32:
    assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass");
    // If stack is realigned we can use aligned stores.
    if (IsStackAligned)
      return Load ?
        (HasVLX    ? X86::VMOVAPSZ256rm :
         HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX :
                     X86::VMOVAPSYrm) :
        (HasVLX    ? X86::VMOVAPSZ256mr :
         HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX :
                     X86::VMOVAPSYmr);
    else
      return Load ?
        (HasVLX    ? X86::VMOVUPSZ256rm :
         HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX :
                     X86::VMOVUPSYrm) :
        (HasVLX    ? X86::VMOVUPSZ256mr :
         HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX :
                     X86::VMOVUPSYmr);
  case 64:
    assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass");
    assert(STI.hasAVX512() && "Using 512-bit register requires AVX512");
    if (IsStackAligned)
      return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
    else
      return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
  case 1024:
    assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
    assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE");
    return Load ? X86::TILELOADD : X86::TILESTORED;
  }
}
 
std::optional<ExtAddrMode>
X86InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
                                      const TargetRegisterInfo *TRI) const {
  const MCInstrDesc &Desc = MemI.getDesc();
  int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
  if (MemRefBegin < 0)
    return std::nullopt;
 
  MemRefBegin += X86II::getOperandBias(Desc);
 
  auto &BaseOp = MemI.getOperand(MemRefBegin + X86::AddrBaseReg);
  if (!BaseOp.isReg()) // Can be an MO_FrameIndex
    return std::nullopt;
 
  const MachineOperand &DispMO = MemI.getOperand(MemRefBegin + X86::AddrDisp);
  // Displacement can be symbolic
  if (!DispMO.isImm())
    return std::nullopt;
 
  ExtAddrMode AM;
  AM.BaseReg = BaseOp.getReg();
  AM.ScaledReg = MemI.getOperand(MemRefBegin + X86::AddrIndexReg).getReg();
  AM.Scale = MemI.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm();
  AM.Displacement = DispMO.getImm();
  return AM;
}
 
bool X86InstrInfo::verifyInstruction(const MachineInstr &MI,
                                     StringRef &ErrInfo) const {
  std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI, nullptr);
  if (!AMOrNone)
    return true;
 
  ExtAddrMode AM = *AMOrNone;
 
  if (AM.ScaledReg != X86::NoRegister) {
    switch (AM.Scale) {
    case 1:
    case 2:
    case 4:
    case 8:
      break;
    default:
      ErrInfo = "Scale factor in address must be 1, 2, 4 or 8";
      return false;
    }
  }
  if (!isInt<32>(AM.Displacement)) {
    ErrInfo = "Displacement in address must fit into 32-bit signed "
              "integer";
    return false;
  }
 
  return true;
}
 
bool X86InstrInfo::getConstValDefinedInReg(const MachineInstr &MI,
                                           const Register Reg,
                                           int64_t &ImmVal) const {
  if (MI.getOpcode() != X86::MOV32ri && MI.getOpcode() != X86::MOV64ri)
    return false;
  // Mov Src can be a global address.
  if (!MI.getOperand(1).isImm() || MI.getOperand(0).getReg() != Reg)
    return false;
  ImmVal = MI.getOperand(1).getImm();
  return true;
}
 
bool X86InstrInfo::preservesZeroValueInReg(
    const MachineInstr *MI, const Register NullValueReg,
    const TargetRegisterInfo *TRI) const {
  if (!MI->modifiesRegister(NullValueReg, TRI))
    return true;
  switch (MI->getOpcode()) {
  // Shift right/left of a null unto itself is still a null, i.e. rax = shl rax
  // X.
  case X86::SHR64ri:
  case X86::SHR32ri:
  case X86::SHL64ri:
  case X86::SHL32ri:
    assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() &&
           "expected for shift opcode!");
    return MI->getOperand(0).getReg() == NullValueReg &&
           MI->getOperand(1).getReg() == NullValueReg;
  // Zero extend of a sub-reg of NullValueReg into itself does not change the
  // null value.
  case X86::MOV32rr:
    return llvm::all_of(MI->operands(), [&](const MachineOperand &MO) {
      return TRI->isSubRegisterEq(NullValueReg, MO.getReg());
    });
  default:
    return false;
  }
  llvm_unreachable("Should be handled above!");
}
 
bool X86InstrInfo::getMemOperandsWithOffsetWidth(
    const MachineInstr &MemOp, SmallVectorImpl<const MachineOperand *> &BaseOps,
    int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
    const TargetRegisterInfo *TRI) const {
  const MCInstrDesc &Desc = MemOp.getDesc();
  int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
  if (MemRefBegin < 0)
    return false;
 
  MemRefBegin += X86II::getOperandBias(Desc);
 
  const MachineOperand *BaseOp =
      &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg);
  if (!BaseOp->isReg()) // Can be an MO_FrameIndex
    return false;
 
  if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1)
    return false;
 
  if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() !=
      X86::NoRegister)
    return false;
 
  const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp);
 
  // Displacement can be symbolic
  if (!DispMO.isImm())
    return false;
 
  Offset = DispMO.getImm();
 
  if (!BaseOp->isReg())
    return false;
 
  OffsetIsScalable = false;
  // FIXME: Relying on memoperands() may not be right thing to do here. Check
  // with X86 maintainers, and fix it accordingly. For now, it is ok, since
  // there is no use of `Width` for X86 back-end at the moment.
  Width =
      !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize() : 0;
  BaseOps.push_back(BaseOp);
  return true;
}
 
static unsigned getStoreRegOpcode(Register SrcReg,
                                  const TargetRegisterClass *RC,
                                  bool IsStackAligned,
                                  const X86Subtarget &STI) {
  return getLoadStoreRegOpcode(SrcReg, RC, IsStackAligned, STI, false);
}
 
static unsigned getLoadRegOpcode(Register DestReg,
                                 const TargetRegisterClass *RC,
                                 bool IsStackAligned, const X86Subtarget &STI) {
  return getLoadStoreRegOpcode(DestReg, RC, IsStackAligned, STI, true);
}
 
static bool isAMXOpcode(unsigned Opc) {
  switch (Opc) {
  default:
    return false;
  case X86::TILELOADD:
  case X86::TILESTORED:
    return true;
  }
}
 
void X86InstrInfo::loadStoreTileReg(MachineBasicBlock &MBB,
                                    MachineBasicBlock::iterator MI,
                                    unsigned Opc, Register Reg, int FrameIdx,
                                    bool isKill) const {
  switch (Opc) {
  default:
    llvm_unreachable("Unexpected special opcode!");
  case X86::TILESTORED: {
    // tilestored %tmm, (%sp, %idx)
    MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
    Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
    BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
    MachineInstr *NewMI =
        addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
            .addReg(Reg, getKillRegState(isKill));
    MachineOperand &MO = NewMI->getOperand(X86::AddrIndexReg);
    MO.setReg(VirtReg);
    MO.setIsKill(true);
    break;
  }
  case X86::TILELOADD: {
    // tileloadd (%sp, %idx), %tmm
    MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
    Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
    BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
    MachineInstr *NewMI = addFrameReference(
        BuildMI(MBB, MI, DebugLoc(), get(Opc), Reg), FrameIdx);
    MachineOperand &MO = NewMI->getOperand(1 + X86::AddrIndexReg);
    MO.setReg(VirtReg);
    MO.setIsKill(true);
    break;
  }
  }
}
 
void X86InstrInfo::storeRegToStackSlot(
    MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg,
    bool isKill, int FrameIdx, const TargetRegisterClass *RC,
    const TargetRegisterInfo *TRI, Register VReg) const {
  const MachineFunction &MF = *MBB.getParent();
  const MachineFrameInfo &MFI = MF.getFrameInfo();
  assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&
         "Stack slot too small for store");
 
  unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
  bool isAligned =
      (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
      (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
 
  unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget);
  if (isAMXOpcode(Opc))
    loadStoreTileReg(MBB, MI, Opc, SrcReg, FrameIdx, isKill);
  else
    addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
        .addReg(SrcReg, getKillRegState(isKill));
}
 
void X86InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
                                        MachineBasicBlock::iterator MI,
                                        Register DestReg, int FrameIdx,
                                        const TargetRegisterClass *RC,
                                        const TargetRegisterInfo *TRI,
                                        Register VReg) const {
  const MachineFunction &MF = *MBB.getParent();
  const MachineFrameInfo &MFI = MF.getFrameInfo();
  assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&
         "Load size exceeds stack slot");
  unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
  bool isAligned =
      (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
      (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
 
  unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget);
  if (isAMXOpcode(Opc))
    loadStoreTileReg(MBB, MI, Opc, DestReg, FrameIdx);
  else
    addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg),
                      FrameIdx);
}
 
bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
                                  Register &SrcReg2, int64_t &CmpMask,
                                  int64_t &CmpValue) const {
  switch (MI.getOpcode()) {
  default: break;
  case X86::CMP64ri32:
  case X86::CMP64ri8:
  case X86::CMP32ri:
  case X86::CMP32ri8:
  case X86::CMP16ri:
  case X86::CMP16ri8:
  case X86::CMP8ri:
    SrcReg = MI.getOperand(0).getReg();
    SrcReg2 = 0;
    if (MI.getOperand(1).isImm()) {
      CmpMask = ~0;
      CmpValue = MI.getOperand(1).getImm();
    } else {
      CmpMask = CmpValue = 0;
    }
    return true;
  // A SUB can be used to perform comparison.
  case X86::SUB64rm:
  case X86::SUB32rm:
  case X86::SUB16rm:
  case X86::SUB8rm:
    SrcReg = MI.getOperand(1).getReg();
    SrcReg2 = 0;
    CmpMask = 0;
    CmpValue = 0;
    return true;
  case X86::SUB64rr:
  case X86::SUB32rr:
  case X86::SUB16rr:
  case X86::SUB8rr:
    SrcReg = MI.getOperand(1).getReg();
    SrcReg2 = MI.getOperand(2).getReg();
    CmpMask = 0;
    CmpValue = 0;
    return true;
  case X86::SUB64ri32:
  case X86::SUB64ri8:
  case X86::SUB32ri:
  case X86::SUB32ri8:
  case X86::SUB16ri:
  case X86::SUB16ri8:
  case X86::SUB8ri:
    SrcReg = MI.getOperand(1).getReg();
    SrcReg2 = 0;
    if (MI.getOperand(2).isImm()) {
      CmpMask = ~0;
      CmpValue = MI.getOperand(2).getImm();
    } else {
      CmpMask = CmpValue = 0;
    }
    return true;
  case X86::CMP64rr:
  case X86::CMP32rr:
  case X86::CMP16rr:
  case X86::CMP8rr:
    SrcReg = MI.getOperand(0).getReg();
    SrcReg2 = MI.getOperand(1).getReg();
    CmpMask = 0;
    CmpValue = 0;
    return true;
  case X86::TEST8rr:
  case X86::TEST16rr:
  case X86::TEST32rr:
  case X86::TEST64rr:
    SrcReg = MI.getOperand(0).getReg();
    if (MI.getOperand(1).getReg() != SrcReg)
      return false;
    // Compare against zero.
    SrcReg2 = 0;
    CmpMask = ~0;
    CmpValue = 0;
    return true;
  }
  return false;
}
 
bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
                                        Register SrcReg, Register SrcReg2,
                                        int64_t ImmMask, int64_t ImmValue,
                                        const MachineInstr &OI, bool *IsSwapped,
                                        int64_t *ImmDelta) const {
  switch (OI.getOpcode()) {
  case X86::CMP64rr:
  case X86::CMP32rr:
  case X86::CMP16rr:
  case X86::CMP8rr:
  case X86::SUB64rr:
  case X86::SUB32rr:
  case X86::SUB16rr:
  case X86::SUB8rr: {
    Register OISrcReg;
    Register OISrcReg2;
    int64_t OIMask;
    int64_t OIValue;
    if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) ||
        OIMask != ImmMask || OIValue != ImmValue)
      return false;
    if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
      *IsSwapped = false;
      return true;
    }
    if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
      *IsSwapped = true;
      return true;
    }
    return false;
  }
  case X86::CMP64ri32:
  case X86::CMP64ri8:
  case X86::CMP32ri:
  case X86::CMP32ri8:
  case X86::CMP16ri:
  case X86::CMP16ri8:
  case X86::CMP8ri:
  case X86::SUB64ri32:
  case X86::SUB64ri8:
  case X86::SUB32ri:
  case X86::SUB32ri8:
  case X86::SUB16ri:
  case X86::SUB16ri8:
  case X86::SUB8ri:
  case X86::TEST64rr:
  case X86::TEST32rr:
  case X86::TEST16rr:
  case X86::TEST8rr: {
    if (ImmMask != 0) {
      Register OISrcReg;
      Register OISrcReg2;
      int64_t OIMask;
      int64_t OIValue;
      if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) &&
          SrcReg == OISrcReg && ImmMask == OIMask) {
        if (OIValue == ImmValue) {
          *ImmDelta = 0;
          return true;
        } else if (static_cast<uint64_t>(ImmValue) ==
                   static_cast<uint64_t>(OIValue) - 1) {
          *ImmDelta = -1;
          return true;
        } else if (static_cast<uint64_t>(ImmValue) ==
                   static_cast<uint64_t>(OIValue) + 1) {
          *ImmDelta = 1;
          return true;
        } else {
          return false;
        }
      }
    }
    return FlagI.isIdenticalTo(OI);
  }
  default:
    return false;
  }
}
 
/// Check whether the definition can be converted
/// to remove a comparison against zero.
inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
                                    bool &ClearsOverflowFlag) {
  NoSignFlag = false;
  ClearsOverflowFlag = false;
 
  // "ELF Handling for Thread-Local Storage" specifies that x86-64 GOTTPOFF, and
  // i386 GOTNTPOFF/INDNTPOFF relocations can convert an ADD to a LEA during
  // Initial Exec to Local Exec relaxation. In these cases, we must not depend
  // on the EFLAGS modification of ADD actually happening in the final binary.
  if (MI.getOpcode() == X86::ADD64rm || MI.getOpcode() == X86::ADD32rm) {
    unsigned Flags = MI.getOperand(5).getTargetFlags();
    if (Flags == X86II::MO_GOTTPOFF || Flags == X86II::MO_INDNTPOFF ||
        Flags == X86II::MO_GOTNTPOFF)
      return false;
  }
 
  switch (MI.getOpcode()) {
  default: return false;
 
  // The shift instructions only modify ZF if their shift count is non-zero.
  // N.B.: The processor truncates the shift count depending on the encoding.
  case X86::SAR8ri:    case X86::SAR16ri:  case X86::SAR32ri:case X86::SAR64ri:
  case X86::SHR8ri:    case X86::SHR16ri:  case X86::SHR32ri:case X86::SHR64ri:
     return getTruncatedShiftCount(MI, 2) != 0;
 
  // Some left shift instructions can be turned into LEA instructions but only
  // if their flags aren't used. Avoid transforming such instructions.
  case X86::SHL8ri:    case X86::SHL16ri:  case X86::SHL32ri:case X86::SHL64ri:{
    unsigned ShAmt = getTruncatedShiftCount(MI, 2);
    if (isTruncatedShiftCountForLEA(ShAmt)) return false;
    return ShAmt != 0;
  }
 
  case X86::SHRD16rri8:case X86::SHRD32rri8:case X86::SHRD64rri8:
  case X86::SHLD16rri8:case X86::SHLD32rri8:case X86::SHLD64rri8:
     return getTruncatedShiftCount(MI, 3) != 0;
 
  case X86::SUB64ri32: case X86::SUB64ri8: case X86::SUB32ri:
  case X86::SUB32ri8:  case X86::SUB16ri:  case X86::SUB16ri8:
  case X86::SUB8ri:    case X86::SUB64rr:  case X86::SUB32rr:
  case X86::SUB16rr:   case X86::SUB8rr:   case X86::SUB64rm:
  case X86::SUB32rm:   case X86::SUB16rm:  case X86::SUB8rm:
  case X86::DEC64r:    case X86::DEC32r:   case X86::DEC16r: case X86::DEC8r:
  case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD32ri:
  case X86::ADD32ri8:  case X86::ADD16ri:  case X86::ADD16ri8:
  case X86::ADD8ri:    case X86::ADD64rr:  case X86::ADD32rr:
  case X86::ADD16rr:   case X86::ADD8rr:   case X86::ADD64rm:
  case X86::ADD32rm:   case X86::ADD16rm:  case X86::ADD8rm:
  case X86::INC64r:    case X86::INC32r:   case X86::INC16r: case X86::INC8r:
  case X86::ADC64ri32: case X86::ADC64ri8: case X86::ADC32ri:
  case X86::ADC32ri8:  case X86::ADC16ri:  case X86::ADC16ri8:
  case X86::ADC8ri:    case X86::ADC64rr:  case X86::ADC32rr:
  case X86::ADC16rr:   case X86::ADC8rr:   case X86::ADC64rm:
  case X86::ADC32rm:   case X86::ADC16rm:  case X86::ADC8rm:
  case X86::SBB64ri32: case X86::SBB64ri8: case X86::SBB32ri:
  case X86::SBB32ri8:  case X86::SBB16ri:  case X86::SBB16ri8:
  case X86::SBB8ri:    case X86::SBB64rr:  case X86::SBB32rr:
  case X86::SBB16rr:   case X86::SBB8rr:   case X86::SBB64rm:
  case X86::SBB32rm:   case X86::SBB16rm:  case X86::SBB8rm:
  case X86::NEG8r:     case X86::NEG16r:   case X86::NEG32r: case X86::NEG64r:
  case X86::SAR8r1:    case X86::SAR16r1:  case X86::SAR32r1:case X86::SAR64r1:
  case X86::SHR8r1:    case X86::SHR16r1:  case X86::SHR32r1:case X86::SHR64r1:
  case X86::SHL8r1:    case X86::SHL16r1:  case X86::SHL32r1:case X86::SHL64r1:
  case X86::LZCNT16rr: case X86::LZCNT16rm:
  case X86::LZCNT32rr: case X86::LZCNT32rm:
  case X86::LZCNT64rr: case X86::LZCNT64rm:
  case X86::POPCNT16rr:case X86::POPCNT16rm:
  case X86::POPCNT32rr:case X86::POPCNT32rm:
  case X86::POPCNT64rr:case X86::POPCNT64rm:
  case X86::TZCNT16rr: case X86::TZCNT16rm:
  case X86::TZCNT32rr: case X86::TZCNT32rm:
  case X86::TZCNT64rr: case X86::TZCNT64rm:
    return true;
  case X86::AND64ri32:   case X86::AND64ri8:  case X86::AND32ri:
  case X86::AND32ri8:    case X86::AND16ri:   case X86::AND16ri8:
  case X86::AND8ri:      case X86::AND64rr:   case X86::AND32rr:
  case X86::AND16rr:     case X86::AND8rr:    case X86::AND64rm:
  case X86::AND32rm:     case X86::AND16rm:   case X86::AND8rm:
  case X86::XOR64ri32:   case X86::XOR64ri8:  case X86::XOR32ri:
  case X86::XOR32ri8:    case X86::XOR16ri:   case X86::XOR16ri8:
  case X86::XOR8ri:      case X86::XOR64rr:   case X86::XOR32rr:
  case X86::XOR16rr:     case X86::XOR8rr:    case X86::XOR64rm:
  case X86::XOR32rm:     case X86::XOR16rm:   case X86::XOR8rm:
  case X86::OR64ri32:    case X86::OR64ri8:   case X86::OR32ri:
  case X86::OR32ri8:     case X86::OR16ri:    case X86::OR16ri8:
  case X86::OR8ri:       case X86::OR64rr:    case X86::OR32rr:
  case X86::OR16rr:      case X86::OR8rr:     case X86::OR64rm:
  case X86::OR32rm:      case X86::OR16rm:    case X86::OR8rm:
  case X86::ANDN32rr:    case X86::ANDN32rm:
  case X86::ANDN64rr:    case X86::ANDN64rm:
  case X86::BLSI32rr:    case X86::BLSI32rm:
  case X86::BLSI64rr:    case X86::BLSI64rm:
  case X86::BLSMSK32rr:  case X86::BLSMSK32rm:
  case X86::BLSMSK64rr:  case X86::BLSMSK64rm:
  case X86::BLSR32rr:    case X86::BLSR32rm:
  case X86::BLSR64rr:    case X86::BLSR64rm:
  case X86::BLCFILL32rr: case X86::BLCFILL32rm:
  case X86::BLCFILL64rr: case X86::BLCFILL64rm:
  case X86::BLCI32rr:    case X86::BLCI32rm:
  case X86::BLCI64rr:    case X86::BLCI64rm:
  case X86::BLCIC32rr:   case X86::BLCIC32rm:
  case X86::BLCIC64rr:   case X86::BLCIC64rm:
  case X86::BLCMSK32rr:  case X86::BLCMSK32rm:
  case X86::BLCMSK64rr:  case X86::BLCMSK64rm:
  case X86::BLCS32rr:    case X86::BLCS32rm:
  case X86::BLCS64rr:    case X86::BLCS64rm:
  case X86::BLSFILL32rr: case X86::BLSFILL32rm:
  case X86::BLSFILL64rr: case X86::BLSFILL64rm:
  case X86::BLSIC32rr:   case X86::BLSIC32rm:
  case X86::BLSIC64rr:   case X86::BLSIC64rm:
  case X86::BZHI32rr:    case X86::BZHI32rm:
  case X86::BZHI64rr:    case X86::BZHI64rm:
  case X86::T1MSKC32rr:  case X86::T1MSKC32rm:
  case X86::T1MSKC64rr:  case X86::T1MSKC64rm:
  case X86::TZMSK32rr:   case X86::TZMSK32rm:
  case X86::TZMSK64rr:   case X86::TZMSK64rm:
    // These instructions clear the overflow flag just like TEST.
    // FIXME: These are not the only instructions in this switch that clear the
    // overflow flag.
    ClearsOverflowFlag = true;
    return true;
  case X86::BEXTR32rr:   case X86::BEXTR64rr:
  case X86::BEXTR32rm:   case X86::BEXTR64rm:
  case X86::BEXTRI32ri:  case X86::BEXTRI32mi:
  case X86::BEXTRI64ri:  case X86::BEXTRI64mi:
    // BEXTR doesn't update the sign flag so we can't use it. It does clear
    // the overflow flag, but that's not useful without the sign flag.
    NoSignFlag = true;
    return true;
  }
}
 
/// Check whether the use can be converted to remove a comparison against zero.
static X86::CondCode isUseDefConvertible(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  default: return X86::COND_INVALID;
  case X86::NEG8r:
  case X86::NEG16r:
  case X86::NEG32r:
  case X86::NEG64r:
    return X86::COND_AE;
  case X86::LZCNT16rr:
  case X86::LZCNT32rr:
  case X86::LZCNT64rr:
    return X86::COND_B;
  case X86::POPCNT16rr:
  case X86::POPCNT32rr:
  case X86::POPCNT64rr:
    return X86::COND_E;
  case X86::TZCNT16rr:
  case X86::TZCNT32rr:
  case X86::TZCNT64rr:
    return X86::COND_B;
  case X86::BSF16rr:
  case X86::BSF32rr:
  case X86::BSF64rr:
  case X86::BSR16rr:
  case X86::BSR32rr:
  case X86::BSR64rr:
    return X86::COND_E;
  case X86::BLSI32rr:
  case X86::BLSI64rr:
    return X86::COND_AE;
  case X86::BLSR32rr:
  case X86::BLSR64rr:
  case X86::BLSMSK32rr:
  case X86::BLSMSK64rr:
    return X86::COND_B;
  // TODO: TBM instructions.
  }
}
 
/// Check if there exists an earlier instruction that
/// operates on the same source operands and sets flags in the same way as
/// Compare; remove Compare if possible.
bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
                                        Register SrcReg2, int64_t CmpMask,
                                        int64_t CmpValue,
                                        const MachineRegisterInfo *MRI) const {
  // Check whether we can replace SUB with CMP.
  switch (CmpInstr.getOpcode()) {
  default: break;
  case X86::SUB64ri32:
  case X86::SUB64ri8:
  case X86::SUB32ri:
  case X86::SUB32ri8:
  case X86::SUB16ri:
  case X86::SUB16ri8:
  case X86::SUB8ri:
  case X86::SUB64rm:
  case X86::SUB32rm:
  case X86::SUB16rm:
  case X86::SUB8rm:
  case X86::SUB64rr:
  case X86::SUB32rr:
  case X86::SUB16rr:
  case X86::SUB8rr: {
    if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
      return false;
    // There is no use of the destination register, we can replace SUB with CMP.
    unsigned NewOpcode = 0;
    switch (CmpInstr.getOpcode()) {
    default: llvm_unreachable("Unreachable!");
    case X86::SUB64rm:   NewOpcode = X86::CMP64rm;   break;
    case X86::SUB32rm:   NewOpcode = X86::CMP32rm;   break;
    case X86::SUB16rm:   NewOpcode = X86::CMP16rm;   break;
    case X86::SUB8rm:    NewOpcode = X86::CMP8rm;    break;
    case X86::SUB64rr:   NewOpcode = X86::CMP64rr;   break;
    case X86::SUB32rr:   NewOpcode = X86::CMP32rr;   break;
    case X86::SUB16rr:   NewOpcode = X86::CMP16rr;   break;
    case X86::SUB8rr:    NewOpcode = X86::CMP8rr;    break;
    case X86::SUB64ri32: NewOpcode = X86::CMP64ri32; break;
    case X86::SUB64ri8:  NewOpcode = X86::CMP64ri8;  break;
    case X86::SUB32ri:   NewOpcode = X86::CMP32ri;   break;
    case X86::SUB32ri8:  NewOpcode = X86::CMP32ri8;  break;
    case X86::SUB16ri:   NewOpcode = X86::CMP16ri;   break;
    case X86::SUB16ri8:  NewOpcode = X86::CMP16ri8;  break;
    case X86::SUB8ri:    NewOpcode = X86::CMP8ri;    break;
    }
    CmpInstr.setDesc(get(NewOpcode));
    CmpInstr.removeOperand(0);
    // Mutating this instruction invalidates any debug data associated with it.
    CmpInstr.dropDebugNumber();
    // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
    if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
        NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
      return false;
  }
  }
 
  // The following code tries to remove the comparison by re-using EFLAGS
  // from earlier instructions.
 
  bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
 
  // Transformation currently requires SSA values.
  if (SrcReg2.isPhysical())
    return false;
  MachineInstr *SrcRegDef = MRI->getVRegDef(SrcReg);
  assert(SrcRegDef && "Must have a definition (SSA)");
 
  MachineInstr *MI = nullptr;
  MachineInstr *Sub = nullptr;
  MachineInstr *Movr0Inst = nullptr;
  bool NoSignFlag = false;
  bool ClearsOverflowFlag = false;
  bool ShouldUpdateCC = false;
  bool IsSwapped = false;
  X86::CondCode NewCC = X86::COND_INVALID;
  int64_t ImmDelta = 0;
 
  // Search backward from CmpInstr for the next instruction defining EFLAGS.
  const TargetRegisterInfo *TRI = &getRegisterInfo();
  MachineBasicBlock &CmpMBB = *CmpInstr.getParent();
  MachineBasicBlock::reverse_iterator From =
      std::next(MachineBasicBlock::reverse_iterator(CmpInstr));
  for (MachineBasicBlock *MBB = &CmpMBB;;) {
    for (MachineInstr &Inst : make_range(From, MBB->rend())) {
      // Try to use EFLAGS from the instruction defining %SrcReg. Example:
      //     %eax = addl ...
      //     ...                // EFLAGS not changed
      //     testl %eax, %eax   // <-- can be removed
      if (&Inst == SrcRegDef) {
        if (IsCmpZero &&
            isDefConvertible(Inst, NoSignFlag, ClearsOverflowFlag)) {
          MI = &Inst;
          break;
        }
 
        // Look back for the following pattern, in which case the test64rr
        // instruction could be erased.
        //
        // Example:
        //  %reg = and32ri %in_reg, 5
        //  ...                         // EFLAGS not changed.
        //  %src_reg = subreg_to_reg 0, %reg, %subreg.sub_index
        //  test64rr %src_reg, %src_reg, implicit-def $eflags
        MachineInstr *AndInstr = nullptr;
        if (IsCmpZero &&
            findRedundantFlagInstr(CmpInstr, Inst, MRI, &AndInstr, TRI,
                                   NoSignFlag, ClearsOverflowFlag)) {
          assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode()));
          MI = AndInstr;
          break;
        }
        // Cannot find other candidates before definition of SrcReg.
        return false;
      }
 
      if (Inst.modifiesRegister(X86::EFLAGS, TRI)) {
        // Try to use EFLAGS produced by an instruction reading %SrcReg.
        // Example:
        //      %eax = ...
        //      ...
        //      popcntl %eax
        //      ...                 // EFLAGS not changed
        //      testl %eax, %eax    // <-- can be removed
        if (IsCmpZero) {
          NewCC = isUseDefConvertible(Inst);
          if (NewCC != X86::COND_INVALID && Inst.getOperand(1).isReg() &&
              Inst.getOperand(1).getReg() == SrcReg) {
            ShouldUpdateCC = true;
            MI = &Inst;
            break;
          }
        }
 
        // Try to use EFLAGS from an instruction with similar flag results.
        // Example:
        //     sub x, y  or  cmp x, y
        //     ...           // EFLAGS not changed
        //     cmp x, y      // <-- can be removed
        if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
                                 Inst, &IsSwapped, &ImmDelta)) {
          Sub = &Inst;
          break;
        }
 
        // MOV32r0 is implemented with xor which clobbers condition code. It is
        // safe to move up, if the definition to EFLAGS is dead and earlier
        // instructions do not read or write EFLAGS.
        if (!Movr0Inst && Inst.getOpcode() == X86::MOV32r0 &&
            Inst.registerDefIsDead(X86::EFLAGS, TRI)) {
          Movr0Inst = &Inst;
          continue;
        }
 
        // Cannot do anything for any other EFLAG changes.
        return false;
      }
    }
 
    if (MI || Sub)
      break;
 
    // Reached begin of basic block. Continue in predecessor if there is
    // exactly one.
    if (MBB->pred_size() != 1)
      return false;
    MBB = *MBB->pred_begin();
    From = MBB->rbegin();
  }
 
  // Scan forward from the instruction after CmpInstr for uses of EFLAGS.
  // It is safe to remove CmpInstr if EFLAGS is redefined or killed.
  // If we are done with the basic block, we need to check whether EFLAGS is
  // live-out.
  bool FlagsMayLiveOut = true;
  SmallVector<std::pair<MachineInstr*, X86::CondCode>, 4> OpsToUpdate;
  MachineBasicBlock::iterator AfterCmpInstr =
      std::next(MachineBasicBlock::iterator(CmpInstr));
  for (MachineInstr &Instr : make_range(AfterCmpInstr, CmpMBB.end())) {
    bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI);
    bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI);
    // We should check the usage if this instruction uses and updates EFLAGS.
    if (!UseEFLAGS && ModifyEFLAGS) {
      // It is safe to remove CmpInstr if EFLAGS is updated again.
      FlagsMayLiveOut = false;
      break;
    }
    if (!UseEFLAGS && !ModifyEFLAGS)
      continue;
 
    // EFLAGS is used by this instruction.
    X86::CondCode OldCC = X86::getCondFromMI(Instr);
    if ((MI || IsSwapped || ImmDelta != 0) && OldCC == X86::COND_INVALID)
      return false;
 
    X86::CondCode ReplacementCC = X86::COND_INVALID;
    if (MI) {
      switch (OldCC) {
      default: break;
      case X86::COND_A: case X86::COND_AE:
      case X86::COND_B: case X86::COND_BE:
        // CF is used, we can't perform this optimization.
        return false;
      case X86::COND_G: case X86::COND_GE:
      case X86::COND_L: case X86::COND_LE:
        // If SF is used, but the instruction doesn't update the SF, then we
        // can't do the optimization.
        if (NoSignFlag)
          return false;
        [[fallthrough]];
      case X86::COND_O: case X86::COND_NO:
        // If OF is used, the instruction needs to clear it like CmpZero does.
        if (!ClearsOverflowFlag)
          return false;
        break;
      case X86::COND_S: case X86::COND_NS:
        // If SF is used, but the instruction doesn't update the SF, then we
        // can't do the optimization.
        if (NoSignFlag)
          return false;
        break;
      }
 
      // If we're updating the condition code check if we have to reverse the
      // condition.
      if (ShouldUpdateCC)
        switch (OldCC) {
        default:
          return false;
        case X86::COND_E:
          ReplacementCC = NewCC;
          break;
        case X86::COND_NE:
          ReplacementCC = GetOppositeBranchCondition(NewCC);
          break;
        }
    } else if (IsSwapped) {
      // If we have SUB(r1, r2) and CMP(r2, r1), the condition code needs
      // to be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
      // We swap the condition code and synthesize the new opcode.
      ReplacementCC = getSwappedCondition(OldCC);
      if (ReplacementCC == X86::COND_INVALID)
        return false;
      ShouldUpdateCC = true;
    } else if (ImmDelta != 0) {
      unsigned BitWidth = TRI->getRegSizeInBits(*MRI->getRegClass(SrcReg));
      // Shift amount for min/max constants to adjust for 8/16/32 instruction
      // sizes.
      switch (OldCC) {
      case X86::COND_L: // x <s (C + 1)  -->  x <=s C
        if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue)
          return false;
        ReplacementCC = X86::COND_LE;
        break;
      case X86::COND_B: // x <u (C + 1)  -->  x <=u C
        if (ImmDelta != 1 || CmpValue == 0)
          return false;
        ReplacementCC = X86::COND_BE;
        break;
      case X86::COND_GE: // x >=s (C + 1)  -->  x >s C
        if (ImmDelta != 1 || APInt::getSignedMinValue(BitWidth) == CmpValue)
          return false;
        ReplacementCC = X86::COND_G;
        break;
      case X86::COND_AE: // x >=u (C + 1)  -->  x >u C
        if (ImmDelta != 1 || CmpValue == 0)
          return false;
        ReplacementCC = X86::COND_A;
        break;
      case X86::COND_G: // x >s (C - 1)  -->  x >=s C
        if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue)
          return false;
        ReplacementCC = X86::COND_GE;
        break;
      case X86::COND_A: // x >u (C - 1)  -->  x >=u C
        if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue)
          return false;
        ReplacementCC = X86::COND_AE;
        break;
      case X86::COND_LE: // x <=s (C - 1)  -->  x <s C
        if (ImmDelta != -1 || APInt::getSignedMaxValue(BitWidth) == CmpValue)
          return false;
        ReplacementCC = X86::COND_L;
        break;
      case X86::COND_BE: // x <=u (C - 1)  -->  x <u C
        if (ImmDelta != -1 || APInt::getMaxValue(BitWidth) == CmpValue)
          return false;
        ReplacementCC = X86::COND_B;
        break;
      default:
        return false;
      }
      ShouldUpdateCC = true;
    }
 
    if (ShouldUpdateCC && ReplacementCC != OldCC) {
      // Push the MachineInstr to OpsToUpdate.
      // If it is safe to remove CmpInstr, the condition code of these
      // instructions will be modified.
      OpsToUpdate.push_back(std::make_pair(&Instr, ReplacementCC));
    }
    if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) {
      // It is safe to remove CmpInstr if EFLAGS is updated again or killed.
      FlagsMayLiveOut = false;
      break;
    }
  }
 
  // If we have to update users but EFLAGS is live-out abort, since we cannot
  // easily find all of the users.
  if ((MI != nullptr || ShouldUpdateCC) && FlagsMayLiveOut) {
    for (MachineBasicBlock *Successor : CmpMBB.successors())
      if (Successor->isLiveIn(X86::EFLAGS))
        return false;
  }
 
  // The instruction to be updated is either Sub or MI.
  assert((MI == nullptr || Sub == nullptr) && "Should not have Sub and MI set");
  Sub = MI != nullptr ? MI : Sub;
  MachineBasicBlock *SubBB = Sub->getParent();
  // Move Movr0Inst to the appropriate place before Sub.
  if (Movr0Inst) {
    // Only move within the same block so we don't accidentally move to a
    // block with higher execution frequency.
    if (&CmpMBB != SubBB)
      return false;
    // Look backwards until we find a def that doesn't use the current EFLAGS.
    MachineBasicBlock::reverse_iterator InsertI = Sub,
                                        InsertE = Sub->getParent()->rend();
    for (; InsertI != InsertE; ++InsertI) {
      MachineInstr *Instr = &*InsertI;
      if (!Instr->readsRegister(X86::EFLAGS, TRI) &&
          Instr->modifiesRegister(X86::EFLAGS, TRI)) {
        Movr0Inst->getParent()->remove(Movr0Inst);
        Instr->getParent()->insert(MachineBasicBlock::iterator(Instr),
                                   Movr0Inst);
        break;
      }
    }
    if (InsertI == InsertE)
      return false;
  }
 
  // Make sure Sub instruction defines EFLAGS and mark the def live.
  MachineOperand *FlagDef = Sub->findRegisterDefOperand(X86::EFLAGS);
  assert(FlagDef && "Unable to locate a def EFLAGS operand");
  FlagDef->setIsDead(false);
 
  CmpInstr.eraseFromParent();
 
  // Modify the condition code of instructions in OpsToUpdate.
  for (auto &Op : OpsToUpdate) {
    Op.first->getOperand(Op.first->getDesc().getNumOperands() - 1)
        .setImm(Op.second);
  }
  // Add EFLAGS to block live-ins between CmpBB and block of flags producer.
  for (MachineBasicBlock *MBB = &CmpMBB; MBB != SubBB;
       MBB = *MBB->pred_begin()) {
    assert(MBB->pred_size() == 1 && "Expected exactly one predecessor");
    if (!MBB->isLiveIn(X86::EFLAGS))
      MBB->addLiveIn(X86::EFLAGS);
  }
  return true;
}
 
/// Try to remove the load by folding it to a register
/// operand at the use. We fold the load instructions if load defines a virtual
/// register, the virtual register is used once in the same BB, and the
/// instructions in-between do not load or store, and have no side effects.
MachineInstr *X86InstrInfo::optimizeLoadInstr(MachineInstr &MI,
                                              const MachineRegisterInfo *MRI,
                                              Register &FoldAsLoadDefReg,
                                              MachineInstr *&DefMI) const {
  // Check whether we can move DefMI here.
  DefMI = MRI->getVRegDef(FoldAsLoadDefReg);
  assert(DefMI);
  bool SawStore = false;
  if (!DefMI->isSafeToMove(nullptr, SawStore))
    return nullptr;
 
  // Collect information about virtual register operands of MI.
  SmallVector<unsigned, 1> SrcOperandIds;
  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
    MachineOperand &MO = MI.getOperand(i);
    if (!MO.isReg())
      continue;
    Register Reg = MO.getReg();
    if (Reg != FoldAsLoadDefReg)
      continue;
    // Do not fold if we have a subreg use or a def.
    if (MO.getSubReg() || MO.isDef())
      return nullptr;
    SrcOperandIds.push_back(i);
  }
  if (SrcOperandIds.empty())
    return nullptr;
 
  // Check whether we can fold the def into SrcOperandId.
  if (MachineInstr *FoldMI = foldMemoryOperand(MI, SrcOperandIds, *DefMI)) {
    FoldAsLoadDefReg = 0;
    return FoldMI;
  }
 
  return nullptr;
}
 
/// Expand a single-def pseudo instruction to a two-addr
/// instruction with two undef reads of the register being defined.
/// This is used for mapping:
///   %xmm4 = V_SET0
/// to:
///   %xmm4 = PXORrr undef %xmm4, undef %xmm4
///
static bool Expand2AddrUndef(MachineInstrBuilder &MIB,
                             const MCInstrDesc &Desc) {
  assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.");
  Register Reg = MIB.getReg(0);
  MIB->setDesc(Desc);
 
  // MachineInstr::addOperand() will insert explicit operands before any
  // implicit operands.
  MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef);
  // But we don't trust that.
  assert(MIB.getReg(1) == Reg &&
         MIB.getReg(2) == Reg && "Misplaced operand");
  return true;
}
 
/// Expand a single-def pseudo instruction to a two-addr
/// instruction with two %k0 reads.
/// This is used for mapping:
///   %k4 = K_SET1
/// to:
///   %k4 = KXNORrr %k0, %k0
static bool Expand2AddrKreg(MachineInstrBuilder &MIB, const MCInstrDesc &Desc,
                            Register Reg) {
  assert(Desc.getNumOperands() == 3 && "Expected two-addr instruction.");
  MIB->setDesc(Desc);
  MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef);
  return true;
}
 
static bool expandMOV32r1(MachineInstrBuilder &MIB, const TargetInstrInfo &TII,
                          bool MinusOne) {
  MachineBasicBlock &MBB = *MIB->getParent();
  const DebugLoc &DL = MIB->getDebugLoc();
  Register Reg = MIB.getReg(0);
 
  // Insert the XOR.
  BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg)
      .addReg(Reg, RegState::Undef)
      .addReg(Reg, RegState::Undef);
 
  // Turn the pseudo into an INC or DEC.
  MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r));
  MIB.addReg(Reg);
 
  return true;
}
 
static bool ExpandMOVImmSExti8(MachineInstrBuilder &MIB,
                               const TargetInstrInfo &TII,
                               const X86Subtarget &Subtarget) {
  MachineBasicBlock &MBB = *MIB->getParent();
  const DebugLoc &DL = MIB->getDebugLoc();
  int64_t Imm = MIB->getOperand(1).getImm();
  assert(Imm != 0 && "Using push/pop for 0 is not efficient.");
  MachineBasicBlock::iterator I = MIB.getInstr();
 
  int StackAdjustment;
 
  if (Subtarget.is64Bit()) {
    assert(MIB->getOpcode() == X86::MOV64ImmSExti8 ||
           MIB->getOpcode() == X86::MOV32ImmSExti8);
 
    // Can't use push/pop lowering if the function might write to the red zone.
    X86MachineFunctionInfo *X86FI =
        MBB.getParent()->getInfo<X86MachineFunctionInfo>();
    if (X86FI->getUsesRedZone()) {
      MIB->setDesc(TII.get(MIB->getOpcode() ==
                           X86::MOV32ImmSExti8 ? X86::MOV32ri : X86::MOV64ri));
      return true;
    }
 
    // 64-bit mode doesn't have 32-bit push/pop, so use 64-bit operations and
    // widen the register if necessary.
    StackAdjustment = 8;
    BuildMI(MBB, I, DL, TII.get(X86::PUSH64i8)).addImm(Imm);
    MIB->setDesc(TII.get(X86::POP64r));
    MIB->getOperand(0)
        .setReg(getX86SubSuperRegister(MIB.getReg(0), 64));
  } else {
    assert(MIB->getOpcode() == X86::MOV32ImmSExti8);
    StackAdjustment = 4;
    BuildMI(MBB, I, DL, TII.get(X86::PUSH32i8)).addImm(Imm);
    MIB->setDesc(TII.get(X86::POP32r));
  }
  MIB->removeOperand(1);
  MIB->addImplicitDefUseOperands(*MBB.getParent());
 
  // Build CFI if necessary.
  MachineFunction &MF = *MBB.getParent();
  const X86FrameLowering *TFL = Subtarget.getFrameLowering();
  bool IsWin64Prologue = MF.getTarget().getMCAsmInfo()->usesWindowsCFI();
  bool NeedsDwarfCFI = !IsWin64Prologue && MF.needsFrameMoves();
  bool EmitCFI = !TFL->hasFP(MF) && NeedsDwarfCFI;
  if (EmitCFI) {
    TFL->BuildCFI(MBB, I, DL,
        MCCFIInstruction::createAdjustCfaOffset(nullptr, StackAdjustment));
    TFL->BuildCFI(MBB, std::next(I), DL,
        MCCFIInstruction::createAdjustCfaOffset(nullptr, -StackAdjustment));
  }
 
  return true;
}
 
// LoadStackGuard has so far only been implemented for 64-bit MachO. Different
// code sequence is needed for other targets.
static void expandLoadStackGuard(MachineInstrBuilder &MIB,
                                 const TargetInstrInfo &TII) {
  MachineBasicBlock &MBB = *MIB->getParent();
  const DebugLoc &DL = MIB->getDebugLoc();
  Register Reg = MIB.getReg(0);
  const GlobalValue *GV =
      cast<GlobalValue>((*MIB->memoperands_begin())->getValue());
  auto Flags = MachineMemOperand::MOLoad |
               MachineMemOperand::MODereferenceable |
               MachineMemOperand::MOInvariant;
  MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand(
      MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, Align(8));
  MachineBasicBlock::iterator I = MIB.getInstr();
 
  BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg).addReg(X86::RIP).addImm(1)
      .addReg(0).addGlobalAddress(GV, 0, X86II::MO_GOTPCREL).addReg(0)
      .addMemOperand(MMO);
  MIB->setDebugLoc(DL);
  MIB->setDesc(TII.get(X86::MOV64rm));
  MIB.addReg(Reg, RegState::Kill).addImm(1).addReg(0).addImm(0).addReg(0);
}
 
static bool expandXorFP(MachineInstrBuilder &MIB, const TargetInstrInfo &TII) {
  MachineBasicBlock &MBB = *MIB->getParent();
  MachineFunction &MF = *MBB.getParent();
  const X86Subtarget &Subtarget = MF.getSubtarget<X86Subtarget>();
  const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
  unsigned XorOp =
      MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr;
  MIB->setDesc(TII.get(XorOp));
  MIB.addReg(TRI->getFrameRegister(MF), RegState::Undef);
  return true;
}
 
// This is used to handle spills for 128/256-bit registers when we have AVX512,
// but not VLX. If it uses an extended register we need to use an instruction
// that loads the lower 128/256-bit, but is available with only AVX512F.
static bool expandNOVLXLoad(MachineInstrBuilder &MIB,
                            const TargetRegisterInfo *TRI,
                            const MCInstrDesc &LoadDesc,
                            const MCInstrDesc &BroadcastDesc,
                            unsigned SubIdx) {
  Register DestReg = MIB.getReg(0);
  // Check if DestReg is XMM16-31 or YMM16-31.
  if (TRI->getEncodingValue(DestReg) < 16) {
    // We can use a normal VEX encoded load.
    MIB->setDesc(LoadDesc);
  } else {
    // Use a 128/256-bit VBROADCAST instruction.
    MIB->setDesc(BroadcastDesc);
    // Change the destination to a 512-bit register.
    DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass);
    MIB->getOperand(0).setReg(DestReg);
  }
  return true;
}
 
// This is used to handle spills for 128/256-bit registers when we have AVX512,
// but not VLX. If it uses an extended register we need to use an instruction
// that stores the lower 128/256-bit, but is available with only AVX512F.
static bool expandNOVLXStore(MachineInstrBuilder &MIB,
                             const TargetRegisterInfo *TRI,
                             const MCInstrDesc &StoreDesc,
                             const MCInstrDesc &ExtractDesc,
                             unsigned SubIdx) {
  Register SrcReg = MIB.getReg(X86::AddrNumOperands);
  // Check if DestReg is XMM16-31 or YMM16-31.
  if (TRI->getEncodingValue(SrcReg) < 16) {
    // We can use a normal VEX encoded store.
    MIB->setDesc(StoreDesc);
  } else {
    // Use a VEXTRACTF instruction.
    MIB->setDesc(ExtractDesc);
    // Change the destination to a 512-bit register.
    SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass);
    MIB->getOperand(X86::AddrNumOperands).setReg(SrcReg);
    MIB.addImm(0x0); // Append immediate to extract from the lower bits.
  }
 
  return true;
}
 
static bool expandSHXDROT(MachineInstrBuilder &MIB, const MCInstrDesc &Desc) {
  MIB->setDesc(Desc);
  int64_t ShiftAmt = MIB->getOperand(2).getImm();
  // Temporarily remove the immediate so we can add another source register.
  MIB->removeOperand(2);
  // Add the register. Don't copy the kill flag if there is one.
  MIB.addReg(MIB.getReg(1),
             getUndefRegState(MIB->getOperand(1).isUndef()));
  // Add back the immediate.
  MIB.addImm(ShiftAmt);
  return true;
}
 
bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
  bool HasAVX = Subtarget.hasAVX();
  MachineInstrBuilder MIB(*MI.getParent()->getParent(), MI);
  switch (MI.getOpcode()) {
  case X86::MOV32r0:
    return Expand2AddrUndef(MIB, get(X86::XOR32rr));
  case X86::MOV32r1:
    return expandMOV32r1(MIB, *this, /*MinusOne=*/ false);
  case X86::MOV32r_1:
    return expandMOV32r1(MIB, *this, /*MinusOne=*/ true);
  case X86::MOV32ImmSExti8:
  case X86::MOV64ImmSExti8:
    return ExpandMOVImmSExti8(MIB, *this, Subtarget);
  case X86::SETB_C32r:
    return Expand2AddrUndef(MIB, get(X86::SBB32rr));
  case X86::SETB_C64r:
    return Expand2AddrUndef(MIB, get(X86::SBB64rr));
  case X86::MMX_SET0:
    return Expand2AddrUndef(MIB, get(X86::MMX_PXORrr));
  case X86::V_SET0:
  case X86::FsFLD0SS:
  case X86::FsFLD0SD:
  case X86::FsFLD0SH:
  case X86::FsFLD0F128:
    return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr));
  case X86::AVX_SET0: {
    assert(HasAVX && "AVX not supported");
    const TargetRegisterInfo *TRI = &getRegisterInfo();
    Register SrcReg = MIB.getReg(0);
    Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm);
    MIB->getOperand(0).setReg(XReg);
    Expand2AddrUndef(MIB, get(X86::VXORPSrr));
    MIB.addReg(SrcReg, RegState::ImplicitDefine);
    return true;
  }
  case X86::AVX512_128_SET0:
  case X86::AVX512_FsFLD0SH:
  case X86::AVX512_FsFLD0SS:
  case X86::AVX512_FsFLD0SD:
  case X86::AVX512_FsFLD0F128: {
    bool HasVLX = Subtarget.hasVLX();
    Register SrcReg = MIB.getReg(0);
    const TargetRegisterInfo *TRI = &getRegisterInfo();
    if (HasVLX || TRI->getEncodingValue(SrcReg) < 16)
      return Expand2AddrUndef(MIB,
                              get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
    // Extended register without VLX. Use a larger XOR.
    SrcReg =
        TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
    MIB->getOperand(0).setReg(SrcReg);
    return Expand2AddrUndef(MIB, get(X86::VPXORDZrr));
  }
  case X86::AVX512_256_SET0:
  case X86::AVX512_512_SET0: {
    bool HasVLX = Subtarget.hasVLX();
    Register SrcReg = MIB.getReg(0);
    const TargetRegisterInfo *TRI = &getRegisterInfo();
    if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) {
      Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm);
      MIB->getOperand(0).setReg(XReg);
      Expand2AddrUndef(MIB,
                       get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
      MIB.addReg(SrcReg, RegState::ImplicitDefine);
      return true;
    }
    if (MI.getOpcode() == X86::AVX512_256_SET0) {
      // No VLX so we must reference a zmm.
      unsigned ZReg =
        TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
      MIB->getOperand(0).setReg(ZReg);
    }
    return Expand2AddrUndef(MIB, get(X86::VPXORDZrr));
  }
  case X86::V_SETALLONES:
    return Expand2AddrUndef(MIB, get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr));
  case X86::AVX2_SETALLONES:
    return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr));
  case X86::AVX1_SETALLONES: {
    Register Reg = MIB.getReg(0);
    // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS.
    MIB->setDesc(get(X86::VCMPPSYrri));
    MIB.addReg(Reg,