77#define DEBUG_TYPE "aarch64-mi-peephole-opt"
93 using OpcodePair = std::pair<unsigned, unsigned>;
95 using SplitAndOpcFunc =
96 std::function<std::optional<OpcodePair>(
T,
unsigned,
T &,
T &)>;
98 std::function<void(
MachineInstr &, OpcodePair,
unsigned,
unsigned,
115 template <
typename T>
117 SplitAndOpcFunc<T> SplitAndOpc, BuildMIFunc BuildInstr);
122 template <
typename T>
123 bool visitADDSUB(
unsigned PosOpc,
unsigned NegOpc,
MachineInstr &
MI);
124 template <
typename T>
125 bool visitADDSSUBS(OpcodePair PosOpcs, OpcodePair NegOpcs,
MachineInstr &
MI);
127 template <
typename T>
139 return "AArch64 MI Peephole Optimization pass";
149char AArch64MIPeepholeOpt::ID = 0;
154 "AArch64 MI Peephole Optimization",
false,
false)
158 T UImm =
static_cast<T>(Imm);
195bool AArch64MIPeepholeOpt::visitAND(
207 return splitTwoPartImm<T>(
210 T &Imm1) -> std::optional<OpcodePair> {
211 if (splitBitmaskImm(
Imm,
RegSize, Imm0, Imm1))
212 return std::make_pair(Opc, Opc);
234 if (
MI.getOperand(3).getImm() != 0)
237 if (
MI.getOperand(1).getReg() != AArch64::WZR)
254 if (SrcMI->
getOpcode() == TargetOpcode::COPY &&
261 if (RC != &AArch64::FPR32RegClass &&
262 ((RC != &AArch64::FPR64RegClass && RC != &AArch64::FPR128RegClass) ||
267 CpySrc =
MRI->createVirtualRegister(&AArch64::FPR32RegClass);
269 TII->get(TargetOpcode::COPY), CpySrc)
277 else if (SrcMI->
getOpcode() <= TargetOpcode::GENERIC_OP_END)
282 MRI->replaceRegWith(DefReg, SrcReg);
283 MRI->clearKillFlags(SrcReg);
285 MI.eraseFromParent();
292 if (
MI.getOperand(1).getReg() !=
MI.getOperand(2).getReg())
296 MI.getOpcode() == AArch64::CSELXr ? AArch64::XZR : AArch64::WZR;
298 MI.getOpcode() == AArch64::CSELXr ? AArch64::ORRXrs : AArch64::ORRWrs;
306 MI.eraseFromParent();
318 if (!
MI.isRegTiedToDefOperand(1))
337 if ((SrcMI->
getOpcode() <= TargetOpcode::GENERIC_OP_END) ||
338 !AArch64::GPR64allRegClass.hasSubClassEq(RC))
344 TII->get(TargetOpcode::SUBREG_TO_REG), DstReg)
346 .
add(
MI.getOperand(2))
347 .
add(
MI.getOperand(3));
350 MI.eraseFromParent();
359 if ((Imm & 0xfff000) == 0 || (Imm & 0xfff) == 0 ||
360 (Imm & ~
static_cast<T>(0xffffff)) != 0)
366 if (
Insn.size() == 1)
370 Imm0 = (Imm >> 12) & 0xfff;
376bool AArch64MIPeepholeOpt::visitADDSUB(
394 if (
MI.getOperand(1).getReg() == AArch64::XZR ||
395 MI.getOperand(1).getReg() == AArch64::WZR)
398 return splitTwoPartImm<T>(
400 [PosOpc, NegOpc](
T Imm,
unsigned RegSize,
T &Imm0,
401 T &Imm1) -> std::optional<OpcodePair> {
403 return std::make_pair(PosOpc, PosOpc);
405 return std::make_pair(NegOpc, NegOpc);
425bool AArch64MIPeepholeOpt::visitADDSSUBS(
430 if (
MI.getOperand(1).getReg() == AArch64::XZR ||
431 MI.getOperand(1).getReg() == AArch64::WZR)
434 return splitTwoPartImm<T>(
438 T &Imm1) -> std::optional<OpcodePair> {
450 if (!NZCVUsed || NZCVUsed->C || NZCVUsed->V)
478 if (L && !
L->isLoopInvariant(
MI))
482 MovMI =
MRI->getUniqueVRegDef(
MI.getOperand(2).getReg());
487 SubregToRegMI =
nullptr;
488 if (MovMI->
getOpcode() == TargetOpcode::SUBREG_TO_REG) {
489 SubregToRegMI = MovMI;
495 if (MovMI->
getOpcode() != AArch64::MOVi32imm &&
496 MovMI->
getOpcode() != AArch64::MOVi64imm)
511bool AArch64MIPeepholeOpt::splitTwoPartImm(
513 SplitAndOpcFunc<T> SplitAndOpc, BuildMIFunc BuildInstr) {
516 "Invalid RegSize for legal immediate peephole optimization");
520 if (!checkMovImmInstr(
MI, MovMI, SubregToRegMI))
532 if (
auto R = SplitAndOpc(Imm,
RegSize, Imm0, Imm1))
545 TII->getRegClass(
TII->get(Opcode.first), 0,
TRI, *MF);
547 TII->getRegClass(
TII->get(Opcode.first), 1,
TRI, *MF);
549 (Opcode.first == Opcode.second)
551 :
TII->getRegClass(
TII->get(Opcode.second), 0,
TRI, *MF);
553 (Opcode.first == Opcode.second)
554 ? FirstInstrOperandRC
555 :
TII->getRegClass(
TII->get(Opcode.second), 1,
TRI, *MF);
560 Register NewTmpReg =
MRI->createVirtualRegister(FirstInstrDstRC);
564 ?
MRI->createVirtualRegister(SecondInstrDstRC)
568 MRI->constrainRegClass(SrcReg, FirstInstrOperandRC);
569 MRI->constrainRegClass(NewTmpReg, SecondInstrOperandRC);
570 if (DstReg != NewDstReg)
571 MRI->constrainRegClass(NewDstReg,
MRI->getRegClass(DstReg));
574 BuildInstr(
MI, Opcode, Imm0, Imm1, SrcReg, NewTmpReg, NewDstReg);
578 if (DstReg != NewDstReg) {
579 MRI->replaceRegWith(DstReg, NewDstReg);
580 MI.getOperand(0).setReg(DstReg);
584 MI.eraseFromParent();
592bool AArch64MIPeepholeOpt::visitINSviGPR(
MachineInstr &
MI,
unsigned Opc) {
609 if (!SrcMI || SrcMI->
getOpcode() != TargetOpcode::COPY)
616 &AArch64::FPR128RegClass) {
626 .
add(
MI.getOperand(1))
627 .
add(
MI.getOperand(2))
633 MI.eraseFromParent();
641 if (!
MI->getOperand(0).isReg() || !
MI->getOperand(0).isDef())
644 if (RC != &AArch64::FPR64RegClass)
646 return MI->getOpcode() > TargetOpcode::GENERIC_OP_END;
658 if (Low64MI->
getOpcode() != AArch64::INSERT_SUBREG)
678 if (!High64MI || High64MI->
getOpcode() != AArch64::INSERT_SUBREG)
681 if (High64MI && High64MI->
getOpcode() == TargetOpcode::COPY)
683 if (!High64MI || (High64MI->
getOpcode() != AArch64::MOVID &&
684 High64MI->
getOpcode() != AArch64::MOVIv2d_ns))
692 MRI->constrainRegClass(NewDef,
MRI->getRegClass(OldDef));
693 MRI->replaceRegWith(OldDef, NewDef);
694 MI.eraseFromParent();
709 MRI->clearKillFlags(OldDef);
710 MRI->clearKillFlags(NewDef);
711 MRI->constrainRegClass(NewDef,
MRI->getRegClass(OldDef));
712 MRI->replaceRegWith(OldDef, NewDef);
713 MI.eraseFromParent();
722 Register InputReg =
MI.getOperand(1).getReg();
723 if (
MI.getOperand(1).getSubReg() != AArch64::sub_32 ||
724 !
MRI->hasOneNonDBGUse(InputReg))
736 if (!SrcMI || SrcMI->
getOpcode() != AArch64::SBFMXri ||
741 MRI->constrainRegClass(SrcReg,
MRI->getRegClass(InputReg));
743 MI.getOperand(1).setReg(SrcReg);
745 for (
auto *DeadMI : DeadInstrs) {
747 DeadMI->eraseFromParent();
759 MLI = &getAnalysis<MachineLoopInfoWrapperPass>().getLI();
762 assert(
MRI->isSSA() &&
"Expected to be run on SSA form!");
764 bool Changed =
false;
768 switch (
MI.getOpcode()) {
771 case AArch64::INSERT_SUBREG:
772 Changed |= visitINSERT(
MI);
774 case AArch64::ANDWrr:
775 Changed |= visitAND<uint32_t>(AArch64::ANDWri,
MI);
777 case AArch64::ANDXrr:
778 Changed |= visitAND<uint64_t>(AArch64::ANDXri,
MI);
780 case AArch64::ORRWrs:
781 Changed |= visitORR(
MI);
783 case AArch64::ADDWrr:
784 Changed |= visitADDSUB<uint32_t>(AArch64::ADDWri, AArch64::SUBWri,
MI);
786 case AArch64::SUBWrr:
787 Changed |= visitADDSUB<uint32_t>(AArch64::SUBWri, AArch64::ADDWri,
MI);
789 case AArch64::ADDXrr:
790 Changed |= visitADDSUB<uint64_t>(AArch64::ADDXri, AArch64::SUBXri,
MI);
792 case AArch64::SUBXrr:
793 Changed |= visitADDSUB<uint64_t>(AArch64::SUBXri, AArch64::ADDXri,
MI);
795 case AArch64::ADDSWrr:
797 visitADDSSUBS<uint32_t>({AArch64::ADDWri, AArch64::ADDSWri},
798 {AArch64::SUBWri, AArch64::SUBSWri},
MI);
800 case AArch64::SUBSWrr:
802 visitADDSSUBS<uint32_t>({AArch64::SUBWri, AArch64::SUBSWri},
803 {AArch64::ADDWri, AArch64::ADDSWri},
MI);
805 case AArch64::ADDSXrr:
807 visitADDSSUBS<uint64_t>({AArch64::ADDXri, AArch64::ADDSXri},
808 {AArch64::SUBXri, AArch64::SUBSXri},
MI);
810 case AArch64::SUBSXrr:
812 visitADDSSUBS<uint64_t>({AArch64::SUBXri, AArch64::SUBSXri},
813 {AArch64::ADDXri, AArch64::ADDSXri},
MI);
815 case AArch64::CSELWr:
816 case AArch64::CSELXr:
817 Changed |= visitCSEL(
MI);
819 case AArch64::INSvi64gpr:
820 Changed |= visitINSviGPR(
MI, AArch64::INSvi64lane);
822 case AArch64::INSvi32gpr:
823 Changed |= visitINSviGPR(
MI, AArch64::INSvi32lane);
825 case AArch64::INSvi16gpr:
826 Changed |= visitINSviGPR(
MI, AArch64::INSvi16lane);
828 case AArch64::INSvi8gpr:
829 Changed |= visitINSviGPR(
MI, AArch64::INSvi8lane);
831 case AArch64::INSvi64lane:
832 Changed |= visitINSvi64lane(
MI);
834 case AArch64::FMOVDr:
835 Changed |= visitFMOVDr(
MI);
838 Changed |= visitCopy(
MI);
848 return new AArch64MIPeepholeOpt();
unsigned const MachineRegisterInfo * MRI
SmallVector< AArch64_IMM::ImmInsnModel, 4 > Insn
static bool is64bitDefwithZeroHigh64bit(MachineInstr *MI, MachineRegisterInfo *MRI)
static bool splitAddSubImm(T Imm, unsigned RegSize, T &Imm0, T &Imm1)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
FunctionPass class - This class is used to implement most global optimizations.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
unsigned getSubReg() const
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Define
Register definition.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createAArch64MIPeepholeOptPass()
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
std::optional< UsedNZCV > examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > *CCUseInstrs=nullptr)
unsigned getRegState(const MachineOperand &RegOp)
Get all register state flags from machine operand RegOp.
void initializeAArch64MIPeepholeOptPass(PassRegistry &)