62#define PASS_KEY "x86-flags-copy-lowering"
63#define DEBUG_TYPE PASS_KEY
65STATISTIC(NumCopiesEliminated,
"Number of copies of EFLAGS eliminated");
66STATISTIC(NumSetCCsInserted,
"Number of setCC instructions inserted");
67STATISTIC(NumTestsInserted,
"Number of test instructions inserted");
68STATISTIC(NumAddsInserted,
"Number of adds instructions inserted");
73using CondRegArray = std::array<unsigned, X86::LAST_VALID_COND + 1>;
100 std::pair<unsigned, bool> getCondOrInverseInReg(
113 CondRegArray &CondRegs);
121 CondRegArray &CondRegs);
133 "X86 EFLAGS copy lowering",
false,
false)
138 return new X86FlagsCopyLoweringPass();
141char X86FlagsCopyLoweringPass::ID = 0;
143void X86FlagsCopyLoweringPass::getAnalysisUsage(
AnalysisUsage &AU)
const {
154enum class FlagArithMnemonic {
167 "by this instruction!");
169#define LLVM_EXPAND_INSTR_SIZES(MNEMONIC, SUFFIX) \
170 case X86::MNEMONIC##8##SUFFIX: \
171 case X86::MNEMONIC##16##SUFFIX: \
172 case X86::MNEMONIC##32##SUFFIX: \
173 case X86::MNEMONIC##64##SUFFIX:
175#define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC) \
176 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr) \
177 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rr_REV) \
178 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, rm) \
179 LLVM_EXPAND_INSTR_SIZES(MNEMONIC, mr) \
180 case X86::MNEMONIC##8ri: \
181 case X86::MNEMONIC##16ri8: \
182 case X86::MNEMONIC##32ri8: \
183 case X86::MNEMONIC##64ri8: \
184 case X86::MNEMONIC##16ri: \
185 case X86::MNEMONIC##32ri: \
186 case X86::MNEMONIC##64ri32: \
187 case X86::MNEMONIC##8mi: \
188 case X86::MNEMONIC##16mi8: \
189 case X86::MNEMONIC##32mi8: \
190 case X86::MNEMONIC##64mi8: \
191 case X86::MNEMONIC##16mi: \
192 case X86::MNEMONIC##32mi: \
193 case X86::MNEMONIC##64mi32: \
194 case X86::MNEMONIC##8i8: \
195 case X86::MNEMONIC##16i16: \
196 case X86::MNEMONIC##32i32: \
197 case X86::MNEMONIC##64i32:
200 return FlagArithMnemonic::ADC;
203 return FlagArithMnemonic::SBB;
205#undef LLVM_EXPAND_ADC_SBB_INSTR
210 return FlagArithMnemonic::RCL;
215 return FlagArithMnemonic::RCR;
217#undef LLVM_EXPAND_INSTR_SIZES
221 return FlagArithMnemonic::SETB;
231 "Split instruction must be in the split block!");
233 "Only designed to split a tail of branch instructions!");
235 "Must split on an actual jCC instruction!");
241 "Must split after an actual jCC instruction!");
243 "Must only have this one terminator prior to the split!");
254 assert(MI.isTerminator() &&
255 "Should only have spliced terminators!");
257 MI.operands(), [&](MachineOperand &MOp) {
258 return MOp.isMBB() && MOp.getMBB() == &UnsplitSucc;
275 if (IsEdgeSplit || *SI != &UnsplitSucc)
284 if (Succ != &UnsplitSucc)
289 "Failed to make the new block a successor!");
297 for (
int OpIdx = 1, NumOps =
MI.getNumOperands(); OpIdx < NumOps;
301 assert(OpMBB.
isMBB() &&
"Block operand to a PHI is not a block!");
306 if (!IsEdgeSplit || Succ != &UnsplitSucc) {
315 MI.addOperand(MF, OpV);
328 case X86::CMOVBE_Fp32:
case X86::CMOVBE_Fp64:
case X86::CMOVBE_Fp80:
330 case X86::CMOVB_Fp32:
case X86::CMOVB_Fp64:
case X86::CMOVB_Fp80:
332 case X86::CMOVE_Fp32:
case X86::CMOVE_Fp64:
case X86::CMOVE_Fp80:
334 case X86::CMOVNBE_Fp32:
case X86::CMOVNBE_Fp64:
case X86::CMOVNBE_Fp80:
336 case X86::CMOVNB_Fp32:
case X86::CMOVNB_Fp64:
case X86::CMOVNB_Fp80:
338 case X86::CMOVNE_Fp32:
case X86::CMOVNE_Fp64:
case X86::CMOVNE_Fp80:
340 case X86::CMOVNP_Fp32:
case X86::CMOVNP_Fp64:
case X86::CMOVNP_Fp80:
342 case X86::CMOVP_Fp32:
case X86::CMOVP_Fp64:
case X86::CMOVP_Fp80:
347bool X86FlagsCopyLoweringPass::runOnMachineFunction(
MachineFunction &MF) {
353 TII = Subtarget->getInstrInfo();
354 TRI = Subtarget->getRegisterInfo();
355 MDT = &getAnalysis<MachineDominatorTree>();
356 PromoteRC = &X86::GR8RegClass;
370 if (
MI.getOpcode() == TargetOpcode::COPY &&
371 MI.getOperand(0).getReg() == X86::EFLAGS)
379 "The input to the copy for EFLAGS should always be a register!");
381 if (CopyDefI.
getOpcode() != TargetOpcode::COPY) {
397 dbgs() <<
"ERROR: Encountered unexpected def of an eflags copy: ";
400 "Cannot lower EFLAGS copy unless it is defined in turn by a copy!");
406 CopyI->eraseFromParent();
409 ++NumCopiesEliminated;
413 assert(DOp.isDef() &&
"Expected register def!");
414 assert(DOp.getReg() == X86::EFLAGS &&
"Unexpected copy def register!");
442 return &
MI != CopyI &&
MI.findRegisterDefOperand(X86::EFLAGS);
447 assert(MDT->dominates(BeginMBB, EndMBB) &&
448 "Only support paths down the dominator tree!");
457 if (!Visited.
insert(PredMBB).second)
459 if (HasEFLAGSClobber(PredMBB->begin(), PredMBB->end()))
464 }
while (!Worklist.
empty());
469 !HasEFLAGSClobber(TestMBB->
begin(), TestPos)) {
476 return MDT->findNearestCommonDominator(LHS, RHS);
482 if (HasEFLAGSClobberPath(HoistMBB, TestMBB))
500 return MI.findRegisterDefOperand(X86::EFLAGS);
503 dbgs() <<
" Using EFLAGS defined by: ";
506 dbgs() <<
" Using live-in flags for BB:\n";
521 CondRegArray CondRegs = collectCondsInRegs(*TestMBB, TestPos);
534 bool FlagsKilled =
false;
545 for (
auto MII = (&UseMBB == &
MBB && !VisitedBlocks.
count(&UseMBB))
546 ? std::next(CopyI->getIterator())
555 if (&
MI == CopyI || &
MI == &CopyDefI) {
557 "Should only encounter these on the second pass over the "
564 if (
MI.findRegisterDefOperand(X86::EFLAGS)) {
593 auto JmpIt =
MI.getIterator();
595 JmpIs.push_back(&*JmpIt);
605 rewriteCMov(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
607 rewriteFCMov(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
609 rewriteSetCC(*TestMBB, TestPos, TestLoc,
MI, *FlagUse, CondRegs);
610 }
else if (
MI.getOpcode() == TargetOpcode::COPY) {
611 rewriteCopy(
MI, *FlagUse, CopyDefI);
614 assert(
MI.findRegisterDefOperand(X86::EFLAGS) &&
615 "Expected a def of EFLAGS for this instruction!");
626 rewriteArithmetic(*TestMBB, TestPos, TestLoc,
MI, *FlagUse,
642 if (SuccMBB->isLiveIn(X86::EFLAGS) &&
643 VisitedBlocks.
insert(SuccMBB).second) {
657 if (SuccMBB == TestMBB || !MDT->dominates(TestMBB, SuccMBB)) {
660 <<
"ERROR: Encountered use that is not dominated by our test "
661 "basic block! Rewriting this would require inserting PHI "
662 "nodes to track the flag state across the CFG.\n\nTest "
665 dbgs() <<
"Use block:\n";
669 "Cannot lower EFLAGS copy when original copy def "
670 "does not dominate all uses.");
673 Blocks.push_back(SuccMBB);
676 SuccMBB->removeLiveIn(X86::EFLAGS);
678 }
while (!
Blocks.empty());
687 if (JmpI->getParent() == LastJmpMBB)
692 rewriteCondJmp(*TestMBB, TestPos, TestLoc, *JmpI, CondRegs);
702 if (
MI.getOpcode() == TargetOpcode::COPY &&
703 (
MI.getOperand(0).getReg() == X86::EFLAGS ||
704 MI.getOperand(1).getReg() == X86::EFLAGS)) {
716CondRegArray X86FlagsCopyLoweringPass::collectCondsInRegs(
718 CondRegArray CondRegs = {};
725 MI.getOperand(0).isReg() &&
MI.getOperand(0).getReg().isVirtual()) {
727 "A non-storing SETcc should always define a register!");
728 CondRegs[
Cond] =
MI.getOperand(0).getReg();
733 if (
MI.findRegisterDefOperand(X86::EFLAGS))
739Register X86FlagsCopyLoweringPass::promoteCondToReg(
743 auto SetI =
BuildMI(TestMBB, TestPos, TestLoc,
751std::pair<unsigned, bool> X86FlagsCopyLoweringPass::getCondOrInverseInReg(
754 unsigned &CondReg = CondRegs[
Cond];
756 if (!CondReg && !InvCondReg)
757 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
760 return {CondReg,
false};
762 return {InvCondReg,
true};
767 const DebugLoc &Loc,
unsigned Reg) {
775void X86FlagsCopyLoweringPass::rewriteArithmetic(
778 CondRegArray &CondRegs) {
787 case FlagArithMnemonic::ADC:
788 case FlagArithMnemonic::RCL:
789 case FlagArithMnemonic::RCR:
790 case FlagArithMnemonic::SBB:
791 case FlagArithMnemonic::SETB:
802 unsigned &CondReg = CondRegs[
Cond];
804 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
809 Register TmpReg =
MRI->createVirtualRegister(PromoteRC);
826 CondRegArray &CondRegs) {
831 std::tie(CondReg, Inverted) =
832 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
852 CondRegArray &CondRegs) {
857 std::tie(CondReg, Inverted) =
858 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
865 auto getFCMOVOpcode = [](
unsigned Opcode,
bool Inverted) {
868 case X86::CMOVBE_Fp32:
case X86::CMOVNBE_Fp32:
869 case X86::CMOVB_Fp32:
case X86::CMOVNB_Fp32:
870 case X86::CMOVE_Fp32:
case X86::CMOVNE_Fp32:
871 case X86::CMOVP_Fp32:
case X86::CMOVNP_Fp32:
872 return Inverted ? X86::CMOVE_Fp32 : X86::CMOVNE_Fp32;
873 case X86::CMOVBE_Fp64:
case X86::CMOVNBE_Fp64:
874 case X86::CMOVB_Fp64:
case X86::CMOVNB_Fp64:
875 case X86::CMOVE_Fp64:
case X86::CMOVNE_Fp64:
876 case X86::CMOVP_Fp64:
case X86::CMOVNP_Fp64:
877 return Inverted ? X86::CMOVE_Fp64 : X86::CMOVNE_Fp64;
878 case X86::CMOVBE_Fp80:
case X86::CMOVNBE_Fp80:
879 case X86::CMOVB_Fp80:
case X86::CMOVNB_Fp80:
880 case X86::CMOVE_Fp80:
case X86::CMOVNE_Fp80:
881 case X86::CMOVP_Fp80:
case X86::CMOVNP_Fp80:
882 return Inverted ? X86::CMOVE_Fp80 : X86::CMOVNE_Fp80;
892void X86FlagsCopyLoweringPass::rewriteCondJmp(
899 std::tie(CondReg, Inverted) =
900 getCondOrInverseInReg(TestMBB, TestPos, TestLoc,
Cond, CondRegs);
918 MRI->replaceRegWith(
MI.getOperand(0).getReg(),
920 MI.eraseFromParent();
928 CondRegArray &CondRegs) {
933 unsigned &CondReg = CondRegs[
Cond];
935 CondReg = promoteCondToReg(TestMBB, TestPos, TestLoc,
Cond);
941 "Cannot have a non-register defined operand to SETcc!");
945 MRI->clearKillFlags(OldReg);
946 MRI->replaceRegWith(OldReg, CondReg);
unsigned const MachineRegisterInfo * MRI
DenseMap< Block *, BlockRelaxAux > Blocks
static const HTTPClientCleanup Cleanup
const HexagonInstrInfo * TII
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
const SmallVectorImpl< MachineOperand > & Cond
static void r1(uint32_t &A, uint32_t &B, uint32_t &C, uint32_t &D, uint32_t &E, int I, uint32_t *Buf)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the make_scope_exit function, which executes user-defined cleanup logic at scope ex...
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static X86::CondCode getCondFromFCMOV(unsigned Opcode)
#define LLVM_EXPAND_INSTR_SIZES(MNEMONIC, SUFFIX)
static MachineBasicBlock & splitBlock(MachineBasicBlock &MBB, MachineInstr &SplitI, const X86InstrInfo &TII)
static FlagArithMnemonic getMnemonicFromOpcode(unsigned Opcode)
#define LLVM_EXPAND_ADC_SBB_INSTR(MNEMONIC)
static constexpr uint32_t Opcode
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
FunctionPass class - This class is used to implement most global optimizations.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
void normalizeSuccProbs()
Normalize probabilities of all successors so that the sum of them becomes one.
instr_iterator instr_begin()
void replaceSuccessor(MachineBasicBlock *Old, MachineBasicBlock *New)
Replace successor OLD with NEW and update probability info.
MachineBasicBlock * getFallThrough(bool JumpToFallThrough=true)
Return the fallthrough block if the block can implicitly transfer control to the block after it by fa...
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
succ_iterator succ_begin()
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
void copySuccessor(const MachineBasicBlock *Orig, succ_iterator I)
Copy a successor (and any probability info) from original block to this block's.
pred_iterator pred_begin()
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
iterator_range< pred_iterator > predecessors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
bool isBranch(QueryType Type=AnyInBundle) const
Returns true if this is a conditional, unconditional, or indirect branch.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand * findRegisterUseOperand(Register Reg, bool isKill=false, const TargetRegisterInfo *TRI=nullptr)
Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsKill(bool Val=true)
void setMBB(MachineBasicBlock *MBB)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0)
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Reg
All possible values of the reg field in the ModR/M byte.
CondCode getCondFromBranch(const MachineInstr &MI)
CondCode GetOppositeBranchCondition(CondCode CC)
GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.
CondCode getCondFromSETCC(const MachineInstr &MI)
CondCode getCondFromCMov(const MachineInstr &MI)
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
detail::scope_exit< std::decay_t< Callable > > make_scope_exit(Callable &&F)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
FunctionPass * createX86FlagsCopyLoweringPass()
Return a pass that lowers EFLAGS copy pseudo instructions.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
OutputIt copy(R &&Range, OutputIt Out)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.