33#define DEBUG_TYPE "x86-domain-reassignment"
35STATISTIC(NumClosuresConverted,
"Number of closures converted by the pass");
42enum RegDomain { NoDomain = -1, GPRDomain, MaskDomain, OtherDomain, NumDomains };
45 return X86::GR64RegClass.hasSubClassEq(RC) ||
46 X86::GR32RegClass.hasSubClassEq(RC) ||
47 X86::GR16RegClass.hasSubClassEq(RC) ||
48 X86::GR8RegClass.hasSubClassEq(RC);
53 return X86::VK16RegClass.hasSubClassEq(RC);
69 if (X86::GR8RegClass.hasSubClassEq(SrcRC))
70 return &X86::VK8RegClass;
71 if (X86::GR16RegClass.hasSubClassEq(SrcRC))
72 return &X86::VK16RegClass;
73 if (X86::GR32RegClass.hasSubClassEq(SrcRC))
74 return &X86::VK32RegClass;
75 if (X86::GR64RegClass.hasSubClassEq(SrcRC))
76 return &X86::VK64RegClass;
82class InstrConverterBase {
87 InstrConverterBase(
unsigned SrcOpcode) : SrcOpcode(SrcOpcode) {}
89 virtual ~InstrConverterBase() =
default;
94 assert(
MI->getOpcode() == SrcOpcode &&
95 "Wrong instruction passed to converter");
113class InstrIgnore :
public InstrConverterBase {
115 InstrIgnore(
unsigned SrcOpcode) : InstrConverterBase(SrcOpcode) {}
119 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
130class InstrReplacer :
public InstrConverterBase {
135 InstrReplacer(
unsigned SrcOpcode,
unsigned DstOpcode)
136 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
140 if (!InstrConverterBase::isLegal(
MI,
TII))
144 for (
const auto &MO :
MI->implicit_operands())
145 if (MO.isReg() && MO.isDef() && !MO.isDead() &&
146 !
TII->get(DstOpcode).hasImplicitDefOfPhysReg(MO.getReg()))
153 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
158 for (
auto &
Op :
MI->explicit_operands())
172class InstrReplacerDstCOPY :
public InstrConverterBase {
176 InstrReplacerDstCOPY(
unsigned SrcOpcode,
unsigned DstOpcode)
177 : InstrConverterBase(SrcOpcode), DstOpcode(DstOpcode) {}
181 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
186 TII->getRegClass(
TII->get(DstOpcode), 0,
MRI->getTargetRegisterInfo(),
193 .
add(
MI->getOperand(0))
208class InstrCOPYReplacer :
public InstrReplacer {
212 InstrCOPYReplacer(
unsigned SrcOpcode, RegDomain DstDomain,
unsigned DstOpcode)
213 : InstrReplacer(SrcOpcode, DstOpcode), DstDomain(DstDomain) {}
217 if (!InstrConverterBase::isLegal(
MI,
TII))
223 if (DstReg.
isPhysical() && (X86::GR8RegClass.contains(DstReg) ||
224 X86::GR16RegClass.contains(DstReg)))
227 if (SrcReg.
isPhysical() && (X86::GR8RegClass.contains(SrcReg) ||
228 X86::GR16RegClass.contains(SrcReg)))
236 assert(
MI->getOpcode() == TargetOpcode::COPY &&
"Expected a COPY");
238 for (
const auto &MO :
MI->operands()) {
242 if (MO.getReg().isPhysical())
245 RegDomain OpDomain =
getDomain(
MRI->getRegClass(MO.getReg()),
246 MRI->getTargetRegisterInfo());
249 if (OpDomain == DstDomain)
257class InstrReplaceWithCopy :
public InstrConverterBase {
262 InstrReplaceWithCopy(
unsigned SrcOpcode,
unsigned SrcOpIdx)
263 : InstrConverterBase(SrcOpcode), SrcOpIdx(SrcOpIdx) {}
267 assert(isLegal(
MI,
TII) &&
"Cannot convert instruction");
269 TII->get(TargetOpcode::COPY))
270 .
add({
MI->getOperand(0),
MI->getOperand(SrcOpIdx)});
282typedef std::pair<int, unsigned> InstrConverterBaseKeyTy;
285 InstrConverterBaseMap;
307 std::bitset<NumDomains> LegalDstDomains;
314 Closure(
unsigned ID, std::initializer_list<RegDomain> LegalDstDomainList) :
ID(
ID) {
315 for (RegDomain
D : LegalDstDomainList)
316 LegalDstDomains.set(
D);
320 void setAllIllegal() { LegalDstDomains.reset(); }
323 bool hasLegalDstDomain()
const {
return LegalDstDomains.any(); }
326 bool isLegal(RegDomain RD)
const {
return LegalDstDomains[RD]; }
329 void setIllegal(RegDomain RD) { LegalDstDomains[RD] =
false; }
331 bool empty()
const {
return Edges.
empty(); }
333 bool insertEdge(
Register Reg) {
return Edges.
insert(Reg).second; }
349 dbgs() <<
"Registers: ";
357 dbgs() <<
"\n" <<
"Instructions:";
365 unsigned getID()
const {
395 return "X86 Domain Reassignment Pass";
400 InstrConverterBaseMap Converters;
403 void initConverters();
406 void buildClosure(Closure &,
Register Reg);
413 void reassign(
const Closure &
C, RegDomain
Domain)
const;
419 bool isReassignmentProfitable(
const Closure &
C, RegDomain
Domain)
const;
422 double calculateCost(
const Closure &
C, RegDomain
Domain)
const;
425char X86DomainReassignment::ID = 0;
429void X86DomainReassignment::visitRegister(Closure &
C,
Register Reg,
432 if (!
Reg.isVirtual())
438 if (!
MRI->hasOneDef(Reg))
441 RegDomain RD =
getDomain(
MRI->getRegClass(Reg),
MRI->getTargetRegisterInfo());
452void X86DomainReassignment::encloseInstr(Closure &
C,
MachineInstr *
MI) {
453 auto I = EnclosedInstrs.find(
MI);
454 if (
I != EnclosedInstrs.end()) {
455 if (
I->second !=
C.getID())
462 EnclosedInstrs[
MI] =
C.getID();
463 C.addInstruction(
MI);
468 for (
int i = 0; i != NumDomains; ++i) {
469 if (
C.isLegal((RegDomain)i)) {
470 auto I = Converters.find({i,
MI->getOpcode()});
471 if (
I == Converters.end() || !
I->second->isLegal(
MI,
TII))
472 C.setIllegal((RegDomain)i);
477double X86DomainReassignment::calculateCost(
const Closure &
C,
478 RegDomain DstDomain)
const {
479 assert(
C.isLegal(DstDomain) &&
"Cannot calculate cost for illegal closure");
482 for (
auto *
MI :
C.instructions())
483 Cost += Converters.find({DstDomain,
MI->getOpcode()})
484 ->second->getExtraCost(
MI,
MRI);
488bool X86DomainReassignment::isReassignmentProfitable(
const Closure &
C,
490 return calculateCost(
C,
Domain) < 0.0;
493void X86DomainReassignment::reassign(
const Closure &
C, RegDomain
Domain)
const {
494 assert(
C.isLegal(
Domain) &&
"Cannot convert illegal closure");
499 for (
auto *
MI :
C.instructions())
500 if (Converters.find({Domain, MI->getOpcode()})
501 ->second->convertInstr(
MI,
TII,
MRI))
507 MRI->setRegClass(Reg, getDstRC(
MRI->getRegClass(Reg),
Domain));
508 for (
auto &MO :
MRI->use_operands(Reg)) {
516 for (
auto *
MI : ToErase)
517 MI->eraseFromParent();
524 if (!
MI.mayLoadOrStore())
529 if (MemOpStart == -1)
533 for (
unsigned MemOpIdx = MemOpStart,
535 MemOpIdx < MemOpEnd; ++MemOpIdx) {
537 if (
Op.isReg() &&
Op.getReg() == Reg)
543void X86DomainReassignment::buildClosure(Closure &
C,
Register Reg) {
545 RegDomain
Domain = NoDomain;
546 visitRegister(
C, Reg,
Domain, Worklist);
547 while (!Worklist.
empty()) {
551 if (!
C.insertEdge(CurReg))
566 for (
int OpIdx = 0; OpIdx < OpEnd; ++OpIdx) {
567 if (OpIdx ==
MemOp) {
573 if (!
Op.isReg() || !
Op.isUse())
575 visitRegister(
C,
Op.getReg(),
Domain, Worklist);
579 for (
auto &
UseMI :
MRI->use_nodbg_instructions(CurReg)) {
588 for (
auto &DefOp :
UseMI.defs()) {
597 visitRegister(
C, DefReg,
Domain, Worklist);
603void X86DomainReassignment::initConverters() {
604 Converters[{MaskDomain, TargetOpcode::PHI}] =
605 std::make_unique<InstrIgnore>(TargetOpcode::PHI);
607 Converters[{MaskDomain, TargetOpcode::IMPLICIT_DEF}] =
608 std::make_unique<InstrIgnore>(TargetOpcode::IMPLICIT_DEF);
610 Converters[{MaskDomain, TargetOpcode::INSERT_SUBREG}] =
611 std::make_unique<InstrReplaceWithCopy>(TargetOpcode::INSERT_SUBREG, 2);
613 Converters[{MaskDomain, TargetOpcode::COPY}] =
614 std::make_unique<InstrCOPYReplacer>(TargetOpcode::COPY, MaskDomain,
617 auto createReplacerDstCOPY = [&](
unsigned From,
unsigned To) {
618 Converters[{MaskDomain,
From}] =
619 std::make_unique<InstrReplacerDstCOPY>(
From, To);
622#define GET_EGPR_IF_ENABLED(OPC) STI->hasEGPR() ? OPC##_EVEX : OPC
639 auto createReplacer = [&](
unsigned From,
unsigned To) {
640 Converters[{MaskDomain,
From}] = std::make_unique<InstrReplacer>(
From, To);
646 createReplacer(X86::SHR16ri, X86::KSHIFTRWri);
647 createReplacer(X86::SHL16ri, X86::KSHIFTLWri);
648 createReplacer(X86::NOT16r, X86::KNOTWrr);
649 createReplacer(X86::OR16rr, X86::KORWrr);
650 createReplacer(X86::AND16rr, X86::KANDWrr);
651 createReplacer(X86::XOR16rr, X86::KXORWrr);
653 bool HasNDD = STI->hasNDD();
655 createReplacer(X86::SHR16ri_ND, X86::KSHIFTRWri);
656 createReplacer(X86::SHL16ri_ND, X86::KSHIFTLWri);
657 createReplacer(X86::NOT16r_ND, X86::KNOTWrr);
658 createReplacer(X86::OR16rr_ND, X86::KORWrr);
659 createReplacer(X86::AND16rr_ND, X86::KANDWrr);
660 createReplacer(X86::XOR16rr_ND, X86::KXORWrr);
673 createReplacer(X86::SHR32ri, X86::KSHIFTRDri);
674 createReplacer(X86::SHR64ri, X86::KSHIFTRQri);
676 createReplacer(X86::SHL32ri, X86::KSHIFTLDri);
677 createReplacer(X86::SHL64ri, X86::KSHIFTLQri);
679 createReplacer(X86::ADD32rr, X86::KADDDrr);
680 createReplacer(X86::ADD64rr, X86::KADDQrr);
682 createReplacer(X86::NOT32r, X86::KNOTDrr);
683 createReplacer(X86::NOT64r, X86::KNOTQrr);
685 createReplacer(X86::OR32rr, X86::KORDrr);
686 createReplacer(X86::OR64rr, X86::KORQrr);
688 createReplacer(X86::AND32rr, X86::KANDDrr);
689 createReplacer(X86::AND64rr, X86::KANDQrr);
691 createReplacer(X86::ANDN32rr, X86::KANDNDrr);
692 createReplacer(X86::ANDN64rr, X86::KANDNQrr);
694 createReplacer(X86::XOR32rr, X86::KXORDrr);
695 createReplacer(X86::XOR64rr, X86::KXORQrr);
698 createReplacer(X86::SHR32ri_ND, X86::KSHIFTRDri);
699 createReplacer(X86::SHL32ri_ND, X86::KSHIFTLDri);
700 createReplacer(X86::ADD32rr_ND, X86::KADDDrr);
701 createReplacer(X86::NOT32r_ND, X86::KNOTDrr);
702 createReplacer(X86::OR32rr_ND, X86::KORDrr);
703 createReplacer(X86::AND32rr_ND, X86::KANDDrr);
704 createReplacer(X86::XOR32rr_ND, X86::KXORDrr);
705 createReplacer(X86::SHR64ri_ND, X86::KSHIFTRQri);
706 createReplacer(X86::SHL64ri_ND, X86::KSHIFTLQri);
707 createReplacer(X86::ADD64rr_ND, X86::KADDQrr);
708 createReplacer(X86::NOT64r_ND, X86::KNOTQrr);
709 createReplacer(X86::OR64rr_ND, X86::KORQrr);
710 createReplacer(X86::AND64rr_ND, X86::KANDQrr);
711 createReplacer(X86::XOR64rr_ND, X86::KXORQrr);
721 createReplacer(X86::ADD8rr, X86::KADDBrr);
722 createReplacer(X86::ADD16rr, X86::KADDWrr);
724 createReplacer(X86::AND8rr, X86::KANDBrr);
730 createReplacer(X86::NOT8r, X86::KNOTBrr);
732 createReplacer(X86::OR8rr, X86::KORBrr);
734 createReplacer(X86::SHR8ri, X86::KSHIFTRBri);
735 createReplacer(X86::SHL8ri, X86::KSHIFTLBri);
742 createReplacer(X86::XOR8rr, X86::KXORBrr);
745 createReplacer(X86::ADD8rr_ND, X86::KADDBrr);
746 createReplacer(X86::ADD16rr_ND, X86::KADDWrr);
747 createReplacer(X86::AND8rr_ND, X86::KANDBrr);
748 createReplacer(X86::NOT8r_ND, X86::KNOTBrr);
749 createReplacer(X86::OR8rr_ND, X86::KORBrr);
750 createReplacer(X86::SHR8ri_ND, X86::KSHIFTRBri);
751 createReplacer(X86::SHL8ri_ND, X86::KSHIFTLBri);
752 createReplacer(X86::XOR8rr_ND, X86::KXORBrr);
755#undef GET_EGPR_IF_ENABLED
765 dbgs() <<
"***** Machine Function before Domain Reassignment *****\n");
774 if (!STI->hasAVX512() || !STI->hasBWI())
778 assert(
MRI->isSSA() &&
"Expected MIR to be in SSA form");
780 TII = STI->getInstrInfo();
782 bool Changed =
false;
784 EnclosedEdges.clear();
785 EnclosedEdges.resize(
MRI->getNumVirtRegs());
786 EnclosedInstrs.clear();
788 std::vector<Closure> Closures;
791 unsigned ClosureID = 0;
792 for (
unsigned Idx = 0;
Idx <
MRI->getNumVirtRegs(); ++
Idx) {
796 if (
MRI->reg_nodbg_empty(Reg))
800 if (!isGPR(
MRI->getRegClass(Reg)))
804 if (EnclosedEdges.test(
Idx))
808 Closure
C(ClosureID++, {MaskDomain});
809 buildClosure(
C, Reg);
812 if (!
C.empty() &&
C.isLegal(MaskDomain))
813 Closures.push_back(std::move(
C));
816 for (Closure &
C : Closures) {
818 if (isReassignmentProfitable(
C, MaskDomain)) {
819 reassign(
C, MaskDomain);
820 ++NumClosuresConverted;
826 dbgs() <<
"***** Machine Function after Domain Reassignment *****\n");
833 "X86 Domain Reassignment Pass",
false,
false)
837 return new X86DomainReassignment();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Expand Atomic instructions
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
#define GET_EGPR_IF_ENABLED(OPC)
static cl::opt< bool > DisableX86DomainReassignment("disable-x86-domain-reassignment", cl::Hidden, cl::desc("X86: Disable Virtual Register Reassignment."), cl::init(false))
static bool usedAsAddr(const MachineInstr &MI, Register Reg, const TargetInstrInfo *TII)
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This class represents an Operation in the Expression.
Implements a dense probed hash-table based set.
FunctionPass class - This class is used to implement most global optimizations.
Describe properties that are true of each instruction in the target description file.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
void print(raw_ostream &OS, const SlotIndexes *=nullptr) const
print - Print out the MachineFunction in a format suitable for debugging to the specified stream.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getNumOperands() const
Retuns the total number of operands.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
static Register index2VirtReg(unsigned Index)
Convert a 0-based index to a virtual register number.
static unsigned virtReg2Index(Register Reg)
Convert a virtual register number to a 0-based index.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
std::pair< iterator, bool > insert(const ValueT &V)
A range adaptor for a pair of iterators.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
Reg
All possible values of the reg field in the ModR/M byte.
int getMemoryOperandNo(uint64_t TSFlags)
unsigned getOperandBias(const MCInstrDesc &Desc)
Compute whether all of the def operands are repeated in the uses and therefore should be skipped.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createX86DomainReassignmentPass()
Return a Machine IR pass that reassigns instruction chains from one domain to another,...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Description of the encoding of one expression Op.