Go to the documentation of this file.
47 #define DEBUG_TYPE "aarch64-a57-fp-load-balancing"
53 cl::desc(
"Always modify dest registers regardless of color"),
60 cl::desc(
"Ignore balance information, always return "
61 "(1: Even, 2: Odd)."),
69 switch (
MI->getOpcode()) {
70 case AArch64::FMULSrr:
71 case AArch64::FNMULSrr:
72 case AArch64::FMULDrr:
73 case AArch64::FNMULDrr:
82 switch (
MI->getOpcode()) {
83 case AArch64::FMSUBSrrr:
84 case AArch64::FMADDSrrr:
85 case AArch64::FNMSUBSrrr:
86 case AArch64::FNMADDSrrr:
87 case AArch64::FMSUBDrrr:
88 case AArch64::FMADDDrrr:
89 case AArch64::FNMSUBDrrr:
90 case AArch64::FNMADDDrrr:
102 enum class Color { Even, Odd };
104 static const char *ColorNames[2] = {
"Even",
"Odd" };
128 return "A57 FP Anti-dependency breaker";
143 std::map<unsigned, Chain*> &Active,
144 std::vector<std::unique_ptr<Chain>> &AllChains);
146 std::map<unsigned, Chain*> &RegChains);
148 Chain *getAndEraseNext(Color PreferredColor, std::vector<Chain*> &L);
155 "AArch64 A57 FP Load-Balancing",
false,
false)
204 : StartInst(
MI), LastInst(
MI), KillInst(nullptr),
205 StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0),
216 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
217 "Chain: broken invariant. A Chain can only be killed after its last "
236 KillIsImmutable = Immutable;
237 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
238 "Chain: broken invariant. A Chain can only be killed after its last "
267 unsigned End = KillInst ? KillInstIdx : LastInstIdx;
268 unsigned OtherEnd =
Other.KillInst ?
271 return StartInstIdx <= OtherEnd &&
Other.StartInstIdx <= End;
276 return StartInstIdx <
Other->StartInstIdx;
281 return (getKill() && isKillImmutable()) || !getKill();
290 StartInst->
print(OS,
true);
292 LastInst->
print(OS,
true);
295 KillInst->
print(OS,
true);
310 if (skipFunction(
F.getFunction()))
316 bool Changed =
false;
319 MRI = &
F.getRegInfo();
320 TRI =
F.getRegInfo().getTargetRegisterInfo();
323 for (
auto &
MBB :
F) {
331 bool Changed =
false;
333 <<
" - scanning instructions...\n");
340 std::map<unsigned, Chain*> ActiveChains;
341 std::vector<std::unique_ptr<Chain>> AllChains;
344 scanInstruction(&
MI, Idx++, ActiveChains, AllChains);
347 <<
" chains created.\n");
357 for (
auto &
I : AllChains)
360 for (
auto &
I : AllChains)
361 for (
auto &J : AllChains)
362 if (
I != J &&
I->rangeOverlapsWith(*J))
363 EC.unionSets(
I.get(), J.get());
364 LLVM_DEBUG(
dbgs() <<
"Created " <<
EC.getNumClasses() <<
" disjoint sets.\n");
370 std::vector<std::vector<Chain*> > V;
371 for (
auto I =
EC.begin(),
E =
EC.end();
I !=
E; ++
I) {
372 std::vector<Chain*> Cs(
EC.member_begin(
I),
EC.member_end());
373 if (Cs.empty())
continue;
380 [](
const std::vector<Chain *> &A,
const std::vector<Chain *> &
B) {
381 return A.front()->startsBefore(
B.front());
403 Chain *AArch64A57FPLoadBalancing::getAndEraseNext(Color PreferredColor,
404 std::vector<Chain*> &L) {
416 const unsigned SizeFuzz = 1;
417 unsigned MinSize = L.front()->size() - SizeFuzz;
418 for (
auto I = L.begin(),
E = L.end();
I !=
E; ++
I) {
419 if ((*I)->size() <= MinSize) {
426 if ((*I)->getPreferredColor() == PreferredColor) {
434 Chain *Ch = L.front();
439 bool AArch64A57FPLoadBalancing::colorChainSet(std::vector<Chain*> GV,
442 bool Changed =
false;
443 LLVM_DEBUG(
dbgs() <<
"colorChainSet(): #sets=" << GV.size() <<
"\n");
454 llvm::sort(GV, [](
const Chain *G1,
const Chain *G2) {
455 if (G1->size() != G2->size())
456 return G1->size() > G2->size();
457 if (G1->requiresFixup() != G2->requiresFixup())
458 return G1->requiresFixup() > G2->requiresFixup();
460 assert((G1 == G2 || (G1->startsBefore(G2) ^ G2->startsBefore(G1))) &&
461 "Starts before not total order!");
462 return G1->startsBefore(G2);
465 Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
466 while (Chain *
G = getAndEraseNext(PreferredColor, GV)) {
468 Color
C = PreferredColor;
471 C =
G->getPreferredColor();
474 <<
", Color=" << ColorNames[(
int)
C] <<
"\n");
479 if (
G->requiresFixup() &&
C !=
G->getPreferredColor()) {
480 C =
G->getPreferredColor();
482 <<
" - not worthwhile changing; "
484 << ColorNames[(
int)
C] <<
"\n");
487 Changed |= colorChain(
G,
C,
MBB);
489 Parity += (
C == Color::Even) ?
G->size() : -
G->size();
490 PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
496 int AArch64A57FPLoadBalancing::scavengeRegister(Chain *
G, Color
C,
501 Units.addLiveOuts(
MBB);
504 while (
I != ChainEnd) {
506 Units.stepBackward(*
I);
511 assert(ChainBegin != ChainEnd &&
"Chain should contain instructions");
514 Units.accumulate(*
I);
515 }
while (
I != ChainBegin);
518 unsigned RegClassID = ChainBegin->getDesc().OpInfo[0].RegClass;
520 for (
auto Reg : Ord) {
521 if (!Units.available(
Reg))
523 if (
C == getColor(
Reg))
530 bool AArch64A57FPLoadBalancing::colorChain(Chain *
G, Color
C,
532 bool Changed =
false;
534 << ColorNames[(
int)
C] <<
")\n");
538 int Reg = scavengeRegister(
G,
C,
MBB);
545 std::map<unsigned, unsigned> Substs;
547 if (!
G->contains(
I) && (&
I !=
G->getKill() ||
G->isKillImmutable()))
552 std::vector<unsigned> ToErase;
553 for (
auto &U :
I.operands()) {
554 if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) {
556 U.setReg(Substs[OrigReg]);
560 ToErase.push_back(OrigReg);
561 }
else if (U.isRegMask()) {
562 for (
auto J : Substs) {
563 if (U.clobbersPhysReg(J.first))
564 ToErase.push_back(J.first);
569 for (
auto J : ToErase)
573 if (&
I !=
G->getKill()) {
577 if (
G->requiresFixup() && &
I ==
G->getLast())
588 assert(Substs.size() == 0 &&
"No substitutions should be left active!");
600 void AArch64A57FPLoadBalancing::scanInstruction(
601 MachineInstr *
MI,
unsigned Idx, std::map<unsigned, Chain *> &ActiveChains,
602 std::vector<std::unique_ptr<Chain>> &AllChains) {
607 for (
auto &
I :
MI->uses())
608 maybeKillChain(
I, Idx, ActiveChains);
609 for (
auto &
I :
MI->defs())
610 maybeKillChain(
I, Idx, ActiveChains);
614 Register DestReg =
MI->getOperand(0).getReg();
619 auto G = std::make_unique<Chain>(
MI, Idx, getColor(DestReg));
620 ActiveChains[DestReg] =
G.get();
627 Register DestReg =
MI->getOperand(0).getReg();
628 Register AccumReg =
MI->getOperand(3).getReg();
630 maybeKillChain(
MI->getOperand(1), Idx, ActiveChains);
631 maybeKillChain(
MI->getOperand(2), Idx, ActiveChains);
632 if (DestReg != AccumReg)
633 maybeKillChain(
MI->getOperand(0), Idx, ActiveChains);
635 if (ActiveChains.find(AccumReg) != ActiveChains.end()) {
644 if (
MI->getOperand(3).isKill()) {
646 LLVM_DEBUG(
dbgs() <<
"Instruction was successfully added to chain.\n");
647 ActiveChains[AccumReg]->add(
MI, Idx, getColor(DestReg));
649 if (DestReg != AccumReg) {
650 ActiveChains[DestReg] = ActiveChains[AccumReg];
651 ActiveChains.erase(AccumReg);
657 dbgs() <<
"Cannot add to chain because accumulator operand wasn't "
658 <<
"marked <kill>!\n");
659 maybeKillChain(
MI->getOperand(3), Idx, ActiveChains);
664 auto G = std::make_unique<Chain>(
MI, Idx, getColor(DestReg));
665 ActiveChains[DestReg] =
G.get();
672 for (
auto &
I :
MI->uses())
673 maybeKillChain(
I, Idx, ActiveChains);
674 for (
auto &
I :
MI->defs())
675 maybeKillChain(
I, Idx, ActiveChains);
680 void AArch64A57FPLoadBalancing::
682 std::map<unsigned, Chain*> &ActiveChains) {
690 if (MO.
isKill() && ActiveChains.find(MO.
getReg()) != ActiveChains.end()) {
695 ActiveChains.erase(MO.
getReg());
699 for (
auto I = ActiveChains.begin(),
E = ActiveChains.end();
704 I->second->setKill(
MI, Idx,
true);
705 ActiveChains.erase(
I++);
713 Color AArch64A57FPLoadBalancing::getColor(
unsigned Reg) {
722 return new AArch64A57FPLoadBalancing();
void initializeAArch64A57FPLoadBalancingPass(PassRegistry &)
MachineBasicBlock::iterator end() const
Return an instruction that can be used as an iterator for the end of the chain.
INITIALIZE_PASS_BEGIN(AArch64A57FPLoadBalancing, DEBUG_TYPE, "AArch64 A57 FP Load-Balancing", false, false) INITIALIZE_PASS_END(AArch64A57FPLoadBalancing
MachineBasicBlock::iterator begin() const
ArrayRef< MCPhysReg > getOrder(const TargetRegisterClass *RC) const
getOrder - Returns the preferred allocation order for RC.
static bool runOnBasicBlock(MachineBasicBlock *MBB, unsigned BasicBlockNum, VRegRenamer &Renamer)
unsigned size() const
Return the number of instructions in the chain.
void add(MachineInstr *MI, unsigned Idx, Color C)
Add a new instruction into the chain.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MachineInstr * getLast() const
Return the last instruction in the chain.
A raw_ostream that writes to an std::string.
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
static cl::opt< bool > TransformAll("aarch64-a57-fp-load-balancing-force-all", cl::desc("Always modify dest registers regardless of color"), cl::init(false), cl::Hidden)
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
uint16_t getEncodingValue(MCRegister RegNo) const
Returns the encoding for RegNo.
Properties which a MachineFunction may have at a given point in time.
FunctionPass * createAArch64A57FPLoadBalancing()
MachineInstr * getStart() const
Return the first instruction in the chain.
A Chain is a sequence of instructions that are linked together by an accumulation operand.
unsigned const TargetRegisterInfo * TRI
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
std::set< MachineInstr * > Insts
All instructions in the chain.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
(vector float) vec_cmpeq(*A, *B) C
Represent the analysis usage information of a pass.
std::string str() const
Return a simple string representation of the chain.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
MachineOperand class - Representation of each machine instruction operand.
MachineFunctionProperties & set(Property P)
bool balanceFPOps() const
Chain(MachineInstr *MI, unsigned Idx, Color C)
A set of register units used to track register liveness.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
void runOnMachineFunction(const MachineFunction &MF)
runOnFunction - Prepare to answer questions about MF.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
const TargetRegisterClass * getRegClass(unsigned i) const
Returns the register class associated with the enumeration value.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Representation of each machine instruction.
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
bool requiresFixup() const
Return true if the group will require a fixup MOV at the end.
initializer< Ty > init(const Ty &Val)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
Register getReg() const
getReg - Returns the register number.
bool startsBefore(const Chain *Other) const
Return true if this chain starts before Other.
Color LastColor
The "color" of LastInst.
MachineInstrBundleIterator< MachineInstr > iterator
void setPreservesCFG()
This function should be called by the pass, iff they do not:
StringRef - Represent a constant reference to a string, i.e.
if(llvm_vc STREQUAL "") set(fake_version_inc "$
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
MachineInstr * StartInst
The important (marker) instructions.
bool rangeOverlapsWith(const Chain &Other) const
Return true if this chain (StartInst..KillInst) overlaps with Other.
unsigned const MachineRegisterInfo * MRI
Wrapper class representing virtual and physical registers.
static bool isMla(MachineInstr *MI)
bool KillIsImmutable
True if KillInst cannot be modified.
AArch64 A57 FP Load Balancing
MachineInstr * getKill() const
Return the "kill" instruction (as set with setKill()) or NULL.
void sort(IteratorTy Start, IteratorTy End)
static bool isMul(MachineInstr *MI)
bool contains(MachineInstr &MI)
Return true if MI is a member of the chain.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isKillImmutable() const
Can the Kill instruction (assuming one exists) be modified?
FunctionPass class - This class is used to implement most global optimizations.
static cl::opt< unsigned > OverrideBalance("aarch64-a57-fp-load-balancing-override", cl::desc("Ignore balance information, always return " "(1: Even, 2: Odd)."), cl::init(0), cl::Hidden)
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
std::string & str()
Flushes the stream contents to the target string and returns the string's reference.
unsigned StartInstIdx
The index, from the start of the basic block, that each marker appears.
void setKill(MachineInstr *MI, unsigned Idx, bool Immutable)
Inform the chain that its last active register (the dest register of LastInst) is killed by MI with n...
Color getPreferredColor()
Return the preferred color of this chain.
Optional< std::vector< StOtherPiece > > Other