47#define DEBUG_TYPE "aarch64-a57-fp-load-balancing"
53 cl::desc(
"Always modify dest registers regardless of color"),
60 cl::desc(
"Ignore balance information, always return "
61 "(1: Even, 2: Odd)."),
69 switch (
MI->getOpcode()) {
70 case AArch64::FMULSrr:
71 case AArch64::FNMULSrr:
72 case AArch64::FMULDrr:
73 case AArch64::FNMULDrr:
82 switch (
MI->getOpcode()) {
83 case AArch64::FMSUBSrrr:
84 case AArch64::FMADDSrrr:
85 case AArch64::FNMSUBSrrr:
86 case AArch64::FNMADDSrrr:
87 case AArch64::FMSUBDrrr:
88 case AArch64::FMADDDrrr:
89 case AArch64::FNMSUBDrrr:
90 case AArch64::FNMADDDrrr:
102enum class Color { Even, Odd };
104static const char *ColorNames[2] = {
"Even",
"Odd" };
124 MachineFunctionProperties::Property::NoVRegs);
128 return "A57 FP Anti-dependency breaker";
143 std::map<unsigned, Chain*> &Active,
144 std::vector<std::unique_ptr<Chain>> &AllChains);
146 std::map<unsigned, Chain*> &RegChains);
148 Chain *getAndEraseNext(Color PreferredColor, std::vector<Chain*> &L);
152char AArch64A57FPLoadBalancing::ID = 0;
155 "AArch64 A57 FP Load-Balancing",
false,
false)
204 : StartInst(
MI), LastInst(
MI), KillInst(nullptr),
205 StartInstIdx(
Idx), LastInstIdx(
Idx), KillInstIdx(0),
216 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
217 "Chain: broken invariant. A Chain can only be killed after its last "
236 KillIsImmutable = Immutable;
237 assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) &&
238 "Chain: broken invariant. A Chain can only be killed after its last "
267 unsigned End = KillInst ? KillInstIdx : LastInstIdx;
268 unsigned OtherEnd =
Other.KillInst ?
271 return StartInstIdx <= OtherEnd &&
Other.StartInstIdx <=
End;
276 return StartInstIdx <
Other->StartInstIdx;
281 return (getKill() && isKillImmutable()) || !getKill();
310 if (skipFunction(
F.getFunction()))
316 bool Changed =
false;
319 MRI = &
F.getRegInfo();
320 TRI =
F.getRegInfo().getTargetRegisterInfo();
323 for (
auto &
MBB :
F) {
331 bool Changed =
false;
333 <<
" - scanning instructions...\n");
340 std::map<unsigned, Chain*> ActiveChains;
341 std::vector<std::unique_ptr<Chain>> AllChains;
344 scanInstruction(&
MI,
Idx++, ActiveChains, AllChains);
347 <<
" chains created.\n");
357 for (
auto &
I : AllChains)
360 for (
auto &
I : AllChains)
361 for (
auto &J : AllChains)
362 if (
I != J &&
I->rangeOverlapsWith(*J))
363 EC.unionSets(
I.get(), J.get());
364 LLVM_DEBUG(
dbgs() <<
"Created " <<
EC.getNumClasses() <<
" disjoint sets.\n");
370 std::vector<std::vector<Chain*> >
V;
371 for (
auto I =
EC.begin(),
E =
EC.end();
I !=
E; ++
I) {
372 std::vector<Chain*> Cs(
EC.member_begin(
I),
EC.member_end());
373 if (Cs.empty())
continue;
374 V.push_back(std::move(Cs));
380 [](
const std::vector<Chain *> &
A,
const std::vector<Chain *> &
B) {
381 return A.front()->startsBefore(
B.front());
398 Changed |= colorChainSet(std::move(
I),
MBB, Parity);
403Chain *AArch64A57FPLoadBalancing::getAndEraseNext(Color PreferredColor,
404 std::vector<Chain*> &L) {
416 const unsigned SizeFuzz = 1;
417 unsigned MinSize =
L.front()->size() - SizeFuzz;
418 for (
auto I =
L.begin(),
E =
L.end();
I !=
E; ++
I) {
419 if ((*I)->size() <= MinSize) {
426 if ((*I)->getPreferredColor() == PreferredColor) {
434 Chain *Ch =
L.front();
439bool AArch64A57FPLoadBalancing::colorChainSet(std::vector<Chain*> GV,
442 bool Changed =
false;
443 LLVM_DEBUG(
dbgs() <<
"colorChainSet(): #sets=" << GV.size() <<
"\n");
454 llvm::sort(GV, [](
const Chain *G1,
const Chain *G2) {
455 if (G1->size() != G2->size())
456 return G1->size() > G2->size();
457 if (G1->requiresFixup() != G2->requiresFixup())
458 return G1->requiresFixup() > G2->requiresFixup();
460 assert((G1 == G2 || (G1->startsBefore(G2) ^ G2->startsBefore(G1))) &&
461 "Starts before not total order!");
462 return G1->startsBefore(G2);
465 Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
466 while (Chain *
G = getAndEraseNext(PreferredColor, GV)) {
468 Color
C = PreferredColor;
471 C =
G->getPreferredColor();
474 <<
", Color=" << ColorNames[(
int)
C] <<
"\n");
479 if (
G->requiresFixup() &&
C !=
G->getPreferredColor()) {
480 C =
G->getPreferredColor();
482 <<
" - not worthwhile changing; "
484 << ColorNames[(
int)
C] <<
"\n");
487 Changed |= colorChain(
G,
C,
MBB);
489 Parity += (
C == Color::Even) ?
G->size() : -
G->size();
490 PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
496int AArch64A57FPLoadBalancing::scavengeRegister(Chain *
G, Color
C,
501 Units.addLiveOuts(
MBB);
504 while (
I != ChainEnd) {
506 Units.stepBackward(*
I);
511 assert(ChainBegin != ChainEnd &&
"Chain should contain instructions");
514 Units.accumulate(*
I);
515 }
while (
I != ChainBegin);
518 unsigned RegClassID = ChainBegin->getDesc().operands()[0].RegClass;
519 auto Ord = RCI.
getOrder(
TRI->getRegClass(RegClassID));
520 for (
auto Reg : Ord) {
521 if (!Units.available(Reg))
523 if (
C == getColor(Reg))
530bool AArch64A57FPLoadBalancing::colorChain(Chain *
G, Color
C,
532 bool Changed =
false;
534 << ColorNames[(
int)
C] <<
")\n");
538 int Reg = scavengeRegister(
G,
C,
MBB);
545 std::map<unsigned, unsigned> Substs;
547 if (!
G->contains(
I) && (&
I !=
G->getKill() ||
G->isKillImmutable()))
552 std::vector<unsigned> ToErase;
553 for (
auto &U :
I.operands()) {
554 if (
U.isReg() &&
U.isUse() && Substs.find(
U.getReg()) != Substs.end()) {
556 U.setReg(Substs[OrigReg]);
560 ToErase.push_back(OrigReg);
561 }
else if (
U.isRegMask()) {
562 for (
auto J : Substs) {
563 if (
U.clobbersPhysReg(J.first))
564 ToErase.push_back(J.first);
569 for (
auto J : ToErase)
573 if (&
I !=
G->getKill()) {
577 if (
G->requiresFixup() && &
I ==
G->getLast())
588 assert(Substs.size() == 0 &&
"No substitutions should be left active!");
600void AArch64A57FPLoadBalancing::scanInstruction(
602 std::vector<std::unique_ptr<Chain>> &AllChains) {
607 for (
auto &
I :
MI->uses())
608 maybeKillChain(
I,
Idx, ActiveChains);
609 for (
auto &
I :
MI->defs())
610 maybeKillChain(
I,
Idx, ActiveChains);
614 Register DestReg =
MI->getOperand(0).getReg();
619 auto G = std::make_unique<Chain>(
MI,
Idx, getColor(DestReg));
620 ActiveChains[DestReg] =
G.get();
621 AllChains.push_back(std::move(
G));
627 Register DestReg =
MI->getOperand(0).getReg();
628 Register AccumReg =
MI->getOperand(3).getReg();
630 maybeKillChain(
MI->getOperand(1),
Idx, ActiveChains);
631 maybeKillChain(
MI->getOperand(2),
Idx, ActiveChains);
632 if (DestReg != AccumReg)
633 maybeKillChain(
MI->getOperand(0),
Idx, ActiveChains);
635 if (ActiveChains.find(AccumReg) != ActiveChains.end()) {
644 if (
MI->getOperand(3).isKill()) {
646 LLVM_DEBUG(
dbgs() <<
"Instruction was successfully added to chain.\n");
647 ActiveChains[AccumReg]->add(
MI,
Idx, getColor(DestReg));
649 if (DestReg != AccumReg) {
650 ActiveChains[DestReg] = ActiveChains[AccumReg];
651 ActiveChains.erase(AccumReg);
657 dbgs() <<
"Cannot add to chain because accumulator operand wasn't "
658 <<
"marked <kill>!\n");
659 maybeKillChain(
MI->getOperand(3),
Idx, ActiveChains);
664 auto G = std::make_unique<Chain>(
MI,
Idx, getColor(DestReg));
665 ActiveChains[DestReg] =
G.get();
666 AllChains.push_back(std::move(
G));
672 for (
auto &
I :
MI->uses())
673 maybeKillChain(
I,
Idx, ActiveChains);
674 for (
auto &
I :
MI->defs())
675 maybeKillChain(
I,
Idx, ActiveChains);
680void AArch64A57FPLoadBalancing::
682 std::map<unsigned, Chain*> &ActiveChains) {
690 if (MO.
isKill() && ActiveChains.find(MO.
getReg()) != ActiveChains.end()) {
695 ActiveChains.erase(MO.
getReg());
699 for (
auto I = ActiveChains.begin(),
E = ActiveChains.end();
704 I->second->setKill(
MI,
Idx,
true);
705 ActiveChains.erase(
I++);
713Color AArch64A57FPLoadBalancing::getColor(
unsigned Reg) {
714 if ((
TRI->getEncodingValue(Reg) % 2) == 0)
722 return new AArch64A57FPLoadBalancing();
AArch64 A57 FP Load Balancing
static bool isMul(MachineInstr *MI)
static cl::opt< unsigned > OverrideBalance("aarch64-a57-fp-load-balancing-override", cl::desc("Ignore balance information, always return " "(1: Even, 2: Odd)."), cl::init(0), cl::Hidden)
static cl::opt< bool > TransformAll("aarch64-a57-fp-load-balancing-force-all", cl::desc("Always modify dest registers regardless of color"), cl::init(false), cl::Hidden)
static bool isMla(MachineInstr *MI)
unsigned const MachineRegisterInfo * MRI
This file implements the BitVector class.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
std::optional< std::vector< StOtherPiece > > Other
Generic implementation of equivalence classes through the use Tarjan's efficient union-find algorithm...
static bool runOnBasicBlock(MachineBasicBlock *MBB, unsigned BasicBlockNum, VRegRenamer &Renamer)
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
A Chain is a sequence of instructions that are linked together by an accumulation operand.
MachineBasicBlock::iterator begin() const
bool isKillImmutable() const
Can the Kill instruction (assuming one exists) be modified?
void add(MachineInstr *MI, unsigned Idx, Color C)
Add a new instruction into the chain.
bool contains(MachineInstr &MI)
Return true if MI is a member of the chain.
Color LastColor
The "color" of LastInst.
bool requiresFixup() const
Return true if the group will require a fixup MOV at the end.
MachineInstr * getLast() const
Return the last instruction in the chain.
bool KillIsImmutable
True if KillInst cannot be modified.
bool rangeOverlapsWith(const Chain &Other) const
Return true if this chain (StartInst..KillInst) overlaps with Other.
MachineInstr * getStart() const
Return the first instruction in the chain.
unsigned size() const
Return the number of instructions in the chain.
MachineBasicBlock::iterator end() const
Return an instruction that can be used as an iterator for the end of the chain.
void setKill(MachineInstr *MI, unsigned Idx, bool Immutable)
Inform the chain that its last active register (the dest register of LastInst) is killed by MI with n...
MachineInstr * getKill() const
Return the "kill" instruction (as set with setKill()) or NULL.
Color getPreferredColor()
Return the preferred color of this chain.
std::string str() const
Return a simple string representation of the chain.
std::set< MachineInstr * > Insts
All instructions in the chain.
Chain(MachineInstr *MI, unsigned Idx, Color C)
bool startsBefore(const Chain *Other) const
Return true if this chain starts before Other.
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
FunctionPass class - This class is used to implement most global optimizations.
A set of register units used to track register liveness.
MachineInstrBundleIterator< MachineInstr > iterator
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
Representation of each machine instruction.
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
bool isRegMask() const
isRegMask - Tests if this is a MO_RegisterMask operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
Register getReg() const
getReg - Returns the register number.
static bool clobbersPhysReg(const uint32_t *RegMask, MCRegister PhysReg)
clobbersPhysReg - Returns true if this RegMask clobbers PhysReg.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
void runOnMachineFunction(const MachineFunction &MF)
runOnFunction - Prepare to answer questions about MF.
ArrayRef< MCPhysReg > getOrder(const TargetRegisterClass *RC) const
getOrder - Returns the preferred allocation order for RC.
Wrapper class representing virtual and physical registers.
StringRef - Represent a constant reference to a string, i.e.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
A raw_ostream that writes to an std::string.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void initializeAArch64A57FPLoadBalancingPass(PassRegistry &)
FunctionPass * createAArch64A57FPLoadBalancing()
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.