30#define DEBUG_TYPE "si-i1-copies"
83 bool lowerCopiesFromI1();
84 bool lowerCopiesToI1();
85 bool cleanConstrainRegs(
bool Changed);
87 return Reg.isVirtual() &&
MRI->getRegClass(Reg) == &AMDGPU::VReg_1RegClass;
96bool Vreg1LoweringHelper::cleanConstrainRegs(
bool Changed) {
99 MRI->constrainRegClass(Reg, &AMDGPU::SReg_1_XEXECRegClass);
100 ConstrainRegs.clear();
127class PhiIncomingAnalysis {
145 return ReachableMap.
find(&
MBB)->second;
152 ReachableMap.
clear();
153 ReachableOrdered.
clear();
154 Predecessors.
clear();
163 if (
MBB == &DefBlock) {
164 ReachableMap[&DefBlock] =
true;
177 while (!
Stack.empty()) {
187 bool HaveReachablePred =
false;
189 if (ReachableMap.
count(Pred)) {
190 HaveReachablePred =
true;
192 Stack.push_back(Pred);
195 if (!HaveReachablePred)
196 ReachableMap[
MBB] =
true;
197 if (HaveReachablePred) {
259 unsigned FoundLoopLevel = ~0
u;
267 : DT(DT), PDT(PDT) {}
271 CommonDominators.
clear();
274 VisitedPostDom =
nullptr;
275 FoundLoopLevel = ~0
u;
291 while (PDNode->
getBlock() != PostDom) {
292 if (PDNode->
getBlock() == VisitedPostDom)
296 if (FoundLoopLevel == Level)
316 if (!inLoopLevel(*Dom, LoopLevel, Incomings)) {
323 if (!inLoopLevel(*Pred, LoopLevel, Incomings))
333 auto DomIt = Visited.
find(&
MBB);
334 if (DomIt != Visited.
end() && DomIt->second <= LoopLevel)
344 void advanceLevel() {
347 if (!VisitedPostDom) {
348 VisitedPostDom = DefBlock;
349 VisitedDom = DefBlock;
350 Stack.push_back(DefBlock);
353 VisitedDom = CommonDominators.
back();
355 for (
unsigned i = 0; i < NextLevel.
size();) {
356 if (PDT.
dominates(VisitedPostDom, NextLevel[i])) {
357 Stack.push_back(NextLevel[i]);
359 NextLevel[i] = NextLevel.
back();
367 unsigned Level = CommonDominators.
size();
368 while (!
Stack.empty()) {
373 Visited[
MBB] = Level;
377 if (Succ == DefBlock) {
378 if (
MBB == VisitedPostDom)
379 FoundLoopLevel = std::min(FoundLoopLevel, Level + 1);
381 FoundLoopLevel = std::min(FoundLoopLevel, Level);
386 if (
MBB == VisitedPostDom)
389 Stack.push_back(Succ);
407char SILowerI1Copies::
ID = 0;
412 return new SILowerI1Copies();
418 return MRI->createVirtualRegister(LaneMaskRegAttrs);
445 MachineFunctionProperties::Property::Selected))
448 Vreg1LoweringHelper Helper(&TheMF, &getAnalysis<MachineDominatorTree>(),
449 &getAnalysis<MachinePostDominatorTree>());
451 bool Changed =
false;
452 Changed |= Helper.lowerCopiesFromI1();
453 Changed |= Helper.lowerPhis();
454 Changed |= Helper.lowerCopiesToI1();
455 return Helper.cleanConstrainRegs(Changed);
462 unsigned Size =
TRI.getRegSizeInBits(Reg,
MRI);
467bool Vreg1LoweringHelper::lowerCopiesFromI1() {
468 bool Changed =
false;
473 if (
MI.getOpcode() != AMDGPU::COPY)
478 if (!isVreg1(SrcReg))
481 if (isLaneMaskReg(DstReg) || isVreg1(DstReg))
491 assert(!
MI.getOperand(0).getSubReg());
493 ConstrainRegs.insert(SrcReg);
504 MI->eraseFromParent();
513 : MF(MF), DT(DT), PDT(PDT) {
522 MovOp = AMDGPU::S_MOV_B32;
523 AndOp = AMDGPU::S_AND_B32;
524 OrOp = AMDGPU::S_OR_B32;
525 XorOp = AMDGPU::S_XOR_B32;
527 OrN2Op = AMDGPU::S_ORN2_B32;
530 MovOp = AMDGPU::S_MOV_B64;
531 AndOp = AMDGPU::S_AND_B64;
532 OrOp = AMDGPU::S_OR_B64;
533 XorOp = AMDGPU::S_XOR_B64;
535 OrN2Op = AMDGPU::S_ORN2_B64;
541 LoopFinder LF(*
DT, *
PDT);
542 PhiIncomingAnalysis PIA(*
PDT,
TII);
547 if (Vreg1Phis.
empty())
554 if (&
MBB != PrevMBB) {
582 std::vector<MachineBasicBlock *> DomBlocks = {&
MBB};
593 unsigned FoundLoopLevel = LF.findLoop(PostDomBound);
597 if (FoundLoopLevel) {
615 PIA.analyze(
MBB, Incomings);
623 if (PIA.isSource(IMBB)) {
644 if (NewReg != DstReg) {
646 MI->eraseFromParent();
654bool Vreg1LoweringHelper::lowerCopiesToI1() {
655 bool Changed =
false;
657 LoopFinder LF(*DT, *PDT);
664 if (
MI.getOpcode() != AMDGPU::IMPLICIT_DEF &&
665 MI.getOpcode() != AMDGPU::COPY)
669 if (!isVreg1(DstReg))
674 if (
MRI->use_empty(DstReg)) {
681 markAsLaneMask(DstReg);
682 initializeLaneMaskRegisterAttributes(DstReg);
684 if (
MI.getOpcode() == AMDGPU::IMPLICIT_DEF)
689 assert(!
MI.getOperand(1).getSubReg());
691 if (!SrcReg.
isVirtual() || (!isLaneMaskReg(SrcReg) && !isVreg1(SrcReg))) {
692 assert(
TII->getRegisterInfo().getRegSizeInBits(SrcReg, *
MRI) == 32);
697 MI.getOperand(1).setReg(TmpReg);
701 MI.getOperand(1).setIsKill(
false);
706 std::vector<MachineBasicBlock *> DomBlocks = {&
MBB};
712 unsigned FoundLoopLevel = LF.findLoop(PostDomBound);
713 if (FoundLoopLevel) {
716 LF.addLoopEntries(FoundLoopLevel,
SSAUpdater, *
MRI, LaneMaskRegAttrs);
718 buildMergeLaneMasks(
MBB,
MI,
DL, DstReg,
725 MI->eraseFromParent();
735 if (
MI->getOpcode() == AMDGPU::IMPLICIT_DEF)
738 if (
MI->getOpcode() != AMDGPU::COPY)
741 Reg =
MI->getOperand(1).getReg();
742 if (!Reg.isVirtual())
751 if (!
MI->getOperand(1).isImm())
754 int64_t Imm =
MI->getOperand(1).getImm();
772 if (MO.isReg() && MO.getReg() == AMDGPU::SCC) {
786 bool TerminatorsUseSCC =
false;
787 for (
auto I = InsertionPt, E =
MBB.
end();
I != E; ++
I) {
790 if (TerminatorsUseSCC || DefsSCC)
794 if (!TerminatorsUseSCC)
797 while (InsertionPt !=
MBB.
begin()) {
811void Vreg1LoweringHelper::markAsLaneMask(
Register DstReg)
const {
812 MRI->setRegClass(DstReg, ST->getBoolRC());
815void Vreg1LoweringHelper::getCandidatesForLowering(
819 if (isVreg1(
MI.getOperand(0).getReg()))
825void Vreg1LoweringHelper::collectIncomingValuesFromPhi(
827 for (
unsigned i = 1; i <
MI->getNumOperands(); i += 2) {
828 assert(i + 1 <
MI->getNumOperands());
829 Register IncomingReg =
MI->getOperand(i).getReg();
833 if (IncomingDef->
getOpcode() == AMDGPU::COPY) {
835 assert(isLaneMaskReg(IncomingReg) || isVreg1(IncomingReg));
837 }
else if (IncomingDef->
getOpcode() == AMDGPU::IMPLICIT_DEF) {
840 assert(IncomingDef->
isPHI() || PhiRegisters.count(IncomingReg));
849 MRI->replaceRegWith(NewReg, OldReg);
857 bool PrevVal =
false;
858 bool PrevConstant = isConstantLaneMask(PrevReg, PrevVal);
860 bool CurConstant = isConstantLaneMask(CurReg, CurVal);
862 if (PrevConstant && CurConstant) {
863 if (PrevVal == CurVal) {
878 if (CurConstant && CurVal) {
879 PrevMaskedReg = PrevReg;
889 if (PrevConstant && PrevVal) {
890 CurMaskedReg = CurReg;
899 if (PrevConstant && !PrevVal) {
902 }
else if (CurConstant && !CurVal) {
905 }
else if (PrevConstant && PrevVal) {
912 .
addReg(CurMaskedReg ? CurMaskedReg : ExecReg);
916void Vreg1LoweringHelper::constrainAsLaneMask(
Incoming &In) {}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void instrDefsUsesSCC(const MachineInstr &MI, bool &Def, bool &Use)
static Register insertUndefLaneMask(MachineBasicBlock *MBB, MachineRegisterInfo *MRI, MachineRegisterInfo::VRegAttrs LaneMaskRegAttrs)
static bool isVRegCompatibleReg(const SIRegisterInfo &TRI, const MachineRegisterInfo &MRI, Register Reg)
Interface definition of the PhiLoweringHelper class that implements lane mask merging algorithm for d...
static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ArrayRef< StringLiteral > StandardNames)
Initialize the set of available library functions based on the specified target triple.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Implements a dense probed hash-table based set.
Base class for the actual dominator tree node.
DomTreeNodeBase * getIDom() const
unsigned getDFSNumIn() const
getDFSNumIn/getDFSNumOut - These return the DFS visitation order for nodes in the dominator tree.
void updateDFSNumbers() const
updateDFSNumbers - Assign In and Out numbers to the nodes while walking dominator tree in dfs order.
FunctionPass class - This class is used to implement most global optimizations.
const SIInstrInfo * getInstrInfo() const override
iterator_range< iterator > phis()
Returns a range that iterates over the phis in the basic block.
void push_back(MachineInstr *MI)
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
iterator_range< pred_iterator > predecessors()
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineBasicBlock * findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B)
findNearestCommonDominator - Find nearest common dominator basic block for basic block A and B.
MachineDomTreeNode * getNode(MachineBasicBlock *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
MachineDomTree & getBase()
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
bool hasProperty(Property P) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const MachineFunctionProperties & getProperties() const
Get the function properties.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineBasicBlock * findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B) const
MachineDomTreeNode * getNode(MachineBasicBlock *BB) const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
MachineSSAUpdater - This class updates SSA form for a set of virtual registers defined in multiple bl...
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
PhiLoweringHelper(MachineFunction *MF, MachineDominatorTree *DT, MachinePostDominatorTree *PDT)
bool isLaneMaskReg(Register Reg) const
MachineRegisterInfo * MRI
MachineDominatorTree * DT
DenseSet< Register > PhiRegisters
virtual void getCandidatesForLowering(SmallVectorImpl< MachineInstr * > &Vreg1Phis) const =0
virtual void constrainAsLaneMask(Incoming &In)=0
virtual void collectIncomingValuesFromPhi(const MachineInstr *MI, SmallVectorImpl< Incoming > &Incomings) const =0
virtual void markAsLaneMask(Register DstReg) const =0
MachinePostDominatorTree * PDT
MachineRegisterInfo::VRegAttrs LaneMaskRegAttrs
MachineBasicBlock::iterator getSaluInsertionAtEnd(MachineBasicBlock &MBB) const
Return a point at the end of the given MBB to insert SALU instructions for lane mask calculation.
void initializeLaneMaskRegisterAttributes(Register LaneMask)
bool isConstantLaneMask(Register Reg, bool &Val) const
virtual void buildMergeLaneMasks(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, Register DstReg, Register PrevReg, Register CurReg)=0
virtual void replaceDstReg(Register NewReg, Register OldReg, MachineBasicBlock *MBB)=0
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Helper class for SSA formation on a set of values defined in multiple blocks.
void Initialize(Type *Ty, StringRef Name)
Reset this object to get ready for a new set of SSA updates with type 'Ty'.
Value * GetValueInMiddleOfBlock(BasicBlock *BB)
Construct SSA form, materializing a value that is live in the middle of the specified block.
void AddAvailableValue(BasicBlock *BB, Value *V)
Indicate that a rewritten value is available in the specified block with the specified value.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
A Use represents the edge between a Value definition and its users.
std::pair< iterator, bool > insert(const ValueT &V)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Register createLaneMaskReg(MachineRegisterInfo *MRI, MachineRegisterInfo::VRegAttrs LaneMaskRegAttrs)
FunctionPass * createSILowerI1CopiesPass()
void initializeSILowerI1CopiesPass(PassRegistry &)
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
MachineBasicBlock * Block
All attributes(register class or bank and low-level type) a virtual register can have.