65#define DEBUG_TYPE "machine-sink"
69 cl::desc(
"Split critical edges during machine sinking"),
74 cl::desc(
"Use block frequency info to find successors to sink"),
78 "machine-sink-split-probability-threshold",
80 "Percentage threshold for splitting single-instruction critical edge. "
81 "If the branch threshold is higher than this threshold, we allow "
82 "speculative execution of up to 1 instruction to avoid branching to "
83 "splitted critical edge"),
87 "machine-sink-load-instrs-threshold",
88 cl::desc(
"Do not try to find alias store for a load if there is a in-path "
89 "block whose instruction number is higher than this threshold."),
93 "machine-sink-load-blocks-threshold",
94 cl::desc(
"Do not try to find alias store for a load if the block number in "
95 "the straight line is higher than this threshold."),
100 cl::desc(
"Sink instructions into cycles to avoid "
105 "machine-sink-cycle-limit",
106 cl::desc(
"The maximum number of instructions considered for cycle sinking."),
109STATISTIC(NumSunk,
"Number of machine instructions sunk");
110STATISTIC(NumCycleSunk,
"Number of machine instructions sunk into a cycle");
113STATISTIC(NumPostRACopySink,
"Number of copies sunk after RA");
139 using AllSuccsCache =
140 std::map<MachineBasicBlock *, SmallVector<MachineBasicBlock *, 4>>;
161 std::map<std::pair<MachineBasicBlock *, MachineBasicBlock *>,
bool>
163 std::map<std::pair<MachineBasicBlock *, MachineBasicBlock *>,
164 std::vector<MachineInstr *>>
168 std::map<MachineBasicBlock *, std::vector<unsigned>> CachedRegisterPressure;
193 CEBCandidates.
clear();
223 AllSuccsCache &AllSuccessors);
233 bool &LocalUse)
const;
235 bool &BreakPHIEdge, AllSuccsCache &AllSuccessors);
244 AllSuccsCache &AllSuccessors);
251 AllSuccsCache &AllSuccessors)
const;
258char MachineSinking::ID = 0;
263 "Machine code sinking",
false,
false)
271bool MachineSinking::PerformTrivialForwardCoalescing(
MachineInstr &
MI,
279 !
MRI->hasOneNonDBGUse(SrcReg))
292 MRI->replaceRegWith(DstReg, SrcReg);
293 MI.eraseFromParent();
297 MRI->clearKillFlags(SrcReg);
307bool MachineSinking::AllUsesDominatedByBlock(
Register Reg,
311 bool &LocalUse)
const {
312 assert(
Reg.isVirtual() &&
"Only makes sense for vregs");
315 if (
MRI->use_nodbg_empty(
Reg))
333 MachineInstr *UseInst = MO.getParent();
334 unsigned OpNo = MO.getOperandNo();
335 MachineBasicBlock *UseBlock = UseInst->getParent();
336 return UseBlock == MBB && UseInst->isPHI() &&
337 UseInst->getOperand(OpNo + 1).getMBB() == DefMBB;
346 unsigned OpNo = &MO - &UseInst->
getOperand(0);
348 if (UseInst->
isPHI()) {
352 }
else if (UseBlock == DefMBB) {
368 assert(
MI.mayLoad() &&
"Expected MI that loads!");
372 if (
MI.memoperands_empty())
377 if (PSV->isGOT() || PSV->isConstantPool())
383void MachineSinking::FindCycleSinkCandidates(
386 for (
auto &
MI : *BB) {
389 LLVM_DEBUG(
dbgs() <<
"CycleSink: Instruction not a candidate for this "
394 LLVM_DEBUG(
dbgs() <<
"CycleSink: Instruction is not cycle invariant\n");
397 bool DontMoveAcrossStore =
true;
398 if (!
MI.isSafeToMove(AA, DontMoveAcrossStore)) {
399 LLVM_DEBUG(
dbgs() <<
"CycleSink: Instruction not safe to move.\n");
403 LLVM_DEBUG(
dbgs() <<
"CycleSink: Dont sink GOT or constant pool loads\n");
406 if (
MI.isConvergent())
415 LLVM_DEBUG(
dbgs() <<
"CycleSink: Instruction added as candidate.\n");
429 DT = &getAnalysis<MachineDominatorTree>();
430 PDT = &getAnalysis<MachinePostDominatorTree>();
431 CI = &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo();
432 MBFI =
UseBlockFreqInfo ? &getAnalysis<MachineBlockFrequencyInfo>() : nullptr;
433 MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
434 AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
437 bool EverMadeChange =
false;
440 bool MadeChange =
false;
443 CEBCandidates.
clear();
449 for (
const auto &Pair : ToSplit) {
450 auto NewSucc = Pair.first->SplitCriticalEdge(Pair.second, *
this);
451 if (NewSucc !=
nullptr) {
465 if (!MadeChange)
break;
466 EverMadeChange =
true;
472 for (
auto *
Cycle : Cycles) {
479 FindCycleSinkCandidates(
Cycle, Preheader, Candidates);
487 LLVM_DEBUG(
dbgs() <<
"CycleSink: Limit reached of instructions to "
492 if (!SinkIntoCycle(
Cycle, *
I))
494 EverMadeChange =
true;
500 HasStoreCache.clear();
501 StoreInstrCache.clear();
504 for (
auto I : RegsToClearKillFlags)
505 MRI->clearKillFlags(
I);
506 RegsToClearKillFlags.clear();
508 return EverMadeChange;
520 bool MadeChange =
false;
523 AllSuccsCache AllSuccessors;
528 bool ProcessedBegin, SawStore =
false;
538 if (
MI.isDebugOrPseudoInstr()) {
539 if (
MI.isDebugValue())
544 bool Joined = PerformTrivialForwardCoalescing(
MI, &
MBB);
556 }
while (!ProcessedBegin);
558 SeenDbgUsers.
clear();
561 CachedRegisterPressure.clear();
569 assert(
MI.isDebugValue() &&
"Expected DBG_VALUE for processing");
572 MI.getDebugLoc()->getInlinedAt());
573 bool SeenBefore = SeenDbgVars.
contains(Var);
577 SeenDbgUsers[MO.
getReg()].push_back(SeenDbgUser(&
MI, SeenBefore));
592 if (!CEBCandidates.
insert(std::make_pair(
From, To)).second)
612 if (
Reg.isPhysical())
618 if (
MRI->hasOneNonDBGUse(
Reg)) {
636 if (!isWorthBreakingCriticalEdge(
MI, FromBB, ToBB))
647 if (FromCycle == ToCycle && FromCycle &&
692 if (Pred != FromBB && !DT->
dominates(ToBB, Pred))
696 ToSplit.insert(std::make_pair(FromBB, ToBB));
701std::vector<unsigned> &
707 auto RP = CachedRegisterPressure.find(&
MBB);
708 if (RP != CachedRegisterPressure.end())
722 if (
MI.isDebugInstr() ||
MI.isPseudoProbe())
726 RPTracker.recedeSkipDebugValues();
727 assert(&*RPTracker.getPos() == &
MI &&
"RPTracker sync error!");
728 RPTracker.recede(RegOpers);
731 RPTracker.closeRegion();
732 auto It = CachedRegisterPressure.insert(
733 std::make_pair(&
MBB, RPTracker.getPressure().MaxSetPressure));
734 return It.first->second;
741 AllSuccsCache &AllSuccessors) {
742 assert (SuccToSinkTo &&
"Invalid SinkTo Candidate BB");
744 if (
MBB == SuccToSinkTo)
757 bool NonPHIUse =
false;
760 if (UseBlock == SuccToSinkTo && !UseInst.
isPHI())
768 bool BreakPHIEdge =
false;
771 FindSuccToSinkTo(
MI, SuccToSinkTo, BreakPHIEdge, AllSuccessors))
772 return isProfitableToSinkTo(
Reg,
MI, SuccToSinkTo, MBB2, AllSuccessors);
782 unsigned Weight =
TRI->getRegClassWeight(RC).RegWeight;
783 const int *PS =
TRI->getRegClassPressureSets(RC);
785 std::vector<unsigned> BBRegisterPressure =
786 getBBRegisterPressure(*SuccToSinkTo);
787 for (; *PS != -1; PS++)
790 if (Weight + BBRegisterPressure[*PS] >=
806 if (
Reg.isPhysical()) {
808 if (MO.
isUse() && !
MRI->isConstantPhysReg(
Reg) && !
TII->isIgnorableUse(MO))
816 bool LocalUse =
false;
817 if (!AllUsesDominatedByBlock(
Reg, SuccToSinkTo,
MBB, BreakPHIEdge,
835 if (isRegisterPressureSetExceedLimit(
MRI->getRegClass(
Reg))) {
836 LLVM_DEBUG(
dbgs() <<
"register pressure exceed limit, not profitable.");
852 AllSuccsCache &AllSuccessors)
const {
854 auto Succs = AllSuccessors.find(
MBB);
855 if (Succs != AllSuccessors.end())
856 return Succs->second;
869 if (DTChild->getIDom()->getBlock() ==
MI.getParent() &&
872 AllSuccs.push_back(DTChild->getBlock());
880 bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0;
881 return HasBlockFreq ? LHSFreq < RHSFreq
885 auto it = AllSuccessors.insert(std::make_pair(
MBB, AllSuccs));
887 return it.first->second;
894 AllSuccsCache &AllSuccessors) {
895 assert (
MBB &&
"Invalid MachineBasicBlock!");
904 if (!MO.
isReg())
continue;
907 if (
Reg == 0)
continue;
909 if (
Reg.isPhysical()) {
914 if (!
MRI->isConstantPhysReg(
Reg) && !
TII->isIgnorableUse(MO))
916 }
else if (!MO.
isDead()) {
922 if (MO.
isUse())
continue;
925 if (!
TII->isSafeToMoveRegClassDefs(
MRI->getRegClass(
Reg)))
933 bool LocalUse =
false;
934 if (!AllUsesDominatedByBlock(
Reg, SuccToSinkTo,
MBB,
935 BreakPHIEdge, LocalUse))
946 GetAllSortedSuccessors(
MI,
MBB, AllSuccessors)) {
947 bool LocalUse =
false;
948 if (AllUsesDominatedByBlock(
Reg, SuccBlock,
MBB,
949 BreakPHIEdge, LocalUse)) {
950 SuccToSinkTo = SuccBlock;
961 if (!isProfitableToSinkTo(
Reg,
MI,
MBB, SuccToSinkTo, AllSuccessors))
968 if (
MBB == SuccToSinkTo)
973 if (SuccToSinkTo && SuccToSinkTo->
isEHPad())
998 auto *
MBB =
MI.getParent();
1003 auto *PredBB = PredMBB->getBasicBlock();
1009 !PredBB->getTerminator()->getMetadata(LLVMContext::MD_make_implicit))
1014 bool OffsetIsScalable;
1015 if (!
TII->getMemOperandWithOffset(
MI, BaseOp,
Offset, OffsetIsScalable,
TRI))
1018 if (!BaseOp->
isReg())
1021 if (!(
MI.mayLoad() && !
MI.isPredicable()))
1024 MachineBranchPredicate MBP;
1025 if (
TII->analyzeBranchPredicate(*PredMBB, MBP,
false))
1028 return MBP.LHS.isReg() && MBP.RHS.isImm() && MBP.RHS.getImm() == 0 &&
1029 (MBP.Predicate == MachineBranchPredicate::PRED_NE ||
1030 MBP.Predicate == MachineBranchPredicate::PRED_EQ) &&
1031 MBP.LHS.getReg() == BaseOp->
getReg();
1048 auto CopyOperands =
TII.isCopyInstr(SinkInst);
1051 SrcMO = CopyOperands->Source;
1052 DstMO = CopyOperands->Destination;
1055 bool PostRA =
MRI.getNumVirtRegs() == 0;
1063 bool arePhysRegs = !
Reg.isVirtual();
1064 if (arePhysRegs != PostRA)
1071 if (DbgMO.getSubReg() != SrcMO->
getSubReg() ||
1072 DbgMO.getSubReg() != DstMO->getSubReg())
1078 if (PostRA &&
Reg != DstMO->getReg())
1082 DbgMO.setReg(SrcMO->
getReg());
1088using MIRegs = std::pair<MachineInstr *, SmallVector<unsigned, 2>>;
1096 if (!SuccToSinkTo.
empty() && InsertPos != SuccToSinkTo.
end())
1098 InsertPos->getDebugLoc()));
1104 SuccToSinkTo.
splice(InsertPos, ParentBlock,
MI,
1111 for (
const auto &DbgValueToSink : DbgValuesToSink) {
1114 SuccToSinkTo.
insert(InsertPos, NewDbgMI);
1116 bool PropagatedAllSunkOps =
true;
1117 for (
unsigned Reg : DbgValueToSink.second) {
1120 PropagatedAllSunkOps =
false;
1125 if (!PropagatedAllSunkOps)
1139 auto BlockPair = std::make_pair(
From, To);
1143 if (HasStoreCache.find(BlockPair) != HasStoreCache.end())
1144 return HasStoreCache[BlockPair];
1146 if (StoreInstrCache.find(BlockPair) != StoreInstrCache.end())
1148 return I->mayAlias(AA,
MI,
false);
1151 bool SawStore =
false;
1152 bool HasAliasedStore =
false;
1161 if (BB == To || BB ==
From)
1165 if (HandledBlocks.
count(BB))
1168 HandledBlocks.
insert(BB);
1171 if (!HandledDomBlocks.
count(BB))
1172 HandledDomBlocks.
insert(BB);
1178 for (
auto *DomBB : HandledDomBlocks) {
1179 if (DomBB != BB && DT->
dominates(DomBB, BB))
1180 HasStoreCache[std::make_pair(DomBB, To)] =
true;
1181 else if(DomBB != BB && DT->
dominates(BB, DomBB))
1182 HasStoreCache[std::make_pair(
From, DomBB)] =
true;
1184 HasStoreCache[BlockPair] =
true;
1191 if (
I.isCall() ||
I.hasOrderedMemoryRef()) {
1192 for (
auto *DomBB : HandledDomBlocks) {
1193 if (DomBB != BB && DT->
dominates(DomBB, BB))
1194 HasStoreCache[std::make_pair(DomBB, To)] =
true;
1195 else if(DomBB != BB && DT->
dominates(BB, DomBB))
1196 HasStoreCache[std::make_pair(
From, DomBB)] =
true;
1198 HasStoreCache[BlockPair] =
true;
1208 if (
I.mayAlias(AA,
MI,
false))
1209 HasAliasedStore =
true;
1210 StoreInstrCache[BlockPair].push_back(&
I);
1217 HasStoreCache[BlockPair] =
false;
1218 return HasAliasedStore;
1227 assert(Preheader &&
"Cycle sink needs a preheader block");
1229 bool CanSink =
true;
1235 LLVM_DEBUG(
dbgs() <<
"CycleSink: Use not in cycle, can't sink.\n");
1250 SinkBlock =
MI.getParent();
1257 LLVM_DEBUG(
dbgs() <<
"CycleSink: Can't find nearest dominator\n");
1261 LLVM_DEBUG(
dbgs() <<
"CycleSink: Setting nearest common dom block: " <<
1270 LLVM_DEBUG(
dbgs() <<
"CycleSink: Not sinking, can't find sink block.\n");
1273 if (SinkBlock == Preheader) {
1275 dbgs() <<
"CycleSink: Not sinking, sink block is the preheader\n");
1280 dbgs() <<
"CycleSink: Not Sinking, block too large to analyse.\n");
1291 RegsToClearKillFlags.insert(MO.
getReg());
1296 assert(!
I.isDebugInstr() &&
"Should not sink debug inst");
1313 if (!
TII->isBasicBlockPrologue(*PI))
1315 for (
auto &MO :
MI.operands()) {
1322 if (
Reg.isPhysical() &&
1323 (
TII->isIgnorableUse(MO) || (
MRI &&
MRI->isConstantPhysReg(
Reg))))
1325 if (PI->modifiesRegister(
Reg,
TRI))
1328 if (PI->readsRegister(
Reg,
TRI))
1331 auto *DefOp = PI->findRegisterDefOperand(
Reg,
false,
true,
TRI);
1332 if (DefOp && !DefOp->isDead())
1342bool MachineSinking::SinkInstruction(
MachineInstr &
MI,
bool &SawStore,
1343 AllSuccsCache &AllSuccessors) {
1349 if (!
MI.isSafeToMove(AA, SawStore))
1354 if (
MI.isConvergent())
1370 bool BreakPHIEdge =
false;
1373 FindSuccToSinkTo(
MI, ParentBlock, BreakPHIEdge, AllSuccessors);
1384 if (
Reg == 0 || !
Reg.isPhysical())
1390 LLVM_DEBUG(
dbgs() <<
"Sink instr " <<
MI <<
"\tinto block " << *SuccToSinkTo);
1397 bool TryBreak =
false;
1399 MI.mayLoad() ? hasStoreBetween(ParentBlock, SuccToSinkTo,
MI) :
true;
1400 if (!
MI.isSafeToMove(AA, Store)) {
1401 LLVM_DEBUG(
dbgs() <<
" *** NOTE: Won't sink load along critical edge.\n");
1407 if (!TryBreak && !DT->
dominates(ParentBlock, SuccToSinkTo)) {
1413 if (!TryBreak && CI->
getCycle(SuccToSinkTo) &&
1428 PostponeSplitCriticalEdge(
MI, ParentBlock, SuccToSinkTo, BreakPHIEdge);
1431 "break critical edge\n");
1441 bool Status = PostponeSplitCriticalEdge(
MI, ParentBlock,
1442 SuccToSinkTo, BreakPHIEdge);
1445 "break critical edge\n");
1454 LLVM_DEBUG(
dbgs() <<
" *** Not sinking: prologue interference\n");
1460 for (
auto &MO :
MI.all_defs()) {
1470 if (
User.getInt()) {
1485 if (
MI.getMF()->getFunction().getSubprogram() &&
MI.isCopy())
1486 SalvageUnsunkDebugUsersOfCopy(
MI, SuccToSinkTo);
1496 RegsToClearKillFlags.insert(MO.
getReg());
1501void MachineSinking::SalvageUnsunkDebugUsersOfCopy(
1512 for (
auto &MO :
MI.all_defs()) {
1521 if (
User.getParent() ==
MI.getParent())
1525 "DBG_VALUE user of vreg, but has no operand for it?");
1532 for (
auto *
User : DbgDefUsers) {
1533 for (
auto &
Reg : DbgUseRegs) {
1534 for (
auto &
DbgOp :
User->getDebugOperandsForReg(
Reg)) {
1535 DbgOp.setReg(
MI.getOperand(1).getReg());
1536 DbgOp.setSubReg(
MI.getOperand(1).getSubReg());
1593 MachineFunctionProperties::Property::NoVRegs);
1613char PostRAMachineSinking::ID = 0;
1617 "PostRA Machine Sink",
false,
false)
1632 for (
auto *
SI : SinkableBBs) {
1633 if (aliasWithRegsInLiveIn(*
SI,
Reg,
TRI)) {
1659 for (
auto DefReg : DefedRegsInCopy) {
1662 if (!BB || (SingleBB && SingleBB != BB))
1673 for (
auto U : UsedOpsInCopy) {
1679 if (UI.killsRegister(SrcReg,
TRI)) {
1680 UI.clearRegisterKills(SrcReg,
TRI);
1694 for (
unsigned DefReg : DefedRegsInCopy)
1697 for (
auto U : UsedOpsInCopy) {
1698 Register SrcReg =
MI->getOperand(U).getReg();
1701 Mask |= (*S).second;
1713 bool HasRegDependency =
false;
1714 for (
unsigned i = 0, e =
MI->getNumOperands(); i != e; ++i) {
1723 HasRegDependency =
true;
1732 }
else if (MO.
isUse()) {
1734 HasRegDependency =
true;
1740 return HasRegDependency;
1752 if (!
SI->livein_empty() &&
SI->pred_size() == 1)
1755 if (SinkableBBs.
empty())
1758 bool Changed =
false;
1762 ModifiedRegUnits.clear();
1763 UsedRegUnits.clear();
1764 SeenDbgInstrs.clear();
1774 if (
MI.isDebugValue() && !
MI.isDebugRef()) {
1776 bool IsValid =
true;
1782 ModifiedRegUnits, UsedRegUnits)) {
1790 MIUnits[*RI].push_back(MO.
getReg());
1794 for (
auto &RegOps : MIUnits)
1795 SeenDbgInstrs[RegOps.first].emplace_back(&
MI,
1796 std::move(RegOps.second));
1801 if (
MI.isDebugOrPseudoInstr())
1808 if (!
MI.isCopy() || !
MI.getOperand(0).isRenamable()) {
1816 ModifiedRegUnits, UsedRegUnits)) {
1822 "Unexpect SrcReg or DefReg");
1833 "Unexpected predecessor");
1839 for (
auto &MO :
MI.all_defs()) {
1841 for (
const auto &
MIRegs : SeenDbgInstrs.lookup(*RI)) {
1842 auto &Regs = DbgValsToSinkMap[
MIRegs.first];
1844 Regs.push_back(
Reg);
1848 auto DbgValsToSink = DbgValsToSinkMap.
takeVector();
1856 dbgs() <<
" *** Not sinking: prologue interference\n");
1867 ++NumPostRACopySink;
1876 bool Changed =
false;
1880 ModifiedRegUnits.init(*
TRI);
1881 UsedRegUnits.init(*
TRI);
1883 Changed |= tryToSinkCopy(BB, MF,
TRI,
TII);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
BlockVerifier::State From
This file defines the DenseSet and SmallDenseSet classes.
This file builds on the ADT/GraphTraits.h file to build generic depth first graph iterator.
const HexagonInstrInfo * TII
iv Induction Variable Users
static cl::opt< unsigned > SinkLoadInstsPerBlockThreshold("machine-sink-load-instrs-threshold", cl::desc("Do not try to find alias store for a load if there is a in-path " "block whose instruction number is higher than this threshold."), cl::init(2000), cl::Hidden)
static void clearKillFlags(MachineInstr *MI, MachineBasicBlock &CurBB, SmallVectorImpl< unsigned > &UsedOpsInCopy, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)
unsigned const TargetRegisterInfo * TRI
static void performSink(MachineInstr &MI, MachineBasicBlock &SuccToSinkTo, MachineBasicBlock::iterator InsertPos, ArrayRef< MIRegs > DbgValuesToSink)
Sink an instruction and its associated debug instructions.
static cl::opt< bool > SplitEdges("machine-sink-split", cl::desc("Split critical edges during machine sinking"), cl::init(true), cl::Hidden)
static bool mayLoadFromGOTOrConstantPool(MachineInstr &MI)
Return true if this machine instruction loads from global offset table or constant pool.
static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
Return true if MI is likely to be usable as a memory operation by the implicit null check optimizatio...
static cl::opt< bool > SinkInstsIntoCycle("sink-insts-to-avoid-spills", cl::desc("Sink instructions into cycles to avoid " "register spills"), cl::init(false), cl::Hidden)
static cl::opt< unsigned > SinkLoadBlocksThreshold("machine-sink-load-blocks-threshold", cl::desc("Do not try to find alias store for a load if the block number in " "the straight line is higher than this threshold."), cl::init(20), cl::Hidden)
static bool hasRegisterDependency(MachineInstr *MI, SmallVectorImpl< unsigned > &UsedOpsInCopy, SmallVectorImpl< unsigned > &DefedRegsInCopy, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits)
static void updateLiveIn(MachineInstr *MI, MachineBasicBlock *SuccBB, SmallVectorImpl< unsigned > &UsedOpsInCopy, SmallVectorImpl< unsigned > &DefedRegsInCopy)
static bool blockPrologueInterferes(MachineBasicBlock *BB, MachineBasicBlock::iterator End, MachineInstr &MI, const TargetRegisterInfo *TRI, const TargetInstrInfo *TII, const MachineRegisterInfo *MRI)
Return true if a target defined block prologue instruction interferes with a sink candidate.
static cl::opt< unsigned > SplitEdgeProbabilityThreshold("machine-sink-split-probability-threshold", cl::desc("Percentage threshold for splitting single-instruction critical edge. " "If the branch threshold is higher than this threshold, we allow " "speculative execution of up to 1 instruction to avoid branching to " "splitted critical edge"), cl::init(40), cl::Hidden)
static bool attemptDebugCopyProp(MachineInstr &SinkInst, MachineInstr &DbgMI, Register Reg)
If the sunk instruction is a copy, try to forward the copy instead of leaving an 'undef' DBG_VALUE in...
static MachineBasicBlock * getSingleLiveInSuccBB(MachineBasicBlock &CurBB, const SmallPtrSetImpl< MachineBasicBlock * > &SinkableBBs, unsigned Reg, const TargetRegisterInfo *TRI)
static cl::opt< unsigned > SinkIntoCycleLimit("machine-sink-cycle-limit", cl::desc("The maximum number of instructions considered for cycle sinking."), cl::init(50), cl::Hidden)
static cl::opt< bool > UseBlockFreqInfo("machine-sink-bfi", cl::desc("Use block frequency info to find successors to sink"), cl::init(true), cl::Hidden)
std::pair< MachineInstr *, SmallVector< unsigned, 2 > > MIRegs
This file implements a map that provides insertion order iteration.
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
This file defines the PointerIntPair class.
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
static bool ProcessBlock(BasicBlock &BB, DominatorTree &DT, LoopInfo &LI, AAResults &AA)
static bool SinkInstruction(Instruction *Inst, SmallPtrSetImpl< Instruction * > &Stores, DominatorTree &DT, LoopInfo &LI, AAResults &AA)
SinkInstruction - Determine whether it is safe to sink the specified machine instruction out of its c...
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
static DILocation * getMergedLocation(DILocation *LocA, DILocation *LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
Identifies a unique instance of a variable.
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Implements a dense probed hash-table based set.
Base class for the actual dominator tree node.
iterator_range< iterator > children()
const_toplevel_iterator toplevel_end() const
unsigned getCycleDepth(const BlockT *Block) const
get the depth for the cycle which containing a given block.
const_toplevel_iterator toplevel_begin() const
CycleT * getCycle(const BlockT *Block) const
Find the innermost cycle containing a given block.
A possibly irreducible generalization of a Loop.
BlockT * getHeader() const
bool isReducible() const
Whether the cycle is a natural loop.
BlockT * getCyclePreheader() const
Return the preheader block for this cycle.
bool contains(const BlockT *Block) const
Return whether Block is contained in the cycle.
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool shouldSink(const MachineInstr &MI) const override
A set of register units used to track register liveness.
static void accumulateUsedDefed(const MachineInstr &MI, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)
For a machine instruction MI, adds all register units used in UsedRegUnits and defined or clobbered i...
bool available(MCPhysReg Reg) const
Returns true if no part of physical register Reg is live.
void addLiveIns(const MachineBasicBlock &MBB)
Adds registers living into block MBB.
MCRegUnitMaskIterator enumerates a list of register units and their associated lane masks for Reg.
bool isValid() const
Returns true if this iterator is not yet at the end.
bool isValid() const
isValid - returns true if this iterator is not yet at the end.
bool isInlineAsmBrIndirectTarget() const
Returns true if this is the indirect dest of an INLINEASM_BR.
unsigned pred_size() const
bool isEHPad() const
Returns true if the block is a landing pad.
instr_iterator instr_begin()
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator SkipPHIsAndLabels(iterator I)
Return the first instruction in MBB after I that is not a PHI or a label.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
unsigned succ_size() const
void sortUniqueLiveIns()
Sorts and uniques the LiveIns vector.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
pred_iterator pred_begin()
instr_iterator instr_end()
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator_range< succ_iterator > successors()
bool isSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB is a successor of this block.
iterator_range< pred_iterator > predecessors()
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
void removeLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Remove the specified register from the live in set.
bool sizeWithoutDebugLargerThan(unsigned Limit) const
MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate machine basic b...
BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const
getblockFreq - Return block frequency.
void onEdgeSplit(const MachineBasicBlock &NewPredecessor, const MachineBasicBlock &NewSuccessor, const MachineBranchProbabilityInfo &MBPI)
incrementally calculate block frequencies when we split edges, to avoid full CFG traversal.
BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const
Legacy analysis pass which computes a MachineCycleInfo.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineBasicBlock * findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B)
findNearestCommonDominator - Find nearest common dominator basic block for basic block A and B.
bool isReachableFromEntry(const MachineBasicBlock *A)
isReachableFromEntry - Return true if A is dominated by the entry block of the function containing it...
MachineDomTreeNode * getNode(MachineBasicBlock *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
Representation of each machine instruction.
static iterator_range< filter_iterator< Operand *, std::function< bool(Operand &Op)> > > getDebugOperandsForReg(Instruction *MI, Register Reg)
Returns a range of all of the operands that correspond to a debug use of Reg.
bool hasDebugOperandForReg(Register Reg) const
Returns whether this debug value has at least one debug operand with the register Reg.
void setDebugValueUndef()
Sets all register debug operands in this debug value instruction to be undef.
const MachineBasicBlock * getParent() const
bool isCopyLike() const
Return true if the instruction behaves like a copy.
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
bool readsReg() const
readsReg - Returns true if this operand reads the previous value of its register.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setIsKill(bool Val=true)
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
VectorType takeVector()
Clear the MapVector and return the underlying vector.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
virtual void releaseMemory()
releaseMemory() - This member can be implemented by a pass if it wants to be able to release its memo...
PointerIntPair - This class implements a pair of a pointer and small integer.
Special value supplied for machine level alias analysis.
Track the current register pressure at some position in the instruction stream, and remember the high...
void runOnMachineFunction(const MachineFunction &MF)
runOnFunction - Prepare to answer questions about MF.
List of registers defined and used by a machine instruction.
void collect(const MachineInstr &MI, const TargetRegisterInfo &TRI, const MachineRegisterInfo &MRI, bool TrackLaneMasks, bool IgnoreDead)
Analyze the given instruction MI and fill in the Uses, Defs and DeadDefs list based on the MachineOpe...
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
A vector that has set insertion semantics.
void clear()
Completely clear the SetVector.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
size_type count(const_arg_type_t< ValueT > V) const
Return 1 if the specified key is in the set, 0 otherwise.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isCycleInvariant(const MachineCycle *Cycle, MachineInstr &I)
char & MachineSinkingID
MachineSinking - This pass performs sinking on machine instructions.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
char & PostRAMachineSinkingID
This pass perform post-ra machine sink for COPY instructions.
void initializeMachineSinkingPass(PassRegistry &)
iterator_range< df_iterator< T > > depth_first(const T &G)
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
TODO: Might pack better if we changed this to a Struct of Arrays, since MachineOperand is width 32,...
static constexpr LaneBitmask getAll()
RegisterPressure computed within a region of instructions delimited by TopPos and BottomPos.
Represents a predicate at the MachineFunction level.