69#include "llvm/Config/llvm-config.h"
99#define DEBUG_TYPE "pipeliner"
101STATISTIC(NumTrytoPipeline,
"Number of loops that we attempt to pipeline");
102STATISTIC(NumPipelined,
"Number of loops software pipelined");
103STATISTIC(NumNodeOrderIssues,
"Number of node order issues found");
104STATISTIC(NumFailBranch,
"Pipeliner abort due to unknown branch");
105STATISTIC(NumFailLoop,
"Pipeliner abort due to unsupported loop");
106STATISTIC(NumFailPreheader,
"Pipeliner abort due to missing preheader");
107STATISTIC(NumFailLargeMaxMII,
"Pipeliner abort due to MaxMII too large");
108STATISTIC(NumFailZeroMII,
"Pipeliner abort due to zero MII");
109STATISTIC(NumFailNoSchedule,
"Pipeliner abort due to no schedule found");
110STATISTIC(NumFailZeroStage,
"Pipeliner abort due to zero stage");
111STATISTIC(NumFailLargeMaxStage,
"Pipeliner abort due to too many stages");
115 cl::desc(
"Enable Software Pipelining"));
124 cl::desc(
"Size limit for the MII."),
130 cl::desc(
"Force pipeliner to use specified II."),
136 cl::desc(
"Maximum stages allowed in the generated scheduled."),
143 cl::desc(
"Prune dependences between unrelated Phi nodes."),
150 cl::desc(
"Prune loop carried order dependences."),
168 cl::desc(
"Instead of emitting the pipelined code, annotate instructions "
169 "with the generated schedule for feeding into the "
170 "-modulo-schedule-test pass"));
175 "Use the experimental peeling code generator for software pipelining"));
182 cl::desc(
"Enable CopyToPhi DAG Mutation"));
187 "pipeliner-force-issue-width",
193unsigned SwingSchedulerDAG::Circuits::MaxPaths = 5;
201 "Modulo Software Pipelining",
false,
false)
211 if (skipFunction(mf.getFunction()))
217 if (mf.getFunction().getAttributes().hasFnAttr(Attribute::OptimizeForSize) &&
221 if (!mf.getSubtarget().enableMachinePipeliner())
226 if (mf.getSubtarget().useDFAforSMS() &&
227 (!mf.getSubtarget().getInstrItineraryData() ||
228 mf.getSubtarget().getInstrItineraryData()->isEmpty()))
232 MLI = &getAnalysis<MachineLoopInfo>();
233 MDT = &getAnalysis<MachineDominatorTree>();
234 ORE = &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
235 TII = MF->getSubtarget().getInstrInfo();
236 RegClassInfo.runOnMachineFunction(*MF);
238 for (
const auto &L : *MLI)
248bool MachinePipeliner::scheduleLoop(
MachineLoop &L) {
249 bool Changed =
false;
250 for (
const auto &InnerLoop : L)
251 Changed |= scheduleLoop(*InnerLoop);
263 setPragmaPipelineOptions(L);
264 if (!canPipelineLoop(L)) {
268 L.getStartLoc(), L.getHeader())
269 <<
"Failed to pipeline loop";
278 Changed = swingModuloScheduler(L);
284void MachinePipeliner::setPragmaPipelineOptions(
MachineLoop &L) {
303 if (LoopID ==
nullptr)
320 if (S->
getString() ==
"llvm.loop.pipeline.initiationinterval") {
322 "Pipeline initiation interval hint metadata should have two operands.");
324 mdconst::extract<ConstantInt>(MD->
getOperand(1))->getZExtValue();
326 }
else if (S->
getString() ==
"llvm.loop.pipeline.disable") {
335bool MachinePipeliner::canPipelineLoop(
MachineLoop &L) {
336 if (
L.getNumBlocks() != 1) {
339 L.getStartLoc(),
L.getHeader())
340 <<
"Not a single basic block: "
341 <<
ore::NV(
"NumBlocks",
L.getNumBlocks());
349 L.getStartLoc(),
L.getHeader())
350 <<
"Disabled by Pragma.";
361 LLVM_DEBUG(
dbgs() <<
"Unable to analyzeBranch, can NOT pipeline Loop\n");
365 L.getStartLoc(),
L.getHeader())
366 <<
"The branch can't be understood";
375 LLVM_DEBUG(
dbgs() <<
"Unable to analyzeLoop, can NOT pipeline Loop\n");
379 L.getStartLoc(),
L.getHeader())
380 <<
"The loop structure is not supported";
385 if (!
L.getLoopPreheader()) {
386 LLVM_DEBUG(
dbgs() <<
"Preheader not found, can NOT pipeline Loop\n");
390 L.getStartLoc(),
L.getHeader())
391 <<
"No loop preheader found";
397 preprocessPhiNodes(*
L.getHeader());
403 SlotIndexes &Slots = *getAnalysis<LiveIntervals>().getSlotIndexes();
408 auto *RC =
MRI.getRegClass(DefOp.
getReg());
410 for (
unsigned i = 1, n = PI.getNumOperands(); i != n; i += 2) {
435bool MachinePipeliner::swingModuloScheduler(
MachineLoop &L) {
436 assert(
L.getBlocks().size() == 1 &&
"SMS works on single blocks only.");
459 return SMS.hasNewSchedule();
472void SwingSchedulerDAG::setMII(
unsigned ResMII,
unsigned RecMII) {
475 else if (II_setByPragma > 0)
476 MII = II_setByPragma;
478 MII = std::max(ResMII, RecMII);
481void SwingSchedulerDAG::setMAX_II() {
484 else if (II_setByPragma > 0)
485 MAX_II = II_setByPragma;
495 addLoopCarriedDependences(AA);
496 updatePhiDependences();
503 findCircuits(NodeSets);
507 unsigned ResMII = calculateResMII();
508 unsigned RecMII = calculateRecMII(NodeSets);
516 setMII(ResMII, RecMII);
520 <<
" (rec=" << RecMII <<
", res=" << ResMII <<
")\n");
526 Pass.ORE->emit([&]() {
529 <<
"Invalid Minimal Initiation Interval: 0";
537 <<
", we don't pipeline large loops\n");
538 NumFailLargeMaxMII++;
539 Pass.ORE->emit([&]() {
542 <<
"Minimal Initiation Interval too large: "
543 <<
ore::NV(
"MII", (
int)MII) <<
" > "
545 <<
"Refer to -pipeliner-max-mii.";
550 computeNodeFunctions(NodeSets);
552 registerPressureFilter(NodeSets);
554 colocateNodeSets(NodeSets);
556 checkNodeSets(NodeSets);
559 for (
auto &
I : NodeSets) {
560 dbgs() <<
" Rec NodeSet ";
567 groupRemainingNodes(NodeSets);
569 removeDuplicateNodes(NodeSets);
572 for (
auto &
I : NodeSets) {
573 dbgs() <<
" NodeSet ";
578 computeNodeOrder(NodeSets);
581 checkValidNodeOrder(Circuits);
584 Scheduled = schedulePipeline(Schedule);
589 Pass.ORE->emit([&]() {
592 <<
"Unable to find schedule";
599 if (numStages == 0) {
602 Pass.ORE->emit([&]() {
605 <<
"No need to pipeline - no overlapped iterations in schedule.";
612 <<
" : too many stages, abort\n");
613 NumFailLargeMaxStage++;
614 Pass.ORE->emit([&]() {
617 <<
"Too many stages in schedule: "
618 <<
ore::NV(
"numStages", (
int)numStages) <<
" > "
620 <<
". Refer to -pipeliner-max-stages.";
625 Pass.ORE->emit([&]() {
628 <<
"Pipelined succesfully!";
633 std::vector<MachineInstr *> OrderedInsts;
637 OrderedInsts.push_back(SU->getInstr());
638 Cycles[SU->getInstr()] =
Cycle;
643 for (
auto &KV : NewMIs) {
644 Cycles[KV.first] = Cycles[KV.second];
645 Stages[KV.first] = Stages[KV.second];
646 NewInstrChanges[KV.first] = InstrChanges[
getSUnit(KV.first)];
653 "Cannot serialize a schedule with InstrChanges!");
672 for (
auto &KV : NewMIs)
683 unsigned &InitVal,
unsigned &LoopVal) {
684 assert(Phi.isPHI() &&
"Expecting a Phi.");
688 for (
unsigned i = 1, e = Phi.getNumOperands(); i != e; i += 2)
689 if (Phi.getOperand(i + 1).getMBB() !=
Loop)
690 InitVal = Phi.getOperand(i).getReg();
692 LoopVal = Phi.getOperand(i).getReg();
694 assert(InitVal != 0 && LoopVal != 0 &&
"Unexpected Phi structure.");
699 for (
unsigned i = 1, e = Phi.getNumOperands(); i != e; i += 2)
700 if (Phi.getOperand(i + 1).getMBB() == LoopBB)
701 return Phi.getOperand(i).getReg();
710 while (!Worklist.
empty()) {
712 for (
const auto &SI : SU->
Succs) {
713 SUnit *SuccSU = SI.getSUnit();
715 if (Visited.
count(SuccSU))
730 return MI.isCall() ||
MI.mayRaiseFPException() ||
731 MI.hasUnmodeledSideEffects() ||
732 (
MI.hasOrderedMemoryRef() &&
733 (!
MI.mayLoad() || !
MI.isDereferenceableInvariantLoad()));
741 if (!
MI->hasOneMemOperand())
747 for (
const Value *V : Objs) {
760void SwingSchedulerDAG::addLoopCarriedDependences(
AliasAnalysis *AA) {
767 PendingLoads.
clear();
768 else if (
MI.mayLoad()) {
773 for (
const auto *V : Objs) {
777 }
else if (
MI.mayStore()) {
782 for (
const auto *V : Objs) {
784 PendingLoads.
find(V);
785 if (
I == PendingLoads.
end())
787 for (
auto *Load :
I->second) {
795 int64_t Offset1, Offset2;
796 bool Offset1IsScalable, Offset2IsScalable;
798 Offset1IsScalable,
TRI) &&
800 Offset2IsScalable,
TRI)) {
802 Offset1IsScalable == Offset2IsScalable &&
803 (
int)Offset1 < (
int)Offset2) {
805 "What happened to the chain edge?");
856void SwingSchedulerDAG::updatePhiDependences() {
864 unsigned HasPhiUse = 0;
865 unsigned HasPhiDef = 0;
889 if (SU->
NodeNum <
I.NodeNum && !
I.isPred(SU))
894 }
else if (MO.isUse()) {
897 if (
DefMI ==
nullptr)
904 ST.adjustSchedDependency(SU, 0, &
I, MO.getOperandNo(), Dep);
910 if (SU->
NodeNum <
I.NodeNum && !
I.isPred(SU))
919 for (
auto &PI :
I.Preds) {
922 if (
I.getInstr()->isPHI()) {
931 for (
int i = 0, e = RemoveDeps.
size(); i != e; ++i)
932 I.removePred(RemoveDeps[i]);
938void SwingSchedulerDAG::changeDependences() {
943 unsigned BasePos = 0, OffsetPos = 0, NewBase = 0;
944 int64_t NewOffset = 0;
945 if (!canUseLastOffsetValue(
I.getInstr(), BasePos, OffsetPos, NewBase,
950 Register OrigBase =
I.getInstr()->getOperand(BasePos).getReg();
970 for (
const SDep &
P :
I.Preds)
971 if (
P.getSUnit() == DefSU)
973 for (
int i = 0, e = Deps.
size(); i != e; i++) {
975 I.removePred(Deps[i]);
979 for (
auto &
P : LastSU->
Preds)
982 for (
int i = 0, e = Deps.
size(); i != e; i++) {
995 InstrChanges[&
I] = std::make_pair(NewBase, NewOffset);
1003struct FuncUnitSorter {
1009 : InstrItins(TSI.getInstrItineraryData()), STI(&TSI) {}
1017 unsigned min = UINT_MAX;
1018 if (InstrItins && !InstrItins->
isEmpty()) {
1021 InstrItins->
endStage(SchedClass))) {
1024 if (numAlternatives < min) {
1025 min = numAlternatives;
1042 if (!PRE.ReleaseAtCycle)
1046 unsigned NumUnits = ProcResource->
NumUnits;
1047 if (NumUnits < min) {
1049 F = PRE.ProcResourceIdx;
1054 llvm_unreachable(
"Should have non-empty InstrItins or hasInstrSchedModel!");
1063 unsigned SchedClass =
MI.getDesc().getSchedClass();
1064 if (InstrItins && !InstrItins->
isEmpty()) {
1067 InstrItins->
endStage(SchedClass))) {
1070 Resources[FuncUnits]++;
1085 if (!PRE.ReleaseAtCycle)
1087 Resources[PRE.ProcResourceIdx]++;
1091 llvm_unreachable(
"Should have non-empty InstrItins or hasInstrSchedModel!");
1097 unsigned MFUs1 = minFuncUnits(IS1, F1);
1098 unsigned MFUs2 = minFuncUnits(IS2, F2);
1101 return MFUs1 > MFUs2;
1113unsigned SwingSchedulerDAG::calculateResMII() {
1116 return RM.calculateResMII();
1125unsigned SwingSchedulerDAG::calculateRecMII(NodeSetType &NodeSets) {
1126 unsigned RecMII = 0;
1128 for (
NodeSet &Nodes : NodeSets) {
1132 unsigned Delay = Nodes.getLatency();
1133 unsigned Distance = 1;
1136 unsigned CurMII = (Delay + Distance - 1) / Distance;
1137 Nodes.setRecMII(CurMII);
1138 if (CurMII > RecMII)
1149 for (
SUnit &SU : SUnits) {
1152 DepsAdded.
push_back(std::make_pair(&SU, Pred));
1154 for (std::pair<SUnit *, SDep> &
P : DepsAdded) {
1158 SUnit *TargetSU =
D.getSUnit();
1159 unsigned Reg =
D.getReg();
1160 unsigned Lat =
D.getLatency();
1169void SwingSchedulerDAG::Circuits::createAdjacencyStructure(
1173 for (
int i = 0, e = SUnits.size(); i != e; ++i) {
1176 for (
auto &SI : SUnits[i].Succs) {
1180 int N =
SI.getSUnit()->NodeNum;
1182 auto Dep = OutputDeps.
find(BackEdge);
1183 if (Dep != OutputDeps.
end()) {
1184 BackEdge = Dep->second;
1185 OutputDeps.
erase(Dep);
1187 OutputDeps[
N] = BackEdge;
1191 if (
SI.getSUnit()->isBoundaryNode() ||
SI.isArtificial() ||
1192 (
SI.getKind() ==
SDep::Anti && !
SI.getSUnit()->getInstr()->isPHI()))
1194 int N =
SI.getSUnit()->NodeNum;
1196 AdjK[i].push_back(
N);
1202 for (
auto &PI : SUnits[i].Preds) {
1203 if (!SUnits[i].getInstr()->mayStore() ||
1206 if (PI.getKind() ==
SDep::Order && PI.getSUnit()->getInstr()->mayLoad()) {
1207 int N = PI.getSUnit()->NodeNum;
1209 AdjK[i].push_back(
N);
1216 for (
auto &OD : OutputDeps)
1217 if (!
Added.test(OD.second)) {
1218 AdjK[OD.first].push_back(OD.second);
1219 Added.set(OD.second);
1225bool SwingSchedulerDAG::Circuits::circuit(
int V,
int S, NodeSetType &NodeSets,
1232 for (
auto W : AdjK[V]) {
1233 if (NumPaths > MaxPaths)
1243 }
else if (!Blocked.test(W)) {
1244 if (circuit(W, S, NodeSets,
1245 Node2Idx->at(W) < Node2Idx->at(V) ?
true : HasBackedge))
1253 for (
auto W : AdjK[V]) {
1264void SwingSchedulerDAG::Circuits::unblock(
int U) {
1267 while (!BU.
empty()) {
1269 assert(SI != BU.
end() &&
"Invalid B set.");
1272 if (Blocked.test(
W->NodeNum))
1273 unblock(
W->NodeNum);
1279void SwingSchedulerDAG::findCircuits(NodeSetType &NodeSets) {
1284 Circuits Cir(
SUnits, Topo);
1286 Cir.createAdjacencyStructure(
this);
1287 for (
int i = 0, e =
SUnits.size(); i != e; ++i) {
1289 Cir.circuit(i, i, NodeSets);
1325 for (
auto &Dep : SU.
Preds) {
1326 SUnit *TmpSU = Dep.getSUnit();
1338 if (PHISUs.
size() == 0 || SrcSUs.
size() == 0)
1346 for (
auto &Dep : PHISUs[
Index]->Succs) {
1350 SUnit *TmpSU = Dep.getSUnit();
1360 if (UseSUs.
size() == 0)
1365 for (
auto *
I : UseSUs) {
1366 for (
auto *Src : SrcSUs) {
1380 if (
D.isArtificial() ||
D.getSUnit()->isBoundaryNode())
1391void SwingSchedulerDAG::computeNodeFunctions(NodeSetType &NodeSets) {
1392 ScheduleInfo.resize(
SUnits.size());
1395 for (
int I : Topo) {
1403 for (
int I : Topo) {
1405 int zeroLatencyDepth = 0;
1409 if (
P.getLatency() == 0)
1414 asap = std::max(asap, (
int)(
getASAP(
pred) +
P.getLatency() -
1417 maxASAP = std::max(maxASAP, asap);
1418 ScheduleInfo[
I].ASAP = asap;
1419 ScheduleInfo[
I].ZeroLatencyDepth = zeroLatencyDepth;
1425 int zeroLatencyHeight = 0;
1440 ScheduleInfo[
I].ALAP = alap;
1441 ScheduleInfo[
I].ZeroLatencyHeight = zeroLatencyHeight;
1446 I.computeNodeSetInfo(
this);
1449 for (
unsigned i = 0; i <
SUnits.size(); i++) {
1450 dbgs() <<
"\tNode " << i <<
":\n";
1471 if (S && S->count(Pred.
getSUnit()) == 0)
1482 if (S && S->count(Succ.
getSUnit()) == 0)
1488 return !Preds.
empty();
1500 if (S && S->count(Succ.
getSUnit()) == 0)
1510 if (S && S->count(Pred.
getSUnit()) == 0)
1516 return !Succs.
empty();
1531 if (!Visited.
insert(Cur).second)
1532 return Path.contains(Cur);
1533 bool FoundPath =
false;
1534 for (
auto &SI : Cur->
Succs)
1537 computePath(SI.getSUnit(), Path, DestNodes, Exclude, Visited);
1538 for (
auto &PI : Cur->
Preds)
1541 computePath(PI.getSUnit(), Path, DestNodes, Exclude, Visited);
1556 for (
SUnit *SU : NS) {
1562 if (Reg.isVirtual())
1564 else if (
MRI.isAllocatable(Reg))
1569 for (
SUnit *SU : NS)
1573 if (Reg.isVirtual()) {
1574 if (!
Uses.count(Reg))
1577 }
else if (
MRI.isAllocatable(Reg)) {
1579 if (!
Uses.count(Unit))
1589void SwingSchedulerDAG::registerPressureFilter(NodeSetType &NodeSets) {
1590 for (
auto &NS : NodeSets) {
1596 RecRPTracker.init(&
MF, &RegClassInfo, &LIS,
BB,
BB->
end(),
false,
true);
1598 RecRPTracker.closeBottom();
1600 std::vector<SUnit *>
SUnits(NS.begin(), NS.end());
1602 return A->NodeNum >
B->NodeNum;
1605 for (
auto &SU :
SUnits) {
1611 RecRPTracker.setPos(std::next(CurInstI));
1615 RecRPTracker.getMaxUpwardPressureDelta(SU->
getInstr(),
nullptr, RPDelta,
1620 dbgs() <<
"Excess register pressure: SU(" << SU->
NodeNum <<
") "
1623 NS.setExceedPressure(SU);
1626 RecRPTracker.recede();
1633void SwingSchedulerDAG::colocateNodeSets(NodeSetType &NodeSets) {
1634 unsigned Colocate = 0;
1635 for (
int i = 0, e = NodeSets.size(); i < e; ++i) {
1640 for (
int j = i + 1;
j <
e; ++
j) {
1661void SwingSchedulerDAG::checkNodeSets(NodeSetType &NodeSets) {
1666 for (
auto &NS : NodeSets) {
1667 if (NS.getRecMII() > 2)
1669 if (NS.getMaxDepth() > MII)
1678void SwingSchedulerDAG::groupRemainingNodes(NodeSetType &NodeSets) {
1706 NodesAdded.
insert(
I.begin(),
I.end());
1715 addConnectedNodes(
I, NewSet, NodesAdded);
1716 if (!NewSet.
empty())
1717 NodeSets.push_back(NewSet);
1724 addConnectedNodes(
I, NewSet, NodesAdded);
1725 if (!NewSet.
empty())
1726 NodeSets.push_back(NewSet);
1731 if (NodesAdded.
count(&SU) == 0) {
1733 addConnectedNodes(&SU, NewSet, NodesAdded);
1734 if (!NewSet.
empty())
1735 NodeSets.push_back(NewSet);
1741void SwingSchedulerDAG::addConnectedNodes(
SUnit *SU,
NodeSet &NewSet,
1745 for (
auto &SI : SU->
Succs) {
1747 if (!
SI.isArtificial() && !
Successor->isBoundaryNode() &&
1749 addConnectedNodes(
Successor, NewSet, NodesAdded);
1751 for (
auto &PI : SU->
Preds) {
1752 SUnit *Predecessor = PI.getSUnit();
1753 if (!PI.isArtificial() && NodesAdded.
count(Predecessor) == 0)
1754 addConnectedNodes(Predecessor, NewSet, NodesAdded);
1763 for (
SUnit *SU : Set1) {
1764 if (Set2.
count(SU) != 0)
1767 return !Result.empty();
1771void SwingSchedulerDAG::fuseRecs(NodeSetType &NodeSets) {
1780 for (
SUnit *SU : *J)
1792void SwingSchedulerDAG::removeDuplicateNodes(NodeSetType &NodeSets) {
1796 J->remove_if([&](
SUnit *SUJ) {
return I->count(SUJ); });
1811void SwingSchedulerDAG::computeNodeOrder(NodeSetType &NodeSets) {
1815 for (
auto &Nodes : NodeSets) {
1820 R.insert(
N.begin(),
N.end());
1824 R.insert(
N.begin(),
N.end());
1832 }
else if (NodeSets.size() == 1) {
1833 for (
const auto &
N : Nodes)
1834 if (
N->Succs.size() == 0)
1840 SUnit *maxASAP =
nullptr;
1841 for (
SUnit *SU : Nodes) {
1851 while (!
R.empty()) {
1852 if (Order == TopDown) {
1856 while (!
R.empty()) {
1857 SUnit *maxHeight =
nullptr;
1870 NodeOrder.insert(maxHeight);
1872 R.remove(maxHeight);
1873 for (
const auto &
I : maxHeight->
Succs) {
1874 if (Nodes.count(
I.getSUnit()) == 0)
1876 if (NodeOrder.contains(
I.getSUnit()))
1880 R.insert(
I.getSUnit());
1883 for (
const auto &
I : maxHeight->
Preds) {
1886 if (Nodes.count(
I.getSUnit()) == 0)
1888 if (NodeOrder.contains(
I.getSUnit()))
1890 R.insert(
I.getSUnit());
1896 if (
pred_L(NodeOrder,
N, &Nodes))
1897 R.insert(
N.begin(),
N.end());
1902 while (!
R.empty()) {
1903 SUnit *maxDepth =
nullptr;
1915 NodeOrder.insert(maxDepth);
1918 if (Nodes.isExceedSU(maxDepth)) {
1921 R.insert(Nodes.getNode(0));
1924 for (
const auto &
I : maxDepth->
Preds) {
1925 if (Nodes.count(
I.getSUnit()) == 0)
1927 if (NodeOrder.contains(
I.getSUnit()))
1929 R.insert(
I.getSUnit());
1932 for (
const auto &
I : maxDepth->
Succs) {
1935 if (Nodes.count(
I.getSUnit()) == 0)
1937 if (NodeOrder.contains(
I.getSUnit()))
1939 R.insert(
I.getSUnit());
1945 if (
succ_L(NodeOrder,
N, &Nodes))
1946 R.insert(
N.begin(),
N.end());
1953 dbgs() <<
"Node order: ";
1954 for (
SUnit *
I : NodeOrder)
1955 dbgs() <<
" " <<
I->NodeNum <<
" ";
1962bool SwingSchedulerDAG::schedulePipeline(
SMSchedule &Schedule) {
1964 if (NodeOrder.empty()){
1969 bool scheduleFound =
false;
1971 for (
unsigned II = MII; II <= MAX_II && !scheduleFound; ++II) {
1983 int EarlyStart = INT_MIN;
1984 int LateStart = INT_MAX;
1987 int SchedEnd = INT_MAX;
1988 int SchedStart = INT_MIN;
1989 Schedule.
computeStart(SU, &EarlyStart, &LateStart, &SchedEnd, &SchedStart,
1998 dbgs() <<
format(
"\tes: %8x ls: %8x me: %8x ms: %8x\n", EarlyStart,
1999 LateStart, SchedEnd, SchedStart);
2002 if (EarlyStart > LateStart || SchedEnd < EarlyStart ||
2003 SchedStart > LateStart)
2004 scheduleFound =
false;
2005 else if (EarlyStart != INT_MIN && LateStart == INT_MAX) {
2006 SchedEnd = std::min(SchedEnd, EarlyStart + (
int)II - 1);
2007 scheduleFound = Schedule.
insert(SU, EarlyStart, SchedEnd, II);
2008 }
else if (EarlyStart == INT_MIN && LateStart != INT_MAX) {
2009 SchedStart = std::max(SchedStart, LateStart - (
int)II + 1);
2010 scheduleFound = Schedule.
insert(SU, LateStart, SchedStart, II);
2011 }
else if (EarlyStart != INT_MIN && LateStart != INT_MAX) {
2013 std::min(SchedEnd, std::min(LateStart, EarlyStart + (
int)II - 1));
2018 scheduleFound = Schedule.
insert(SU, SchedEnd, EarlyStart, II);
2020 scheduleFound = Schedule.
insert(SU, EarlyStart, SchedEnd, II);
2023 scheduleFound = Schedule.
insert(SU, FirstCycle +
getASAP(SU),
2024 FirstCycle +
getASAP(SU) + II - 1, II);
2031 scheduleFound =
false;
2035 dbgs() <<
"\tCan't schedule\n";
2037 }
while (++NI != NE && scheduleFound);
2053 if (scheduleFound) {
2059 if (scheduleFound) {
2061 Pass.ORE->emit([&]() {
2064 <<
"Schedule found with Initiation Interval: "
2066 <<
", MaxStageCount: "
2077bool SwingSchedulerDAG::computeDelta(
MachineInstr &
MI,
unsigned &Delta) {
2081 bool OffsetIsScalable;
2086 if (OffsetIsScalable)
2089 if (!BaseOp->
isReg())
2097 if (BaseDef && BaseDef->
isPHI()) {
2121 unsigned &OffsetPos,
2127 unsigned BasePosLd, OffsetPosLd;
2130 Register BaseReg =
MI->getOperand(BasePosLd).getReg();
2135 if (!Phi || !
Phi->isPHI())
2144 if (!PrevDef || PrevDef ==
MI)
2150 unsigned BasePos1 = 0, OffsetPos1 = 0;
2156 int64_t LoadOffset =
MI->getOperand(OffsetPosLd).getImm();
2166 BasePos = BasePosLd;
2167 OffsetPos = OffsetPosLd;
2179 InstrChanges.find(SU);
2180 if (It != InstrChanges.end()) {
2181 std::pair<unsigned, int64_t> RegAndOffset = It->second;
2182 unsigned BasePos, OffsetPos;
2185 Register BaseReg =
MI->getOperand(BasePos).getReg();
2191 if (BaseStageNum < DefStageNum) {
2193 int OffsetDiff = DefStageNum - BaseStageNum;
2194 if (DefCycleNum < BaseCycleNum) {
2200 MI->getOperand(OffsetPos).getImm() + RegAndOffset.second * OffsetDiff;
2215 while (Def->isPHI()) {
2216 if (!Visited.
insert(Def).second)
2218 for (
unsigned i = 1, e = Def->getNumOperands(); i < e; i += 2)
2219 if (Def->getOperand(i + 1).getMBB() ==
BB) {
2246 assert(SI !=
nullptr && DI !=
nullptr &&
"Expecting SUnit with an MI.");
2255 if (!DI->
mayStore() || !SI->mayLoad())
2258 unsigned DeltaS, DeltaD;
2259 if (!computeDelta(*SI, DeltaS) || !computeDelta(*DI, DeltaD))
2263 int64_t OffsetS, OffsetD;
2264 bool OffsetSIsScalable, OffsetDIsScalable;
2272 assert(!OffsetSIsScalable && !OffsetDIsScalable &&
2273 "Expected offsets to be byte offsets");
2277 if (!DefS || !DefD || !DefS->
isPHI() || !DefD->
isPHI())
2280 unsigned InitValS = 0;
2281 unsigned LoopValS = 0;
2282 unsigned InitValD = 0;
2283 unsigned LoopValD = 0;
2308 if (DeltaS != DeltaD || DeltaS < AccessSizeS || DeltaD < AccessSizeD)
2311 return (OffsetS + (int64_t)AccessSizeS < OffsetD + (int64_t)AccessSizeD);
2314void SwingSchedulerDAG::postProcessDAG() {
2315 for (
auto &M : Mutations)
2325 bool forward =
true;
2327 dbgs() <<
"Trying to insert node between " << StartCycle <<
" and "
2328 << EndCycle <<
" II: " << II <<
"\n";
2330 if (StartCycle > EndCycle)
2334 int termCycle = forward ? EndCycle + 1 : EndCycle - 1;
2335 for (
int curCycle = StartCycle; curCycle != termCycle;
2336 forward ? ++curCycle : --curCycle) {
2341 dbgs() <<
"\tinsert at cycle " << curCycle <<
" ";
2346 ProcItinResources.reserveResources(*SU, curCycle);
2347 ScheduledInstrs[curCycle].push_back(SU);
2348 InstrToCycle.insert(std::make_pair(SU, curCycle));
2349 if (curCycle > LastCycle)
2350 LastCycle = curCycle;
2351 if (curCycle < FirstCycle)
2352 FirstCycle = curCycle;
2356 dbgs() <<
"\tfailed to insert at cycle " << curCycle <<
" ";
2368 int EarlyCycle = INT_MAX;
2369 while (!Worklist.
empty()) {
2372 if (Visited.
count(PrevSU))
2374 std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(PrevSU);
2375 if (it == InstrToCycle.end())
2377 EarlyCycle = std::min(EarlyCycle, it->second);
2378 for (
const auto &PI : PrevSU->
Preds)
2391 int LateCycle = INT_MIN;
2392 while (!Worklist.
empty()) {
2397 std::map<SUnit *, int>::const_iterator it = InstrToCycle.find(SuccSU);
2398 if (it == InstrToCycle.end())
2400 LateCycle = std::max(LateCycle, it->second);
2401 for (
const auto &SI : SuccSU->
Succs)
2413 for (
auto &
P : SU->
Preds)
2414 if (DAG->
isBackedge(SU,
P) &&
P.getSUnit()->getInstr()->isPHI())
2415 for (
auto &S :
P.getSUnit()->Succs)
2417 return P.getSUnit();
2424 int *MinEnd,
int *MaxStart,
int II,
2429 for (
int cycle =
getFirstCycle(); cycle <= LastCycle; ++cycle) {
2435 for (
unsigned i = 0, e = (
unsigned)SU->
Preds.size(); i != e; ++i) {
2441 *MaxEarlyStart = std::max(*MaxEarlyStart, EarlyStart);
2444 *MinEnd = std::min(*MinEnd,
End);
2449 *MinLateStart = std::min(*MinLateStart, LateStart);
2457 *MinLateStart = std::min(*MinLateStart, cycle);
2459 for (
unsigned i = 0, e = (
unsigned)SU->
Succs.size(); i != e; ++i) {
2460 if (SU->
Succs[i].getSUnit() ==
I) {
2465 *MinLateStart = std::min(*MinLateStart, LateStart);
2468 *MaxStart = std::max(*MaxStart, Start);
2473 *MaxEarlyStart = std::max(*MaxEarlyStart, EarlyStart);
2485 std::deque<SUnit *> &Insts) {
2487 bool OrderBeforeUse =
false;
2488 bool OrderAfterDef =
false;
2489 bool OrderBeforeDef =
false;
2490 unsigned MoveDef = 0;
2491 unsigned MoveUse = 0;
2495 for (std::deque<SUnit *>::iterator
I = Insts.begin(),
E = Insts.end();
I !=
E;
2498 if (!MO.isReg() || !MO.getReg().isVirtual())
2502 unsigned BasePos, OffsetPos;
2503 if (ST.getInstrInfo()->getBaseAndOffsetPosition(*
MI, BasePos, OffsetPos))
2504 if (
MI->getOperand(BasePos).getReg() == Reg)
2508 std::tie(Reads,
Writes) =
2509 (*I)->getInstr()->readsWritesVirtualRegister(Reg);
2511 OrderBeforeUse =
true;
2516 OrderAfterDef =
true;
2520 OrderBeforeUse =
true;
2524 OrderAfterDef =
true;
2528 OrderBeforeUse =
true;
2532 OrderAfterDef =
true;
2537 OrderBeforeUse =
true;
2543 OrderBeforeDef =
true;
2550 for (
auto &S : SU->
Succs) {
2554 OrderBeforeUse =
true;
2562 OrderBeforeUse =
true;
2563 if ((MoveUse == 0) || (Pos < MoveUse))
2567 for (
auto &
P : SU->
Preds) {
2568 if (
P.getSUnit() != *
I)
2571 OrderAfterDef =
true;
2578 if (OrderAfterDef && OrderBeforeUse && MoveUse == MoveDef)
2579 OrderBeforeUse =
false;
2584 OrderBeforeUse = !OrderAfterDef || (MoveUse > MoveDef);
2588 if (OrderBeforeUse && OrderAfterDef) {
2589 SUnit *UseSU = Insts.at(MoveUse);
2590 SUnit *DefSU = Insts.at(MoveDef);
2591 if (MoveUse > MoveDef) {
2592 Insts.erase(Insts.begin() + MoveUse);
2593 Insts.erase(Insts.begin() + MoveDef);
2595 Insts.erase(Insts.begin() + MoveDef);
2596 Insts.erase(Insts.begin() + MoveUse);
2606 Insts.push_front(SU);
2608 Insts.push_back(SU);
2615 assert(Phi.isPHI() &&
"Expecting a Phi.");
2620 unsigned InitVal = 0;
2621 unsigned LoopVal = 0;
2622 getPhiRegs(Phi, Phi.getParent(), InitVal, LoopVal);
2630 return (LoopCycle > DefCycle) || (LoopStage <= DefStage);
2647 if (!Phi || !Phi->isPHI() || Phi->getParent() != Def->getParent())
2653 if (DMO.getReg() == LoopReg)
2665 for (
auto &SU : SSD->
SUnits)
2669 while (!Worklist.
empty()) {
2671 if (DoNotPipeline.
count(SU))
2674 DoNotPipeline.
insert(SU);
2675 for (
auto &Dep : SU->
Preds)
2678 for (
auto &Dep : SU->
Succs)
2682 return DoNotPipeline;
2691 int NewLastCycle = INT_MIN;
2696 NewLastCycle = std::max(NewLastCycle, InstrToCycle[&SU]);
2702 for (
auto &Dep : SU.
Preds)
2703 NewCycle = std::max(InstrToCycle[Dep.getSUnit()], NewCycle);
2705 int OldCycle = InstrToCycle[&SU];
2706 if (OldCycle != NewCycle) {
2707 InstrToCycle[&SU] = NewCycle;
2712 <<
") is not pipelined; moving from cycle " << OldCycle
2713 <<
" to " << NewCycle <<
" Instr:" << *SU.
getInstr());
2715 NewLastCycle = std::max(NewLastCycle, NewCycle);
2717 LastCycle = NewLastCycle;
2734 int CycleDef = InstrToCycle[&SU];
2735 assert(StageDef != -1 &&
"Instruction should have been scheduled.");
2736 for (
auto &SI : SU.
Succs)
2737 if (SI.isAssignedRegDep() && !SI.getSUnit()->isBoundaryNode())
2741 if (InstrToCycle[SI.getSUnit()] <= CycleDef)
2758void SwingSchedulerDAG::checkValidNodeOrder(
const NodeSetType &Circuits)
const {
2761 typedef std::pair<SUnit *, unsigned> UnitIndex;
2762 std::vector<UnitIndex> Indices(NodeOrder.size(), std::make_pair(
nullptr, 0));
2764 for (
unsigned i = 0, s = NodeOrder.size(); i < s; ++i)
2765 Indices.push_back(std::make_pair(NodeOrder[i], i));
2767 auto CompareKey = [](UnitIndex i1, UnitIndex i2) {
2768 return std::get<0>(i1) < std::get<0>(i2);
2781 for (
unsigned i = 0, s = NodeOrder.size(); i < s; ++i) {
2782 SUnit *SU = NodeOrder[i];
2785 bool PredBefore =
false;
2786 bool SuccBefore =
false;
2795 unsigned PredIndex = std::get<1>(
2811 unsigned SuccIndex = std::get<1>(
2824 Circuits, [SU](
const NodeSet &Circuit) {
return Circuit.
count(SU); });
2829 NumNodeOrderIssues++;
2833 <<
" are scheduled before node " << SU->
NodeNum
2840 dbgs() <<
"Invalid node order found!\n";
2851 unsigned OverlapReg = 0;
2852 unsigned NewBaseReg = 0;
2853 for (
SUnit *SU : Instrs) {
2855 for (
unsigned i = 0, e =
MI->getNumOperands(); i < e; ++i) {
2863 InstrChanges.find(SU);
2864 if (It != InstrChanges.end()) {
2865 unsigned BasePos, OffsetPos;
2871 MI->getOperand(OffsetPos).getImm() - It->second.second;
2884 unsigned TiedUseIdx = 0;
2885 if (
MI->isRegTiedToUseOperand(i, &TiedUseIdx)) {
2887 OverlapReg =
MI->getOperand(TiedUseIdx).getReg();
2889 NewBaseReg =
MI->getOperand(i).getReg();
2904 std::deque<SUnit *> &cycleInstrs =
2905 ScheduledInstrs[cycle + (stage * InitiationInterval)];
2907 ScheduledInstrs[cycle].push_front(SU);
2913 for (
int cycle =
getFinalCycle() + 1; cycle <= LastCycle; ++cycle)
2914 ScheduledInstrs.erase(cycle);
2924 std::deque<SUnit *> &cycleInstrs = ScheduledInstrs[
Cycle];
2925 std::deque<SUnit *> newOrderPhi;
2926 for (
SUnit *SU : cycleInstrs) {
2928 newOrderPhi.push_back(SU);
2930 std::deque<SUnit *> newOrderI;
2931 for (
SUnit *SU : cycleInstrs) {
2936 cycleInstrs.swap(newOrderPhi);
2945 os <<
"Num nodes " <<
size() <<
" rec " << RecMII <<
" mov " << MaxMOV
2946 <<
" depth " << MaxDepth <<
" col " << Colocate <<
"\n";
2947 for (
const auto &
I : Nodes)
2948 os <<
" SU(" <<
I->NodeNum <<
") " << *(
I->getInstr());
2952#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2959 for (
SUnit *CI : cycleInstrs->second) {
2961 os <<
"(" << CI->
NodeNum <<
") ";
2972void ResourceManager::dumpMRT()
const {
2976 std::stringstream SS;
2978 SS << std::setw(4) <<
"Slot";
2980 SS << std::setw(3) <<
I;
2981 SS << std::setw(7) <<
"#Mops"
2983 for (
int Slot = 0; Slot < InitiationInterval; ++Slot) {
2984 SS << std::setw(4) << Slot;
2986 SS << std::setw(3) << MRT[Slot][
I];
2987 SS << std::setw(7) << NumScheduledMops[Slot] <<
"\n";
2996 unsigned ProcResourceID = 0;
3001 "Too many kinds of resources, unsupported");
3007 if (
Desc.SubUnitsIdxBegin)
3009 Masks[
I] = 1ULL << ProcResourceID;
3015 if (!
Desc.SubUnitsIdxBegin)
3017 Masks[
I] = 1ULL << ProcResourceID;
3018 for (
unsigned U = 0; U <
Desc.NumUnits; ++U)
3019 Masks[
I] |= Masks[
Desc.SubUnitsIdxBegin[U]];
3024 dbgs() <<
"ProcResourceDesc:\n";
3027 dbgs() <<
format(
" %16s(%2d): Mask: 0x%08x, NumUnits:%2d\n",
3028 ProcResource->
Name,
I, Masks[
I],
3031 dbgs() <<
" -----------------\n";
3039 dbgs() <<
"canReserveResources:\n";
3042 return DFAResources[positiveModulo(
Cycle, InitiationInterval)]
3048 dbgs() <<
"No valid Schedule Class Desc for schedClass!\n";
3054 reserveResources(SCDesc,
Cycle);
3055 bool Result = !isOverbooked();
3056 unreserveResources(SCDesc,
Cycle);
3065 dbgs() <<
"reserveResources:\n";
3068 return DFAResources[positiveModulo(
Cycle, InitiationInterval)]
3074 dbgs() <<
"No valid Schedule Class Desc for schedClass!\n";
3080 reserveResources(SCDesc,
Cycle);
3085 dbgs() <<
"reserveResources: done!\n\n";
3096 ++MRT[positiveModulo(
C, InitiationInterval)][PRE.ProcResourceIdx];
3099 ++NumScheduledMops[positiveModulo(
C, InitiationInterval)];
3108 --MRT[positiveModulo(
C, InitiationInterval)][PRE.ProcResourceIdx];
3111 --NumScheduledMops[positiveModulo(
C, InitiationInterval)];
3114bool ResourceManager::isOverbooked()
const {
3116 for (
int Slot = 0;
Slot < InitiationInterval; ++
Slot) {
3119 if (MRT[Slot][
I] >
Desc->NumUnits)
3122 if (NumScheduledMops[Slot] > IssueWidth)
3128int ResourceManager::calculateResMIIDFA()
const {
3133 FuncUnitSorter FUS = FuncUnitSorter(*ST);
3135 FUS.calcCriticalResources(*SU.
getInstr());
3146 while (!FuncUnitOrder.empty()) {
3148 FuncUnitOrder.pop();
3155 unsigned ReservedCycles = 0;
3156 auto *RI = Resources.
begin();
3157 auto *RE = Resources.
end();
3159 dbgs() <<
"Trying to reserve resource for " << NumCycles
3160 <<
" cycles for \n";
3163 for (
unsigned C = 0;
C < NumCycles; ++
C)
3165 if ((*RI)->canReserveResources(*
MI)) {
3166 (*RI)->reserveResources(*
MI);
3173 <<
", NumCycles:" << NumCycles <<
"\n");
3175 for (
unsigned C = ReservedCycles;
C < NumCycles; ++
C) {
3177 <<
"NewResource created to reserve resources"
3180 assert(NewResource->canReserveResources(*
MI) &&
"Reserve error.");
3181 NewResource->reserveResources(*
MI);
3182 Resources.
push_back(std::unique_ptr<DFAPacketizer>(NewResource));
3186 int Resmii = Resources.
size();
3193 return calculateResMIIDFA();
3212 <<
" WriteProcRes: ";
3223 dbgs() <<
Desc->Name <<
": " << PRE.ReleaseAtCycle <<
", ";
3226 ResourceCount[PRE.ProcResourceIdx] += PRE.ReleaseAtCycle;
3231 int Result = (NumMops + IssueWidth - 1) / IssueWidth;
3234 dbgs() <<
"#Mops: " << NumMops <<
", "
3235 <<
"IssueWidth: " << IssueWidth <<
", "
3236 <<
"Cycles: " << Result <<
"\n";
3241 std::stringstream SS;
3242 SS << std::setw(2) <<
"ID" << std::setw(16) <<
"Name" << std::setw(10)
3243 <<
"Units" << std::setw(10) <<
"Consumed" << std::setw(10) <<
"Cycles"
3250 int Cycles = (ResourceCount[
I] +
Desc->NumUnits - 1) /
Desc->NumUnits;
3253 std::stringstream SS;
3254 SS << std::setw(2) <<
I << std::setw(16) <<
Desc->Name << std::setw(10)
3255 <<
Desc->NumUnits << std::setw(10) << ResourceCount[
I]
3256 << std::setw(10) << Cycles <<
"\n";
3260 if (Cycles > Result)
3267 InitiationInterval = II;
3268 DFAResources.clear();
3269 DFAResources.resize(II);
3270 for (
auto &
I : DFAResources)
3271 I.reset(ST->getInstrInfo()->CreateTargetScheduleState(*ST));
3274 NumScheduledMops.
clear();
3275 NumScheduledMops.
resize(II);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
This file contains the simple types necessary to represent the attributes associated with functions a...
This file implements the BitVector class.
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
This file declares an analysis pass that computes CycleInfo for LLVM IR, specialized from GenericCycl...
This file defines the DenseMap class.
SmallVector< uint32_t, 0 > Writes
Rewrite Partial Register Uses
const HexagonInstrInfo * TII
A common definition of LaneBitmask for use in TableGen and CodeGen.
static cl::opt< int > SwpForceII("pipeliner-force-ii", cl::desc("Force pipeliner to use specified II."), cl::Hidden, cl::init(-1))
A command line argument to force pipeliner to use specified initial interval.
static cl::opt< bool > ExperimentalCodeGen("pipeliner-experimental-cg", cl::Hidden, cl::init(false), cl::desc("Use the experimental peeling code generator for software pipelining"))
static bool pred_L(SetVector< SUnit * > &NodeOrder, SmallSetVector< SUnit *, 8 > &Preds, const NodeSet *S=nullptr)
Compute the Pred_L(O) set, as defined in the paper.
static unsigned getLoopPhiReg(MachineInstr &Phi, MachineBasicBlock *LoopBB)
Return the Phi register value that comes the loop block.
static cl::opt< bool > SwpDebugResource("pipeliner-dbg-res", cl::Hidden, cl::init(false))
static void computeLiveOuts(MachineFunction &MF, RegPressureTracker &RPTracker, NodeSet &NS)
Compute the live-out registers for the instructions in a node-set.
static cl::opt< bool > EmitTestAnnotations("pipeliner-annotate-for-testing", cl::Hidden, cl::init(false), cl::desc("Instead of emitting the pipelined code, annotate instructions " "with the generated schedule for feeding into the " "-modulo-schedule-test pass"))
static bool isIntersect(SmallSetVector< SUnit *, 8 > &Set1, const NodeSet &Set2, SmallSetVector< SUnit *, 8 > &Result)
Return true if Set1 contains elements in Set2.
static cl::opt< bool > SwpIgnoreRecMII("pipeliner-ignore-recmii", cl::ReallyHidden, cl::desc("Ignore RecMII"))
static cl::opt< int > SwpLoopLimit("pipeliner-max", cl::Hidden, cl::init(-1))
static bool succ_L(SetVector< SUnit * > &NodeOrder, SmallSetVector< SUnit *, 8 > &Succs, const NodeSet *S=nullptr)
Compute the Succ_L(O) set, as defined in the paper.
Modulo Software Pipelining
static cl::opt< bool > SwpPruneLoopCarried("pipeliner-prune-loop-carried", cl::desc("Prune loop carried order dependences."), cl::Hidden, cl::init(true))
A command line option to disable the pruning of loop carried order dependences.
static bool isDependenceBarrier(MachineInstr &MI)
Return true if the instruction causes a chain between memory references before and after it.
static cl::opt< int > SwpMaxMii("pipeliner-max-mii", cl::desc("Size limit for the MII."), cl::Hidden, cl::init(27))
A command line argument to limit minimum initial interval for pipelining.
static void swapAntiDependences(std::vector< SUnit > &SUnits)
Swap all the anti dependences in the DAG.
static bool isSuccOrder(SUnit *SUa, SUnit *SUb)
Return true if SUb can be reached from SUa following the chain edges.
static cl::opt< int > SwpMaxStages("pipeliner-max-stages", cl::desc("Maximum stages allowed in the generated scheduled."), cl::Hidden, cl::init(3))
A command line argument to limit the number of stages in the pipeline.
static cl::opt< bool > EnableSWPOptSize("enable-pipeliner-opt-size", cl::desc("Enable SWP at Os."), cl::Hidden, cl::init(false))
A command line option to enable SWP at -Os.
static cl::opt< bool > SwpShowResMask("pipeliner-show-mask", cl::Hidden, cl::init(false))
static void getPhiRegs(MachineInstr &Phi, MachineBasicBlock *Loop, unsigned &InitVal, unsigned &LoopVal)
Return the register values for the operands of a Phi instruction.
static cl::opt< bool > EnableSWP("enable-pipeliner", cl::Hidden, cl::init(true), cl::desc("Enable Software Pipelining"))
A command line option to turn software pipelining on or off.
static bool ignoreDependence(const SDep &D, bool isPred)
Return true for DAG nodes that we ignore when computing the cost functions.
static cl::opt< bool > SwpPruneDeps("pipeliner-prune-deps", cl::desc("Prune dependences between unrelated Phi nodes."), cl::Hidden, cl::init(true))
A command line option to disable the pruning of chain dependences due to an unrelated Phi.
static SUnit * multipleIterations(SUnit *SU, SwingSchedulerDAG *DAG)
If an instruction has a use that spans multiple iterations, then return true.
static bool computePath(SUnit *Cur, SetVector< SUnit * > &Path, SetVector< SUnit * > &DestNodes, SetVector< SUnit * > &Exclude, SmallPtrSet< SUnit *, 8 > &Visited)
Return true if there is a path from the specified node to any of the nodes in DestNodes.
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
This file provides utility analysis objects describing memory locations.
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file defines the PriorityQueue class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines generic set operations that may be used on set's of different types,...
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
static unsigned getSize(unsigned Kind)
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB)
A trivial helper function to check to see if the specified pointers are no-alias.
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
bool erase(const KeyT &Val)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
A possibly irreducible generalization of a Loop.
Itinerary data supplied by a subtarget to be used by a target.
const InstrStage * beginStage(unsigned ItinClassIndx) const
Return the first stage of the itinerary.
const InstrStage * endStage(unsigned ItinClassIndx) const
Return the last+1 stage of the itinerary.
bool isEmpty() const
Returns true if there are no itineraries.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
BlockT * getHeader() const
Represents a single loop in the control flow graph.
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
unsigned getSchedClass() const
Return the scheduling class for this instruction.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Generic base class for all target subtargets.
const MCWriteProcResEntry * getWriteProcResEnd(const MCSchedClassDesc *SC) const
const MCWriteProcResEntry * getWriteProcResBegin(const MCSchedClassDesc *SC) const
Return an iterator at the first process resource consumed by the given scheduling class.
const MCSchedModel & getSchedModel() const
Get the machine model for this subtarget's CPU.
const MDOperand & getOperand(unsigned I) const
unsigned getNumOperands() const
Return number of MDNode operands.
StringRef getString() const
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
instr_iterator instr_end()
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void deleteMachineInstr(MachineInstr *MI)
DeleteMachineInstr - Delete the given MachineInstr.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineInstr * CloneMachineInstr(const MachineInstr *Orig)
Create a new MachineInstr which is a copy of Orig, identical in all ways except the instruction has n...
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool mayRaiseFPException() const
Return true if this instruction could possibly raise a floating-point exception.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool isRegSequence() const
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
bool isIdenticalTo(const MachineInstr &Other, MICheckType Check=CheckDefs) const
Return true if this instruction is identical to Other.
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
bool isPseudo(QueryType Type=IgnoreBundle) const
Return true if this is a pseudo instruction that doesn't correspond to a real machine instruction.
const MachineOperand & getOperand(unsigned i) const
iterator_range< filtered_mop_iterator > all_defs()
Returns an iterator range over all operands that are (explicit or implicit) register defs.
A description of a memory reference used in the backend.
AAMDNodes getAAInfo() const
Return the AA tags for the memory reference.
const Value * getValue() const
Return the base address of the memory access.
int64_t getOffset() const
For normal values, this is a byte offset added to the base address.
MachineOperand class - Representation of each machine instruction operand.
void setSubReg(unsigned subReg)
unsigned getSubReg() const
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
The main class in the implementation of the target independent software pipeliner pass.
const TargetInstrInfo * TII
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineOptimizationRemarkEmitter * ORE
RegisterClassInfo RegClassInfo
defusechain_iterator - This class provides iterator support for machine operands in the function that...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
use_instr_iterator use_instr_begin(Register RegNo) const
static use_instr_iterator use_instr_end()
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
static MemoryLocation getAfter(const Value *Ptr, const AAMDNodes &AATags=AAMDNodes())
Return a location that may access any location after Ptr, while remaining within the underlying objec...
The ModuloScheduleExpander takes a ModuloSchedule and expands it in-place, rewriting the old loop and...
void cleanup()
Performs final cleanup after expansion.
void expand()
Performs the actual expansion.
Expander that simply annotates each scheduled instruction with a post-instr symbol that can be consum...
void annotate()
Performs the annotation.
Represents a schedule for a single-block loop.
A NodeSet contains a set of SUnit DAG nodes with additional information that assigns a priority to th...
SUnit * getNode(unsigned i) const
void print(raw_ostream &os) const
void setRecMII(unsigned mii)
unsigned count(SUnit *SU) const
void setColocate(unsigned c)
int compareRecMII(NodeSet &RHS)
LLVM_DUMP_METHOD void dump() const
Pass interface - Implemented by all 'passes'.
AnalysisType & getAnalysis() const
getAnalysis<AnalysisType>() - This function is used by subclasses to get to the analysis information ...
A reimplementation of ModuloScheduleExpander.
PriorityQueue - This class behaves like std::priority_queue and provides a few additional convenience...
Track the current register pressure at some position in the instruction stream, and remember the high...
void addLiveRegs(ArrayRef< RegisterMaskPair > Regs)
Force liveness of virtual registers or physical register units.
Wrapper class representing virtual and physical registers.
static constexpr bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
int calculateResMII() const
void initProcResourceVectors(const MCSchedModel &SM, SmallVectorImpl< uint64_t > &Masks)
void init(int II)
Initialize resources with the initiation interval II.
bool canReserveResources(SUnit &SU, int Cycle)
Check if the resources occupied by a machine instruction are available in the current state.
Kind getKind() const
Returns an enum value representing the kind of the dependence.
Kind
These are the different kinds of scheduling dependencies.
@ Output
A register output-dependence (aka WAW).
@ Order
Any other ordering dependency.
@ Anti
A register anti-dependence (aka WAR).
@ Data
Regular data dependence (aka true-dependence).
void setLatency(unsigned Lat)
Sets the latency for this edge.
@ Barrier
An unknown scheduling barrier.
@ Artificial
Arbitrary strong DAG edge (no real dependence).
unsigned getLatency() const
Returns the latency value for this edge, which roughly means the minimum number of cycles that must e...
bool isArtificial() const
Tests if this is an Order dependence that is marked as "artificial", meaning it isn't necessary for c...
This class represents the scheduled code.
int earliestCycleInChain(const SDep &Dep)
Return the cycle of the earliest scheduled instruction in the dependence chain.
void setInitiationInterval(int ii)
Set the initiation interval for this schedule.
SmallSet< SUnit *, 8 > computeUnpipelineableNodes(SwingSchedulerDAG *SSD, TargetInstrInfo::PipelinerLoopInfo *PLI)
Determine transitive dependences of unpipelineable instructions.
bool isLoopCarriedDefOfUse(SwingSchedulerDAG *SSD, MachineInstr *Def, MachineOperand &MO)
Return true if the instruction is a definition that is loop carried and defines the use on the next i...
void dump() const
Utility function used for debugging to print the schedule.
bool insert(SUnit *SU, int StartCycle, int EndCycle, int II)
Try to schedule the node at the specified StartCycle and continue until the node is schedule or the E...
unsigned getMaxStageCount()
Return the maximum stage count needed for this schedule.
void print(raw_ostream &os) const
Print the schedule information to the given output.
int latestCycleInChain(const SDep &Dep)
Return the cycle of the latest scheduled instruction in the dependence chain.
int stageScheduled(SUnit *SU) const
Return the stage for a scheduled instruction.
void computeStart(SUnit *SU, int *MaxEarlyStart, int *MinLateStart, int *MinEnd, int *MaxStart, int II, SwingSchedulerDAG *DAG)
Compute the scheduling start slot for the instruction.
bool isValidSchedule(SwingSchedulerDAG *SSD)
int getInitiationInterval() const
Return the initiation interval for this schedule.
std::deque< SUnit * > & getInstructions(int cycle)
Return the instructions that are scheduled at the specified cycle.
int getFirstCycle() const
Return the first cycle in the completed schedule.
unsigned cycleScheduled(SUnit *SU) const
Return the cycle for a scheduled instruction.
bool isLoopCarried(SwingSchedulerDAG *SSD, MachineInstr &Phi)
Return true if the scheduled Phi has a loop carried operand.
bool normalizeNonPipelinedInstructions(SwingSchedulerDAG *SSD, TargetInstrInfo::PipelinerLoopInfo *PLI)
int getFinalCycle() const
Return the last cycle in the finalized schedule.
void orderDependence(SwingSchedulerDAG *SSD, SUnit *SU, std::deque< SUnit * > &Insts)
Order the instructions within a cycle so that the definitions occur before the uses.
void finalizeSchedule(SwingSchedulerDAG *SSD)
After the schedule has been formed, call this function to combine the instructions from the different...
Scheduling unit. This is a node in the scheduling DAG.
bool isInstr() const
Returns true if this SUnit refers to a machine instruction as opposed to an SDNode.
unsigned NodeNum
Entry # of node in the node vector.
void setInstr(MachineInstr *MI)
Assigns the instruction for the SUnit.
void removePred(const SDep &D)
Removes the specified edge as a pred of the current node if it exists.
bool isPred(const SUnit *N) const
Tests if node N is a predecessor of this node.
unsigned short Latency
Node latency.
bool isBoundaryNode() const
Boundary nodes are placeholders for the boundary of the scheduling region.
bool hasPhysRegDefs
Has physreg defs that are being used.
SmallVector< SDep, 4 > Succs
All sunit successors.
SmallVector< SDep, 4 > Preds
All sunit predecessors.
bool addPred(const SDep &D, bool Required=true)
Adds the specified edge as a pred of the current node if not already.
MachineInstr * getInstr() const
Returns the representative MachineInstr for this SUnit.
A ScheduleDAG for scheduling lists of MachineInstr.
DenseMap< MachineInstr *, SUnit * > MISUnitMap
After calling BuildSchedGraph, each machine instruction in the current scheduling region is mapped to...
virtual void finishBlock()
Cleans up after scheduling in the given block.
MachineBasicBlock * BB
The block in which to insert instructions.
const MCSchedClassDesc * getSchedClass(SUnit *SU) const
Resolves and cache a resolved scheduling class for an SUnit.
void dumpNode(const SUnit &SU) const override
UndefValue * UnknownValue
For an unanalyzable memory access, this Value is used in maps.
void buildSchedGraph(AAResults *AA, RegPressureTracker *RPTracker=nullptr, PressureDiffs *PDiffs=nullptr, LiveIntervals *LIS=nullptr, bool TrackLaneMasks=false)
Builds SUnits for the current region.
SUnit * getSUnit(MachineInstr *MI) const
Returns an existing SUnit for this MI, or nullptr.
void dump() const override
void RemovePred(SUnit *M, SUnit *N)
Updates the topological ordering to accommodate an edge to be removed from the specified node N from ...
void InitDAGTopologicalSorting()
Creates the initial topological ordering from the DAG to be scheduled.
void AddPred(SUnit *Y, SUnit *X)
Updates the topological ordering to accommodate an edge to be added from SUnit X to SUnit Y.
bool IsReachable(const SUnit *SU, const SUnit *TargetSU)
Checks if SU is reachable from TargetSU.
MachineRegisterInfo & MRI
Virtual/real register map.
const TargetInstrInfo * TII
Target instruction information.
std::vector< SUnit > SUnits
The scheduling units.
const TargetRegisterInfo * TRI
Target processor register info.
MachineFunction & MF
Machine function.
A vector that has set insertion semantics.
size_type size() const
Determine the number of elements in the SetVector.
typename vector_type::const_iterator iterator
void clear()
Completely clear the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
SlotIndex insertMachineInstrInMaps(MachineInstr &MI, bool Late=false)
Insert the given machine instruction into the mapping.
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
bool contains(const T &V) const
Check if the SmallSet contains the given element.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class builds the dependence graph for the instructions in a loop, and attempts to schedule the i...
unsigned getDepth(SUnit *Node)
The depth, in the dependence graph, for a node.
unsigned getInstrBaseReg(SUnit *SU)
Return the new base register that was stored away for the changed instruction.
int getASAP(SUnit *Node)
Return the earliest time an instruction may be scheduled.
void applyInstrChange(MachineInstr *MI, SMSchedule &Schedule)
Apply changes to the instruction if needed.
void finishBlock() override
Clean up after the software pipeliner runs.
void fixupRegisterOverlaps(std::deque< SUnit * > &Instrs)
Attempt to fix the degenerate cases when the instruction serialization causes the register lifetimes ...
int getZeroLatencyDepth(SUnit *Node)
The maximum unweighted length of a path from an arbitrary node to the given node in which each edge h...
bool isLoopCarriedDep(SUnit *Source, const SDep &Dep, bool isSucc=true)
Return true for an order or output dependence that is loop carried potentially.
unsigned getDistance(SUnit *U, SUnit *V, const SDep &Dep)
The distance function, which indicates that operation V of iteration I depends on operations U of ite...
void schedule() override
We override the schedule function in ScheduleDAGInstrs to implement the scheduling part of the Swing ...
int getMOV(SUnit *Node)
The mobility function, which the number of slots in which an instruction may be scheduled.
int getZeroLatencyHeight(SUnit *Node)
The maximum unweighted length of a path from the given node to an arbitrary node in which each edge h...
bool isBackedge(SUnit *Source, const SDep &Dep)
Return true if the dependence is a back-edge in the data dependence graph.
unsigned getHeight(SUnit *Node)
The height, in the dependence graph, for a node.
int getALAP(SUnit *Node)
Return the latest time an instruction my be scheduled.
Object returned by analyzeLoopForPipelining.
virtual bool shouldIgnoreForPipelining(const MachineInstr *MI) const =0
Return true if the given instruction should not be pipelined and should be ignored.
virtual bool shouldUseSchedule(SwingSchedulerDAG &SSD, SMSchedule &SMS)
Return true if the proposed schedule should used.
virtual std::unique_ptr< PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const
Analyze loop L, which must be a single-basic-block loop, and if the conditions can be understood enou...
bool isZeroCost(unsigned Opcode) const
Return true for pseudo instructions that don't consume any machine resources in their current form.
virtual bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify=false) const
Analyze the branching code at the end of MBB, returning true if it cannot be understood (e....
virtual DFAPacketizer * CreateTargetScheduleState(const TargetSubtargetInfo &) const
Create machine specific model for scheduling.
virtual bool isPostIncrement(const MachineInstr &MI) const
Return true for post-incremented instructions.
virtual bool getBaseAndOffsetPosition(const MachineInstr &MI, unsigned &BasePos, unsigned &OffsetPos) const
Return true if the instruction contains a base register and offset.
virtual bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const
Sometimes, it is possible for the target to tell, even without aliasing information,...
virtual bool getIncrementValue(const MachineInstr &MI, int &Value) const
If the instruction is an increment of a constant value, return the amount.
bool getMemOperandWithOffset(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset, bool &OffsetIsScalable, const TargetRegisterInfo *TRI) const
Get the base operand and byte offset of an instruction that reads/writes memory.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const char * getRegPressureSetName(unsigned Idx) const =0
Get the name of this register unit pressure set.
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
static Type * getVoidTy(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
LLVM Value Representation.
This class implements an extremely fast bulk output stream that can only output to a stream.
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< PhiNode * > Phi
NodeAddr< DefNode * > Def
This is an optimization pass for GlobalISel generic memory operations.
void stable_sort(R &&Range)
int popcount(T Value) noexcept
Count the number of set bits in a value.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool set_is_subset(const S1Ty &S1, const S2Ty &S2)
set_is_subset(A, B) - Return true iff A in B
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
auto reverse(ContainerTy &&C)
void sort(IteratorTy Start, IteratorTy End)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
void getUnderlyingObjects(const Value *V, SmallVectorImpl< const Value * > &Objects, LoopInfo *LI=nullptr, unsigned MaxLookup=6)
This method is similar to getUnderlyingObject except that it can look through phi and select instruct...
unsigned getRegState(const MachineOperand &RegOp)
Get all register state flags from machine operand RegOp.
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
void erase_value(Container &C, ValueType V)
Wrapper function to remove a value from a container:
char & MachinePipelinerID
This pass performs software pipelining on machine instructions.
cl::opt< bool > SwpEnableCopyToPhi
bool isIdentifiedObject(const Value *V)
Return true if this pointer refers to a distinct and identifiable object.
cl::opt< int > SwpForceIssueWidth
A command line argument to force pipeliner to use specified issue width.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Description of the encoding of one expression Op.
These values represent a non-pipelined step in the execution of an instruction.
RegisterPressure computed within a region of instructions delimited by TopIdx and BottomIdx.
static constexpr LaneBitmask getNone()
Define a kind of processor resource that will be modeled by the scheduler.
Summarize the scheduling resources required for an instruction of a particular scheduling class.
Machine model for scheduling, bundling, and heuristics.
const MCSchedClassDesc * getSchedClassDesc(unsigned SchedClassIdx) const
unsigned getNumProcResourceKinds() const
bool hasInstrSchedModel() const
Does this machine model include instruction-level scheduling.
const MCProcResourceDesc * getProcResource(unsigned ProcResourceIdx) const
Identify one of the processor resource kinds consumed by a particular scheduling class for the specif...
MachineInstr * LoopInductionVar
SmallVector< MachineOperand, 4 > BrCond
MachineInstr * LoopCompare
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > LoopPipelinerInfo
Store the effects of a change in pressure on things that MI scheduler cares about.
std::vector< unsigned > MaxSetPressure
Map of max reg pressure indexed by pressure set ID, not class ID.