Bug Summary

File:lib/Target/AMDGPU/GCNSchedStrategy.cpp
Warning:line 531, column 7
Forming reference to null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name GCNSchedStrategy.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-9~svn362543/lib/Target/AMDGPU -I /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn362543/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn362543/build-llvm/lib/Target/AMDGPU -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn362543=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-06-05-060531-1271-1 -x c++ /build/llvm-toolchain-snapshot-9~svn362543/lib/Target/AMDGPU/GCNSchedStrategy.cpp -faddrsig
1//===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// This contains a MachineSchedStrategy implementation for maximizing wave
11/// occupancy on GCN hardware.
12//===----------------------------------------------------------------------===//
13
14#include "GCNSchedStrategy.h"
15#include "AMDGPUSubtarget.h"
16#include "SIInstrInfo.h"
17#include "SIMachineFunctionInfo.h"
18#include "SIRegisterInfo.h"
19#include "llvm/CodeGen/RegisterClassInfo.h"
20#include "llvm/Support/MathExtras.h"
21
22#define DEBUG_TYPE"machine-scheduler" "machine-scheduler"
23
24using namespace llvm;
25
26GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
27 const MachineSchedContext *C) :
28 GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
29
30void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
31 GenericScheduler::initialize(DAG);
32
33 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
34
35 MF = &DAG->MF;
36
37 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
38
39 // FIXME: This is also necessary, because some passes that run after
40 // scheduling and before regalloc increase register pressure.
41 const int ErrorMargin = 3;
42
43 SGPRExcessLimit = Context->RegClassInfo
44 ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
45 VGPRExcessLimit = Context->RegClassInfo
46 ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
47 if (TargetOccupancy) {
48 SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
49 VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
50 } else {
51 SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
52 SRI->getSGPRPressureSet());
53 VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
54 SRI->getVGPRPressureSet());
55 }
56
57 SGPRCriticalLimit -= ErrorMargin;
58 VGPRCriticalLimit -= ErrorMargin;
59}
60
61void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
62 bool AtTop, const RegPressureTracker &RPTracker,
63 const SIRegisterInfo *SRI,
64 unsigned SGPRPressure,
65 unsigned VGPRPressure) {
66
67 Cand.SU = SU;
68 Cand.AtTop = AtTop;
69
70 // getDownwardPressure() and getUpwardPressure() make temporary changes to
71 // the tracker, so we need to pass those function a non-const copy.
72 RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
73
74 std::vector<unsigned> Pressure;
75 std::vector<unsigned> MaxPressure;
76
77 if (AtTop)
78 TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
79 else {
80 // FIXME: I think for bottom up scheduling, the register pressure is cached
81 // and can be retrieved by DAG->getPressureDif(SU).
82 TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
83 }
84
85 unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
86 unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
87
88 // If two instructions increase the pressure of different register sets
89 // by the same amount, the generic scheduler will prefer to schedule the
90 // instruction that increases the set with the least amount of registers,
91 // which in our case would be SGPRs. This is rarely what we want, so
92 // when we report excess/critical register pressure, we do it either
93 // only for VGPRs or only for SGPRs.
94
95 // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
96 const unsigned MaxVGPRPressureInc = 16;
97 bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
98 bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
99
100
101 // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
102 // to increase the likelihood we don't go over the limits. We should improve
103 // the analysis to look through dependencies to find the path with the least
104 // register pressure.
105
106 // We only need to update the RPDelata for instructions that increase
107 // register pressure. Instructions that decrease or keep reg pressure
108 // the same will be marked as RegExcess in tryCandidate() when they
109 // are compared with instructions that increase the register pressure.
110 if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
111 Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
112 Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
113 }
114
115 if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
116 Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
117 Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
118 }
119
120 // Register pressure is considered 'CRITICAL' if it is approaching a value
121 // that would reduce the wave occupancy for the execution unit. When
122 // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
123 // has the same cost, so we don't need to prefer one over the other.
124
125 int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
126 int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
127
128 if (SGPRDelta >= 0 || VGPRDelta >= 0) {
129 if (SGPRDelta > VGPRDelta) {
130 Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
131 Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
132 } else {
133 Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
134 Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
135 }
136 }
137}
138
139// This function is mostly cut and pasted from
140// GenericScheduler::pickNodeFromQueue()
141void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
142 const CandPolicy &ZonePolicy,
143 const RegPressureTracker &RPTracker,
144 SchedCandidate &Cand) {
145 const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
146 ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
147 unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
148 unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
149 ReadyQueue &Q = Zone.Available;
150 for (SUnit *SU : Q) {
151
152 SchedCandidate TryCand(ZonePolicy);
153 initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
154 SGPRPressure, VGPRPressure);
155 // Pass SchedBoundary only when comparing nodes from the same boundary.
156 SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
157 GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
158 if (TryCand.Reason != NoCand) {
159 // Initialize resource delta if needed in case future heuristics query it.
160 if (TryCand.ResDelta == SchedResourceDelta())
161 TryCand.initResourceDelta(Zone.DAG, SchedModel);
162 Cand.setBest(TryCand);
163 }
164 }
165}
166
167// This function is mostly cut and pasted from
168// GenericScheduler::pickNodeBidirectional()
169SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
170 // Schedule as far as possible in the direction of no choice. This is most
171 // efficient, but also provides the best heuristics for CriticalPSets.
172 if (SUnit *SU = Bot.pickOnlyChoice()) {
173 IsTopNode = false;
174 return SU;
175 }
176 if (SUnit *SU = Top.pickOnlyChoice()) {
177 IsTopNode = true;
178 return SU;
179 }
180 // Set the bottom-up policy based on the state of the current bottom zone and
181 // the instructions outside the zone, including the top zone.
182 CandPolicy BotPolicy;
183 setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
184 // Set the top-down policy based on the state of the current top zone and
185 // the instructions outside the zone, including the bottom zone.
186 CandPolicy TopPolicy;
187 setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
188
189 // See if BotCand is still valid (because we previously scheduled from Top).
190 LLVM_DEBUG(dbgs() << "Picking from Bot:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking from Bot:\n"
; } } while (false);
191 if (!BotCand.isValid() || BotCand.SU->isScheduled ||
192 BotCand.Policy != BotPolicy) {
193 BotCand.reset(CandPolicy());
194 pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
195 assert(BotCand.Reason != NoCand && "failed to find the first candidate")((BotCand.Reason != NoCand && "failed to find the first candidate"
) ? static_cast<void> (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find the first candidate\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 195, __PRETTY_FUNCTION__));
196 } else {
197 LLVM_DEBUG(traceCandidate(BotCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { traceCandidate(BotCand); } } while (
false);
198 }
199
200 // Check if the top Q has a better candidate.
201 LLVM_DEBUG(dbgs() << "Picking from Top:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking from Top:\n"
; } } while (false);
202 if (!TopCand.isValid() || TopCand.SU->isScheduled ||
203 TopCand.Policy != TopPolicy) {
204 TopCand.reset(CandPolicy());
205 pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
206 assert(TopCand.Reason != NoCand && "failed to find the first candidate")((TopCand.Reason != NoCand && "failed to find the first candidate"
) ? static_cast<void> (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find the first candidate\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 206, __PRETTY_FUNCTION__));
207 } else {
208 LLVM_DEBUG(traceCandidate(TopCand))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { traceCandidate(TopCand); } } while (
false);
209 }
210
211 // Pick best from BotCand and TopCand.
212 LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Top Cand: "; traceCandidate
(TopCand); dbgs() << "Bot Cand: "; traceCandidate(BotCand
);; } } while (false)
213 dbgs() << "Bot Cand: "; traceCandidate(BotCand);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Top Cand: "; traceCandidate
(TopCand); dbgs() << "Bot Cand: "; traceCandidate(BotCand
);; } } while (false);
214 SchedCandidate Cand;
215 if (TopCand.Reason == BotCand.Reason) {
216 Cand = BotCand;
217 GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
218 TopCand.Reason = NoCand;
219 GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
220 if (TopCand.Reason != NoCand) {
221 Cand.setBest(TopCand);
222 } else {
223 TopCand.Reason = TopReason;
224 }
225 } else {
226 if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
227 Cand = TopCand;
228 } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
229 Cand = BotCand;
230 } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
231 Cand = TopCand;
232 } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
233 Cand = BotCand;
234 } else {
235 if (BotCand.Reason > TopCand.Reason) {
236 Cand = TopCand;
237 } else {
238 Cand = BotCand;
239 }
240 }
241 }
242 LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Picking: "; traceCandidate
(Cand);; } } while (false);
243
244 IsTopNode = Cand.AtTop;
245 return Cand.SU;
246}
247
248// This function is mostly cut and pasted from
249// GenericScheduler::pickNode()
250SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
251 if (DAG->top() == DAG->bottom()) {
252 assert(Top.Available.empty() && Top.Pending.empty() &&((Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage") ? static_cast<void> (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 253, __PRETTY_FUNCTION__))
253 Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage")((Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() &&
"ReadyQ garbage") ? static_cast<void> (0) : __assert_fail
("Top.Available.empty() && Top.Pending.empty() && Bot.Available.empty() && Bot.Pending.empty() && \"ReadyQ garbage\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 253, __PRETTY_FUNCTION__));
254 return nullptr;
255 }
256 SUnit *SU;
257 do {
258 if (RegionPolicy.OnlyTopDown) {
259 SU = Top.pickOnlyChoice();
260 if (!SU) {
261 CandPolicy NoPolicy;
262 TopCand.reset(NoPolicy);
263 pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
264 assert(TopCand.Reason != NoCand && "failed to find a candidate")((TopCand.Reason != NoCand && "failed to find a candidate"
) ? static_cast<void> (0) : __assert_fail ("TopCand.Reason != NoCand && \"failed to find a candidate\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 264, __PRETTY_FUNCTION__));
265 SU = TopCand.SU;
266 }
267 IsTopNode = true;
268 } else if (RegionPolicy.OnlyBottomUp) {
269 SU = Bot.pickOnlyChoice();
270 if (!SU) {
271 CandPolicy NoPolicy;
272 BotCand.reset(NoPolicy);
273 pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
274 assert(BotCand.Reason != NoCand && "failed to find a candidate")((BotCand.Reason != NoCand && "failed to find a candidate"
) ? static_cast<void> (0) : __assert_fail ("BotCand.Reason != NoCand && \"failed to find a candidate\""
, "/build/llvm-toolchain-snapshot-9~svn362543/lib/Target/AMDGPU/GCNSchedStrategy.cpp"
, 274, __PRETTY_FUNCTION__));
275 SU = BotCand.SU;
276 }
277 IsTopNode = false;
278 } else {
279 SU = pickNodeBidirectional(IsTopNode);
280 }
281 } while (SU->isScheduled);
282
283 if (SU->isTopReady())
284 Top.removeReady(SU);
285 if (SU->isBottomReady())
286 Bot.removeReady(SU);
287
288 LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling SU(" <<
SU->NodeNum << ") " << *SU->getInstr(); } }
while (false)
289 << *SU->getInstr())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling SU(" <<
SU->NodeNum << ") " << *SU->getInstr(); } }
while (false);
290 return SU;
291}
292
293GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
294 std::unique_ptr<MachineSchedStrategy> S) :
295 ScheduleDAGMILive(C, std::move(S)),
296 ST(MF.getSubtarget<GCNSubtarget>()),
297 MFI(*MF.getInfo<SIMachineFunctionInfo>()),
298 StartingOccupancy(MFI.getOccupancy()),
299 MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
300
301 LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Starting occupancy is "
<< StartingOccupancy << ".\n"; } } while (false);
302}
303
304void GCNScheduleDAGMILive::schedule() {
305 if (Stage == 0) {
306 // Just record regions at the first pass.
307 Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
308 return;
309 }
310
311 std::vector<MachineInstr*> Unsched;
312 Unsched.reserve(NumRegionInstrs);
313 for (auto &I : *this) {
314 Unsched.push_back(&I);
315 }
316
317 GCNRegPressure PressureBefore;
318 if (LIS) {
319 PressureBefore = Pressure[RegionIdx];
320
321 LLVM_DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
322 GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
323 dbgs() << "Region live-in pressure: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
324 llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
325 dbgs() << "Region register pressure: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false)
326 PressureBefore.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure before scheduling:\nRegion live-ins:"
; GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI
); dbgs() << "Region live-in pressure: "; llvm::getRegPressure
(MRI, LiveIns[RegionIdx]).print(dbgs()); dbgs() << "Region register pressure: "
; PressureBefore.print(dbgs()); } } while (false);
327 }
328
329 ScheduleDAGMILive::schedule();
330 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
331
332 if (!LIS)
333 return;
334
335 // Check the results of scheduling.
336 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
337 auto PressureAfter = getRealRegPressure();
338
339 LLVM_DEBUG(dbgs() << "Pressure after scheduling: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure after scheduling: "
; PressureAfter.print(dbgs()); } } while (false)
340 PressureAfter.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure after scheduling: "
; PressureAfter.print(dbgs()); } } while (false);
341
342 if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
343 PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
344 Pressure[RegionIdx] = PressureAfter;
345 LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Pressure in desired limits, done.\n"
; } } while (false);
346 return;
347 }
348 unsigned Occ = MFI.getOccupancy();
349 unsigned WavesAfter = std::min(Occ, PressureAfter.getOccupancy(ST));
350 unsigned WavesBefore = std::min(Occ, PressureBefore.getOccupancy(ST));
351 LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBeforedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false)
352 << ", after " << WavesAfter << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy before scheduling: "
<< WavesBefore << ", after " << WavesAfter
<< ".\n"; } } while (false);
353
354 // We could not keep current target occupancy because of the just scheduled
355 // region. Record new occupancy for next scheduling cycle.
356 unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
357 // Allow memory bound functions to drop to 4 waves if not limited by an
358 // attribute.
359 if (WavesAfter < WavesBefore && WavesAfter < MinOccupancy &&
360 WavesAfter >= MFI.getMinAllowedOccupancy()) {
361 LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Function is memory bound, allow occupancy drop up to "
<< MFI.getMinAllowedOccupancy() << " waves\n"; }
} while (false)
362 << MFI.getMinAllowedOccupancy() << " waves\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Function is memory bound, allow occupancy drop up to "
<< MFI.getMinAllowedOccupancy() << " waves\n"; }
} while (false);
363 NewOccupancy = WavesAfter;
364 }
365 if (NewOccupancy < MinOccupancy) {
366 MinOccupancy = NewOccupancy;
367 MFI.limitOccupancy(MinOccupancy);
368 LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false)
369 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Occupancy lowered for the function to "
<< MinOccupancy << ".\n"; } } while (false);
370 }
371
372 if (WavesAfter >= MinOccupancy) {
373 Pressure[RegionIdx] = PressureAfter;
374 return;
375 }
376
377 LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Attempting to revert scheduling.\n"
; } } while (false);
378 RegionEnd = RegionBegin;
379 for (MachineInstr *MI : Unsched) {
380 if (MI->isDebugInstr())
381 continue;
382
383 if (MI->getIterator() != RegionEnd) {
384 BB->remove(MI);
385 BB->insert(RegionEnd, MI);
386 if (!MI->isDebugInstr())
387 LIS->handleMove(*MI, true);
388 }
389 // Reset read-undef flags and update them later.
390 for (auto &Op : MI->operands())
391 if (Op.isReg() && Op.isDef())
392 Op.setIsUndef(false);
393 RegisterOperands RegOpers;
394 RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
395 if (!MI->isDebugInstr()) {
396 if (ShouldTrackLaneMasks) {
397 // Adjust liveness and add missing dead+read-undef flags.
398 SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
399 RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
400 } else {
401 // Adjust for missing dead-def flags.
402 RegOpers.detectDeadDefs(*MI, *LIS);
403 }
404 }
405 RegionEnd = MI->getIterator();
406 ++RegionEnd;
407 LLVM_DEBUG(dbgs() << "Scheduling " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Scheduling " <<
*MI; } } while (false);
408 }
409 RegionBegin = Unsched.front()->getIterator();
410 Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
411
412 placeDebugValues();
413}
414
415GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
416 GCNDownwardRPTracker RPTracker(*LIS);
417 RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
418 return RPTracker.moveMaxPressure();
419}
420
421void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
422 GCNDownwardRPTracker RPTracker(*LIS);
423
424 // If the block has the only successor then live-ins of that successor are
425 // live-outs of the current block. We can reuse calculated live set if the
426 // successor will be sent to scheduling past current block.
427 const MachineBasicBlock *OnlySucc = nullptr;
428 if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
429 SlotIndexes *Ind = LIS->getSlotIndexes();
430 if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
431 OnlySucc = *MBB->succ_begin();
432 }
433
434 // Scheduler sends regions from the end of the block upwards.
435 size_t CurRegion = RegionIdx;
436 for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
437 if (Regions[CurRegion].first->getParent() != MBB)
438 break;
439 --CurRegion;
440
441 auto I = MBB->begin();
442 auto LiveInIt = MBBLiveIns.find(MBB);
443 if (LiveInIt != MBBLiveIns.end()) {
444 auto LiveIn = std::move(LiveInIt->second);
445 RPTracker.reset(*MBB->begin(), &LiveIn);
446 MBBLiveIns.erase(LiveInIt);
447 } else {
448 I = Regions[CurRegion].first;
449 RPTracker.reset(*I);
450 }
451
452 for ( ; ; ) {
453 I = RPTracker.getNext();
454
455 if (Regions[CurRegion].first == I) {
456 LiveIns[CurRegion] = RPTracker.getLiveRegs();
457 RPTracker.clearMaxPressure();
458 }
459
460 if (Regions[CurRegion].second == I) {
461 Pressure[CurRegion] = RPTracker.moveMaxPressure();
462 if (CurRegion-- == RegionIdx)
463 break;
464 }
465 RPTracker.advanceToNext();
466 RPTracker.advanceBeforeNext();
467 }
468
469 if (OnlySucc) {
470 if (I != MBB->end()) {
471 RPTracker.advanceToNext();
472 RPTracker.advance(MBB->end());
473 }
474 RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
475 RPTracker.advanceBeforeNext();
476 MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
477 }
478}
479
480void GCNScheduleDAGMILive::finalizeSchedule() {
481 GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
482 LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "All regions recorded, starting actual scheduling.\n"
; } } while (false);
1
Assuming 'DebugFlag' is 0
→
2
←
Loop condition is false. Exiting loop
→
483
484 LiveIns.resize(Regions.size());
485 Pressure.resize(Regions.size());
486
487 do {
488 Stage++;
489 RegionIdx = 0;
490 MachineBasicBlock *MBB = nullptr;
3
←
'MBB' initialized to a null pointer value
→
491
492 if (Stage > 1) {
4
←
Assuming the condition is false
→
5
←
Taking false branch
→
493 // Retry function scheduling if we found resulting occupancy and it is
494 // lower than used for first pass scheduling. This will give more freedom
495 // to schedule low register pressure blocks.
496 // Code is partially copied from MachineSchedulerBase::scheduleRegions().
497
498 if (!LIS || StartingOccupancy <= MinOccupancy)
499 break;
500
501 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
502 dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
503 << "Retrying function scheduling with lowest recorded occupancy "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false)
504 << MinOccupancy << ".\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "Retrying function scheduling with lowest recorded occupancy "
<< MinOccupancy << ".\n"; } } while (false);
505
506 S.setTargetOccupancy(MinOccupancy);
507 }
508
509 for (auto Region : Regions) {
6
←
Assuming '__begin2' is not equal to '__end2'
→
510 RegionBegin = Region.first;
511 RegionEnd = Region.second;
512
513 if (RegionBegin->getParent() != MBB) {
7
←
Assuming the condition is false
→
8
←
Taking false branch
→
514 if (MBB) finishBlock();
515 MBB = RegionBegin->getParent();
516 startBlock(MBB);
517 if (Stage == 1)
518 computeBlockPressure(MBB);
519 }
520
521 unsigned NumRegionInstrs = std::distance(begin(), end());
522 enterRegion(MBB, begin(), end(), NumRegionInstrs);
523
524 // Skip empty scheduling regions (0 or 1 schedulable instructions).
525 if (begin() == end() || begin() == std::prev(end())) {
9
←
Taking false branch
→
526 exitRegion();
527 continue;
528 }
529
530 LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << "********** MI Scheduling **********\n"
; } } while (false);
10
←
Assuming 'DebugFlag' is not equal to 0
→
11
←
Assuming the condition is false
→
12
←
Taking false branch
→
13
←
Loop condition is false. Exiting loop
→
531 LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*MBB) << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
14
←
Assuming 'DebugFlag' is not equal to 0
→
15
←
Assuming the condition is true
→
16
←
Taking true branch
→
17
←
Forming reference to null pointer
532 << MBB->getName() << "\n From: " << *begin()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
533 << " To: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
534 if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
535 else dbgs() << "End";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false)
536 dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("machine-scheduler")) { dbgs() << MF.getName() <<
":" << printMBBReference(*MBB) << " " << MBB
->getName() << "\n From: " << *begin() <<
" To: "; if (RegionEnd != MBB->end()) dbgs() << *
RegionEnd; else dbgs() << "End"; dbgs() << " RegionInstrs: "
<< NumRegionInstrs << '\n'; } } while (false);
537
538 schedule();
539
540 exitRegion();
541 ++RegionIdx;
542 }
543 finishBlock();
544
545 } while (Stage < 2);
546}