LLVM 19.0.0git
ExecuteStage.cpp
Go to the documentation of this file.
1//===---------------------- ExecuteStage.cpp --------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9///
10/// This file defines the execution stage of an instruction pipeline.
11///
12/// The ExecuteStage is responsible for managing the hardware scheduler
13/// and issuing notifications that an instruction has been executed.
14///
15//===----------------------------------------------------------------------===//
16
19#include "llvm/Support/Debug.h"
20
21#define DEBUG_TYPE "llvm-mca"
22
23namespace llvm {
24namespace mca {
25
27 switch (Status) {
38 }
39
40 llvm_unreachable("Don't know how to process this StallKind!");
41}
42
44 if (Scheduler::Status S = HWS.isAvailable(IR)) {
46 notifyEvent<HWStallEvent>(HWStallEvent(ET, IR));
47 return false;
48 }
49
50 return true;
51}
52
53Error ExecuteStage::issueInstruction(InstRef &IR) {
57
58 HWS.issueInstruction(IR, Used, Pending, Ready);
59 Instruction &IS = *IR.getInstruction();
60 NumIssuedOpcodes += IS.getNumMicroOps();
61
62 notifyReservedOrReleasedBuffers(IR, /* Reserved */ false);
63
65 if (IS.isExecuted()) {
67 // FIXME: add a buffer of executed instructions.
69 return S;
70 }
71
72 for (const InstRef &I : Pending)
74
75 for (const InstRef &I : Ready)
77 return ErrorSuccess();
78}
79
80Error ExecuteStage::issueReadyInstructions() {
81 InstRef IR = HWS.select();
82 while (IR) {
83 if (Error Err = issueInstruction(IR))
84 return Err;
85
86 // Select the next instruction to issue.
87 IR = HWS.select();
88 }
89
90 return ErrorSuccess();
91}
92
98
99 HWS.cycleEvent(Freed, Executed, Pending, Ready);
100 NumDispatchedOpcodes = 0;
101 NumIssuedOpcodes = 0;
102
103 for (const ResourceRef &RR : Freed)
105
106 for (InstRef &IR : Executed) {
108 // FIXME: add a buffer of executed instructions.
109 if (Error S = moveToTheNextStage(IR))
110 return S;
111 }
112
113 for (const InstRef &IR : Pending)
115
116 for (const InstRef &IR : Ready)
118
119 return issueReadyInstructions();
120}
121
123 if (!EnablePressureEvents)
124 return ErrorSuccess();
125
126 // Always conservatively report any backpressure events if the dispatch logic
127 // was stalled due to unavailable scheduler resources.
128 if (!HWS.hadTokenStall() && NumDispatchedOpcodes <= NumIssuedOpcodes)
129 return ErrorSuccess();
130
132 uint64_t Mask = HWS.analyzeResourcePressure(Insts);
133 if (Mask) {
134 LLVM_DEBUG(dbgs() << "[E] Backpressure increased because of unavailable "
135 "pipeline resources: "
136 << format_hex(Mask, 16) << '\n');
138 notifyEvent(Ev);
139 }
140
143 HWS.analyzeDataDependencies(RegDeps, MemDeps);
144 if (RegDeps.size()) {
146 dbgs() << "[E] Backpressure increased by register dependencies\n");
148 notifyEvent(Ev);
149 }
150
151 if (MemDeps.size()) {
152 LLVM_DEBUG(dbgs() << "[E] Backpressure increased by memory dependencies\n");
154 notifyEvent(Ev);
155 }
156
157 return ErrorSuccess();
158}
159
160#ifndef NDEBUG
162 const Instruction &Inst = *IR.getInstruction();
163 assert(Inst.isEliminated() && "Instruction was not eliminated!");
164 assert(Inst.isReady() && "Instruction in an inconsistent state!");
165
166 // Ensure that instructions eliminated at register renaming stage are in a
167 // consistent state.
168 assert(!Inst.getMayLoad() && !Inst.getMayStore() &&
169 "Cannot eliminate a memory op!");
170}
171#endif
172
173Error ExecuteStage::handleInstructionEliminated(InstRef &IR) {
174#ifndef NDEBUG
176#endif
180 IR.getInstruction()->forceExecuted();
182 return moveToTheNextStage(IR);
183}
184
185// Schedule the instruction for execution on the hardware.
187 assert(isAvailable(IR) && "Scheduler is not available!");
188
189#ifndef NDEBUG
190 // Ensure that the HWS has not stored this instruction in its queues.
191 HWS.instructionCheck(IR);
192#endif
193
194 if (IR.getInstruction()->isEliminated())
195 return handleInstructionEliminated(IR);
196
197 // Reserve a slot in each buffered resource. Also, mark units with
198 // BufferSize=0 as reserved. Resources with a buffer size of zero will only
199 // be released after MCIS is issued, and all the ReleaseAtCycles for those
200 // units have been consumed.
201 bool IsReadyInstruction = HWS.dispatch(IR);
202 const Instruction &Inst = *IR.getInstruction();
203 unsigned NumMicroOps = Inst.getNumMicroOps();
204 NumDispatchedOpcodes += NumMicroOps;
205 notifyReservedOrReleasedBuffers(IR, /* Reserved */ true);
206
207 if (!IsReadyInstruction) {
208 if (Inst.isPending())
210 return ErrorSuccess();
211 }
212
214
215 // If we did not return early, then the scheduler is ready for execution.
217
218 // If we cannot issue immediately, the HWS will add IR to its ready queue for
219 // execution later, so we must return early here.
220 if (!HWS.mustIssueImmediately(IR))
221 return ErrorSuccess();
222
223 // Issue IR to the underlying pipelines.
224 return issueInstruction(IR);
225}
226
228 LLVM_DEBUG(dbgs() << "[E] Instruction Executed: #" << IR << '\n');
229 notifyEvent<HWInstructionEvent>(
231}
232
234 LLVM_DEBUG(dbgs() << "[E] Instruction Pending: #" << IR << '\n');
235 notifyEvent<HWInstructionEvent>(
237}
238
240 LLVM_DEBUG(dbgs() << "[E] Instruction Ready: #" << IR << '\n');
241 notifyEvent<HWInstructionEvent>(
243}
244
246 LLVM_DEBUG(dbgs() << "[E] Resource Available: [" << RR.first << '.'
247 << RR.second << "]\n");
248 for (HWEventListener *Listener : getListeners())
249 Listener->onResourceAvailable(RR);
250}
251
253 const InstRef &IR, MutableArrayRef<ResourceUse> Used) const {
254 LLVM_DEBUG({
255 dbgs() << "[E] Instruction Issued: #" << IR << '\n';
256 for (const ResourceUse &Use : Used) {
257 assert(Use.second.getDenominator() == 1 && "Invalid cycles!");
258 dbgs() << "[E] Resource Used: [" << Use.first.first << '.'
259 << Use.first.second << "], ";
260 dbgs() << "cycles: " << Use.second.getNumerator() << '\n';
261 }
262 });
263
264 // Replace resource masks with valid resource processor IDs.
265 for (ResourceUse &Use : Used)
266 Use.first.first = HWS.getResourceID(Use.first.first);
267
268 notifyEvent<HWInstructionEvent>(HWInstructionIssuedEvent(IR, Used));
269}
270
272 bool Reserved) const {
273 uint64_t UsedBuffers = IR.getInstruction()->getDesc().UsedBuffers;
274 if (!UsedBuffers)
275 return;
276
277 SmallVector<unsigned, 4> BufferIDs(llvm::popcount(UsedBuffers), 0);
278 for (unsigned I = 0, E = BufferIDs.size(); I < E; ++I) {
279 uint64_t CurrentBufferMask = UsedBuffers & (-UsedBuffers);
280 BufferIDs[I] = HWS.getResourceID(CurrentBufferMask);
281 UsedBuffers ^= CurrentBufferMask;
282 }
283
284 if (Reserved) {
285 for (HWEventListener *Listener : getListeners())
286 Listener->onReservedBuffers(IR, BufferIDs);
287 return;
288 }
289
290 for (HWEventListener *Listener : getListeners())
291 Listener->onReleasedBuffers(IR, BufferIDs);
292}
293
294} // namespace mca
295} // namespace llvm
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the execution stage of a default instruction pipeline.
Legalize the Machine IR a function s Machine IR
Definition: Legalizer.cpp:81
#define I(x, y, z)
Definition: MD5.cpp:58
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
Subclass of Error for the sole purpose of identifying the success path in the type system.
Definition: Error.h:332
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
size_t size() const
Definition: SmallVector.h:91
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void notifyResourceAvailable(const ResourceRef &RR) const
Error cycleStart() override
Called once at the start of each cycle.
bool isAvailable(const InstRef &IR) const override
Returns true if it can execute IR during this cycle.
Error cycleEnd() override
Called once at the end of each cycle.
void notifyInstructionExecuted(const InstRef &IR) const
void notifyInstructionIssued(const InstRef &IR, MutableArrayRef< ResourceUse > Used) const
void notifyReservedOrReleasedBuffers(const InstRef &IR, bool Reserved) const
void notifyInstructionReady(const InstRef &IR) const
Error execute(InstRef &IR) override
The primary action that this stage performs on instruction IR.
void notifyInstructionPending(const InstRef &IR) const
An InstRef contains both a SourceMgr index and Instruction pair.
Definition: Instruction.h:720
unsigned getNumMicroOps() const
Definition: Instruction.h:542
An instruction propagated through the simulated instruction pipeline.
Definition: Instruction.h:600
bool isEliminated() const
Definition: Instruction.h:691
bool isExecuted() const
Definition: Instruction.h:689
bool isReady() const
Definition: Instruction.h:687
bool isPending() const
Definition: Instruction.h:686
InstRef select()
Select the next instruction to issue from the ReadySet.
Definition: Scheduler.cpp:192
unsigned getResourceID(uint64_t Mask) const
Convert a resource mask into a valid llvm processor resource identifier.
Definition: Scheduler.h:234
void issueInstruction(InstRef &IR, SmallVectorImpl< std::pair< ResourceRef, ReleaseAtCycles > > &Used, SmallVectorImpl< InstRef > &Pending, SmallVectorImpl< InstRef > &Ready)
Issue an instruction and populates a vector of used pipeline resources, and a vector of instructions ...
Definition: Scheduler.cpp:99
Status isAvailable(const InstRef &IR)
Check if the instruction in 'IR' can be dispatched during this cycle.
Definition: Scheduler.cpp:40
void analyzeDataDependencies(SmallVectorImpl< InstRef > &RegDeps, SmallVectorImpl< InstRef > &MemDeps)
This method is called by the ExecuteStage at the end of each cycle to identify bottlenecks caused by ...
Definition: Scheduler.cpp:248
bool dispatch(InstRef &IR)
Reserves buffer and LSUnit queue resources that are necessary to issue this instruction.
Definition: Scheduler.cpp:300
void cycleEvent(SmallVectorImpl< ResourceRef > &Freed, SmallVectorImpl< InstRef > &Executed, SmallVectorImpl< InstRef > &Pending, SmallVectorImpl< InstRef > &Ready)
This routine notifies the Scheduler that a new cycle just started.
Definition: Scheduler.cpp:264
bool mustIssueImmediately(const InstRef &IR) const
Returns true if IR has to be issued immediately, or if IR is a zero latency instruction.
Definition: Scheduler.cpp:290
uint64_t analyzeResourcePressure(SmallVectorImpl< InstRef > &Insts)
Returns a mask of busy resources, and populates vector Insts with instructions that could not be issu...
Definition: Scheduler.cpp:243
void instructionCheck(const InstRef &IR) const
Definition: Scheduler.h:270
bool hadTokenStall() const
Definition: Scheduler.h:261
Error moveToTheNextStage(InstRef &IR)
Called when an instruction is ready to move the next pipeline stage.
Definition: Stage.h:73
void notifyEvent(const EventT &Event) const
Notify listeners of a particular hardware event.
Definition: Stage.h:82
const std::set< HWEventListener * > & getListeners() const
Definition: Stage.h:35
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
std::pair< ResourceRef, ReleaseAtCycles > ResourceUse
HWStallEvent::GenericEventType toHWStallEventType(Scheduler::Status Status)
std::pair< uint64_t, uint64_t > ResourceRef
A resource unit identifier.
static void verifyInstructionEliminated(const InstRef &IR)
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition: bit.h:385
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
FormattedNumber format_hex(uint64_t N, unsigned Width, bool Upper=false)
format_hex - Output N as a fixed width hexadecimal.
Definition: Format.h:187