LLVM 19.0.0git
SIInsertWaitcnts.cpp
Go to the documentation of this file.
1//===- SIInsertWaitcnts.cpp - Insert Wait Instructions --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Insert wait instructions for memory reads and writes.
11///
12/// Memory reads and writes are issued asynchronously, so we need to insert
13/// S_WAITCNT instructions when we want to access any of their results or
14/// overwrite any register that's used asynchronously.
15///
16/// TODO: This pass currently keeps one timeline per hardware counter. A more
17/// finely-grained approach that keeps one timeline per event type could
18/// sometimes get away with generating weaker s_waitcnt instructions. For
19/// example, when both SMEM and LDS are in flight and we need to wait for
20/// the i-th-last LDS instruction, then an lgkmcnt(i) is actually sufficient,
21/// but the pass will currently generate a conservative lgkmcnt(0) because
22/// multiple event types are in flight.
23//
24//===----------------------------------------------------------------------===//
25
26#include "AMDGPU.h"
27#include "GCNSubtarget.h"
31#include "llvm/ADT/MapVector.h"
33#include "llvm/ADT/Sequence.h"
40using namespace llvm;
41
42#define DEBUG_TYPE "si-insert-waitcnts"
43
44DEBUG_COUNTER(ForceExpCounter, DEBUG_TYPE"-forceexp",
45 "Force emit s_waitcnt expcnt(0) instrs");
46DEBUG_COUNTER(ForceLgkmCounter, DEBUG_TYPE"-forcelgkm",
47 "Force emit s_waitcnt lgkmcnt(0) instrs");
48DEBUG_COUNTER(ForceVMCounter, DEBUG_TYPE"-forcevm",
49 "Force emit s_waitcnt vmcnt(0) instrs");
50
52 "amdgpu-waitcnt-forcezero",
53 cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
54 cl::init(false), cl::Hidden);
55
56namespace {
57// Class of object that encapsulates latest instruction counter score
58// associated with the operand. Used for determining whether
59// s_waitcnt instruction needs to be emitted.
60
61enum InstCounterType {
62 LOAD_CNT = 0, // VMcnt prior to gfx12.
63 DS_CNT, // LKGMcnt prior to gfx12.
64 EXP_CNT, //
65 STORE_CNT, // VScnt in gfx10/gfx11.
66 NUM_NORMAL_INST_CNTS,
67 SAMPLE_CNT = NUM_NORMAL_INST_CNTS, // gfx12+ only.
68 BVH_CNT, // gfx12+ only.
69 KM_CNT, // gfx12+ only.
70 NUM_EXTENDED_INST_CNTS,
71 NUM_INST_CNTS = NUM_EXTENDED_INST_CNTS
72};
73} // namespace
74
75namespace llvm {
76template <> struct enum_iteration_traits<InstCounterType> {
77 static constexpr bool is_iterable = true;
78};
79} // namespace llvm
80
81namespace {
82// Return an iterator over all counters between LOAD_CNT (the first counter)
83// and \c MaxCounter (exclusive, default value yields an enumeration over
84// all counters).
85auto inst_counter_types(InstCounterType MaxCounter = NUM_INST_CNTS) {
86 return enum_seq(LOAD_CNT, MaxCounter);
87}
88
89using RegInterval = std::pair<int, int>;
90
91struct HardwareLimits {
92 unsigned LoadcntMax; // Corresponds to VMcnt prior to gfx12.
93 unsigned ExpcntMax;
94 unsigned DscntMax; // Corresponds to LGKMcnt prior to gfx12.
95 unsigned StorecntMax; // Corresponds to VScnt in gfx10/gfx11.
96 unsigned SamplecntMax; // gfx12+ only.
97 unsigned BvhcntMax; // gfx12+ only.
98 unsigned KmcntMax; // gfx12+ only.
99};
100
101struct RegisterEncoding {
102 unsigned VGPR0;
103 unsigned VGPRL;
104 unsigned SGPR0;
105 unsigned SGPRL;
106};
107
108enum WaitEventType {
109 VMEM_ACCESS, // vector-memory read & write
110 VMEM_READ_ACCESS, // vector-memory read
111 VMEM_SAMPLER_READ_ACCESS, // vector-memory SAMPLER read (gfx12+ only)
112 VMEM_BVH_READ_ACCESS, // vector-memory BVH read (gfx12+ only)
113 VMEM_WRITE_ACCESS, // vector-memory write that is not scratch
114 SCRATCH_WRITE_ACCESS, // vector-memory write that may be scratch
115 LDS_ACCESS, // lds read & write
116 GDS_ACCESS, // gds read & write
117 SQ_MESSAGE, // send message
118 SMEM_ACCESS, // scalar-memory read & write
119 EXP_GPR_LOCK, // export holding on its data src
120 GDS_GPR_LOCK, // GDS holding on its data and addr src
121 EXP_POS_ACCESS, // write to export position
122 EXP_PARAM_ACCESS, // write to export parameter
123 VMW_GPR_LOCK, // vector-memory write holding on its data src
124 EXP_LDS_ACCESS, // read by ldsdir counting as export
125 NUM_WAIT_EVENTS,
126};
127
128// The mapping is:
129// 0 .. SQ_MAX_PGM_VGPRS-1 real VGPRs
130// SQ_MAX_PGM_VGPRS .. NUM_ALL_VGPRS-1 extra VGPR-like slots
131// NUM_ALL_VGPRS .. NUM_ALL_VGPRS+SQ_MAX_PGM_SGPRS-1 real SGPRs
132// We reserve a fixed number of VGPR slots in the scoring tables for
133// special tokens like SCMEM_LDS (needed for buffer load to LDS).
134enum RegisterMapping {
135 SQ_MAX_PGM_VGPRS = 512, // Maximum programmable VGPRs across all targets.
136 AGPR_OFFSET = 256, // Maximum programmable ArchVGPRs across all targets.
137 SQ_MAX_PGM_SGPRS = 256, // Maximum programmable SGPRs across all targets.
138 NUM_EXTRA_VGPRS = 9, // Reserved slots for DS.
139 // Artificial register slots to track LDS writes into specific LDS locations
140 // if a location is known. When slots are exhausted or location is
141 // unknown use the first slot. The first slot is also always updated in
142 // addition to known location's slot to properly generate waits if dependent
143 // instruction's location is unknown.
144 EXTRA_VGPR_LDS = 0,
145 NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_EXTRA_VGPRS, // Where SGPR starts.
146};
147
148// Enumerate different types of result-returning VMEM operations. Although
149// s_waitcnt orders them all with a single vmcnt counter, in the absence of
150// s_waitcnt only instructions of the same VmemType are guaranteed to write
151// their results in order -- so there is no need to insert an s_waitcnt between
152// two instructions of the same type that write the same vgpr.
153enum VmemType {
154 // BUF instructions and MIMG instructions without a sampler.
155 VMEM_NOSAMPLER,
156 // MIMG instructions with a sampler.
157 VMEM_SAMPLER,
158 // BVH instructions
159 VMEM_BVH,
160 NUM_VMEM_TYPES
161};
162
163// Maps values of InstCounterType to the instruction that waits on that
164// counter. Only used if GCNSubtarget::hasExtendedWaitCounts()
165// returns true.
166static const unsigned instrsForExtendedCounterTypes[NUM_EXTENDED_INST_CNTS] = {
167 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT, AMDGPU::S_WAIT_EXPCNT,
168 AMDGPU::S_WAIT_STORECNT, AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
169 AMDGPU::S_WAIT_KMCNT};
170
171static bool updateVMCntOnly(const MachineInstr &Inst) {
172 return SIInstrInfo::isVMEM(Inst) || SIInstrInfo::isFLATGlobal(Inst) ||
174}
175
176#ifndef NDEBUG
177static bool isNormalMode(InstCounterType MaxCounter) {
178 return MaxCounter == NUM_NORMAL_INST_CNTS;
179}
180#endif // NDEBUG
181
182VmemType getVmemType(const MachineInstr &Inst) {
183 assert(updateVMCntOnly(Inst));
184 if (!SIInstrInfo::isMIMG(Inst) && !SIInstrInfo::isVIMAGE(Inst) &&
186 return VMEM_NOSAMPLER;
188 const AMDGPU::MIMGBaseOpcodeInfo *BaseInfo =
190 // We have to make an additional check for isVSAMPLE here since some
191 // instructions don't have a sampler, but are still classified as sampler
192 // instructions for the purposes of e.g. waitcnt.
193 return BaseInfo->BVH ? VMEM_BVH
194 : (BaseInfo->Sampler || SIInstrInfo::isVSAMPLE(Inst)) ? VMEM_SAMPLER
195 : VMEM_NOSAMPLER;
196}
197
198unsigned &getCounterRef(AMDGPU::Waitcnt &Wait, InstCounterType T) {
199 switch (T) {
200 case LOAD_CNT:
201 return Wait.LoadCnt;
202 case EXP_CNT:
203 return Wait.ExpCnt;
204 case DS_CNT:
205 return Wait.DsCnt;
206 case STORE_CNT:
207 return Wait.StoreCnt;
208 case SAMPLE_CNT:
209 return Wait.SampleCnt;
210 case BVH_CNT:
211 return Wait.BvhCnt;
212 case KM_CNT:
213 return Wait.KmCnt;
214 default:
215 llvm_unreachable("bad InstCounterType");
216 }
217}
218
219void addWait(AMDGPU::Waitcnt &Wait, InstCounterType T, unsigned Count) {
220 unsigned &WC = getCounterRef(Wait, T);
221 WC = std::min(WC, Count);
222}
223
224void setNoWait(AMDGPU::Waitcnt &Wait, InstCounterType T) {
225 getCounterRef(Wait, T) = ~0u;
226}
227
228unsigned getWait(AMDGPU::Waitcnt &Wait, InstCounterType T) {
229 return getCounterRef(Wait, T);
230}
231
232// Mapping from event to counter according to the table masks.
233InstCounterType eventCounter(const unsigned *masks, WaitEventType E) {
234 for (auto T : inst_counter_types()) {
235 if (masks[T] & (1 << E))
236 return T;
237 }
238 llvm_unreachable("event type has no associated counter");
239}
240
241// This objects maintains the current score brackets of each wait counter, and
242// a per-register scoreboard for each wait counter.
243//
244// We also maintain the latest score for every event type that can change the
245// waitcnt in order to know if there are multiple types of events within
246// the brackets. When multiple types of event happen in the bracket,
247// wait count may get decreased out of order, therefore we need to put in
248// "s_waitcnt 0" before use.
249class WaitcntBrackets {
250public:
251 WaitcntBrackets(const GCNSubtarget *SubTarget, InstCounterType MaxCounter,
252 HardwareLimits Limits, RegisterEncoding Encoding,
253 const unsigned *WaitEventMaskForInst,
254 InstCounterType SmemAccessCounter)
255 : ST(SubTarget), MaxCounter(MaxCounter), Limits(Limits),
256 Encoding(Encoding), WaitEventMaskForInst(WaitEventMaskForInst),
257 SmemAccessCounter(SmemAccessCounter) {}
258
259 unsigned getWaitCountMax(InstCounterType T) const {
260 switch (T) {
261 case LOAD_CNT:
262 return Limits.LoadcntMax;
263 case DS_CNT:
264 return Limits.DscntMax;
265 case EXP_CNT:
266 return Limits.ExpcntMax;
267 case STORE_CNT:
268 return Limits.StorecntMax;
269 case SAMPLE_CNT:
270 return Limits.SamplecntMax;
271 case BVH_CNT:
272 return Limits.BvhcntMax;
273 case KM_CNT:
274 return Limits.KmcntMax;
275 default:
276 break;
277 }
278 return 0;
279 }
280
281 unsigned getScoreLB(InstCounterType T) const {
282 assert(T < NUM_INST_CNTS);
283 return ScoreLBs[T];
284 }
285
286 unsigned getScoreUB(InstCounterType T) const {
287 assert(T < NUM_INST_CNTS);
288 return ScoreUBs[T];
289 }
290
291 unsigned getScoreRange(InstCounterType T) const {
292 return getScoreUB(T) - getScoreLB(T);
293 }
294
295 unsigned getRegScore(int GprNo, InstCounterType T) const {
296 if (GprNo < NUM_ALL_VGPRS) {
297 return VgprScores[T][GprNo];
298 }
299 assert(T == SmemAccessCounter);
300 return SgprScores[GprNo - NUM_ALL_VGPRS];
301 }
302
303 bool merge(const WaitcntBrackets &Other);
304
305 RegInterval getRegInterval(const MachineInstr *MI,
307 const SIRegisterInfo *TRI, unsigned OpNo) const;
308
309 bool counterOutOfOrder(InstCounterType T) const;
310 void simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const;
311 void simplifyWaitcnt(InstCounterType T, unsigned &Count) const;
312 void determineWait(InstCounterType T, int RegNo, AMDGPU::Waitcnt &Wait) const;
313 void applyWaitcnt(const AMDGPU::Waitcnt &Wait);
314 void applyWaitcnt(InstCounterType T, unsigned Count);
315 void updateByEvent(const SIInstrInfo *TII, const SIRegisterInfo *TRI,
316 const MachineRegisterInfo *MRI, WaitEventType E,
318
319 unsigned hasPendingEvent() const { return PendingEvents; }
320 unsigned hasPendingEvent(WaitEventType E) const {
321 return PendingEvents & (1 << E);
322 }
323 unsigned hasPendingEvent(InstCounterType T) const {
324 unsigned HasPending = PendingEvents & WaitEventMaskForInst[T];
325 assert((HasPending != 0) == (getScoreRange(T) != 0));
326 return HasPending;
327 }
328
329 bool hasMixedPendingEvents(InstCounterType T) const {
330 unsigned Events = hasPendingEvent(T);
331 // Return true if more than one bit is set in Events.
332 return Events & (Events - 1);
333 }
334
335 bool hasPendingFlat() const {
336 return ((LastFlat[DS_CNT] > ScoreLBs[DS_CNT] &&
337 LastFlat[DS_CNT] <= ScoreUBs[DS_CNT]) ||
338 (LastFlat[LOAD_CNT] > ScoreLBs[LOAD_CNT] &&
339 LastFlat[LOAD_CNT] <= ScoreUBs[LOAD_CNT]));
340 }
341
342 void setPendingFlat() {
343 LastFlat[LOAD_CNT] = ScoreUBs[LOAD_CNT];
344 LastFlat[DS_CNT] = ScoreUBs[DS_CNT];
345 }
346
347 // Return true if there might be pending writes to the specified vgpr by VMEM
348 // instructions with types different from V.
349 bool hasOtherPendingVmemTypes(int GprNo, VmemType V) const {
350 assert(GprNo < NUM_ALL_VGPRS);
351 return VgprVmemTypes[GprNo] & ~(1 << V);
352 }
353
354 void clearVgprVmemTypes(int GprNo) {
355 assert(GprNo < NUM_ALL_VGPRS);
356 VgprVmemTypes[GprNo] = 0;
357 }
358
359 void setStateOnFunctionEntryOrReturn() {
360 setScoreUB(STORE_CNT, getScoreUB(STORE_CNT) + getWaitCountMax(STORE_CNT));
361 PendingEvents |= WaitEventMaskForInst[STORE_CNT];
362 }
363
364 ArrayRef<const MachineInstr *> getLDSDMAStores() const {
365 return LDSDMAStores;
366 }
367
368 void print(raw_ostream &);
369 void dump() { print(dbgs()); }
370
371private:
372 struct MergeInfo {
373 unsigned OldLB;
374 unsigned OtherLB;
375 unsigned MyShift;
376 unsigned OtherShift;
377 };
378 static bool mergeScore(const MergeInfo &M, unsigned &Score,
379 unsigned OtherScore);
380
381 void setScoreLB(InstCounterType T, unsigned Val) {
382 assert(T < NUM_INST_CNTS);
383 ScoreLBs[T] = Val;
384 }
385
386 void setScoreUB(InstCounterType T, unsigned Val) {
387 assert(T < NUM_INST_CNTS);
388 ScoreUBs[T] = Val;
389
390 if (T != EXP_CNT)
391 return;
392
393 if (getScoreRange(EXP_CNT) > getWaitCountMax(EXP_CNT))
394 ScoreLBs[EXP_CNT] = ScoreUBs[EXP_CNT] - getWaitCountMax(EXP_CNT);
395 }
396
397 void setRegScore(int GprNo, InstCounterType T, unsigned Val) {
398 if (GprNo < NUM_ALL_VGPRS) {
399 VgprUB = std::max(VgprUB, GprNo);
400 VgprScores[T][GprNo] = Val;
401 } else {
402 assert(T == SmemAccessCounter);
403 SgprUB = std::max(SgprUB, GprNo - NUM_ALL_VGPRS);
404 SgprScores[GprNo - NUM_ALL_VGPRS] = Val;
405 }
406 }
407
408 void setExpScore(const MachineInstr *MI, const SIInstrInfo *TII,
410 unsigned OpNo, unsigned Val);
411
412 const GCNSubtarget *ST = nullptr;
413 InstCounterType MaxCounter = NUM_EXTENDED_INST_CNTS;
414 HardwareLimits Limits = {};
415 RegisterEncoding Encoding = {};
416 const unsigned *WaitEventMaskForInst;
417 InstCounterType SmemAccessCounter;
418 unsigned ScoreLBs[NUM_INST_CNTS] = {0};
419 unsigned ScoreUBs[NUM_INST_CNTS] = {0};
420 unsigned PendingEvents = 0;
421 // Remember the last flat memory operation.
422 unsigned LastFlat[NUM_INST_CNTS] = {0};
423 // wait_cnt scores for every vgpr.
424 // Keep track of the VgprUB and SgprUB to make merge at join efficient.
425 int VgprUB = -1;
426 int SgprUB = -1;
427 unsigned VgprScores[NUM_INST_CNTS][NUM_ALL_VGPRS] = {{0}};
428 // Wait cnt scores for every sgpr, only DS_CNT (corresponding to LGKMcnt
429 // pre-gfx12) or KM_CNT (gfx12+ only) are relevant.
430 unsigned SgprScores[SQ_MAX_PGM_SGPRS] = {0};
431 // Bitmask of the VmemTypes of VMEM instructions that might have a pending
432 // write to each vgpr.
433 unsigned char VgprVmemTypes[NUM_ALL_VGPRS] = {0};
434 // Store representative LDS DMA operations. The only useful info here is
435 // alias info. One store is kept per unique AAInfo.
436 SmallVector<const MachineInstr *, NUM_EXTRA_VGPRS - 1> LDSDMAStores;
437};
438
439// This abstracts the logic for generating and updating S_WAIT* instructions
440// away from the analysis that determines where they are needed. This was
441// done because the set of counters and instructions for waiting on them
442// underwent a major shift with gfx12, sufficiently so that having this
443// abstraction allows the main analysis logic to be simpler than it would
444// otherwise have had to become.
445class WaitcntGenerator {
446protected:
447 const GCNSubtarget *ST = nullptr;
448 const SIInstrInfo *TII = nullptr;
450 InstCounterType MaxCounter;
451 bool OptNone;
452
453public:
454 WaitcntGenerator() {}
455 WaitcntGenerator(const MachineFunction &MF, InstCounterType MaxCounter)
456 : ST(&MF.getSubtarget<GCNSubtarget>()), TII(ST->getInstrInfo()),
457 IV(AMDGPU::getIsaVersion(ST->getCPU())), MaxCounter(MaxCounter),
458 OptNone(MF.getFunction().hasOptNone() ||
459 MF.getTarget().getOptLevel() == CodeGenOptLevel::None) {}
460
461 // Return true if the current function should be compiled with no
462 // optimization.
463 bool isOptNone() const { return OptNone; }
464
465 // Edits an existing sequence of wait count instructions according
466 // to an incoming Waitcnt value, which is itself updated to reflect
467 // any new wait count instructions which may need to be generated by
468 // WaitcntGenerator::createNewWaitcnt(). It will return true if any edits
469 // were made.
470 //
471 // This editing will usually be merely updated operands, but it may also
472 // delete instructions if the incoming Wait value indicates they are not
473 // needed. It may also remove existing instructions for which a wait
474 // is needed if it can be determined that it is better to generate new
475 // instructions later, as can happen on gfx12.
476 virtual bool
477 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
478 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
480
481 // Transform a soft waitcnt into a normal one.
482 bool promoteSoftWaitCnt(MachineInstr *Waitcnt) const;
483
484 // Generates new wait count instructions according to the value of
485 // Wait, returning true if any new instructions were created.
486 virtual bool createNewWaitcnt(MachineBasicBlock &Block,
489
490 // Returns an array of bit masks which can be used to map values in
491 // WaitEventType to corresponding counter values in InstCounterType.
492 virtual const unsigned *getWaitEventMask() const = 0;
493
494 // Returns a new waitcnt with all counters except VScnt set to 0. If
495 // IncludeVSCnt is true, VScnt is set to 0, otherwise it is set to ~0u.
496 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const = 0;
497
498 virtual ~WaitcntGenerator() = default;
499
500 // Create a mask value from the initializer list of wait event types.
501 static constexpr unsigned
502 eventMask(std::initializer_list<WaitEventType> Events) {
503 unsigned Mask = 0;
504 for (auto &E : Events)
505 Mask |= 1 << E;
506
507 return Mask;
508 }
509};
510
511class WaitcntGeneratorPreGFX12 : public WaitcntGenerator {
512public:
513 WaitcntGeneratorPreGFX12() {}
514 WaitcntGeneratorPreGFX12(const MachineFunction &MF)
515 : WaitcntGenerator(MF, NUM_NORMAL_INST_CNTS) {}
516
517 bool
518 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
519 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
520 MachineBasicBlock::instr_iterator It) const override;
521
522 bool createNewWaitcnt(MachineBasicBlock &Block,
524 AMDGPU::Waitcnt Wait) override;
525
526 const unsigned *getWaitEventMask() const override {
527 assert(ST);
528
529 static const unsigned WaitEventMaskForInstPreGFX12[NUM_INST_CNTS] = {
530 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS,
531 VMEM_BVH_READ_ACCESS}),
532 eventMask({SMEM_ACCESS, LDS_ACCESS, GDS_ACCESS, SQ_MESSAGE}),
533 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
534 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
535 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
536 0,
537 0,
538 0};
539
540 return WaitEventMaskForInstPreGFX12;
541 }
542
543 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
544};
545
546class WaitcntGeneratorGFX12Plus : public WaitcntGenerator {
547public:
548 WaitcntGeneratorGFX12Plus() {}
549 WaitcntGeneratorGFX12Plus(const MachineFunction &MF,
550 InstCounterType MaxCounter)
551 : WaitcntGenerator(MF, MaxCounter) {}
552
553 bool
554 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
555 MachineInstr &OldWaitcntInstr, AMDGPU::Waitcnt &Wait,
556 MachineBasicBlock::instr_iterator It) const override;
557
558 bool createNewWaitcnt(MachineBasicBlock &Block,
560 AMDGPU::Waitcnt Wait) override;
561
562 const unsigned *getWaitEventMask() const override {
563 assert(ST);
564
565 static const unsigned WaitEventMaskForInstGFX12Plus[NUM_INST_CNTS] = {
566 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS}),
567 eventMask({LDS_ACCESS, GDS_ACCESS}),
568 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
569 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
570 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
571 eventMask({VMEM_SAMPLER_READ_ACCESS}),
572 eventMask({VMEM_BVH_READ_ACCESS}),
573 eventMask({SMEM_ACCESS, SQ_MESSAGE})};
574
575 return WaitEventMaskForInstGFX12Plus;
576 }
577
578 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(bool IncludeVSCnt) const override;
579};
580
581class SIInsertWaitcnts : public MachineFunctionPass {
582private:
583 const GCNSubtarget *ST = nullptr;
584 const SIInstrInfo *TII = nullptr;
585 const SIRegisterInfo *TRI = nullptr;
586 const MachineRegisterInfo *MRI = nullptr;
587
589 DenseMap<MachineBasicBlock *, bool> PreheadersToFlush;
590 MachineLoopInfo *MLI;
592 AliasAnalysis *AA = nullptr;
593
594 struct BlockInfo {
595 std::unique_ptr<WaitcntBrackets> Incoming;
596 bool Dirty = true;
597 };
598
599 InstCounterType SmemAccessCounter;
600
602
603 // ForceEmitZeroWaitcnts: force all waitcnts insts to be s_waitcnt 0
604 // because of amdgpu-waitcnt-forcezero flag
605 bool ForceEmitZeroWaitcnts;
606 bool ForceEmitWaitcnt[NUM_INST_CNTS];
607
608 // In any given run of this pass, WCG will point to one of these two
609 // generator objects, which must have been re-initialised before use
610 // from a value made using a subtarget constructor.
611 WaitcntGeneratorPreGFX12 WCGPreGFX12;
612 WaitcntGeneratorGFX12Plus WCGGFX12Plus;
613
614 WaitcntGenerator *WCG = nullptr;
615
616 // S_ENDPGM instructions before which we should insert a DEALLOC_VGPRS
617 // message.
618 DenseSet<MachineInstr *> ReleaseVGPRInsts;
619
620 InstCounterType MaxCounter = NUM_NORMAL_INST_CNTS;
621
622public:
623 static char ID;
624
625 SIInsertWaitcnts() : MachineFunctionPass(ID) {
626 (void)ForceExpCounter;
627 (void)ForceLgkmCounter;
628 (void)ForceVMCounter;
629 }
630
631 bool shouldFlushVmCnt(MachineLoop *ML, WaitcntBrackets &Brackets);
632 bool isPreheaderToFlush(MachineBasicBlock &MBB,
633 WaitcntBrackets &ScoreBrackets);
634 bool isVMEMOrFlatVMEM(const MachineInstr &MI) const;
635 bool runOnMachineFunction(MachineFunction &MF) override;
636
637 StringRef getPassName() const override {
638 return "SI insert wait instructions";
639 }
640
641 void getAnalysisUsage(AnalysisUsage &AU) const override {
642 AU.setPreservesCFG();
648 }
649
650 bool isForceEmitWaitcnt() const {
651 for (auto T : inst_counter_types())
652 if (ForceEmitWaitcnt[T])
653 return true;
654 return false;
655 }
656
657 void setForceEmitWaitcnt() {
658// For non-debug builds, ForceEmitWaitcnt has been initialized to false;
659// For debug builds, get the debug counter info and adjust if need be
660#ifndef NDEBUG
661 if (DebugCounter::isCounterSet(ForceExpCounter) &&
662 DebugCounter::shouldExecute(ForceExpCounter)) {
663 ForceEmitWaitcnt[EXP_CNT] = true;
664 } else {
665 ForceEmitWaitcnt[EXP_CNT] = false;
666 }
667
668 if (DebugCounter::isCounterSet(ForceLgkmCounter) &&
669 DebugCounter::shouldExecute(ForceLgkmCounter)) {
670 ForceEmitWaitcnt[DS_CNT] = true;
671 ForceEmitWaitcnt[KM_CNT] = true;
672 } else {
673 ForceEmitWaitcnt[DS_CNT] = false;
674 ForceEmitWaitcnt[KM_CNT] = false;
675 }
676
677 if (DebugCounter::isCounterSet(ForceVMCounter) &&
678 DebugCounter::shouldExecute(ForceVMCounter)) {
679 ForceEmitWaitcnt[LOAD_CNT] = true;
680 ForceEmitWaitcnt[SAMPLE_CNT] = true;
681 ForceEmitWaitcnt[BVH_CNT] = true;
682 } else {
683 ForceEmitWaitcnt[LOAD_CNT] = false;
684 ForceEmitWaitcnt[SAMPLE_CNT] = false;
685 ForceEmitWaitcnt[BVH_CNT] = false;
686 }
687#endif // NDEBUG
688 }
689
690 // Return the appropriate VMEM_*_ACCESS type for Inst, which must be a VMEM or
691 // FLAT instruction.
692 WaitEventType getVmemWaitEventType(const MachineInstr &Inst) const {
693 // Maps VMEM access types to their corresponding WaitEventType.
694 static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
695 VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
696
698 // LDS DMA loads are also stores, but on the LDS side. On the VMEM side
699 // these should use VM_CNT.
700 if (!ST->hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
701 return VMEM_ACCESS;
702 if (Inst.mayStore() && !SIInstrInfo::isAtomicRet(Inst)) {
703 // FLAT and SCRATCH instructions may access scratch. Other VMEM
704 // instructions do not.
705 if (SIInstrInfo::isFLAT(Inst) && mayAccessScratchThroughFlat(Inst))
706 return SCRATCH_WRITE_ACCESS;
707 return VMEM_WRITE_ACCESS;
708 }
709 if (!ST->hasExtendedWaitCounts() || SIInstrInfo::isFLAT(Inst))
710 return VMEM_READ_ACCESS;
711 return VmemReadMapping[getVmemType(Inst)];
712 }
713
714 bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
715 bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;
716 bool mayAccessScratchThroughFlat(const MachineInstr &MI) const;
717 bool generateWaitcntInstBefore(MachineInstr &MI,
718 WaitcntBrackets &ScoreBrackets,
719 MachineInstr *OldWaitcntInstr,
720 bool FlushVmCnt);
721 bool generateWaitcnt(AMDGPU::Waitcnt Wait,
723 MachineBasicBlock &Block, WaitcntBrackets &ScoreBrackets,
724 MachineInstr *OldWaitcntInstr);
725 void updateEventWaitcntAfter(MachineInstr &Inst,
726 WaitcntBrackets *ScoreBrackets);
727 bool insertWaitcntInBlock(MachineFunction &MF, MachineBasicBlock &Block,
728 WaitcntBrackets &ScoreBrackets);
729};
730
731} // end anonymous namespace
732
733RegInterval WaitcntBrackets::getRegInterval(const MachineInstr *MI,
735 const SIRegisterInfo *TRI,
736 unsigned OpNo) const {
737 const MachineOperand &Op = MI->getOperand(OpNo);
738 if (!TRI->isInAllocatableClass(Op.getReg()))
739 return {-1, -1};
740
741 // A use via a PW operand does not need a waitcnt.
742 // A partial write is not a WAW.
743 assert(!Op.getSubReg() || !Op.isUndef());
744
745 RegInterval Result;
746
747 unsigned Reg = TRI->getEncodingValue(AMDGPU::getMCReg(Op.getReg(), *ST)) &
749
750 if (TRI->isVectorRegister(*MRI, Op.getReg())) {
751 assert(Reg >= Encoding.VGPR0 && Reg <= Encoding.VGPRL);
752 Result.first = Reg - Encoding.VGPR0;
753 if (TRI->isAGPR(*MRI, Op.getReg()))
754 Result.first += AGPR_OFFSET;
755 assert(Result.first >= 0 && Result.first < SQ_MAX_PGM_VGPRS);
756 } else if (TRI->isSGPRReg(*MRI, Op.getReg())) {
757 assert(Reg >= Encoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
758 Result.first = Reg - Encoding.SGPR0 + NUM_ALL_VGPRS;
759 assert(Result.first >= NUM_ALL_VGPRS &&
760 Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS);
761 }
762 // TODO: Handle TTMP
763 // else if (TRI->isTTMP(*MRI, Reg.getReg())) ...
764 else
765 return {-1, -1};
766
767 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Op.getReg());
768 unsigned Size = TRI->getRegSizeInBits(*RC);
769 Result.second = Result.first + ((Size + 16) / 32);
770
771 return Result;
772}
773
774void WaitcntBrackets::setExpScore(const MachineInstr *MI,
775 const SIInstrInfo *TII,
776 const SIRegisterInfo *TRI,
777 const MachineRegisterInfo *MRI, unsigned OpNo,
778 unsigned Val) {
779 RegInterval Interval = getRegInterval(MI, MRI, TRI, OpNo);
780 assert(TRI->isVectorRegister(*MRI, MI->getOperand(OpNo).getReg()));
781 for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
782 setRegScore(RegNo, EXP_CNT, Val);
783 }
784}
785
786void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
787 const SIRegisterInfo *TRI,
789 WaitEventType E, MachineInstr &Inst) {
790 InstCounterType T = eventCounter(WaitEventMaskForInst, E);
791
792 unsigned UB = getScoreUB(T);
793 unsigned CurrScore = UB + 1;
794 if (CurrScore == 0)
795 report_fatal_error("InsertWaitcnt score wraparound");
796 // PendingEvents and ScoreUB need to be update regardless if this event
797 // changes the score of a register or not.
798 // Examples including vm_cnt when buffer-store or lgkm_cnt when send-message.
799 PendingEvents |= 1 << E;
800 setScoreUB(T, CurrScore);
801
802 if (T == EXP_CNT) {
803 // Put score on the source vgprs. If this is a store, just use those
804 // specific register(s).
805 if (TII->isDS(Inst) && (Inst.mayStore() || Inst.mayLoad())) {
806 int AddrOpIdx =
807 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::addr);
808 // All GDS operations must protect their address register (same as
809 // export.)
810 if (AddrOpIdx != -1) {
811 setExpScore(&Inst, TII, TRI, MRI, AddrOpIdx, CurrScore);
812 }
813
814 if (Inst.mayStore()) {
815 if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data0)) {
816 setExpScore(
817 &Inst, TII, TRI, MRI,
818 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0),
819 CurrScore);
820 }
821 if (AMDGPU::hasNamedOperand(Inst.getOpcode(), AMDGPU::OpName::data1)) {
822 setExpScore(&Inst, TII, TRI, MRI,
824 AMDGPU::OpName::data1),
825 CurrScore);
826 }
827 } else if (SIInstrInfo::isAtomicRet(Inst) && !SIInstrInfo::isGWS(Inst) &&
828 Inst.getOpcode() != AMDGPU::DS_APPEND &&
829 Inst.getOpcode() != AMDGPU::DS_CONSUME &&
830 Inst.getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
831 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
832 const MachineOperand &Op = Inst.getOperand(I);
833 if (Op.isReg() && !Op.isDef() &&
834 TRI->isVectorRegister(*MRI, Op.getReg())) {
835 setExpScore(&Inst, TII, TRI, MRI, I, CurrScore);
836 }
837 }
838 }
839 } else if (TII->isFLAT(Inst)) {
840 if (Inst.mayStore()) {
841 setExpScore(
842 &Inst, TII, TRI, MRI,
843 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
844 CurrScore);
845 } else if (SIInstrInfo::isAtomicRet(Inst)) {
846 setExpScore(
847 &Inst, TII, TRI, MRI,
848 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
849 CurrScore);
850 }
851 } else if (TII->isMIMG(Inst)) {
852 if (Inst.mayStore()) {
853 setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
854 } else if (SIInstrInfo::isAtomicRet(Inst)) {
855 setExpScore(
856 &Inst, TII, TRI, MRI,
857 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
858 CurrScore);
859 }
860 } else if (TII->isMTBUF(Inst)) {
861 if (Inst.mayStore()) {
862 setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
863 }
864 } else if (TII->isMUBUF(Inst)) {
865 if (Inst.mayStore()) {
866 setExpScore(&Inst, TII, TRI, MRI, 0, CurrScore);
867 } else if (SIInstrInfo::isAtomicRet(Inst)) {
868 setExpScore(
869 &Inst, TII, TRI, MRI,
870 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data),
871 CurrScore);
872 }
873 } else if (TII->isLDSDIR(Inst)) {
874 // LDSDIR instructions attach the score to the destination.
875 setExpScore(
876 &Inst, TII, TRI, MRI,
877 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::vdst),
878 CurrScore);
879 } else {
880 if (TII->isEXP(Inst)) {
881 // For export the destination registers are really temps that
882 // can be used as the actual source after export patching, so
883 // we need to treat them like sources and set the EXP_CNT
884 // score.
885 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
886 MachineOperand &DefMO = Inst.getOperand(I);
887 if (DefMO.isReg() && DefMO.isDef() &&
888 TRI->isVGPR(*MRI, DefMO.getReg())) {
889 setRegScore(
890 TRI->getEncodingValue(AMDGPU::getMCReg(DefMO.getReg(), *ST)),
891 EXP_CNT, CurrScore);
892 }
893 }
894 }
895 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
896 MachineOperand &MO = Inst.getOperand(I);
897 if (MO.isReg() && !MO.isDef() &&
898 TRI->isVectorRegister(*MRI, MO.getReg())) {
899 setExpScore(&Inst, TII, TRI, MRI, I, CurrScore);
900 }
901 }
902 }
903 } else /* LGKM_CNT || EXP_CNT || VS_CNT || NUM_INST_CNTS */ {
904 // Match the score to the destination registers.
905 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
906 auto &Op = Inst.getOperand(I);
907 if (!Op.isReg() || !Op.isDef())
908 continue;
909 RegInterval Interval = getRegInterval(&Inst, MRI, TRI, I);
910 if (T == LOAD_CNT || T == SAMPLE_CNT || T == BVH_CNT) {
911 if (Interval.first >= NUM_ALL_VGPRS)
912 continue;
913 if (updateVMCntOnly(Inst)) {
914 // updateVMCntOnly should only leave us with VGPRs
915 // MUBUF, MTBUF, MIMG, FlatGlobal, and FlatScratch only have VGPR/AGPR
916 // defs. That's required for a sane index into `VgprMemTypes` below
917 assert(TRI->isVectorRegister(*MRI, Op.getReg()));
918 VmemType V = getVmemType(Inst);
919 for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo)
920 VgprVmemTypes[RegNo] |= 1 << V;
921 }
922 }
923 for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
924 setRegScore(RegNo, T, CurrScore);
925 }
926 }
927 if (Inst.mayStore() &&
928 (TII->isDS(Inst) || TII->mayWriteLDSThroughDMA(Inst))) {
929 // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
930 // written can be accessed. A load from LDS to VMEM does not need a wait.
931 unsigned Slot = 0;
932 for (const auto *MemOp : Inst.memoperands()) {
933 if (!MemOp->isStore() ||
934 MemOp->getAddrSpace() != AMDGPUAS::LOCAL_ADDRESS)
935 continue;
936 // Comparing just AA info does not guarantee memoperands are equal
937 // in general, but this is so for LDS DMA in practice.
938 auto AAI = MemOp->getAAInfo();
939 // Alias scope information gives a way to definitely identify an
940 // original memory object and practically produced in the module LDS
941 // lowering pass. If there is no scope available we will not be able
942 // to disambiguate LDS aliasing as after the module lowering all LDS
943 // is squashed into a single big object. Do not attempt to use one of
944 // the limited LDSDMAStores for something we will not be able to use
945 // anyway.
946 if (!AAI || !AAI.Scope)
947 break;
948 for (unsigned I = 0, E = LDSDMAStores.size(); I != E && !Slot; ++I) {
949 for (const auto *MemOp : LDSDMAStores[I]->memoperands()) {
950 if (MemOp->isStore() && AAI == MemOp->getAAInfo()) {
951 Slot = I + 1;
952 break;
953 }
954 }
955 }
956 if (Slot || LDSDMAStores.size() == NUM_EXTRA_VGPRS - 1)
957 break;
958 LDSDMAStores.push_back(&Inst);
959 Slot = LDSDMAStores.size();
960 break;
961 }
962 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS + Slot, T, CurrScore);
963 if (Slot)
964 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS, T, CurrScore);
965 }
966 }
967}
968
969void WaitcntBrackets::print(raw_ostream &OS) {
970 OS << '\n';
971 for (auto T : inst_counter_types(MaxCounter)) {
972 unsigned SR = getScoreRange(T);
973
974 switch (T) {
975 case LOAD_CNT:
976 OS << " " << (ST->hasExtendedWaitCounts() ? "LOAD" : "VM") << "_CNT("
977 << SR << "): ";
978 break;
979 case DS_CNT:
980 OS << " " << (ST->hasExtendedWaitCounts() ? "DS" : "LGKM") << "_CNT("
981 << SR << "): ";
982 break;
983 case EXP_CNT:
984 OS << " EXP_CNT(" << SR << "): ";
985 break;
986 case STORE_CNT:
987 OS << " " << (ST->hasExtendedWaitCounts() ? "STORE" : "VS") << "_CNT("
988 << SR << "): ";
989 break;
990 case SAMPLE_CNT:
991 OS << " SAMPLE_CNT(" << SR << "): ";
992 break;
993 case BVH_CNT:
994 OS << " BVH_CNT(" << SR << "): ";
995 break;
996 case KM_CNT:
997 OS << " KM_CNT(" << SR << "): ";
998 break;
999 default:
1000 OS << " UNKNOWN(" << SR << "): ";
1001 break;
1002 }
1003
1004 if (SR != 0) {
1005 // Print vgpr scores.
1006 unsigned LB = getScoreLB(T);
1007
1008 for (int J = 0; J <= VgprUB; J++) {
1009 unsigned RegScore = getRegScore(J, T);
1010 if (RegScore <= LB)
1011 continue;
1012 unsigned RelScore = RegScore - LB - 1;
1013 if (J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS) {
1014 OS << RelScore << ":v" << J << " ";
1015 } else {
1016 OS << RelScore << ":ds ";
1017 }
1018 }
1019 // Also need to print sgpr scores for lgkm_cnt.
1020 if (T == SmemAccessCounter) {
1021 for (int J = 0; J <= SgprUB; J++) {
1022 unsigned RegScore = getRegScore(J + NUM_ALL_VGPRS, T);
1023 if (RegScore <= LB)
1024 continue;
1025 unsigned RelScore = RegScore - LB - 1;
1026 OS << RelScore << ":s" << J << " ";
1027 }
1028 }
1029 }
1030 OS << '\n';
1031 }
1032 OS << '\n';
1033}
1034
1035/// Simplify the waitcnt, in the sense of removing redundant counts, and return
1036/// whether a waitcnt instruction is needed at all.
1037void WaitcntBrackets::simplifyWaitcnt(AMDGPU::Waitcnt &Wait) const {
1038 simplifyWaitcnt(LOAD_CNT, Wait.LoadCnt);
1039 simplifyWaitcnt(EXP_CNT, Wait.ExpCnt);
1040 simplifyWaitcnt(DS_CNT, Wait.DsCnt);
1041 simplifyWaitcnt(STORE_CNT, Wait.StoreCnt);
1042 simplifyWaitcnt(SAMPLE_CNT, Wait.SampleCnt);
1043 simplifyWaitcnt(BVH_CNT, Wait.BvhCnt);
1044 simplifyWaitcnt(KM_CNT, Wait.KmCnt);
1045}
1046
1047void WaitcntBrackets::simplifyWaitcnt(InstCounterType T,
1048 unsigned &Count) const {
1049 // The number of outstanding events for this type, T, can be calculated
1050 // as (UB - LB). If the current Count is greater than or equal to the number
1051 // of outstanding events, then the wait for this counter is redundant.
1052 if (Count >= getScoreRange(T))
1053 Count = ~0u;
1054}
1055
1056void WaitcntBrackets::determineWait(InstCounterType T, int RegNo,
1057 AMDGPU::Waitcnt &Wait) const {
1058 unsigned ScoreToWait = getRegScore(RegNo, T);
1059
1060 // If the score of src_operand falls within the bracket, we need an
1061 // s_waitcnt instruction.
1062 const unsigned LB = getScoreLB(T);
1063 const unsigned UB = getScoreUB(T);
1064 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1065 if ((T == LOAD_CNT || T == DS_CNT) && hasPendingFlat() &&
1066 !ST->hasFlatLgkmVMemCountInOrder()) {
1067 // If there is a pending FLAT operation, and this is a VMem or LGKM
1068 // waitcnt and the target can report early completion, then we need
1069 // to force a waitcnt 0.
1070 addWait(Wait, T, 0);
1071 } else if (counterOutOfOrder(T)) {
1072 // Counter can get decremented out-of-order when there
1073 // are multiple types event in the bracket. Also emit an s_wait counter
1074 // with a conservative value of 0 for the counter.
1075 addWait(Wait, T, 0);
1076 } else {
1077 // If a counter has been maxed out avoid overflow by waiting for
1078 // MAX(CounterType) - 1 instead.
1079 unsigned NeededWait = std::min(UB - ScoreToWait, getWaitCountMax(T) - 1);
1080 addWait(Wait, T, NeededWait);
1081 }
1082 }
1083}
1084
1085void WaitcntBrackets::applyWaitcnt(const AMDGPU::Waitcnt &Wait) {
1086 applyWaitcnt(LOAD_CNT, Wait.LoadCnt);
1087 applyWaitcnt(EXP_CNT, Wait.ExpCnt);
1088 applyWaitcnt(DS_CNT, Wait.DsCnt);
1089 applyWaitcnt(STORE_CNT, Wait.StoreCnt);
1090 applyWaitcnt(SAMPLE_CNT, Wait.SampleCnt);
1091 applyWaitcnt(BVH_CNT, Wait.BvhCnt);
1092 applyWaitcnt(KM_CNT, Wait.KmCnt);
1093}
1094
1095void WaitcntBrackets::applyWaitcnt(InstCounterType T, unsigned Count) {
1096 const unsigned UB = getScoreUB(T);
1097 if (Count >= UB)
1098 return;
1099 if (Count != 0) {
1100 if (counterOutOfOrder(T))
1101 return;
1102 setScoreLB(T, std::max(getScoreLB(T), UB - Count));
1103 } else {
1104 setScoreLB(T, UB);
1105 PendingEvents &= ~WaitEventMaskForInst[T];
1106 }
1107}
1108
1109// Where there are multiple types of event in the bracket of a counter,
1110// the decrement may go out of order.
1111bool WaitcntBrackets::counterOutOfOrder(InstCounterType T) const {
1112 // Scalar memory read always can go out of order.
1113 if (T == SmemAccessCounter && hasPendingEvent(SMEM_ACCESS))
1114 return true;
1115 return hasMixedPendingEvents(T);
1116}
1117
1118INITIALIZE_PASS_BEGIN(SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false,
1119 false)
1122INITIALIZE_PASS_END(SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false,
1123 false)
1124
1125char SIInsertWaitcnts::ID = 0;
1126
1127char &llvm::SIInsertWaitcntsID = SIInsertWaitcnts::ID;
1128
1130 return new SIInsertWaitcnts();
1131}
1132
1134 unsigned NewEnc) {
1135 int OpIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OpName);
1136 assert(OpIdx >= 0);
1137
1138 MachineOperand &MO = MI.getOperand(OpIdx);
1139
1140 if (NewEnc == MO.getImm())
1141 return false;
1142
1143 MO.setImm(NewEnc);
1144 return true;
1145}
1146
1147/// Determine if \p MI is a gfx12+ single-counter S_WAIT_*CNT instruction,
1148/// and if so, which counter it is waiting on.
1149static std::optional<InstCounterType> counterTypeForInstr(unsigned Opcode) {
1150 switch (Opcode) {
1151 case AMDGPU::S_WAIT_LOADCNT:
1152 return LOAD_CNT;
1153 case AMDGPU::S_WAIT_EXPCNT:
1154 return EXP_CNT;
1155 case AMDGPU::S_WAIT_STORECNT:
1156 return STORE_CNT;
1157 case AMDGPU::S_WAIT_SAMPLECNT:
1158 return SAMPLE_CNT;
1159 case AMDGPU::S_WAIT_BVHCNT:
1160 return BVH_CNT;
1161 case AMDGPU::S_WAIT_DSCNT:
1162 return DS_CNT;
1163 case AMDGPU::S_WAIT_KMCNT:
1164 return KM_CNT;
1165 default:
1166 return {};
1167 }
1168}
1169
1170bool WaitcntGenerator::promoteSoftWaitCnt(MachineInstr *Waitcnt) const {
1171 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Waitcnt->getOpcode());
1172 if (Opcode == Waitcnt->getOpcode())
1173 return false;
1174
1175 Waitcnt->setDesc(TII->get(Opcode));
1176 return true;
1177}
1178
1179/// Combine consecutive S_WAITCNT and S_WAITCNT_VSCNT instructions that
1180/// precede \p It and follow \p OldWaitcntInstr and apply any extra waits
1181/// from \p Wait that were added by previous passes. Currently this pass
1182/// conservatively assumes that these preexisting waits are required for
1183/// correctness.
1184bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1185 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1187 assert(ST);
1188 assert(isNormalMode(MaxCounter));
1189
1190 bool Modified = false;
1191 MachineInstr *WaitcntInstr = nullptr;
1192 MachineInstr *WaitcntVsCntInstr = nullptr;
1193
1194 for (auto &II :
1195 make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
1196 if (II.isMetaInstruction())
1197 continue;
1198
1199 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(II.getOpcode());
1200 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1201
1202 // Update required wait count. If this is a soft waitcnt (= it was added
1203 // by an earlier pass), it may be entirely removed.
1204 if (Opcode == AMDGPU::S_WAITCNT) {
1205 unsigned IEnc = II.getOperand(0).getImm();
1206 AMDGPU::Waitcnt OldWait = AMDGPU::decodeWaitcnt(IV, IEnc);
1207 if (TrySimplify)
1208 ScoreBrackets.simplifyWaitcnt(OldWait);
1209 Wait = Wait.combined(OldWait);
1210
1211 // Merge consecutive waitcnt of the same type by erasing multiples.
1212 if (WaitcntInstr || (!Wait.hasWaitExceptStoreCnt() && TrySimplify)) {
1213 II.eraseFromParent();
1214 Modified = true;
1215 } else
1216 WaitcntInstr = &II;
1217 } else {
1218 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1219 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1220
1221 unsigned OldVSCnt =
1222 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1223 if (TrySimplify)
1224 ScoreBrackets.simplifyWaitcnt(InstCounterType::STORE_CNT, OldVSCnt);
1225 Wait.StoreCnt = std::min(Wait.StoreCnt, OldVSCnt);
1226
1227 if (WaitcntVsCntInstr || (!Wait.hasWaitStoreCnt() && TrySimplify)) {
1228 II.eraseFromParent();
1229 Modified = true;
1230 } else
1231 WaitcntVsCntInstr = &II;
1232 }
1233 }
1234
1235 if (WaitcntInstr) {
1236 Modified |= updateOperandIfDifferent(*WaitcntInstr, AMDGPU::OpName::simm16,
1238 Modified |= promoteSoftWaitCnt(WaitcntInstr);
1239
1240 ScoreBrackets.applyWaitcnt(LOAD_CNT, Wait.LoadCnt);
1241 ScoreBrackets.applyWaitcnt(EXP_CNT, Wait.ExpCnt);
1242 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt);
1243 Wait.LoadCnt = ~0u;
1244 Wait.ExpCnt = ~0u;
1245 Wait.DsCnt = ~0u;
1246
1247 LLVM_DEBUG(It == WaitcntInstr->getParent()->end()
1248 ? dbgs()
1249 << "applyPreexistingWaitcnt\n"
1250 << "New Instr at block end: " << *WaitcntInstr << '\n'
1251 : dbgs() << "applyPreexistingWaitcnt\n"
1252 << "Old Instr: " << *It
1253 << "New Instr: " << *WaitcntInstr << '\n');
1254 }
1255
1256 if (WaitcntVsCntInstr) {
1257 Modified |= updateOperandIfDifferent(*WaitcntVsCntInstr,
1258 AMDGPU::OpName::simm16, Wait.StoreCnt);
1259 Modified |= promoteSoftWaitCnt(WaitcntVsCntInstr);
1260
1261 ScoreBrackets.applyWaitcnt(STORE_CNT, Wait.StoreCnt);
1262 Wait.StoreCnt = ~0u;
1263
1264 LLVM_DEBUG(It == WaitcntVsCntInstr->getParent()->end()
1265 ? dbgs() << "applyPreexistingWaitcnt\n"
1266 << "New Instr at block end: " << *WaitcntVsCntInstr
1267 << '\n'
1268 : dbgs() << "applyPreexistingWaitcnt\n"
1269 << "Old Instr: " << *It
1270 << "New Instr: " << *WaitcntVsCntInstr << '\n');
1271 }
1272
1273 return Modified;
1274}
1275
1276/// Generate S_WAITCNT and/or S_WAITCNT_VSCNT instructions for any
1277/// required counters in \p Wait
1278bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1281 assert(ST);
1282 assert(isNormalMode(MaxCounter));
1283
1284 bool Modified = false;
1285 const DebugLoc &DL = Block.findDebugLoc(It);
1286
1287 // Waits for VMcnt, LKGMcnt and/or EXPcnt are encoded together into a
1288 // single instruction while VScnt has its own instruction.
1289 if (Wait.hasWaitExceptStoreCnt()) {
1290 unsigned Enc = AMDGPU::encodeWaitcnt(IV, Wait);
1291 [[maybe_unused]] auto SWaitInst =
1292 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAITCNT)).addImm(Enc);
1293 Modified = true;
1294
1295 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
1296 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1297 dbgs() << "New Instr: " << *SWaitInst << '\n');
1298 }
1299
1300 if (Wait.hasWaitStoreCnt()) {
1301 assert(ST->hasVscnt());
1302
1303 [[maybe_unused]] auto SWaitInst =
1304 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAITCNT_VSCNT))
1305 .addReg(AMDGPU::SGPR_NULL, RegState::Undef)
1306 .addImm(Wait.StoreCnt);
1307 Modified = true;
1308
1309 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
1310 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1311 dbgs() << "New Instr: " << *SWaitInst << '\n');
1312 }
1313
1314 return Modified;
1315}
1316
1318WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1319 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt && ST->hasVscnt() ? 0 : ~0u);
1320}
1321
1323WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(bool IncludeVSCnt) const {
1324 return AMDGPU::Waitcnt(0, 0, 0, IncludeVSCnt ? 0 : ~0u, 0, 0, 0);
1325}
1326
1327/// Combine consecutive S_WAIT_*CNT instructions that precede \p It and
1328/// follow \p OldWaitcntInstr and apply any extra waits from \p Wait that
1329/// were added by previous passes. Currently this pass conservatively
1330/// assumes that these preexisting waits are required for correctness.
1331bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1332 WaitcntBrackets &ScoreBrackets, MachineInstr &OldWaitcntInstr,
1334 assert(ST);
1335 assert(!isNormalMode(MaxCounter));
1336
1337 bool Modified = false;
1338 MachineInstr *CombinedLoadDsCntInstr = nullptr;
1339 MachineInstr *CombinedStoreDsCntInstr = nullptr;
1340 MachineInstr *WaitInstrs[NUM_EXTENDED_INST_CNTS] = {};
1341
1342 for (auto &II :
1343 make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
1344 if (II.isMetaInstruction())
1345 continue;
1346
1347 MachineInstr **UpdatableInstr;
1348
1349 // Update required wait count. If this is a soft waitcnt (= it was added
1350 // by an earlier pass), it may be entirely removed.
1351
1352 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(II.getOpcode());
1353 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1354
1355 // Don't crash if the programmer used legacy waitcnt intrinsics, but don't
1356 // attempt to do more than that either.
1357 if (Opcode == AMDGPU::S_WAITCNT)
1358 continue;
1359
1360 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1361 unsigned OldEnc =
1362 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1364 if (TrySimplify)
1365 ScoreBrackets.simplifyWaitcnt(OldWait);
1366 Wait = Wait.combined(OldWait);
1367 UpdatableInstr = &CombinedLoadDsCntInstr;
1368 } else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1369 unsigned OldEnc =
1370 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1372 if (TrySimplify)
1373 ScoreBrackets.simplifyWaitcnt(OldWait);
1374 Wait = Wait.combined(OldWait);
1375 UpdatableInstr = &CombinedStoreDsCntInstr;
1376 } else {
1377 std::optional<InstCounterType> CT = counterTypeForInstr(Opcode);
1378 assert(CT.has_value());
1379 unsigned OldCnt =
1380 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1381 if (TrySimplify)
1382 ScoreBrackets.simplifyWaitcnt(CT.value(), OldCnt);
1383 addWait(Wait, CT.value(), OldCnt);
1384 UpdatableInstr = &WaitInstrs[CT.value()];
1385 }
1386
1387 // Merge consecutive waitcnt of the same type by erasing multiples.
1388 if (!*UpdatableInstr) {
1389 *UpdatableInstr = &II;
1390 } else {
1391 II.eraseFromParent();
1392 Modified = true;
1393 }
1394 }
1395
1396 if (CombinedLoadDsCntInstr) {
1397 // Only keep an S_WAIT_LOADCNT_DSCNT if both counters actually need
1398 // to be waited for. Otherwise, let the instruction be deleted so
1399 // the appropriate single counter wait instruction can be inserted
1400 // instead, when new S_WAIT_*CNT instructions are inserted by
1401 // createNewWaitcnt(). As a side effect, resetting the wait counts will
1402 // cause any redundant S_WAIT_LOADCNT or S_WAIT_DSCNT to be removed by
1403 // the loop below that deals with single counter instructions.
1404 if (Wait.LoadCnt != ~0u && Wait.DsCnt != ~0u) {
1405 unsigned NewEnc = AMDGPU::encodeLoadcntDscnt(IV, Wait);
1406 Modified |= updateOperandIfDifferent(*CombinedLoadDsCntInstr,
1407 AMDGPU::OpName::simm16, NewEnc);
1408 Modified |= promoteSoftWaitCnt(CombinedLoadDsCntInstr);
1409 ScoreBrackets.applyWaitcnt(LOAD_CNT, Wait.LoadCnt);
1410 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt);
1411 Wait.LoadCnt = ~0u;
1412 Wait.DsCnt = ~0u;
1413
1414 LLVM_DEBUG(It == OldWaitcntInstr.getParent()->end()
1415 ? dbgs() << "applyPreexistingWaitcnt\n"
1416 << "New Instr at block end: "
1417 << *CombinedLoadDsCntInstr << '\n'
1418 : dbgs() << "applyPreexistingWaitcnt\n"
1419 << "Old Instr: " << *It << "New Instr: "
1420 << *CombinedLoadDsCntInstr << '\n');
1421 } else {
1422 CombinedLoadDsCntInstr->eraseFromParent();
1423 Modified = true;
1424 }
1425 }
1426
1427 if (CombinedStoreDsCntInstr) {
1428 // Similarly for S_WAIT_STORECNT_DSCNT.
1429 if (Wait.StoreCnt != ~0u && Wait.DsCnt != ~0u) {
1430 unsigned NewEnc = AMDGPU::encodeStorecntDscnt(IV, Wait);
1431 Modified |= updateOperandIfDifferent(*CombinedStoreDsCntInstr,
1432 AMDGPU::OpName::simm16, NewEnc);
1433 Modified |= promoteSoftWaitCnt(CombinedStoreDsCntInstr);
1434 ScoreBrackets.applyWaitcnt(STORE_CNT, Wait.StoreCnt);
1435 ScoreBrackets.applyWaitcnt(DS_CNT, Wait.DsCnt);
1436 Wait.StoreCnt = ~0u;
1437 Wait.DsCnt = ~0u;
1438
1439 LLVM_DEBUG(It == OldWaitcntInstr.getParent()->end()
1440 ? dbgs() << "applyPreexistingWaitcnt\n"
1441 << "New Instr at block end: "
1442 << *CombinedStoreDsCntInstr << '\n'
1443 : dbgs() << "applyPreexistingWaitcnt\n"
1444 << "Old Instr: " << *It << "New Instr: "
1445 << *CombinedStoreDsCntInstr << '\n');
1446 } else {
1447 CombinedStoreDsCntInstr->eraseFromParent();
1448 Modified = true;
1449 }
1450 }
1451
1452 // Look for an opportunity to convert existing S_WAIT_LOADCNT,
1453 // S_WAIT_STORECNT and S_WAIT_DSCNT into new S_WAIT_LOADCNT_DSCNT
1454 // or S_WAIT_STORECNT_DSCNT. This is achieved by selectively removing
1455 // instructions so that createNewWaitcnt() will create new combined
1456 // instructions to replace them.
1457
1458 if (Wait.DsCnt != ~0u) {
1459 // This is a vector of addresses in WaitInstrs pointing to instructions
1460 // that should be removed if they are present.
1462
1463 // If it's known that both DScnt and either LOADcnt or STOREcnt (but not
1464 // both) need to be waited for, ensure that there are no existing
1465 // individual wait count instructions for these.
1466
1467 if (Wait.LoadCnt != ~0u) {
1468 WaitsToErase.push_back(&WaitInstrs[LOAD_CNT]);
1469 WaitsToErase.push_back(&WaitInstrs[DS_CNT]);
1470 } else if (Wait.StoreCnt != ~0u) {
1471 WaitsToErase.push_back(&WaitInstrs[STORE_CNT]);
1472 WaitsToErase.push_back(&WaitInstrs[DS_CNT]);
1473 }
1474
1475 for (MachineInstr **WI : WaitsToErase) {
1476 if (!*WI)
1477 continue;
1478
1479 (*WI)->eraseFromParent();
1480 *WI = nullptr;
1481 Modified = true;
1482 }
1483 }
1484
1485 for (auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1486 if (!WaitInstrs[CT])
1487 continue;
1488
1489 unsigned NewCnt = getWait(Wait, CT);
1490 if (NewCnt != ~0u) {
1491 Modified |= updateOperandIfDifferent(*WaitInstrs[CT],
1492 AMDGPU::OpName::simm16, NewCnt);
1493 Modified |= promoteSoftWaitCnt(WaitInstrs[CT]);
1494
1495 ScoreBrackets.applyWaitcnt(CT, NewCnt);
1496 setNoWait(Wait, CT);
1497
1498 LLVM_DEBUG(It == OldWaitcntInstr.getParent()->end()
1499 ? dbgs() << "applyPreexistingWaitcnt\n"
1500 << "New Instr at block end: " << *WaitInstrs[CT]
1501 << '\n'
1502 : dbgs() << "applyPreexistingWaitcnt\n"
1503 << "Old Instr: " << *It
1504 << "New Instr: " << *WaitInstrs[CT] << '\n');
1505 } else {
1506 WaitInstrs[CT]->eraseFromParent();
1507 Modified = true;
1508 }
1509 }
1510
1511 return Modified;
1512}
1513
1514/// Generate S_WAIT_*CNT instructions for any required counters in \p Wait
1515bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
1518 assert(ST);
1519 assert(!isNormalMode(MaxCounter));
1520
1521 bool Modified = false;
1522 const DebugLoc &DL = Block.findDebugLoc(It);
1523
1524 // Check for opportunities to use combined wait instructions.
1525 if (Wait.DsCnt != ~0u) {
1526 MachineInstr *SWaitInst = nullptr;
1527
1528 if (Wait.LoadCnt != ~0u) {
1529 unsigned Enc = AMDGPU::encodeLoadcntDscnt(IV, Wait);
1530
1531 SWaitInst = BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAIT_LOADCNT_DSCNT))
1532 .addImm(Enc);
1533
1534 Wait.LoadCnt = ~0u;
1535 Wait.DsCnt = ~0u;
1536 } else if (Wait.StoreCnt != ~0u) {
1537 unsigned Enc = AMDGPU::encodeStorecntDscnt(IV, Wait);
1538
1539 SWaitInst =
1540 BuildMI(Block, It, DL, TII->get(AMDGPU::S_WAIT_STORECNT_DSCNT))
1541 .addImm(Enc);
1542
1543 Wait.StoreCnt = ~0u;
1544 Wait.DsCnt = ~0u;
1545 }
1546
1547 if (SWaitInst) {
1548 Modified = true;
1549
1550 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
1551 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1552 dbgs() << "New Instr: " << *SWaitInst << '\n');
1553 }
1554 }
1555
1556 // Generate an instruction for any remaining counter that needs
1557 // waiting for.
1558
1559 for (auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1560 unsigned Count = getWait(Wait, CT);
1561 if (Count == ~0u)
1562 continue;
1563
1564 [[maybe_unused]] auto SWaitInst =
1565 BuildMI(Block, It, DL, TII->get(instrsForExtendedCounterTypes[CT]))
1566 .addImm(Count);
1567
1568 Modified = true;
1569
1570 LLVM_DEBUG(dbgs() << "generateWaitcnt\n";
1571 if (It != Block.instr_end()) dbgs() << "Old Instr: " << *It;
1572 dbgs() << "New Instr: " << *SWaitInst << '\n');
1573 }
1574
1575 return Modified;
1576}
1577
1578static bool readsVCCZ(const MachineInstr &MI) {
1579 unsigned Opc = MI.getOpcode();
1580 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
1581 !MI.getOperand(1).isUndef();
1582}
1583
1584/// \returns true if the callee inserts an s_waitcnt 0 on function entry.
1586 // Currently all conventions wait, but this may not always be the case.
1587 //
1588 // TODO: If IPRA is enabled, and the callee is isSafeForNoCSROpt, it may make
1589 // senses to omit the wait and do it in the caller.
1590 return true;
1591}
1592
1593/// \returns true if the callee is expected to wait for any outstanding waits
1594/// before returning.
1596 return true;
1597}
1598
1599/// Generate s_waitcnt instruction to be placed before cur_Inst.
1600/// Instructions of a given type are returned in order,
1601/// but instructions of different types can complete out of order.
1602/// We rely on this in-order completion
1603/// and simply assign a score to the memory access instructions.
1604/// We keep track of the active "score bracket" to determine
1605/// if an access of a memory read requires an s_waitcnt
1606/// and if so what the value of each counter is.
1607/// The "score bracket" is bound by the lower bound and upper bound
1608/// scores (*_score_LB and *_score_ub respectively).
1609/// If FlushVmCnt is true, that means that we want to generate a s_waitcnt to
1610/// flush the vmcnt counter here.
1611bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
1612 WaitcntBrackets &ScoreBrackets,
1613 MachineInstr *OldWaitcntInstr,
1614 bool FlushVmCnt) {
1615 setForceEmitWaitcnt();
1616
1617 if (MI.isMetaInstruction())
1618 return false;
1619
1621
1622 // FIXME: This should have already been handled by the memory legalizer.
1623 // Removing this currently doesn't affect any lit tests, but we need to
1624 // verify that nothing was relying on this. The number of buffer invalidates
1625 // being handled here should not be expanded.
1626 if (MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 ||
1627 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC ||
1628 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL ||
1629 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV ||
1630 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) {
1631 Wait.LoadCnt = 0;
1632 }
1633
1634 // All waits must be resolved at call return.
1635 // NOTE: this could be improved with knowledge of all call sites or
1636 // with knowledge of the called routines.
1637 if (MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1638 MI.getOpcode() == AMDGPU::SI_RETURN ||
1639 MI.getOpcode() == AMDGPU::S_SETPC_B64_return ||
1640 (MI.isReturn() && MI.isCall() && !callWaitsOnFunctionEntry(MI))) {
1641 Wait = Wait.combined(WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
1642 }
1643 // Identify S_ENDPGM instructions which may have to wait for outstanding VMEM
1644 // stores. In this case it can be useful to send a message to explicitly
1645 // release all VGPRs before the stores have completed, but it is only safe to
1646 // do this if:
1647 // * there are no outstanding scratch stores
1648 // * we are not in Dynamic VGPR mode
1649 else if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
1650 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
1651 if (ST->getGeneration() >= AMDGPUSubtarget::GFX11 && !WCG->isOptNone() &&
1652 ScoreBrackets.getScoreRange(STORE_CNT) != 0 &&
1653 !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS))
1654 ReleaseVGPRInsts.insert(&MI);
1655 }
1656 // Resolve vm waits before gs-done.
1657 else if ((MI.getOpcode() == AMDGPU::S_SENDMSG ||
1658 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) &&
1659 ST->hasLegacyGeometry() &&
1660 ((MI.getOperand(0).getImm() & AMDGPU::SendMsg::ID_MASK_PreGFX11_) ==
1662 Wait.LoadCnt = 0;
1663 }
1664
1665 // Export & GDS instructions do not read the EXEC mask until after the export
1666 // is granted (which can occur well after the instruction is issued).
1667 // The shader program must flush all EXP operations on the export-count
1668 // before overwriting the EXEC mask.
1669 else {
1670 if (MI.modifiesRegister(AMDGPU::EXEC, TRI)) {
1671 // Export and GDS are tracked individually, either may trigger a waitcnt
1672 // for EXEC.
1673 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
1674 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
1675 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
1676 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
1677 Wait.ExpCnt = 0;
1678 }
1679 }
1680
1681 if (MI.isCall() && callWaitsOnFunctionEntry(MI)) {
1682 // The function is going to insert a wait on everything in its prolog.
1683 // This still needs to be careful if the call target is a load (e.g. a GOT
1684 // load). We also need to check WAW dependency with saved PC.
1686
1687 int CallAddrOpIdx =
1688 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
1689
1690 if (MI.getOperand(CallAddrOpIdx).isReg()) {
1691 RegInterval CallAddrOpInterval =
1692 ScoreBrackets.getRegInterval(&MI, MRI, TRI, CallAddrOpIdx);
1693
1694 for (int RegNo = CallAddrOpInterval.first;
1695 RegNo < CallAddrOpInterval.second; ++RegNo)
1696 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait);
1697
1698 int RtnAddrOpIdx =
1699 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst);
1700 if (RtnAddrOpIdx != -1) {
1701 RegInterval RtnAddrOpInterval =
1702 ScoreBrackets.getRegInterval(&MI, MRI, TRI, RtnAddrOpIdx);
1703
1704 for (int RegNo = RtnAddrOpInterval.first;
1705 RegNo < RtnAddrOpInterval.second; ++RegNo)
1706 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait);
1707 }
1708 }
1709 } else {
1710 // FIXME: Should not be relying on memoperands.
1711 // Look at the source operands of every instruction to see if
1712 // any of them results from a previous memory operation that affects
1713 // its current usage. If so, an s_waitcnt instruction needs to be
1714 // emitted.
1715 // If the source operand was defined by a load, add the s_waitcnt
1716 // instruction.
1717 //
1718 // Two cases are handled for destination operands:
1719 // 1) If the destination operand was defined by a load, add the s_waitcnt
1720 // instruction to guarantee the right WAW order.
1721 // 2) If a destination operand that was used by a recent export/store ins,
1722 // add s_waitcnt on exp_cnt to guarantee the WAR order.
1723
1724 for (const MachineMemOperand *Memop : MI.memoperands()) {
1725 const Value *Ptr = Memop->getValue();
1726 if (Memop->isStore() && SLoadAddresses.count(Ptr)) {
1727 addWait(Wait, SmemAccessCounter, 0);
1728 if (PDT->dominates(MI.getParent(), SLoadAddresses.find(Ptr)->second))
1729 SLoadAddresses.erase(Ptr);
1730 }
1731 unsigned AS = Memop->getAddrSpace();
1733 continue;
1734 // No need to wait before load from VMEM to LDS.
1735 if (TII->mayWriteLDSThroughDMA(MI))
1736 continue;
1737
1738 // LOAD_CNT is only relevant to vgpr or LDS.
1739 unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
1740 bool FoundAliasingStore = false;
1741 // Only objects with alias scope info were added to LDSDMAScopes array.
1742 // In the absense of the scope info we will not be able to disambiguate
1743 // aliasing here. There is no need to try searching for a corresponding
1744 // store slot. This is conservatively correct because in that case we
1745 // will produce a wait using the first (general) LDS DMA wait slot which
1746 // will wait on all of them anyway.
1747 if (Ptr && Memop->getAAInfo() && Memop->getAAInfo().Scope) {
1748 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
1749 for (unsigned I = 0, E = LDSDMAStores.size(); I != E; ++I) {
1750 if (MI.mayAlias(AA, *LDSDMAStores[I], true)) {
1751 FoundAliasingStore = true;
1752 ScoreBrackets.determineWait(LOAD_CNT, RegNo + I + 1, Wait);
1753 }
1754 }
1755 }
1756 if (!FoundAliasingStore)
1757 ScoreBrackets.determineWait(LOAD_CNT, RegNo, Wait);
1758 if (Memop->isStore()) {
1759 ScoreBrackets.determineWait(EXP_CNT, RegNo, Wait);
1760 }
1761 }
1762
1763 // Loop over use and def operands.
1764 for (unsigned I = 0, E = MI.getNumOperands(); I != E; ++I) {
1765 MachineOperand &Op = MI.getOperand(I);
1766 if (!Op.isReg())
1767 continue;
1768
1769 // If the instruction does not read tied source, skip the operand.
1770 if (Op.isTied() && Op.isUse() && TII->doesNotReadTiedSource(MI))
1771 continue;
1772
1773 RegInterval Interval = ScoreBrackets.getRegInterval(&MI, MRI, TRI, I);
1774
1775 const bool IsVGPR = TRI->isVectorRegister(*MRI, Op.getReg());
1776 for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
1777 if (IsVGPR) {
1778 // RAW always needs an s_waitcnt. WAW needs an s_waitcnt unless the
1779 // previous write and this write are the same type of VMEM
1780 // instruction, in which case they're guaranteed to write their
1781 // results in order anyway.
1782 if (Op.isUse() || !updateVMCntOnly(MI) ||
1783 ScoreBrackets.hasOtherPendingVmemTypes(RegNo,
1784 getVmemType(MI))) {
1785 ScoreBrackets.determineWait(LOAD_CNT, RegNo, Wait);
1786 ScoreBrackets.determineWait(SAMPLE_CNT, RegNo, Wait);
1787 ScoreBrackets.determineWait(BVH_CNT, RegNo, Wait);
1788 ScoreBrackets.clearVgprVmemTypes(RegNo);
1789 }
1790 if (Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
1791 ScoreBrackets.determineWait(EXP_CNT, RegNo, Wait);
1792 }
1793 ScoreBrackets.determineWait(DS_CNT, RegNo, Wait);
1794 } else {
1795 ScoreBrackets.determineWait(SmemAccessCounter, RegNo, Wait);
1796 }
1797 }
1798 }
1799 }
1800 }
1801
1802 // The subtarget may have an implicit S_WAITCNT 0 before barriers. If it does
1803 // not, we need to ensure the subtarget is capable of backing off barrier
1804 // instructions in case there are any outstanding memory operations that may
1805 // cause an exception. Otherwise, insert an explicit S_WAITCNT 0 here.
1806 if (TII->isBarrierStart(MI.getOpcode()) &&
1807 !ST->hasAutoWaitcntBeforeBarrier() && !ST->supportsBackOffBarrier()) {
1808 Wait = Wait.combined(WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/true));
1809 }
1810
1811 // TODO: Remove this work-around, enable the assert for Bug 457939
1812 // after fixing the scheduler. Also, the Shader Compiler code is
1813 // independent of target.
1814 if (readsVCCZ(MI) && ST->hasReadVCCZBug()) {
1815 if (ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1816 Wait.DsCnt = 0;
1817 }
1818 }
1819
1820 // Verify that the wait is actually needed.
1821 ScoreBrackets.simplifyWaitcnt(Wait);
1822
1823 if (ForceEmitZeroWaitcnts)
1824 Wait = WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false);
1825
1826 if (ForceEmitWaitcnt[LOAD_CNT])
1827 Wait.LoadCnt = 0;
1828 if (ForceEmitWaitcnt[EXP_CNT])
1829 Wait.ExpCnt = 0;
1830 if (ForceEmitWaitcnt[DS_CNT])
1831 Wait.DsCnt = 0;
1832 if (ForceEmitWaitcnt[SAMPLE_CNT])
1833 Wait.SampleCnt = 0;
1834 if (ForceEmitWaitcnt[BVH_CNT])
1835 Wait.BvhCnt = 0;
1836 if (ForceEmitWaitcnt[KM_CNT])
1837 Wait.KmCnt = 0;
1838
1839 if (FlushVmCnt) {
1840 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
1841 Wait.LoadCnt = 0;
1842 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
1843 Wait.SampleCnt = 0;
1844 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
1845 Wait.BvhCnt = 0;
1846 }
1847
1848 return generateWaitcnt(Wait, MI.getIterator(), *MI.getParent(), ScoreBrackets,
1849 OldWaitcntInstr);
1850}
1851
1852bool SIInsertWaitcnts::generateWaitcnt(AMDGPU::Waitcnt Wait,
1855 WaitcntBrackets &ScoreBrackets,
1856 MachineInstr *OldWaitcntInstr) {
1857 bool Modified = false;
1858
1859 if (OldWaitcntInstr)
1860 // Try to merge the required wait with preexisting waitcnt instructions.
1861 // Also erase redundant waitcnt.
1862 Modified =
1863 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr, Wait, It);
1864
1865 // Any counts that could have been applied to any existing waitcnt
1866 // instructions will have been done so, now deal with any remaining.
1867 ScoreBrackets.applyWaitcnt(Wait);
1868
1869 // ExpCnt can be merged into VINTERP.
1870 if (Wait.ExpCnt != ~0u && It != Block.instr_end() &&
1872 MachineOperand *WaitExp =
1873 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp);
1874 if (Wait.ExpCnt < WaitExp->getImm()) {
1875 WaitExp->setImm(Wait.ExpCnt);
1876 Modified = true;
1877 }
1878 Wait.ExpCnt = ~0u;
1879
1880 LLVM_DEBUG(dbgs() << "generateWaitcnt\n"
1881 << "Update Instr: " << *It);
1882 }
1883
1884 if (WCG->createNewWaitcnt(Block, It, Wait))
1885 Modified = true;
1886
1887 return Modified;
1888}
1889
1890// This is a flat memory operation. Check to see if it has memory tokens other
1891// than LDS. Other address spaces supported by flat memory operations involve
1892// global memory.
1893bool SIInsertWaitcnts::mayAccessVMEMThroughFlat(const MachineInstr &MI) const {
1894 assert(TII->isFLAT(MI));
1895
1896 // All flat instructions use the VMEM counter.
1897 assert(TII->usesVM_CNT(MI));
1898
1899 // If there are no memory operands then conservatively assume the flat
1900 // operation may access VMEM.
1901 if (MI.memoperands_empty())
1902 return true;
1903
1904 // See if any memory operand specifies an address space that involves VMEM.
1905 // Flat operations only supported FLAT, LOCAL (LDS), or address spaces
1906 // involving VMEM such as GLOBAL, CONSTANT, PRIVATE (SCRATCH), etc. The REGION
1907 // (GDS) address space is not supported by flat operations. Therefore, simply
1908 // return true unless only the LDS address space is found.
1909 for (const MachineMemOperand *Memop : MI.memoperands()) {
1910 unsigned AS = Memop->getAddrSpace();
1912 if (AS != AMDGPUAS::LOCAL_ADDRESS)
1913 return true;
1914 }
1915
1916 return false;
1917}
1918
1919// This is a flat memory operation. Check to see if it has memory tokens for
1920// either LDS or FLAT.
1921bool SIInsertWaitcnts::mayAccessLDSThroughFlat(const MachineInstr &MI) const {
1922 assert(TII->isFLAT(MI));
1923
1924 // Flat instruction such as SCRATCH and GLOBAL do not use the lgkm counter.
1925 if (!TII->usesLGKM_CNT(MI))
1926 return false;
1927
1928 // If in tgsplit mode then there can be no use of LDS.
1929 if (ST->isTgSplitEnabled())
1930 return false;
1931
1932 // If there are no memory operands then conservatively assume the flat
1933 // operation may access LDS.
1934 if (MI.memoperands_empty())
1935 return true;
1936
1937 // See if any memory operand specifies an address space that involves LDS.
1938 for (const MachineMemOperand *Memop : MI.memoperands()) {
1939 unsigned AS = Memop->getAddrSpace();
1941 return true;
1942 }
1943
1944 return false;
1945}
1946
1947// This is a flat memory operation. Check to see if it has memory tokens for
1948// either scratch or FLAT.
1949bool SIInsertWaitcnts::mayAccessScratchThroughFlat(
1950 const MachineInstr &MI) const {
1951 assert(TII->isFLAT(MI));
1952
1953 // SCRATCH instructions always access scratch.
1954 if (TII->isFLATScratch(MI))
1955 return true;
1956
1957 // GLOBAL instructions never access scratch.
1958 if (TII->isFLATGlobal(MI))
1959 return false;
1960
1961 // If there are no memory operands then conservatively assume the flat
1962 // operation may access scratch.
1963 if (MI.memoperands_empty())
1964 return true;
1965
1966 // See if any memory operand specifies an address space that involves scratch.
1967 return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) {
1968 unsigned AS = Memop->getAddrSpace();
1969 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
1970 });
1971}
1972
1974 auto Opc = Inst.getOpcode();
1975 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
1976 Opc == AMDGPU::GLOBAL_WBINV;
1977}
1978
1979void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
1980 WaitcntBrackets *ScoreBrackets) {
1981 // Now look at the instruction opcode. If it is a memory access
1982 // instruction, update the upper-bound of the appropriate counter's
1983 // bracket and the destination operand scores.
1984 // TODO: Use the (TSFlags & SIInstrFlags::DS_CNT) property everywhere.
1985
1986 if (TII->isDS(Inst) && TII->usesLGKM_CNT(Inst)) {
1987 if (TII->isAlwaysGDS(Inst.getOpcode()) ||
1988 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
1989 ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_ACCESS, Inst);
1990 ScoreBrackets->updateByEvent(TII, TRI, MRI, GDS_GPR_LOCK, Inst);
1991 } else {
1992 ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst);
1993 }
1994 } else if (TII->isFLAT(Inst)) {
1995 // TODO: Track this properly.
1996 if (isCacheInvOrWBInst(Inst))
1997 return;
1998
1999 assert(Inst.mayLoadOrStore());
2000
2001 int FlatASCount = 0;
2002
2003 if (mayAccessVMEMThroughFlat(Inst)) {
2004 ++FlatASCount;
2005 ScoreBrackets->updateByEvent(TII, TRI, MRI, getVmemWaitEventType(Inst),
2006 Inst);
2007 }
2008
2009 if (mayAccessLDSThroughFlat(Inst)) {
2010 ++FlatASCount;
2011 ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst);
2012 }
2013
2014 // A Flat memory operation must access at least one address space.
2015 assert(FlatASCount);
2016
2017 // This is a flat memory operation that access both VMEM and LDS, so note it
2018 // - it will require that both the VM and LGKM be flushed to zero if it is
2019 // pending when a VM or LGKM dependency occurs.
2020 if (FlatASCount > 1)
2021 ScoreBrackets->setPendingFlat();
2022 } else if (SIInstrInfo::isVMEM(Inst) &&
2024 ScoreBrackets->updateByEvent(TII, TRI, MRI, getVmemWaitEventType(Inst),
2025 Inst);
2026
2027 if (ST->vmemWriteNeedsExpWaitcnt() &&
2028 (Inst.mayStore() || SIInstrInfo::isAtomicRet(Inst))) {
2029 ScoreBrackets->updateByEvent(TII, TRI, MRI, VMW_GPR_LOCK, Inst);
2030 }
2031 } else if (TII->isSMRD(Inst)) {
2032 ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst);
2033 } else if (Inst.isCall()) {
2034 if (callWaitsOnFunctionReturn(Inst)) {
2035 // Act as a wait on everything
2036 ScoreBrackets->applyWaitcnt(
2037 WCG->getAllZeroWaitcnt(/*IncludeVSCnt=*/false));
2038 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2039 } else {
2040 // May need to way wait for anything.
2041 ScoreBrackets->applyWaitcnt(AMDGPU::Waitcnt());
2042 }
2043 } else if (SIInstrInfo::isLDSDIR(Inst)) {
2044 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_LDS_ACCESS, Inst);
2045 } else if (TII->isVINTERP(Inst)) {
2046 int64_t Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
2047 ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
2048 } else if (SIInstrInfo::isEXP(Inst)) {
2049 unsigned Imm = TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
2051 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_PARAM_ACCESS, Inst);
2052 else if (Imm >= AMDGPU::Exp::ET_POS0 && Imm <= AMDGPU::Exp::ET_POS_LAST)
2053 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_POS_ACCESS, Inst);
2054 else
2055 ScoreBrackets->updateByEvent(TII, TRI, MRI, EXP_GPR_LOCK, Inst);
2056 } else {
2057 switch (Inst.getOpcode()) {
2058 case AMDGPU::S_SENDMSG:
2059 case AMDGPU::S_SENDMSG_RTN_B32:
2060 case AMDGPU::S_SENDMSG_RTN_B64:
2061 case AMDGPU::S_SENDMSGHALT:
2062 ScoreBrackets->updateByEvent(TII, TRI, MRI, SQ_MESSAGE, Inst);
2063 break;
2064 case AMDGPU::S_MEMTIME:
2065 case AMDGPU::S_MEMREALTIME:
2066 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
2067 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
2068 case AMDGPU::S_BARRIER_LEAVE:
2069 case AMDGPU::S_GET_BARRIER_STATE_M0:
2070 case AMDGPU::S_GET_BARRIER_STATE_IMM:
2071 ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst);
2072 break;
2073 }
2074 }
2075}
2076
2077bool WaitcntBrackets::mergeScore(const MergeInfo &M, unsigned &Score,
2078 unsigned OtherScore) {
2079 unsigned MyShifted = Score <= M.OldLB ? 0 : Score + M.MyShift;
2080 unsigned OtherShifted =
2081 OtherScore <= M.OtherLB ? 0 : OtherScore + M.OtherShift;
2082 Score = std::max(MyShifted, OtherShifted);
2083 return OtherShifted > MyShifted;
2084}
2085
2086/// Merge the pending events and associater score brackets of \p Other into
2087/// this brackets status.
2088///
2089/// Returns whether the merge resulted in a change that requires tighter waits
2090/// (i.e. the merged brackets strictly dominate the original brackets).
2091bool WaitcntBrackets::merge(const WaitcntBrackets &Other) {
2092 bool StrictDom = false;
2093
2094 VgprUB = std::max(VgprUB, Other.VgprUB);
2095 SgprUB = std::max(SgprUB, Other.SgprUB);
2096
2097 for (auto T : inst_counter_types(MaxCounter)) {
2098 // Merge event flags for this counter
2099 const unsigned OldEvents = PendingEvents & WaitEventMaskForInst[T];
2100 const unsigned OtherEvents = Other.PendingEvents & WaitEventMaskForInst[T];
2101 if (OtherEvents & ~OldEvents)
2102 StrictDom = true;
2103 PendingEvents |= OtherEvents;
2104
2105 // Merge scores for this counter
2106 const unsigned MyPending = ScoreUBs[T] - ScoreLBs[T];
2107 const unsigned OtherPending = Other.ScoreUBs[T] - Other.ScoreLBs[T];
2108 const unsigned NewUB = ScoreLBs[T] + std::max(MyPending, OtherPending);
2109 if (NewUB < ScoreLBs[T])
2110 report_fatal_error("waitcnt score overflow");
2111
2112 MergeInfo M;
2113 M.OldLB = ScoreLBs[T];
2114 M.OtherLB = Other.ScoreLBs[T];
2115 M.MyShift = NewUB - ScoreUBs[T];
2116 M.OtherShift = NewUB - Other.ScoreUBs[T];
2117
2118 ScoreUBs[T] = NewUB;
2119
2120 StrictDom |= mergeScore(M, LastFlat[T], Other.LastFlat[T]);
2121
2122 for (int J = 0; J <= VgprUB; J++)
2123 StrictDom |= mergeScore(M, VgprScores[T][J], Other.VgprScores[T][J]);
2124
2125 if (T == SmemAccessCounter) {
2126 for (int J = 0; J <= SgprUB; J++)
2127 StrictDom |= mergeScore(M, SgprScores[J], Other.SgprScores[J]);
2128 }
2129 }
2130
2131 for (int J = 0; J <= VgprUB; J++) {
2132 unsigned char NewVmemTypes = VgprVmemTypes[J] | Other.VgprVmemTypes[J];
2133 StrictDom |= NewVmemTypes != VgprVmemTypes[J];
2134 VgprVmemTypes[J] = NewVmemTypes;
2135 }
2136
2137 return StrictDom;
2138}
2139
2140static bool isWaitInstr(MachineInstr &Inst) {
2141 unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(Inst.getOpcode());
2142 return Opcode == AMDGPU::S_WAITCNT ||
2143 (Opcode == AMDGPU::S_WAITCNT_VSCNT && Inst.getOperand(0).isReg() &&
2144 Inst.getOperand(0).getReg() == AMDGPU::SGPR_NULL) ||
2145 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2146 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2147 counterTypeForInstr(Opcode).has_value();
2148}
2149
2150// Generate s_waitcnt instructions where needed.
2151bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
2153 WaitcntBrackets &ScoreBrackets) {
2154 bool Modified = false;
2155
2156 LLVM_DEBUG({
2157 dbgs() << "*** Block" << Block.getNumber() << " ***";
2158 ScoreBrackets.dump();
2159 });
2160
2161 // Track the correctness of vccz through this basic block. There are two
2162 // reasons why it might be incorrect; see ST->hasReadVCCZBug() and
2163 // ST->partialVCCWritesUpdateVCCZ().
2164 bool VCCZCorrect = true;
2165 if (ST->hasReadVCCZBug()) {
2166 // vccz could be incorrect at a basic block boundary if a predecessor wrote
2167 // to vcc and then issued an smem load.
2168 VCCZCorrect = false;
2169 } else if (!ST->partialVCCWritesUpdateVCCZ()) {
2170 // vccz could be incorrect at a basic block boundary if a predecessor wrote
2171 // to vcc_lo or vcc_hi.
2172 VCCZCorrect = false;
2173 }
2174
2175 // Walk over the instructions.
2176 MachineInstr *OldWaitcntInstr = nullptr;
2177
2178 for (MachineBasicBlock::instr_iterator Iter = Block.instr_begin(),
2179 E = Block.instr_end();
2180 Iter != E;) {
2181 MachineInstr &Inst = *Iter;
2182
2183 // Track pre-existing waitcnts that were added in earlier iterations or by
2184 // the memory legalizer.
2185 if (isWaitInstr(Inst)) {
2186 if (!OldWaitcntInstr)
2187 OldWaitcntInstr = &Inst;
2188 ++Iter;
2189 continue;
2190 }
2191
2192 bool FlushVmCnt = Block.getFirstTerminator() == Inst &&
2193 isPreheaderToFlush(Block, ScoreBrackets);
2194
2195 // Generate an s_waitcnt instruction to be placed before Inst, if needed.
2196 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
2197 FlushVmCnt);
2198 OldWaitcntInstr = nullptr;
2199
2200 // Restore vccz if it's not known to be correct already.
2201 bool RestoreVCCZ = !VCCZCorrect && readsVCCZ(Inst);
2202
2203 // Don't examine operands unless we need to track vccz correctness.
2204 if (ST->hasReadVCCZBug() || !ST->partialVCCWritesUpdateVCCZ()) {
2205 if (Inst.definesRegister(AMDGPU::VCC_LO, /*TRI=*/nullptr) ||
2206 Inst.definesRegister(AMDGPU::VCC_HI, /*TRI=*/nullptr)) {
2207 // Up to gfx9, writes to vcc_lo and vcc_hi don't update vccz.
2208 if (!ST->partialVCCWritesUpdateVCCZ())
2209 VCCZCorrect = false;
2210 } else if (Inst.definesRegister(AMDGPU::VCC, /*TRI=*/nullptr)) {
2211 // There is a hardware bug on CI/SI where SMRD instruction may corrupt
2212 // vccz bit, so when we detect that an instruction may read from a
2213 // corrupt vccz bit, we need to:
2214 // 1. Insert s_waitcnt lgkm(0) to wait for all outstanding SMRD
2215 // operations to complete.
2216 // 2. Restore the correct value of vccz by writing the current value
2217 // of vcc back to vcc.
2218 if (ST->hasReadVCCZBug() &&
2219 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
2220 // Writes to vcc while there's an outstanding smem read may get
2221 // clobbered as soon as any read completes.
2222 VCCZCorrect = false;
2223 } else {
2224 // Writes to vcc will fix any incorrect value in vccz.
2225 VCCZCorrect = true;
2226 }
2227 }
2228 }
2229
2230 if (TII->isSMRD(Inst)) {
2231 for (const MachineMemOperand *Memop : Inst.memoperands()) {
2232 // No need to handle invariant loads when avoiding WAR conflicts, as
2233 // there cannot be a vector store to the same memory location.
2234 if (!Memop->isInvariant()) {
2235 const Value *Ptr = Memop->getValue();
2236 SLoadAddresses.insert(std::pair(Ptr, Inst.getParent()));
2237 }
2238 }
2239 if (ST->hasReadVCCZBug()) {
2240 // This smem read could complete and clobber vccz at any time.
2241 VCCZCorrect = false;
2242 }
2243 }
2244
2245 updateEventWaitcntAfter(Inst, &ScoreBrackets);
2246
2247 if (ST->isPreciseMemoryEnabled() && Inst.mayLoadOrStore()) {
2248 AMDGPU::Waitcnt Wait = WCG->getAllZeroWaitcnt(
2249 Inst.mayStore() && !SIInstrInfo::isAtomicRet(Inst));
2250 ScoreBrackets.simplifyWaitcnt(Wait);
2251 Modified |= generateWaitcnt(Wait, std::next(Inst.getIterator()), Block,
2252 ScoreBrackets, /*OldWaitcntInstr=*/nullptr);
2253 }
2254
2255 LLVM_DEBUG({
2256 Inst.print(dbgs());
2257 ScoreBrackets.dump();
2258 });
2259
2260 // TODO: Remove this work-around after fixing the scheduler and enable the
2261 // assert above.
2262 if (RestoreVCCZ) {
2263 // Restore the vccz bit. Any time a value is written to vcc, the vcc
2264 // bit is updated, so we can restore the bit by reading the value of
2265 // vcc and then writing it back to the register.
2266 BuildMI(Block, Inst, Inst.getDebugLoc(),
2267 TII->get(ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
2268 TRI->getVCC())
2269 .addReg(TRI->getVCC());
2270 VCCZCorrect = true;
2271 Modified = true;
2272 }
2273
2274 ++Iter;
2275 }
2276
2277 // Flush the LOADcnt, SAMPLEcnt and BVHcnt counters at the end of the block if
2278 // needed.
2280 if (Block.getFirstTerminator() == Block.end() &&
2281 isPreheaderToFlush(Block, ScoreBrackets)) {
2282 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
2283 Wait.LoadCnt = 0;
2284 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
2285 Wait.SampleCnt = 0;
2286 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
2287 Wait.BvhCnt = 0;
2288 }
2289
2290 // Combine or remove any redundant waitcnts at the end of the block.
2291 Modified |= generateWaitcnt(Wait, Block.instr_end(), Block, ScoreBrackets,
2292 OldWaitcntInstr);
2293
2294 return Modified;
2295}
2296
2297// Return true if the given machine basic block is a preheader of a loop in
2298// which we want to flush the vmcnt counter, and false otherwise.
2299bool SIInsertWaitcnts::isPreheaderToFlush(MachineBasicBlock &MBB,
2300 WaitcntBrackets &ScoreBrackets) {
2301 auto [Iterator, IsInserted] = PreheadersToFlush.try_emplace(&MBB, false);
2302 if (!IsInserted)
2303 return Iterator->second;
2304
2306 if (!Succ)
2307 return false;
2308
2309 MachineLoop *Loop = MLI->getLoopFor(Succ);
2310 if (!Loop)
2311 return false;
2312
2313 if (Loop->getLoopPreheader() == &MBB &&
2314 shouldFlushVmCnt(Loop, ScoreBrackets)) {
2315 Iterator->second = true;
2316 return true;
2317 }
2318
2319 return false;
2320}
2321
2322bool SIInsertWaitcnts::isVMEMOrFlatVMEM(const MachineInstr &MI) const {
2323 return SIInstrInfo::isVMEM(MI) ||
2324 (SIInstrInfo::isFLAT(MI) && mayAccessVMEMThroughFlat(MI));
2325}
2326
2327// Return true if it is better to flush the vmcnt counter in the preheader of
2328// the given loop. We currently decide to flush in two situations:
2329// 1. The loop contains vmem store(s), no vmem load and at least one use of a
2330// vgpr containing a value that is loaded outside of the loop. (Only on
2331// targets with no vscnt counter).
2332// 2. The loop contains vmem load(s), but the loaded values are not used in the
2333// loop, and at least one use of a vgpr containing a value that is loaded
2334// outside of the loop.
2335bool SIInsertWaitcnts::shouldFlushVmCnt(MachineLoop *ML,
2336 WaitcntBrackets &Brackets) {
2337 bool HasVMemLoad = false;
2338 bool HasVMemStore = false;
2339 bool UsesVgprLoadedOutside = false;
2340 DenseSet<Register> VgprUse;
2341 DenseSet<Register> VgprDef;
2342
2343 for (MachineBasicBlock *MBB : ML->blocks()) {
2344 for (MachineInstr &MI : *MBB) {
2345 if (isVMEMOrFlatVMEM(MI)) {
2346 if (MI.mayLoad())
2347 HasVMemLoad = true;
2348 if (MI.mayStore())
2349 HasVMemStore = true;
2350 }
2351 for (unsigned I = 0; I < MI.getNumOperands(); I++) {
2352 MachineOperand &Op = MI.getOperand(I);
2353 if (!Op.isReg() || !TRI->isVectorRegister(*MRI, Op.getReg()))
2354 continue;
2355 RegInterval Interval = Brackets.getRegInterval(&MI, MRI, TRI, I);
2356 // Vgpr use
2357 if (Op.isUse()) {
2358 for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
2359 // If we find a register that is loaded inside the loop, 1. and 2.
2360 // are invalidated and we can exit.
2361 if (VgprDef.contains(RegNo))
2362 return false;
2363 VgprUse.insert(RegNo);
2364 // If at least one of Op's registers is in the score brackets, the
2365 // value is likely loaded outside of the loop.
2366 if (Brackets.getRegScore(RegNo, LOAD_CNT) >
2367 Brackets.getScoreLB(LOAD_CNT) ||
2368 Brackets.getRegScore(RegNo, SAMPLE_CNT) >
2369 Brackets.getScoreLB(SAMPLE_CNT) ||
2370 Brackets.getRegScore(RegNo, BVH_CNT) >
2371 Brackets.getScoreLB(BVH_CNT)) {
2372 UsesVgprLoadedOutside = true;
2373 break;
2374 }
2375 }
2376 }
2377 // VMem load vgpr def
2378 else if (isVMEMOrFlatVMEM(MI) && MI.mayLoad() && Op.isDef())
2379 for (int RegNo = Interval.first; RegNo < Interval.second; ++RegNo) {
2380 // If we find a register that is loaded inside the loop, 1. and 2.
2381 // are invalidated and we can exit.
2382 if (VgprUse.contains(RegNo))
2383 return false;
2384 VgprDef.insert(RegNo);
2385 }
2386 }
2387 }
2388 }
2389 if (!ST->hasVscnt() && HasVMemStore && !HasVMemLoad && UsesVgprLoadedOutside)
2390 return true;
2391 return HasVMemLoad && UsesVgprLoadedOutside;
2392}
2393
2394bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
2395 ST = &MF.getSubtarget<GCNSubtarget>();
2396 TII = ST->getInstrInfo();
2397 TRI = &TII->getRegisterInfo();
2398 MRI = &MF.getRegInfo();
2400 MLI = &getAnalysis<MachineLoopInfo>();
2401 PDT = &getAnalysis<MachinePostDominatorTreeWrapperPass>().getPostDomTree();
2402 if (auto AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
2403 AA = &AAR->getAAResults();
2404
2406
2407 if (ST->hasExtendedWaitCounts()) {
2408 MaxCounter = NUM_EXTENDED_INST_CNTS;
2409 WCGGFX12Plus = WaitcntGeneratorGFX12Plus(MF, MaxCounter);
2410 WCG = &WCGGFX12Plus;
2411 } else {
2412 MaxCounter = NUM_NORMAL_INST_CNTS;
2413 WCGPreGFX12 = WaitcntGeneratorPreGFX12(MF);
2414 WCG = &WCGPreGFX12;
2415 }
2416
2417 ForceEmitZeroWaitcnts = ForceEmitZeroFlag;
2418 for (auto T : inst_counter_types())
2419 ForceEmitWaitcnt[T] = false;
2420
2421 const unsigned *WaitEventMaskForInst = WCG->getWaitEventMask();
2422
2423 SmemAccessCounter = eventCounter(WaitEventMaskForInst, SMEM_ACCESS);
2424
2425 HardwareLimits Limits = {};
2426 if (ST->hasExtendedWaitCounts()) {
2427 Limits.LoadcntMax = AMDGPU::getLoadcntBitMask(IV);
2428 Limits.DscntMax = AMDGPU::getDscntBitMask(IV);
2429 } else {
2430 Limits.LoadcntMax = AMDGPU::getVmcntBitMask(IV);
2431 Limits.DscntMax = AMDGPU::getLgkmcntBitMask(IV);
2432 }
2433 Limits.ExpcntMax = AMDGPU::getExpcntBitMask(IV);
2434 Limits.StorecntMax = AMDGPU::getStorecntBitMask(IV);
2435 Limits.SamplecntMax = AMDGPU::getSamplecntBitMask(IV);
2436 Limits.BvhcntMax = AMDGPU::getBvhcntBitMask(IV);
2437 Limits.KmcntMax = AMDGPU::getKmcntBitMask(IV);
2438
2439 unsigned NumVGPRsMax = ST->getAddressableNumVGPRs();
2440 unsigned NumSGPRsMax = ST->getAddressableNumSGPRs();
2441 assert(NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
2442 assert(NumSGPRsMax <= SQ_MAX_PGM_SGPRS);
2443
2444 RegisterEncoding Encoding = {};
2445 Encoding.VGPR0 =
2446 TRI->getEncodingValue(AMDGPU::VGPR0) & AMDGPU::HWEncoding::REG_IDX_MASK;
2447 Encoding.VGPRL = Encoding.VGPR0 + NumVGPRsMax - 1;
2448 Encoding.SGPR0 =
2449 TRI->getEncodingValue(AMDGPU::SGPR0) & AMDGPU::HWEncoding::REG_IDX_MASK;
2450 Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1;
2451
2452 BlockInfos.clear();
2453 bool Modified = false;
2454
2455 MachineBasicBlock &EntryBB = MF.front();
2457
2458 if (!MFI->isEntryFunction()) {
2459 // Wait for any outstanding memory operations that the input registers may
2460 // depend on. We can't track them and it's better to do the wait after the
2461 // costly call sequence.
2462
2463 // TODO: Could insert earlier and schedule more liberally with operations
2464 // that only use caller preserved registers.
2465 for (MachineBasicBlock::iterator E = EntryBB.end();
2466 I != E && (I->isPHI() || I->isMetaInstruction()); ++I)
2467 ;
2468
2469 if (ST->hasExtendedWaitCounts()) {
2470 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAIT_LOADCNT_DSCNT))
2471 .addImm(0);
2472 for (auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
2473 if (CT == LOAD_CNT || CT == DS_CNT || CT == STORE_CNT)
2474 continue;
2475
2476 BuildMI(EntryBB, I, DebugLoc(),
2477 TII->get(instrsForExtendedCounterTypes[CT]))
2478 .addImm(0);
2479 }
2480 } else {
2481 BuildMI(EntryBB, I, DebugLoc(), TII->get(AMDGPU::S_WAITCNT)).addImm(0);
2482 }
2483
2484 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(
2485 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2486 SmemAccessCounter);
2487 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
2488 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
2489
2490 Modified = true;
2491 }
2492
2493 // Keep iterating over the blocks in reverse post order, inserting and
2494 // updating s_waitcnt where needed, until a fix point is reached.
2496 BlockInfos.insert({MBB, BlockInfo()});
2497
2498 std::unique_ptr<WaitcntBrackets> Brackets;
2499 bool Repeat;
2500 do {
2501 Repeat = false;
2502
2503 for (auto BII = BlockInfos.begin(), BIE = BlockInfos.end(); BII != BIE;
2504 ++BII) {
2505 MachineBasicBlock *MBB = BII->first;
2506 BlockInfo &BI = BII->second;
2507 if (!BI.Dirty)
2508 continue;
2509
2510 if (BI.Incoming) {
2511 if (!Brackets)
2512 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
2513 else
2514 *Brackets = *BI.Incoming;
2515 } else {
2516 if (!Brackets)
2517 Brackets = std::make_unique<WaitcntBrackets>(
2518 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2519 SmemAccessCounter);
2520 else
2521 *Brackets = WaitcntBrackets(ST, MaxCounter, Limits, Encoding,
2522 WaitEventMaskForInst, SmemAccessCounter);
2523 }
2524
2525 Modified |= insertWaitcntInBlock(MF, *MBB, *Brackets);
2526 BI.Dirty = false;
2527
2528 if (Brackets->hasPendingEvent()) {
2529 BlockInfo *MoveBracketsToSucc = nullptr;
2530 for (MachineBasicBlock *Succ : MBB->successors()) {
2531 auto SuccBII = BlockInfos.find(Succ);
2532 BlockInfo &SuccBI = SuccBII->second;
2533 if (!SuccBI.Incoming) {
2534 SuccBI.Dirty = true;
2535 if (SuccBII <= BII)
2536 Repeat = true;
2537 if (!MoveBracketsToSucc) {
2538 MoveBracketsToSucc = &SuccBI;
2539 } else {
2540 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
2541 }
2542 } else if (SuccBI.Incoming->merge(*Brackets)) {
2543 SuccBI.Dirty = true;
2544 if (SuccBII <= BII)
2545 Repeat = true;
2546 }
2547 }
2548 if (MoveBracketsToSucc)
2549 MoveBracketsToSucc->Incoming = std::move(Brackets);
2550 }
2551 }
2552 } while (Repeat);
2553
2554 if (ST->hasScalarStores()) {
2556 bool HaveScalarStores = false;
2557
2558 for (MachineBasicBlock &MBB : MF) {
2559 for (MachineInstr &MI : MBB) {
2560 if (!HaveScalarStores && TII->isScalarStore(MI))
2561 HaveScalarStores = true;
2562
2563 if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
2564 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
2565 EndPgmBlocks.push_back(&MBB);
2566 }
2567 }
2568
2569 if (HaveScalarStores) {
2570 // If scalar writes are used, the cache must be flushed or else the next
2571 // wave to reuse the same scratch memory can be clobbered.
2572 //
2573 // Insert s_dcache_wb at wave termination points if there were any scalar
2574 // stores, and only if the cache hasn't already been flushed. This could
2575 // be improved by looking across blocks for flushes in postdominating
2576 // blocks from the stores but an explicitly requested flush is probably
2577 // very rare.
2578 for (MachineBasicBlock *MBB : EndPgmBlocks) {
2579 bool SeenDCacheWB = false;
2580
2581 for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
2582 I != E; ++I) {
2583 if (I->getOpcode() == AMDGPU::S_DCACHE_WB)
2584 SeenDCacheWB = true;
2585 else if (TII->isScalarStore(*I))
2586 SeenDCacheWB = false;
2587
2588 // FIXME: It would be better to insert this before a waitcnt if any.
2589 if ((I->getOpcode() == AMDGPU::S_ENDPGM ||
2590 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
2591 !SeenDCacheWB) {
2592 Modified = true;
2593 BuildMI(*MBB, I, I->getDebugLoc(), TII->get(AMDGPU::S_DCACHE_WB));
2594 }
2595 }
2596 }
2597 }
2598 }
2599
2600 // Insert DEALLOC_VGPR messages before previously identified S_ENDPGM
2601 // instructions.
2602 for (MachineInstr *MI : ReleaseVGPRInsts) {
2603 if (ST->requiresNopBeforeDeallocVGPRs()) {
2604 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(), TII->get(AMDGPU::S_NOP))
2605 .addImm(0);
2606 }
2607 BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
2608 TII->get(AMDGPU::S_SENDMSG))
2610 Modified = true;
2611 }
2612 ReleaseVGPRInsts.clear();
2613
2614 return Modified;
2615}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Provides AMDGPU specific target descriptions.
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
Definition: DebugCounter.h:190
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
std::optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1293
static Function * getFunction(Constant *C)
Definition: Evaluator.cpp:236
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static bool isOptNone(const MachineFunction &MF)
IRTranslator LLVM IR MI
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
std::pair< uint64_t, uint64_t > Interval
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:55
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:59
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:52
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static bool callWaitsOnFunctionReturn(const MachineInstr &MI)
static bool isCacheInvOrWBInst(MachineInstr &Inst)
static bool callWaitsOnFunctionEntry(const MachineInstr &MI)
static bool updateOperandIfDifferent(MachineInstr &MI, uint16_t OpName, unsigned NewEnc)
static bool isWaitInstr(MachineInstr &Inst)
static std::optional< InstCounterType > counterTypeForInstr(unsigned Opcode)
Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is wa...
static bool readsVCCZ(const MachineInstr &MI)
static cl::opt< bool > ForceEmitZeroFlag("amdgpu-waitcnt-forcezero", cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"), cl::init(false), cl::Hidden)
#define DEBUG_TYPE
SI Insert Waitcnts
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
Provides some synthesis utilities to produce sequences of values.
static const uint32_t IV[8]
Definition: blake3_impl.h:78
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Represent the analysis usage information of a pass.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:269
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This class represents an Operation in the Expression.
static bool isCounterSet(unsigned ID)
Definition: DebugCounter.h:96
static bool shouldExecute(unsigned CounterName)
Definition: DebugCounter.h:87
A debug info location.
Definition: DebugLoc.h:33
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Definition: DenseMap.h:235
bool erase(const KeyT &Val)
Definition: DenseMap.h:345
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
bool dominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
dominates - Returns true iff A dominates B.
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:44
const MachineBasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Instructions::iterator instr_iterator
iterator_range< succ_iterator > successors()
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:566
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:343
bool isCall(QueryType Type=AnyInBundle) const
Definition: MachineInstr.h:947
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:569
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:779
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:495
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:576
MachineLoop * getLoopFor(const MachineBasicBlock *BB) const
Return the innermost loop that BB lives in.
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
Definition: MapVector.h:36
iterator end()
Definition: MapVector.h:71
iterator find(const KeyT &Key)
Definition: MapVector.h:167
iterator begin()
Definition: MapVector.h:69
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: MapVector.h:141
void clear()
Definition: MapVector.h:88
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:432
static bool isFLATScratch(const MachineInstr &MI)
Definition: SIInstrInfo.h:636
static bool isEXP(const MachineInstr &MI)
Definition: SIInstrInfo.h:649
static bool mayWriteLDSThroughDMA(const MachineInstr &MI)
Definition: SIInstrInfo.h:691
static bool isVIMAGE(const MachineInstr &MI)
Definition: SIInstrInfo.h:588
static bool isLDSDIR(const MachineInstr &MI)
Definition: SIInstrInfo.h:833
static bool isGWS(const MachineInstr &MI)
Definition: SIInstrInfo.h:570
static bool isFLATGlobal(const MachineInstr &MI)
Definition: SIInstrInfo.h:628
static bool isVSAMPLE(const MachineInstr &MI)
Definition: SIInstrInfo.h:596
static bool isAtomicRet(const MachineInstr &MI)
Definition: SIInstrInfo.h:673
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
Definition: SIInstrInfo.h:947
static bool isVINTERP(const MachineInstr &MI)
Definition: SIInstrInfo.h:841
static bool isMIMG(const MachineInstr &MI)
Definition: SIInstrInfo.h:580
static bool isFLAT(const MachineInstr &MI)
Definition: SIInstrInfo.h:612
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
LLVM Value Representation.
Definition: Value.h:74
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:185
self_iterator getIterator()
Definition: ilist_node.h:109
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
unsigned getStorecntBitMask(const IsaVersion &Version)
IsaVersion getIsaVersion(StringRef GPU)
unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt)
Encodes Vmcnt, Expcnt and Lgkmcnt into Waitcnt for given isa Version.
unsigned getSamplecntBitMask(const IsaVersion &Version)
unsigned getKmcntBitMask(const IsaVersion &Version)
unsigned getVmcntBitMask(const IsaVersion &Version)
Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
unsigned getLgkmcntBitMask(const IsaVersion &Version)
unsigned getBvhcntBitMask(const IsaVersion &Version)
unsigned getExpcntBitMask(const IsaVersion &Version)
unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt)
static unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt, unsigned Dscnt)
bool getMUBUFIsBufferInv(unsigned Opc)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getLoadcntBitMask(const IsaVersion &Version)
static unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt, unsigned Dscnt)
unsigned getDscntBitMask(const IsaVersion &Version)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
Definition: Sequence.h:337
@ Wait
Definition: Threading.h:61
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
char & SIInsertWaitcntsID
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
FunctionPass * createSIInsertWaitcntsPass()
Instruction set architecture version.
Definition: TargetParser.h:127
Represents the counter values to wait for in an s_waitcnt instruction.
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
static constexpr bool is_iterable
Definition: Sequence.h:100