42#define DEBUG_TYPE "si-insert-waitcnts"
45 "Force emit s_waitcnt expcnt(0) instrs");
47 "Force emit s_waitcnt lgkmcnt(0) instrs");
49 "Force emit s_waitcnt vmcnt(0) instrs");
52 "amdgpu-waitcnt-forcezero",
53 cl::desc(
"Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"),
67 SAMPLE_CNT = NUM_NORMAL_INST_CNTS,
70 NUM_EXTENDED_INST_CNTS,
71 NUM_INST_CNTS = NUM_EXTENDED_INST_CNTS
85auto inst_counter_types(InstCounterType MaxCounter = NUM_INST_CNTS) {
86 return enum_seq(LOAD_CNT, MaxCounter);
89using RegInterval = std::pair<int, int>;
91struct HardwareLimits {
96 unsigned SamplecntMax;
101struct RegisterEncoding {
111 VMEM_SAMPLER_READ_ACCESS,
112 VMEM_BVH_READ_ACCESS,
114 SCRATCH_WRITE_ACCESS,
134enum RegisterMapping {
135 SQ_MAX_PGM_VGPRS = 512,
137 SQ_MAX_PGM_SGPRS = 256,
145 NUM_ALL_VGPRS = SQ_MAX_PGM_VGPRS + NUM_EXTRA_VGPRS,
166static const unsigned instrsForExtendedCounterTypes[NUM_EXTENDED_INST_CNTS] = {
167 AMDGPU::S_WAIT_LOADCNT, AMDGPU::S_WAIT_DSCNT, AMDGPU::S_WAIT_EXPCNT,
168 AMDGPU::S_WAIT_STORECNT, AMDGPU::S_WAIT_SAMPLECNT, AMDGPU::S_WAIT_BVHCNT,
169 AMDGPU::S_WAIT_KMCNT};
177static bool isNormalMode(InstCounterType MaxCounter) {
178 return MaxCounter == NUM_NORMAL_INST_CNTS;
183 assert(updateVMCntOnly(Inst));
186 return VMEM_NOSAMPLER;
193 return BaseInfo->
BVH ? VMEM_BVH
207 return Wait.StoreCnt;
209 return Wait.SampleCnt;
220 unsigned &WC = getCounterRef(
Wait,
T);
221 WC = std::min(WC, Count);
225 getCounterRef(
Wait,
T) = ~0
u;
229 return getCounterRef(
Wait,
T);
233InstCounterType eventCounter(
const unsigned *masks, WaitEventType E) {
234 for (
auto T : inst_counter_types()) {
235 if (masks[
T] & (1 << E))
249class WaitcntBrackets {
251 WaitcntBrackets(
const GCNSubtarget *SubTarget, InstCounterType MaxCounter,
252 HardwareLimits Limits, RegisterEncoding Encoding,
253 const unsigned *WaitEventMaskForInst,
254 InstCounterType SmemAccessCounter)
255 :
ST(SubTarget), MaxCounter(MaxCounter), Limits(Limits),
256 Encoding(Encoding), WaitEventMaskForInst(WaitEventMaskForInst),
257 SmemAccessCounter(SmemAccessCounter) {}
259 unsigned getWaitCountMax(InstCounterType
T)
const {
262 return Limits.LoadcntMax;
264 return Limits.DscntMax;
266 return Limits.ExpcntMax;
268 return Limits.StorecntMax;
270 return Limits.SamplecntMax;
272 return Limits.BvhcntMax;
274 return Limits.KmcntMax;
281 unsigned getScoreLB(InstCounterType
T)
const {
286 unsigned getScoreUB(InstCounterType
T)
const {
291 unsigned getScoreRange(InstCounterType
T)
const {
292 return getScoreUB(
T) - getScoreLB(
T);
295 unsigned getRegScore(
int GprNo, InstCounterType
T)
const {
296 if (GprNo < NUM_ALL_VGPRS) {
297 return VgprScores[
T][GprNo];
299 assert(
T == SmemAccessCounter);
300 return SgprScores[GprNo - NUM_ALL_VGPRS];
309 bool counterOutOfOrder(InstCounterType
T)
const;
311 void simplifyWaitcnt(InstCounterType
T,
unsigned &Count)
const;
314 void applyWaitcnt(InstCounterType
T,
unsigned Count);
319 unsigned hasPendingEvent()
const {
return PendingEvents; }
320 unsigned hasPendingEvent(WaitEventType E)
const {
321 return PendingEvents & (1 << E);
323 unsigned hasPendingEvent(InstCounterType
T)
const {
324 unsigned HasPending = PendingEvents & WaitEventMaskForInst[
T];
325 assert((HasPending != 0) == (getScoreRange(
T) != 0));
329 bool hasMixedPendingEvents(InstCounterType
T)
const {
330 unsigned Events = hasPendingEvent(
T);
332 return Events & (Events - 1);
335 bool hasPendingFlat()
const {
336 return ((LastFlat[DS_CNT] > ScoreLBs[DS_CNT] &&
337 LastFlat[DS_CNT] <= ScoreUBs[DS_CNT]) ||
338 (LastFlat[LOAD_CNT] > ScoreLBs[LOAD_CNT] &&
339 LastFlat[LOAD_CNT] <= ScoreUBs[LOAD_CNT]));
342 void setPendingFlat() {
343 LastFlat[LOAD_CNT] = ScoreUBs[LOAD_CNT];
344 LastFlat[DS_CNT] = ScoreUBs[DS_CNT];
349 bool hasOtherPendingVmemTypes(
int GprNo, VmemType V)
const {
350 assert(GprNo < NUM_ALL_VGPRS);
351 return VgprVmemTypes[GprNo] & ~(1 <<
V);
354 void clearVgprVmemTypes(
int GprNo) {
355 assert(GprNo < NUM_ALL_VGPRS);
356 VgprVmemTypes[GprNo] = 0;
359 void setStateOnFunctionEntryOrReturn() {
360 setScoreUB(STORE_CNT, getScoreUB(STORE_CNT) + getWaitCountMax(STORE_CNT));
361 PendingEvents |= WaitEventMaskForInst[STORE_CNT];
378 static bool mergeScore(
const MergeInfo &M,
unsigned &Score,
379 unsigned OtherScore);
381 void setScoreLB(InstCounterType
T,
unsigned Val) {
386 void setScoreUB(InstCounterType
T,
unsigned Val) {
393 if (getScoreRange(EXP_CNT) > getWaitCountMax(EXP_CNT))
397 void setRegScore(
int GprNo, InstCounterType
T,
unsigned Val) {
398 if (GprNo < NUM_ALL_VGPRS) {
399 VgprUB = std::max(VgprUB, GprNo);
400 VgprScores[
T][GprNo] = Val;
402 assert(
T == SmemAccessCounter);
403 SgprUB = std::max(SgprUB, GprNo - NUM_ALL_VGPRS);
404 SgprScores[GprNo - NUM_ALL_VGPRS] = Val;
410 unsigned OpNo,
unsigned Val);
413 InstCounterType MaxCounter = NUM_EXTENDED_INST_CNTS;
414 HardwareLimits Limits = {};
415 RegisterEncoding Encoding = {};
416 const unsigned *WaitEventMaskForInst;
417 InstCounterType SmemAccessCounter;
418 unsigned ScoreLBs[NUM_INST_CNTS] = {0};
419 unsigned ScoreUBs[NUM_INST_CNTS] = {0};
420 unsigned PendingEvents = 0;
422 unsigned LastFlat[NUM_INST_CNTS] = {0};
427 unsigned VgprScores[NUM_INST_CNTS][NUM_ALL_VGPRS] = {{0}};
430 unsigned SgprScores[SQ_MAX_PGM_SGPRS] = {0};
433 unsigned char VgprVmemTypes[NUM_ALL_VGPRS] = {0};
445class WaitcntGenerator {
450 InstCounterType MaxCounter;
454 WaitcntGenerator() {}
455 WaitcntGenerator(
const MachineFunction &MF, InstCounterType MaxCounter)
463 bool isOptNone()
const {
return OptNone; }
477 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
492 virtual const unsigned *getWaitEventMask()
const = 0;
496 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const = 0;
498 virtual ~WaitcntGenerator() =
default;
501 static constexpr unsigned
502 eventMask(std::initializer_list<WaitEventType> Events) {
504 for (
auto &E : Events)
511class WaitcntGeneratorPreGFX12 :
public WaitcntGenerator {
513 WaitcntGeneratorPreGFX12() {}
515 : WaitcntGenerator(MF, NUM_NORMAL_INST_CNTS) {}
518 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
526 const unsigned *getWaitEventMask()
const override {
529 static const unsigned WaitEventMaskForInstPreGFX12[NUM_INST_CNTS] = {
530 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS,
531 VMEM_BVH_READ_ACCESS}),
532 eventMask({SMEM_ACCESS, LDS_ACCESS, GDS_ACCESS, SQ_MESSAGE}),
533 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
534 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
535 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
540 return WaitEventMaskForInstPreGFX12;
543 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const override;
546class WaitcntGeneratorGFX12Plus :
public WaitcntGenerator {
548 WaitcntGeneratorGFX12Plus() {}
550 InstCounterType MaxCounter)
551 : WaitcntGenerator(MF, MaxCounter) {}
554 applyPreexistingWaitcnt(WaitcntBrackets &ScoreBrackets,
562 const unsigned *getWaitEventMask()
const override {
565 static const unsigned WaitEventMaskForInstGFX12Plus[NUM_INST_CNTS] = {
566 eventMask({VMEM_ACCESS, VMEM_READ_ACCESS}),
567 eventMask({LDS_ACCESS, GDS_ACCESS}),
568 eventMask({EXP_GPR_LOCK, GDS_GPR_LOCK, VMW_GPR_LOCK, EXP_PARAM_ACCESS,
569 EXP_POS_ACCESS, EXP_LDS_ACCESS}),
570 eventMask({VMEM_WRITE_ACCESS, SCRATCH_WRITE_ACCESS}),
571 eventMask({VMEM_SAMPLER_READ_ACCESS}),
572 eventMask({VMEM_BVH_READ_ACCESS}),
573 eventMask({SMEM_ACCESS, SQ_MESSAGE})};
575 return WaitEventMaskForInstGFX12Plus;
578 virtual AMDGPU::Waitcnt getAllZeroWaitcnt(
bool IncludeVSCnt)
const override;
595 std::unique_ptr<WaitcntBrackets>
Incoming;
599 InstCounterType SmemAccessCounter;
605 bool ForceEmitZeroWaitcnts;
606 bool ForceEmitWaitcnt[NUM_INST_CNTS];
611 WaitcntGeneratorPreGFX12 WCGPreGFX12;
612 WaitcntGeneratorGFX12Plus WCGGFX12Plus;
614 WaitcntGenerator *WCG =
nullptr;
620 InstCounterType MaxCounter = NUM_NORMAL_INST_CNTS;
626 (void)ForceExpCounter;
627 (void)ForceLgkmCounter;
628 (void)ForceVMCounter;
631 bool shouldFlushVmCnt(
MachineLoop *
ML, WaitcntBrackets &Brackets);
633 WaitcntBrackets &ScoreBrackets);
638 return "SI insert wait instructions";
650 bool isForceEmitWaitcnt()
const {
651 for (
auto T : inst_counter_types())
652 if (ForceEmitWaitcnt[
T])
657 void setForceEmitWaitcnt() {
663 ForceEmitWaitcnt[
EXP_CNT] =
true;
665 ForceEmitWaitcnt[
EXP_CNT] =
false;
670 ForceEmitWaitcnt[DS_CNT] =
true;
671 ForceEmitWaitcnt[KM_CNT] =
true;
673 ForceEmitWaitcnt[DS_CNT] =
false;
674 ForceEmitWaitcnt[KM_CNT] =
false;
679 ForceEmitWaitcnt[LOAD_CNT] =
true;
680 ForceEmitWaitcnt[SAMPLE_CNT] =
true;
681 ForceEmitWaitcnt[BVH_CNT] =
true;
683 ForceEmitWaitcnt[LOAD_CNT] =
false;
684 ForceEmitWaitcnt[SAMPLE_CNT] =
false;
685 ForceEmitWaitcnt[BVH_CNT] =
false;
692 WaitEventType getVmemWaitEventType(
const MachineInstr &Inst)
const {
694 static const WaitEventType VmemReadMapping[NUM_VMEM_TYPES] = {
695 VMEM_READ_ACCESS, VMEM_SAMPLER_READ_ACCESS, VMEM_BVH_READ_ACCESS};
706 return SCRATCH_WRITE_ACCESS;
707 return VMEM_WRITE_ACCESS;
710 return VMEM_READ_ACCESS;
711 return VmemReadMapping[getVmemType(Inst)];
718 WaitcntBrackets &ScoreBrackets,
726 WaitcntBrackets *ScoreBrackets);
728 WaitcntBrackets &ScoreBrackets);
733RegInterval WaitcntBrackets::getRegInterval(
const MachineInstr *
MI,
736 unsigned OpNo)
const {
738 if (!
TRI->isInAllocatableClass(
Op.getReg()))
750 if (
TRI->isVectorRegister(*
MRI,
Op.getReg())) {
751 assert(Reg >= Encoding.VGPR0 && Reg <= Encoding.VGPRL);
754 Result.first += AGPR_OFFSET;
756 }
else if (
TRI->isSGPRReg(*
MRI,
Op.getReg())) {
757 assert(Reg >= Encoding.SGPR0 && Reg < SQ_MAX_PGM_SGPRS);
758 Result.first =
Reg - Encoding.SGPR0 + NUM_ALL_VGPRS;
760 Result.first < SQ_MAX_PGM_SGPRS + NUM_ALL_VGPRS);
768 unsigned Size =
TRI->getRegSizeInBits(*RC);
780 assert(
TRI->isVectorRegister(*
MRI,
MI->getOperand(OpNo).getReg()));
782 setRegScore(RegNo, EXP_CNT, Val);
790 InstCounterType
T = eventCounter(WaitEventMaskForInst, E);
792 unsigned UB = getScoreUB(
T);
793 unsigned CurrScore = UB + 1;
799 PendingEvents |= 1 << E;
800 setScoreUB(
T, CurrScore);
810 if (AddrOpIdx != -1) {
811 setExpScore(&Inst,
TII,
TRI,
MRI, AddrOpIdx, CurrScore);
824 AMDGPU::OpName::data1),
829 Inst.
getOpcode() != AMDGPU::DS_CONSUME &&
830 Inst.
getOpcode() != AMDGPU::DS_ORDERED_COUNT) {
833 if (
Op.isReg() && !
Op.isDef() &&
834 TRI->isVectorRegister(*
MRI,
Op.getReg())) {
835 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
839 }
else if (
TII->isFLAT(Inst)) {
851 }
else if (
TII->isMIMG(Inst)) {
853 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
860 }
else if (
TII->isMTBUF(Inst)) {
862 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
864 }
else if (
TII->isMUBUF(Inst)) {
866 setExpScore(&Inst,
TII,
TRI,
MRI, 0, CurrScore);
873 }
else if (
TII->isLDSDIR(Inst)) {
880 if (
TII->isEXP(Inst)) {
899 setExpScore(&Inst,
TII,
TRI,
MRI,
I, CurrScore);
904 }
else if (Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORD ||
905 Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORDX2 ||
906 Inst.
getOpcode() == AMDGPU::BUFFER_STORE_DWORDX4) {
912 setRegScore(RegNo + NUM_ALL_VGPRS, t, CurrScore);
919 if (!
Op.isReg() || !
Op.isDef())
922 if (
T == LOAD_CNT ||
T == SAMPLE_CNT ||
T == BVH_CNT) {
923 if (
Interval.first >= NUM_ALL_VGPRS)
925 if (updateVMCntOnly(Inst)) {
930 VmemType
V = getVmemType(Inst);
932 VgprVmemTypes[RegNo] |= 1 <<
V;
936 setRegScore(RegNo,
T, CurrScore);
940 (
TII->isDS(Inst) ||
TII->mayWriteLDSThroughDMA(Inst))) {
945 if (!
MemOp->isStore() ||
950 auto AAI =
MemOp->getAAInfo();
958 if (!AAI || !AAI.Scope)
960 for (
unsigned I = 0, E = LDSDMAStores.size();
I != E && !Slot; ++
I) {
961 for (
const auto *
MemOp : LDSDMAStores[
I]->memoperands()) {
962 if (
MemOp->isStore() && AAI ==
MemOp->getAAInfo()) {
968 if (Slot || LDSDMAStores.size() == NUM_EXTRA_VGPRS - 1)
970 LDSDMAStores.push_back(&Inst);
971 Slot = LDSDMAStores.size();
974 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS + Slot,
T, CurrScore);
976 setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS,
T, CurrScore);
983 for (
auto T : inst_counter_types(MaxCounter)) {
984 unsigned SR = getScoreRange(
T);
988 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"LOAD" :
"VM") <<
"_CNT("
992 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"DS" :
"LGKM") <<
"_CNT("
996 OS <<
" EXP_CNT(" << SR <<
"): ";
999 OS <<
" " << (
ST->hasExtendedWaitCounts() ?
"STORE" :
"VS") <<
"_CNT("
1003 OS <<
" SAMPLE_CNT(" << SR <<
"): ";
1006 OS <<
" BVH_CNT(" << SR <<
"): ";
1009 OS <<
" KM_CNT(" << SR <<
"): ";
1012 OS <<
" UNKNOWN(" << SR <<
"): ";
1018 unsigned LB = getScoreLB(
T);
1020 for (
int J = 0; J <= VgprUB; J++) {
1021 unsigned RegScore = getRegScore(J,
T);
1024 unsigned RelScore = RegScore - LB - 1;
1025 if (J < SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS) {
1026 OS << RelScore <<
":v" << J <<
" ";
1028 OS << RelScore <<
":ds ";
1032 if (
T == SmemAccessCounter) {
1033 for (
int J = 0; J <= SgprUB; J++) {
1034 unsigned RegScore = getRegScore(J + NUM_ALL_VGPRS,
T);
1037 unsigned RelScore = RegScore - LB - 1;
1038 OS << RelScore <<
":s" << J <<
" ";
1050 simplifyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1051 simplifyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1052 simplifyWaitcnt(DS_CNT,
Wait.DsCnt);
1053 simplifyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1054 simplifyWaitcnt(SAMPLE_CNT,
Wait.SampleCnt);
1055 simplifyWaitcnt(BVH_CNT,
Wait.BvhCnt);
1056 simplifyWaitcnt(KM_CNT,
Wait.KmCnt);
1059void WaitcntBrackets::simplifyWaitcnt(InstCounterType
T,
1060 unsigned &Count)
const {
1064 if (Count >= getScoreRange(
T))
1068void WaitcntBrackets::determineWait(InstCounterType
T,
int RegNo,
1070 unsigned ScoreToWait = getRegScore(RegNo,
T);
1074 const unsigned LB = getScoreLB(
T);
1075 const unsigned UB = getScoreUB(
T);
1076 if ((UB >= ScoreToWait) && (ScoreToWait > LB)) {
1077 if ((
T == LOAD_CNT ||
T == DS_CNT) && hasPendingFlat() &&
1078 !
ST->hasFlatLgkmVMemCountInOrder()) {
1082 addWait(
Wait,
T, 0);
1083 }
else if (counterOutOfOrder(
T)) {
1087 addWait(
Wait,
T, 0);
1091 unsigned NeededWait = std::min(UB - ScoreToWait, getWaitCountMax(
T) - 1);
1092 addWait(
Wait,
T, NeededWait);
1098 applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1099 applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1100 applyWaitcnt(DS_CNT,
Wait.DsCnt);
1101 applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1102 applyWaitcnt(SAMPLE_CNT,
Wait.SampleCnt);
1103 applyWaitcnt(BVH_CNT,
Wait.BvhCnt);
1104 applyWaitcnt(KM_CNT,
Wait.KmCnt);
1107void WaitcntBrackets::applyWaitcnt(InstCounterType
T,
unsigned Count) {
1108 const unsigned UB = getScoreUB(
T);
1112 if (counterOutOfOrder(
T))
1114 setScoreLB(
T, std::max(getScoreLB(
T), UB - Count));
1117 PendingEvents &= ~WaitEventMaskForInst[
T];
1123bool WaitcntBrackets::counterOutOfOrder(InstCounterType
T)
const {
1125 if (
T == SmemAccessCounter && hasPendingEvent(SMEM_ACCESS))
1127 return hasMixedPendingEvents(
T);
1137char SIInsertWaitcnts::
ID = 0;
1142 return new SIInsertWaitcnts();
1152 if (NewEnc == MO.
getImm())
1163 case AMDGPU::S_WAIT_LOADCNT:
1165 case AMDGPU::S_WAIT_EXPCNT:
1167 case AMDGPU::S_WAIT_STORECNT:
1169 case AMDGPU::S_WAIT_SAMPLECNT:
1171 case AMDGPU::S_WAIT_BVHCNT:
1173 case AMDGPU::S_WAIT_DSCNT:
1175 case AMDGPU::S_WAIT_KMCNT:
1182bool WaitcntGenerator::promoteSoftWaitCnt(
MachineInstr *Waitcnt)
const {
1196bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
1197 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
1200 assert(isNormalMode(MaxCounter));
1208 if (II.isMetaInstruction())
1212 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1216 if (Opcode == AMDGPU::S_WAITCNT) {
1217 unsigned IEnc = II.getOperand(0).getImm();
1220 ScoreBrackets.simplifyWaitcnt(OldWait);
1224 if (WaitcntInstr || (!
Wait.hasWaitExceptStoreCnt() && TrySimplify)) {
1225 II.eraseFromParent();
1230 assert(Opcode == AMDGPU::S_WAITCNT_VSCNT);
1231 assert(II.getOperand(0).getReg() == AMDGPU::SGPR_NULL);
1234 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1236 ScoreBrackets.simplifyWaitcnt(InstCounterType::STORE_CNT, OldVSCnt);
1237 Wait.StoreCnt = std::min(
Wait.StoreCnt, OldVSCnt);
1239 if (WaitcntVsCntInstr || (!
Wait.hasWaitStoreCnt() && TrySimplify)) {
1240 II.eraseFromParent();
1243 WaitcntVsCntInstr = &II;
1250 Modified |= promoteSoftWaitCnt(WaitcntInstr);
1252 ScoreBrackets.applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1253 ScoreBrackets.applyWaitcnt(EXP_CNT,
Wait.ExpCnt);
1254 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1261 <<
"applyPreexistingWaitcnt\n"
1262 <<
"New Instr at block end: " << *WaitcntInstr <<
'\n'
1263 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1264 <<
"Old Instr: " << *It
1265 <<
"New Instr: " << *WaitcntInstr <<
'\n');
1268 if (WaitcntVsCntInstr) {
1270 AMDGPU::OpName::simm16,
Wait.StoreCnt);
1271 Modified |= promoteSoftWaitCnt(WaitcntVsCntInstr);
1273 ScoreBrackets.applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1274 Wait.StoreCnt = ~0
u;
1277 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1278 <<
"New Instr at block end: " << *WaitcntVsCntInstr
1280 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1281 <<
"Old Instr: " << *It
1282 <<
"New Instr: " << *WaitcntVsCntInstr <<
'\n');
1290bool WaitcntGeneratorPreGFX12::createNewWaitcnt(
1294 assert(isNormalMode(MaxCounter));
1301 if (
Wait.hasWaitExceptStoreCnt()) {
1303 [[maybe_unused]]
auto SWaitInst =
1308 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1309 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1312 if (
Wait.hasWaitStoreCnt()) {
1315 [[maybe_unused]]
auto SWaitInst =
1322 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1323 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1330WaitcntGeneratorPreGFX12::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1335WaitcntGeneratorGFX12Plus::getAllZeroWaitcnt(
bool IncludeVSCnt)
const {
1343bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
1344 WaitcntBrackets &ScoreBrackets,
MachineInstr &OldWaitcntInstr,
1347 assert(!isNormalMode(MaxCounter));
1356 if (II.isMetaInstruction())
1365 bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
1369 if (Opcode == AMDGPU::S_WAITCNT)
1372 if (Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT) {
1374 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1377 ScoreBrackets.simplifyWaitcnt(OldWait);
1379 UpdatableInstr = &CombinedLoadDsCntInstr;
1380 }
else if (Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT) {
1382 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1385 ScoreBrackets.simplifyWaitcnt(OldWait);
1387 UpdatableInstr = &CombinedStoreDsCntInstr;
1392 TII->getNamedOperand(II, AMDGPU::OpName::simm16)->getImm();
1394 ScoreBrackets.simplifyWaitcnt(CT.value(), OldCnt);
1395 addWait(
Wait, CT.value(), OldCnt);
1396 UpdatableInstr = &WaitInstrs[CT.value()];
1400 if (!*UpdatableInstr) {
1401 *UpdatableInstr = &II;
1408 if (CombinedLoadDsCntInstr) {
1416 if (
Wait.LoadCnt != ~0u &&
Wait.DsCnt != ~0u) {
1419 AMDGPU::OpName::simm16, NewEnc);
1420 Modified |= promoteSoftWaitCnt(CombinedLoadDsCntInstr);
1421 ScoreBrackets.applyWaitcnt(LOAD_CNT,
Wait.LoadCnt);
1422 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1427 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1428 <<
"New Instr at block end: "
1429 << *CombinedLoadDsCntInstr <<
'\n'
1430 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1431 <<
"Old Instr: " << *It <<
"New Instr: "
1432 << *CombinedLoadDsCntInstr <<
'\n');
1439 if (CombinedStoreDsCntInstr) {
1441 if (
Wait.StoreCnt != ~0u &&
Wait.DsCnt != ~0u) {
1444 AMDGPU::OpName::simm16, NewEnc);
1445 Modified |= promoteSoftWaitCnt(CombinedStoreDsCntInstr);
1446 ScoreBrackets.applyWaitcnt(STORE_CNT,
Wait.StoreCnt);
1447 ScoreBrackets.applyWaitcnt(DS_CNT,
Wait.DsCnt);
1448 Wait.StoreCnt = ~0
u;
1452 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1453 <<
"New Instr at block end: "
1454 << *CombinedStoreDsCntInstr <<
'\n'
1455 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1456 <<
"Old Instr: " << *It <<
"New Instr: "
1457 << *CombinedStoreDsCntInstr <<
'\n');
1470 if (
Wait.DsCnt != ~0u) {
1479 if (
Wait.LoadCnt != ~0u) {
1480 WaitsToErase.
push_back(&WaitInstrs[LOAD_CNT]);
1481 WaitsToErase.
push_back(&WaitInstrs[DS_CNT]);
1482 }
else if (
Wait.StoreCnt != ~0u) {
1483 WaitsToErase.
push_back(&WaitInstrs[STORE_CNT]);
1484 WaitsToErase.
push_back(&WaitInstrs[DS_CNT]);
1491 (*WI)->eraseFromParent();
1497 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1498 if (!WaitInstrs[CT])
1501 unsigned NewCnt = getWait(
Wait, CT);
1502 if (NewCnt != ~0u) {
1504 AMDGPU::OpName::simm16, NewCnt);
1505 Modified |= promoteSoftWaitCnt(WaitInstrs[CT]);
1507 ScoreBrackets.applyWaitcnt(CT, NewCnt);
1508 setNoWait(
Wait, CT);
1511 ?
dbgs() <<
"applyPreexistingWaitcnt\n"
1512 <<
"New Instr at block end: " << *WaitInstrs[CT]
1514 :
dbgs() <<
"applyPreexistingWaitcnt\n"
1515 <<
"Old Instr: " << *It
1516 <<
"New Instr: " << *WaitInstrs[CT] <<
'\n');
1527bool WaitcntGeneratorGFX12Plus::createNewWaitcnt(
1531 assert(!isNormalMode(MaxCounter));
1537 if (
Wait.DsCnt != ~0u) {
1540 if (
Wait.LoadCnt != ~0u) {
1548 }
else if (
Wait.StoreCnt != ~0u) {
1555 Wait.StoreCnt = ~0
u;
1563 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1564 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1571 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
1572 unsigned Count = getWait(
Wait, CT);
1576 [[maybe_unused]]
auto SWaitInst =
1583 if (It !=
Block.instr_end())
dbgs() <<
"Old Instr: " << *It;
1584 dbgs() <<
"New Instr: " << *SWaitInst <<
'\n');
1591 unsigned Opc =
MI.getOpcode();
1592 return (Opc == AMDGPU::S_CBRANCH_VCCNZ || Opc == AMDGPU::S_CBRANCH_VCCZ) &&
1593 !
MI.getOperand(1).isUndef();
1623bool SIInsertWaitcnts::generateWaitcntInstBefore(
MachineInstr &
MI,
1624 WaitcntBrackets &ScoreBrackets,
1627 setForceEmitWaitcnt();
1629 if (
MI.isMetaInstruction())
1638 if (
MI.getOpcode() == AMDGPU::BUFFER_WBINVL1 ||
1639 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_SC ||
1640 MI.getOpcode() == AMDGPU::BUFFER_WBINVL1_VOL ||
1641 MI.getOpcode() == AMDGPU::BUFFER_GL0_INV ||
1642 MI.getOpcode() == AMDGPU::BUFFER_GL1_INV) {
1649 if (
MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1650 MI.getOpcode() == AMDGPU::SI_RETURN ||
1651 MI.getOpcode() == AMDGPU::S_SETPC_B64_return ||
1653 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
false));
1661 else if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
1662 MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
1664 ScoreBrackets.getScoreRange(STORE_CNT) != 0 &&
1665 !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS))
1669 else if ((
MI.getOpcode() == AMDGPU::S_SENDMSG ||
1670 MI.getOpcode() == AMDGPU::S_SENDMSGHALT) &&
1671 ST->hasLegacyGeometry() &&
1677 else if (
MI.getOpcode() == SC_FENCE) {
1678 const unsigned int group_size =
1679 context->shader_info->GetMaxThreadGroupSize();
1681 const bool group_is_multi_wave =
1682 (group_size == 0 || group_size > target_info->GetWaveFrontSize());
1683 const bool fence_is_global = !((SCInstInternalMisc*)Inst)->IsGroupFence();
1685 for (
unsigned int i = 0; i < Inst->NumSrcOperands(); i++) {
1686 SCRegType src_type = Inst->GetSrcType(i);
1689 if (group_is_multi_wave ||
1690 context->OptFlagIsOn(OPT_R1100_LDSMEM_FENCE_CHICKEN_BIT)) {
1691 EmitWaitcnt |= ScoreBrackets->updateByWait(DS_CNT,
1692 ScoreBrackets->getScoreUB(DS_CNT));
1694 if (target_info->HasBufferLoadToLDS()) {
1695 EmitWaitcnt |= ScoreBrackets->updateByWait(LOAD_CNT,
1696 ScoreBrackets->getScoreUB(LOAD_CNT));
1702 if (group_is_multi_wave || fence_is_global) {
1703 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
1704 ScoreBrackets->getScoreUB(EXP_CNT));
1705 EmitWaitcnt |= ScoreBrackets->updateByWait(DS_CNT,
1706 ScoreBrackets->getScoreUB(DS_CNT));
1714 if (group_is_multi_wave || fence_is_global) {
1715 EmitWaitcnt |= ScoreBrackets->updateByWait(EXP_CNT,
1716 ScoreBrackets->getScoreUB(EXP_CNT));
1717 EmitWaitcnt |= ScoreBrackets->updateByWait(LOAD_CNT,
1718 ScoreBrackets->getScoreUB(LOAD_CNT));
1735 if (
MI.modifiesRegister(AMDGPU::EXEC,
TRI)) {
1738 if (ScoreBrackets.hasPendingEvent(EXP_GPR_LOCK) ||
1739 ScoreBrackets.hasPendingEvent(EXP_PARAM_ACCESS) ||
1740 ScoreBrackets.hasPendingEvent(EXP_POS_ACCESS) ||
1741 ScoreBrackets.hasPendingEvent(GDS_GPR_LOCK)) {
1755 if (
MI.getOperand(CallAddrOpIdx).isReg()) {
1756 RegInterval CallAddrOpInterval =
1757 ScoreBrackets.getRegInterval(&
MI,
MRI,
TRI, CallAddrOpIdx);
1759 for (
int RegNo = CallAddrOpInterval.first;
1760 RegNo < CallAddrOpInterval.second; ++RegNo)
1761 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1765 if (RtnAddrOpIdx != -1) {
1766 RegInterval RtnAddrOpInterval =
1767 ScoreBrackets.getRegInterval(&
MI,
MRI,
TRI, RtnAddrOpIdx);
1769 for (
int RegNo = RtnAddrOpInterval.first;
1770 RegNo < RtnAddrOpInterval.second; ++RegNo)
1771 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1790 const Value *
Ptr = Memop->getValue();
1791 if (Memop->isStore() && SLoadAddresses.
count(
Ptr)) {
1792 addWait(
Wait, SmemAccessCounter, 0);
1796 unsigned AS = Memop->getAddrSpace();
1800 if (
TII->mayWriteLDSThroughDMA(
MI))
1804 unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
1805 bool FoundAliasingStore =
false;
1812 if (
Ptr && Memop->getAAInfo() && Memop->getAAInfo().Scope) {
1813 const auto &LDSDMAStores = ScoreBrackets.getLDSDMAStores();
1814 for (
unsigned I = 0, E = LDSDMAStores.size();
I != E; ++
I) {
1815 if (
MI.mayAlias(AA, *LDSDMAStores[
I],
true)) {
1816 FoundAliasingStore =
true;
1817 ScoreBrackets.determineWait(LOAD_CNT, RegNo +
I + 1,
Wait);
1821 if (!FoundAliasingStore)
1822 ScoreBrackets.determineWait(LOAD_CNT, RegNo,
Wait);
1823 if (Memop->isStore()) {
1824 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1829 for (
unsigned I = 0, E =
MI.getNumOperands();
I != E; ++
I) {
1835 if (
Op.isTied() &&
Op.isUse() &&
TII->doesNotReadTiedSource(
MI))
1840 const bool IsVGPR =
TRI->isVectorRegister(*
MRI,
Op.getReg());
1847 if (
Op.isUse() || !updateVMCntOnly(
MI) ||
1848 ScoreBrackets.hasOtherPendingVmemTypes(RegNo,
1850 ScoreBrackets.determineWait(LOAD_CNT, RegNo,
Wait);
1851 ScoreBrackets.determineWait(SAMPLE_CNT, RegNo,
Wait);
1852 ScoreBrackets.determineWait(BVH_CNT, RegNo,
Wait);
1853 ScoreBrackets.clearVgprVmemTypes(RegNo);
1855 if (
Op.isDef() || ScoreBrackets.hasPendingEvent(EXP_LDS_ACCESS)) {
1856 ScoreBrackets.determineWait(EXP_CNT, RegNo,
Wait);
1858 ScoreBrackets.determineWait(DS_CNT, RegNo,
Wait);
1860 ScoreBrackets.determineWait(SmemAccessCounter, RegNo,
Wait);
1871 if (
TII->isBarrierStart(
MI.getOpcode()) &&
1872 !
ST->hasAutoWaitcntBeforeBarrier() && !
ST->supportsBackOffBarrier()) {
1873 Wait =
Wait.combined(WCG->getAllZeroWaitcnt(
true));
1880 if (ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
1886 ScoreBrackets.simplifyWaitcnt(
Wait);
1888 if (ForceEmitZeroWaitcnts)
1889 Wait = WCG->getAllZeroWaitcnt(
false);
1891 if (ForceEmitWaitcnt[LOAD_CNT])
1893 if (ForceEmitWaitcnt[EXP_CNT])
1895 if (ForceEmitWaitcnt[DS_CNT])
1897 if (ForceEmitWaitcnt[SAMPLE_CNT])
1899 if (ForceEmitWaitcnt[BVH_CNT])
1901 if (ForceEmitWaitcnt[KM_CNT])
1905 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
1907 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
1909 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
1913 return generateWaitcnt(
Wait,
MI.getIterator(), *
MI.getParent(), ScoreBrackets,
1920 WaitcntBrackets &ScoreBrackets,
1924 if (OldWaitcntInstr)
1928 WCG->applyPreexistingWaitcnt(ScoreBrackets, *OldWaitcntInstr,
Wait, It);
1932 ScoreBrackets.applyWaitcnt(
Wait);
1935 if (
Wait.ExpCnt != ~0u && It !=
Block.instr_end() &&
1938 TII->getNamedOperand(*It, AMDGPU::OpName::waitexp);
1946 <<
"Update Instr: " << *It);
1949 if (WCG->createNewWaitcnt(
Block, It,
Wait))
1958bool SIInsertWaitcnts::mayAccessVMEMThroughFlat(
const MachineInstr &
MI)
const {
1966 if (
MI.memoperands_empty())
1975 unsigned AS = Memop->getAddrSpace();
1986bool SIInsertWaitcnts::mayAccessLDSThroughFlat(
const MachineInstr &
MI)
const {
1990 if (!
TII->usesLGKM_CNT(
MI))
1994 if (
ST->isTgSplitEnabled())
1999 if (
MI.memoperands_empty())
2004 unsigned AS = Memop->getAddrSpace();
2014bool SIInsertWaitcnts::mayAccessScratchThroughFlat(
2019 if (
TII->isFLATScratch(
MI))
2023 if (
TII->isFLATGlobal(
MI))
2028 if (
MI.memoperands_empty())
2033 unsigned AS = Memop->getAddrSpace();
2034 return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
2040 return Opc == AMDGPU::GLOBAL_INV || Opc == AMDGPU::GLOBAL_WB ||
2041 Opc == AMDGPU::GLOBAL_WBINV;
2044void SIInsertWaitcnts::updateEventWaitcntAfter(
MachineInstr &Inst,
2045 WaitcntBrackets *ScoreBrackets) {
2051 if (
TII->isDS(Inst) &&
TII->usesLGKM_CNT(Inst)) {
2053 TII->hasModifiersSet(Inst, AMDGPU::OpName::gds)) {
2054 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_ACCESS, Inst);
2055 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, GDS_GPR_LOCK, Inst);
2057 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
2059 }
else if (
TII->isFLAT(Inst)) {
2066 int FlatASCount = 0;
2068 if (mayAccessVMEMThroughFlat(Inst)) {
2070 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
2074 if (mayAccessLDSThroughFlat(Inst)) {
2076 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, LDS_ACCESS, Inst);
2085 if (FlatASCount > 1)
2086 ScoreBrackets->setPendingFlat();
2089 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, getVmemWaitEventType(Inst),
2092 if (
ST->vmemWriteNeedsExpWaitcnt() &&
2094 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, VMW_GPR_LOCK, Inst);
2096 }
else if (
TII->isSMRD(Inst)) {
2097 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
2098 }
else if (Inst.
isCall()) {
2101 ScoreBrackets->applyWaitcnt(
2102 WCG->getAllZeroWaitcnt(
false));
2103 ScoreBrackets->setStateOnFunctionEntryOrReturn();
2109 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_LDS_ACCESS, Inst);
2110 }
else if (
TII->isVINTERP(Inst)) {
2111 int64_t
Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::waitexp)->getImm();
2112 ScoreBrackets->applyWaitcnt(EXP_CNT, Imm);
2114 unsigned Imm =
TII->getNamedOperand(Inst, AMDGPU::OpName::tgt)->getImm();
2116 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_PARAM_ACCESS, Inst);
2118 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_POS_ACCESS, Inst);
2120 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, EXP_GPR_LOCK, Inst);
2123 case AMDGPU::S_SENDMSG:
2124 case AMDGPU::S_SENDMSG_RTN_B32:
2125 case AMDGPU::S_SENDMSG_RTN_B64:
2126 case AMDGPU::S_SENDMSGHALT:
2127 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SQ_MESSAGE, Inst);
2129 case AMDGPU::S_MEMTIME:
2130 case AMDGPU::S_MEMREALTIME:
2131 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
2132 case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
2133 case AMDGPU::S_BARRIER_LEAVE:
2134 case AMDGPU::S_GET_BARRIER_STATE_M0:
2135 case AMDGPU::S_GET_BARRIER_STATE_IMM:
2136 ScoreBrackets->updateByEvent(
TII,
TRI,
MRI, SMEM_ACCESS, Inst);
2142bool WaitcntBrackets::mergeScore(
const MergeInfo &M,
unsigned &Score,
2143 unsigned OtherScore) {
2144 unsigned MyShifted = Score <=
M.OldLB ? 0 : Score +
M.MyShift;
2145 unsigned OtherShifted =
2146 OtherScore <=
M.OtherLB ? 0 : OtherScore +
M.OtherShift;
2147 Score = std::max(MyShifted, OtherShifted);
2148 return OtherShifted > MyShifted;
2156bool WaitcntBrackets::merge(
const WaitcntBrackets &
Other) {
2157 bool StrictDom =
false;
2159 VgprUB = std::max(VgprUB,
Other.VgprUB);
2160 SgprUB = std::max(SgprUB,
Other.SgprUB);
2162 for (
auto T : inst_counter_types(MaxCounter)) {
2164 const unsigned OldEvents = PendingEvents & WaitEventMaskForInst[
T];
2165 const unsigned OtherEvents =
Other.PendingEvents & WaitEventMaskForInst[
T];
2166 if (OtherEvents & ~OldEvents)
2168 PendingEvents |= OtherEvents;
2171 const unsigned MyPending = ScoreUBs[
T] - ScoreLBs[
T];
2172 const unsigned OtherPending =
Other.ScoreUBs[
T] -
Other.ScoreLBs[
T];
2173 const unsigned NewUB = ScoreLBs[
T] + std::max(MyPending, OtherPending);
2174 if (NewUB < ScoreLBs[
T])
2178 M.OldLB = ScoreLBs[
T];
2179 M.OtherLB =
Other.ScoreLBs[
T];
2180 M.MyShift = NewUB - ScoreUBs[
T];
2181 M.OtherShift = NewUB -
Other.ScoreUBs[
T];
2183 ScoreUBs[
T] = NewUB;
2185 StrictDom |= mergeScore(M, LastFlat[
T],
Other.LastFlat[
T]);
2187 for (
int J = 0; J <= VgprUB; J++)
2188 StrictDom |= mergeScore(M, VgprScores[
T][J],
Other.VgprScores[
T][J]);
2190 if (
T == SmemAccessCounter) {
2191 for (
int J = 0; J <= SgprUB; J++)
2192 StrictDom |= mergeScore(M, SgprScores[J],
Other.SgprScores[J]);
2196 for (
int J = 0; J <= VgprUB; J++) {
2197 unsigned char NewVmemTypes = VgprVmemTypes[J] |
Other.VgprVmemTypes[J];
2198 StrictDom |= NewVmemTypes != VgprVmemTypes[J];
2199 VgprVmemTypes[J] = NewVmemTypes;
2207 return Opcode == AMDGPU::S_WAITCNT ||
2210 Opcode == AMDGPU::S_WAIT_LOADCNT_DSCNT ||
2211 Opcode == AMDGPU::S_WAIT_STORECNT_DSCNT ||
2218 WaitcntBrackets &ScoreBrackets) {
2222 dbgs() <<
"*** Block" <<
Block.getNumber() <<
" ***";
2223 ScoreBrackets.dump();
2229 bool VCCZCorrect =
true;
2230 if (
ST->hasReadVCCZBug()) {
2233 VCCZCorrect =
false;
2234 }
else if (!
ST->partialVCCWritesUpdateVCCZ()) {
2237 VCCZCorrect =
false;
2244 E =
Block.instr_end();
2251 if (!OldWaitcntInstr)
2252 OldWaitcntInstr = &Inst;
2257 bool FlushVmCnt =
Block.getFirstTerminator() == Inst &&
2258 isPreheaderToFlush(
Block, ScoreBrackets);
2261 Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr,
2263 OldWaitcntInstr =
nullptr;
2266 bool RestoreVCCZ = !VCCZCorrect &&
readsVCCZ(Inst);
2269 if (
ST->hasReadVCCZBug() || !
ST->partialVCCWritesUpdateVCCZ()) {
2273 if (!
ST->partialVCCWritesUpdateVCCZ())
2274 VCCZCorrect =
false;
2283 if (
ST->hasReadVCCZBug() &&
2284 ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) {
2287 VCCZCorrect =
false;
2295 if (
TII->isSMRD(Inst)) {
2299 if (!Memop->isInvariant()) {
2300 const Value *
Ptr = Memop->getValue();
2304 if (
ST->hasReadVCCZBug()) {
2306 VCCZCorrect =
false;
2310 updateEventWaitcntAfter(Inst, &ScoreBrackets);
2316 if (RequireCheckResourceType(Inst, context)) {
2318 ScoreBrackets->setScoreLB(LOAD_CNT,
2319 ScoreBrackets->getScoreUB(LOAD_CNT));
2326 ScoreBrackets.simplifyWaitcnt(
Wait);
2328 ScoreBrackets,
nullptr);
2333 ScoreBrackets.dump();
2343 TII->get(
ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
2356 if (
Block.getFirstTerminator() ==
Block.end() &&
2357 isPreheaderToFlush(
Block, ScoreBrackets)) {
2358 if (ScoreBrackets.hasPendingEvent(LOAD_CNT))
2360 if (ScoreBrackets.hasPendingEvent(SAMPLE_CNT))
2362 if (ScoreBrackets.hasPendingEvent(BVH_CNT))
2376 WaitcntBrackets &ScoreBrackets) {
2377 auto [Iterator, IsInserted] = PreheadersToFlush.
try_emplace(&
MBB,
false);
2379 return Iterator->second;
2390 shouldFlushVmCnt(
Loop, ScoreBrackets)) {
2391 Iterator->second =
true;
2398bool SIInsertWaitcnts::isVMEMOrFlatVMEM(
const MachineInstr &
MI)
const {
2412 WaitcntBrackets &Brackets) {
2413 bool HasVMemLoad =
false;
2414 bool HasVMemStore =
false;
2415 bool UsesVgprLoadedOutside =
false;
2421 if (isVMEMOrFlatVMEM(
MI)) {
2425 HasVMemStore =
true;
2427 for (
unsigned I = 0;
I <
MI.getNumOperands();
I++) {
2429 if (!
Op.isReg() || !
TRI->isVectorRegister(*
MRI,
Op.getReg()))
2442 if (Brackets.getRegScore(RegNo, LOAD_CNT) >
2443 Brackets.getScoreLB(LOAD_CNT) ||
2444 Brackets.getRegScore(RegNo, SAMPLE_CNT) >
2445 Brackets.getScoreLB(SAMPLE_CNT) ||
2446 Brackets.getRegScore(RegNo, BVH_CNT) >
2447 Brackets.getScoreLB(BVH_CNT)) {
2448 UsesVgprLoadedOutside =
true;
2454 else if (isVMEMOrFlatVMEM(
MI) &&
MI.mayLoad() &&
Op.isDef())
2465 if (!
ST->hasVscnt() && HasVMemStore && !HasVMemLoad && UsesVgprLoadedOutside)
2467 return HasVMemLoad && UsesVgprLoadedOutside;
2472 TII =
ST->getInstrInfo();
2473 TRI = &
TII->getRegisterInfo();
2476 MLI = &getAnalysis<MachineLoopInfo>();
2477 PDT = &getAnalysis<MachinePostDominatorTree>();
2478 if (
auto AAR = getAnalysisIfAvailable<AAResultsWrapperPass>())
2479 AA = &AAR->getAAResults();
2483 if (
ST->hasExtendedWaitCounts()) {
2484 MaxCounter = NUM_EXTENDED_INST_CNTS;
2485 WCGGFX12Plus = WaitcntGeneratorGFX12Plus(MF, MaxCounter);
2486 WCG = &WCGGFX12Plus;
2488 MaxCounter = NUM_NORMAL_INST_CNTS;
2489 WCGPreGFX12 = WaitcntGeneratorPreGFX12(MF);
2494 for (
auto T : inst_counter_types())
2495 ForceEmitWaitcnt[
T] =
false;
2497 const unsigned *WaitEventMaskForInst = WCG->getWaitEventMask();
2499 SmemAccessCounter = eventCounter(WaitEventMaskForInst, SMEM_ACCESS);
2501 HardwareLimits Limits = {};
2502 if (
ST->hasExtendedWaitCounts()) {
2515 unsigned NumVGPRsMax =
ST->getAddressableNumVGPRs();
2516 unsigned NumSGPRsMax =
ST->getAddressableNumSGPRs();
2517 assert(NumVGPRsMax <= SQ_MAX_PGM_VGPRS);
2518 assert(NumSGPRsMax <= SQ_MAX_PGM_SGPRS);
2520 RegisterEncoding Encoding = {};
2523 Encoding.VGPRL = Encoding.VGPR0 + NumVGPRsMax - 1;
2526 Encoding.SGPRL = Encoding.SGPR0 + NumSGPRsMax - 1;
2542 I != E && (
I->isPHI() ||
I->isMetaInstruction()); ++
I)
2545 if (
ST->hasExtendedWaitCounts()) {
2548 for (
auto CT : inst_counter_types(NUM_EXTENDED_INST_CNTS)) {
2549 if (CT == LOAD_CNT || CT == DS_CNT || CT == STORE_CNT)
2553 TII->get(instrsForExtendedCounterTypes[CT]))
2560 auto NonKernelInitialState = std::make_unique<WaitcntBrackets>(
2561 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2563 NonKernelInitialState->setStateOnFunctionEntryOrReturn();
2564 BlockInfos[&EntryBB].Incoming = std::move(NonKernelInitialState);
2574 std::unique_ptr<WaitcntBrackets> Brackets;
2579 for (
auto BII = BlockInfos.
begin(), BIE = BlockInfos.
end(); BII != BIE;
2582 BlockInfo &BI = BII->second;
2588 Brackets = std::make_unique<WaitcntBrackets>(*BI.Incoming);
2590 *Brackets = *BI.Incoming;
2593 Brackets = std::make_unique<WaitcntBrackets>(
2594 ST, MaxCounter, Limits, Encoding, WaitEventMaskForInst,
2597 *Brackets = WaitcntBrackets(ST, MaxCounter, Limits, Encoding,
2598 WaitEventMaskForInst, SmemAccessCounter);
2601 Modified |= insertWaitcntInBlock(MF, *
MBB, *Brackets);
2604 if (Brackets->hasPendingEvent()) {
2605 BlockInfo *MoveBracketsToSucc =
nullptr;
2607 auto SuccBII = BlockInfos.
find(Succ);
2608 BlockInfo &SuccBI = SuccBII->second;
2609 if (!SuccBI.Incoming) {
2610 SuccBI.Dirty =
true;
2613 if (!MoveBracketsToSucc) {
2614 MoveBracketsToSucc = &SuccBI;
2616 SuccBI.Incoming = std::make_unique<WaitcntBrackets>(*Brackets);
2618 }
else if (SuccBI.Incoming->merge(*Brackets)) {
2619 SuccBI.Dirty =
true;
2624 if (MoveBracketsToSucc)
2625 MoveBracketsToSucc->Incoming = std::move(Brackets);
2630 if (
ST->hasScalarStores()) {
2632 bool HaveScalarStores =
false;
2636 if (!HaveScalarStores &&
TII->isScalarStore(
MI))
2637 HaveScalarStores =
true;
2639 if (
MI.getOpcode() == AMDGPU::S_ENDPGM ||
2640 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG)
2645 if (HaveScalarStores) {
2655 bool SeenDCacheWB =
false;
2659 if (
I->getOpcode() == AMDGPU::S_DCACHE_WB)
2660 SeenDCacheWB =
true;
2661 else if (
TII->isScalarStore(*
I))
2662 SeenDCacheWB =
false;
2665 if ((
I->getOpcode() == AMDGPU::S_ENDPGM ||
2666 I->getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG) &&
2679 if (
ST->requiresNopBeforeDeallocVGPRs()) {
2684 TII->get(AMDGPU::S_SENDMSG))
2688 ReleaseVGPRInsts.clear();
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Provides AMDGPU specific target descriptions.
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
Analysis containing CSE Info
This file provides an implementation of debug counters.
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
std::optional< std::vector< StOtherPiece > > Other
static Function * getFunction(Constant *C)
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
static bool isOptNone(const MachineFunction &MF)
static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B)
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
std::pair< uint64_t, uint64_t > Interval
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static bool callWaitsOnFunctionReturn(const MachineInstr &MI)
static bool isCacheInvOrWBInst(MachineInstr &Inst)
static bool callWaitsOnFunctionEntry(const MachineInstr &MI)
static bool updateOperandIfDifferent(MachineInstr &MI, uint16_t OpName, unsigned NewEnc)
static bool isWaitInstr(MachineInstr &Inst)
static std::optional< InstCounterType > counterTypeForInstr(unsigned Opcode)
Determine if MI is a gfx12+ single-counter S_WAIT_*CNT instruction, and if so, which counter it is wa...
static bool readsVCCZ(const MachineInstr &MI)
static cl::opt< bool > ForceEmitZeroFlag("amdgpu-waitcnt-forcezero", cl::desc("Force all waitcnt instrs to be emitted as s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)"), cl::init(false), cl::Hidden)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Provides some synthesis utilities to produce sequences of values.
static const uint32_t IV[8]
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
bool isEntryFunction() const
Represent the analysis usage information of a pass.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
This class represents an Operation in the Expression.
static bool isCounterSet(unsigned ID)
static bool shouldExecute(unsigned CounterName)
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
bool erase(const KeyT &Val)
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
FunctionPass class - This class is used to implement most global optimizations.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Represents a single loop in the control flow graph.
const MachineBasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Instructions::iterator instr_iterator
iterator_range< succ_iterator > successors()
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isCall(QueryType Type=AnyInBundle) const
unsigned getNumOperands() const
Retuns the total number of operands.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr fully defines the specified register.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineLoop * getLoopFor(const MachineBasicBlock *BB) const
Return the innermost loop that BB lives in.
A description of a memory reference used in the backend.
MachineOperand class - Representation of each machine instruction operand.
void setImm(int64_t immVal)
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static bool isVMEM(const MachineInstr &MI)
static bool isFLATScratch(const MachineInstr &MI)
static bool isEXP(const MachineInstr &MI)
static bool mayWriteLDSThroughDMA(const MachineInstr &MI)
static bool isVIMAGE(const MachineInstr &MI)
static bool isLDSDIR(const MachineInstr &MI)
static bool isGWS(const MachineInstr &MI)
static bool isFLATGlobal(const MachineInstr &MI)
static bool isVSAMPLE(const MachineInstr &MI)
static bool isAtomicRet(const MachineInstr &MI)
static unsigned getNonSoftWaitcntOpcode(unsigned Opcode)
static bool isVINTERP(const MachineInstr &MI)
static bool isMIMG(const MachineInstr &MI)
static bool isFLAT(const MachineInstr &MI)
This class keeps track of the SPI_SP_INPUT_ADDR config register, which tells the hardware which inter...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ ID_DEALLOC_VGPRS_GFX11Plus
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
void decodeWaitcnt(const IsaVersion &Version, unsigned Waitcnt, unsigned &Vmcnt, unsigned &Expcnt, unsigned &Lgkmcnt)
Decodes Vmcnt, Expcnt and Lgkmcnt from given Waitcnt for given isa Version, and writes decoded values...
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
unsigned getStorecntBitMask(const IsaVersion &Version)
IsaVersion getIsaVersion(StringRef GPU)
unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt)
Encodes Vmcnt, Expcnt and Lgkmcnt into Waitcnt for given isa Version.
unsigned getSamplecntBitMask(const IsaVersion &Version)
unsigned getKmcntBitMask(const IsaVersion &Version)
unsigned getVmcntBitMask(const IsaVersion &Version)
Waitcnt decodeStorecntDscnt(const IsaVersion &Version, unsigned StorecntDscnt)
LLVM_READONLY bool hasNamedOperand(uint64_t Opcode, uint64_t NamedIdx)
unsigned getLgkmcntBitMask(const IsaVersion &Version)
unsigned getBvhcntBitMask(const IsaVersion &Version)
unsigned getExpcntBitMask(const IsaVersion &Version)
unsigned getMCReg(unsigned Reg, const MCSubtargetInfo &STI)
If Reg is a pseudo reg, return the correct hardware register given STI otherwise return Reg.
Waitcnt decodeLoadcntDscnt(const IsaVersion &Version, unsigned LoadcntDscnt)
static unsigned encodeStorecntDscnt(const IsaVersion &Version, unsigned Storecnt, unsigned Dscnt)
bool getMUBUFIsBufferInv(unsigned Opc)
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
unsigned getLoadcntBitMask(const IsaVersion &Version)
static unsigned encodeLoadcntDscnt(const IsaVersion &Version, unsigned Loadcnt, unsigned Dscnt)
unsigned getDscntBitMask(const IsaVersion &Version)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Undef
Value of the register doesn't matter.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
auto enum_seq(EnumT Begin, EnumT End)
Iterate over an enum type from Begin up to - but not including - End.
static StringRef getCPU(StringRef CPU)
Processes a CPU name.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
char & SIInsertWaitcntsID
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
CodeGenOptLevel
Code generation optimization level.
FunctionPass * createSIInsertWaitcntsPass()
Instruction set architecture version.
Represents the counter values to wait for in an s_waitcnt instruction.
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...
static constexpr bool is_iterable