30#define DEBUG_TYPE "si-memory-legalizer"
31#define PASS_NAME "SI Memory Legalizer"
35 cl::desc(
"Use this to skip inserting cache invalidating instructions."));
57enum class SIAtomicScope {
68enum class SIAtomicAddrSpace {
88class SIMemOpInfo final {
91 friend class SIMemOpAccess;
95 SIAtomicScope
Scope = SIAtomicScope::SYSTEM;
96 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE;
97 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE;
98 bool IsCrossAddressSpaceOrdering =
false;
100 bool IsNonTemporal =
false;
102 SIMemOpInfo(
AtomicOrdering Ordering = AtomicOrdering::SequentiallyConsistent,
103 SIAtomicScope Scope = SIAtomicScope::SYSTEM,
104 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::ATOMIC,
105 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::ALL,
106 bool IsCrossAddressSpaceOrdering =
true,
108 AtomicOrdering::SequentiallyConsistent,
109 bool IsVolatile =
false,
110 bool IsNonTemporal =
false)
111 : Ordering(Ordering), FailureOrdering(FailureOrdering),
112 Scope(
Scope), OrderingAddrSpace(OrderingAddrSpace),
113 InstrAddrSpace(InstrAddrSpace),
114 IsCrossAddressSpaceOrdering(IsCrossAddressSpaceOrdering),
116 IsNonTemporal(IsNonTemporal) {
118 if (Ordering == AtomicOrdering::NotAtomic) {
119 assert(Scope == SIAtomicScope::NONE &&
120 OrderingAddrSpace == SIAtomicAddrSpace::NONE &&
121 !IsCrossAddressSpaceOrdering &&
122 FailureOrdering == AtomicOrdering::NotAtomic);
126 assert(Scope != SIAtomicScope::NONE &&
127 (OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) !=
128 SIAtomicAddrSpace::NONE &&
129 (InstrAddrSpace & SIAtomicAddrSpace::ATOMIC) !=
130 SIAtomicAddrSpace::NONE);
135 if ((OrderingAddrSpace == InstrAddrSpace) &&
137 this->IsCrossAddressSpaceOrdering =
false;
141 if ((InstrAddrSpace & ~SIAtomicAddrSpace::SCRATCH) ==
142 SIAtomicAddrSpace::NONE) {
143 this->Scope = std::min(Scope, SIAtomicScope::SINGLETHREAD);
144 }
else if ((InstrAddrSpace &
145 ~(SIAtomicAddrSpace::SCRATCH | SIAtomicAddrSpace::LDS)) ==
146 SIAtomicAddrSpace::NONE) {
147 this->Scope = std::min(Scope, SIAtomicScope::WORKGROUP);
148 }
else if ((InstrAddrSpace &
149 ~(SIAtomicAddrSpace::SCRATCH | SIAtomicAddrSpace::LDS |
150 SIAtomicAddrSpace::GDS)) == SIAtomicAddrSpace::NONE) {
151 this->Scope = std::min(Scope, SIAtomicScope::AGENT);
171 return FailureOrdering;
176 SIAtomicAddrSpace getInstrAddrSpace()
const {
177 return InstrAddrSpace;
182 SIAtomicAddrSpace getOrderingAddrSpace()
const {
183 return OrderingAddrSpace;
188 bool getIsCrossAddressSpaceOrdering()
const {
189 return IsCrossAddressSpaceOrdering;
194 bool isVolatile()
const {
200 bool isNonTemporal()
const {
201 return IsNonTemporal;
206 bool isAtomic()
const {
207 return Ordering != AtomicOrdering::NotAtomic;
212class SIMemOpAccess final {
218 const char *Msg)
const;
224 std::optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>>
225 toSIAtomicScope(
SyncScope::ID SSID, SIAtomicAddrSpace InstrAddrSpace)
const;
228 SIAtomicAddrSpace toSIAtomicAddrSpace(
unsigned AS)
const;
232 std::optional<SIMemOpInfo>
241 std::optional<SIMemOpInfo>
246 std::optional<SIMemOpInfo>
251 std::optional<SIMemOpInfo>
256 std::optional<SIMemOpInfo>
260class SICacheControl {
284 static std::unique_ptr<SICacheControl> create(
const GCNSubtarget &ST);
291 SIAtomicAddrSpace AddrSpace)
const = 0;
298 SIAtomicAddrSpace AddrSpace)
const = 0;
305 SIAtomicAddrSpace AddrSpace)
const = 0;
311 SIAtomicAddrSpace AddrSpace,
312 SIMemOp
Op,
bool IsVolatile,
313 bool IsNonTemporal)
const = 0;
324 SIAtomicAddrSpace AddrSpace,
326 bool IsCrossAddrSpaceOrdering,
327 Position Pos)
const = 0;
336 SIAtomicAddrSpace AddrSpace,
337 Position Pos)
const = 0;
347 SIAtomicAddrSpace AddrSpace,
348 bool IsCrossAddrSpaceOrdering,
349 Position Pos)
const = 0;
352 virtual ~SICacheControl() =
default;
354 virtual bool tryForceStoreSC0SC1(
const SIMemOpInfo &MOI,
360class SIGfx6CacheControl :
public SICacheControl {
377 SIGfx6CacheControl(
const GCNSubtarget &ST) : SICacheControl(
ST) {}
381 SIAtomicAddrSpace AddrSpace)
const override;
385 SIAtomicAddrSpace AddrSpace)
const override;
389 SIAtomicAddrSpace AddrSpace)
const override;
392 SIAtomicAddrSpace AddrSpace, SIMemOp
Op,
394 bool IsNonTemporal)
const override;
398 SIAtomicAddrSpace AddrSpace,
400 bool IsCrossAddrSpaceOrdering,
401 Position Pos)
const override;
405 SIAtomicAddrSpace AddrSpace,
406 Position Pos)
const override;
410 SIAtomicAddrSpace AddrSpace,
411 bool IsCrossAddrSpaceOrdering,
412 Position Pos)
const override;
415class SIGfx7CacheControl :
public SIGfx6CacheControl {
418 SIGfx7CacheControl(
const GCNSubtarget &ST) : SIGfx6CacheControl(
ST) {}
422 SIAtomicAddrSpace AddrSpace,
423 Position Pos)
const override;
427class SIGfx90ACacheControl :
public SIGfx7CacheControl {
430 SIGfx90ACacheControl(
const GCNSubtarget &ST) : SIGfx7CacheControl(
ST) {}
434 SIAtomicAddrSpace AddrSpace)
const override;
438 SIAtomicAddrSpace AddrSpace)
const override;
442 SIAtomicAddrSpace AddrSpace)
const override;
445 SIAtomicAddrSpace AddrSpace, SIMemOp
Op,
447 bool IsNonTemporal)
const override;
451 SIAtomicAddrSpace AddrSpace,
453 bool IsCrossAddrSpaceOrdering,
454 Position Pos)
const override;
458 SIAtomicAddrSpace AddrSpace,
459 Position Pos)
const override;
463 SIAtomicAddrSpace AddrSpace,
464 bool IsCrossAddrSpaceOrdering,
465 Position Pos)
const override;
468class SIGfx940CacheControl :
public SIGfx90ACacheControl {
491 SIGfx940CacheControl(
const GCNSubtarget &ST) : SIGfx90ACacheControl(
ST) {};
495 SIAtomicAddrSpace AddrSpace)
const override;
499 SIAtomicAddrSpace AddrSpace)
const override;
503 SIAtomicAddrSpace AddrSpace)
const override;
506 SIAtomicAddrSpace AddrSpace, SIMemOp
Op,
508 bool IsNonTemporal)
const override;
511 SIAtomicAddrSpace AddrSpace, Position Pos)
const override;
514 SIAtomicAddrSpace AddrSpace,
bool IsCrossAddrSpaceOrdering,
515 Position Pos)
const override;
517 bool tryForceStoreSC0SC1(
const SIMemOpInfo &MOI,
519 bool Changed =
false;
520 if (
ST.hasForceStoreSC0SC1() &&
521 (MOI.getInstrAddrSpace() & (SIAtomicAddrSpace::SCRATCH |
522 SIAtomicAddrSpace::GLOBAL |
523 SIAtomicAddrSpace::OTHER)) !=
524 SIAtomicAddrSpace::NONE) {
525 Changed |= enableSC0Bit(
MI);
526 Changed |= enableSC1Bit(
MI);
532class SIGfx10CacheControl :
public SIGfx7CacheControl {
543 SIGfx10CacheControl(
const GCNSubtarget &ST) : SIGfx7CacheControl(
ST) {}
547 SIAtomicAddrSpace AddrSpace)
const override;
550 SIAtomicAddrSpace AddrSpace, SIMemOp
Op,
552 bool IsNonTemporal)
const override;
556 SIAtomicAddrSpace AddrSpace,
558 bool IsCrossAddrSpaceOrdering,
559 Position Pos)
const override;
563 SIAtomicAddrSpace AddrSpace,
564 Position Pos)
const override;
567class SIGfx11CacheControl :
public SIGfx10CacheControl {
569 SIGfx11CacheControl(
const GCNSubtarget &ST) : SIGfx10CacheControl(
ST) {}
573 SIAtomicAddrSpace AddrSpace)
const override;
576 SIAtomicAddrSpace AddrSpace, SIMemOp
Op,
578 bool IsNonTemporal)
const override;
585 std::unique_ptr<SICacheControl>
CC =
nullptr;
588 std::list<MachineBasicBlock::iterator> AtomicPseudoMIs;
598 bool removeAtomicPseudoMIs();
602 bool expandLoad(
const SIMemOpInfo &MOI,
606 bool expandStore(
const SIMemOpInfo &MOI,
610 bool expandAtomicFence(
const SIMemOpInfo &MOI,
614 bool expandAtomicCmpxchgOrRmw(
const SIMemOpInfo &MOI,
637 const char *Msg)
const {
638 const Function &
Func =
MI->getParent()->getParent()->getFunction();
640 Func.getContext().diagnose(Diag);
643std::optional<std::tuple<SIAtomicScope, SIAtomicAddrSpace, bool>>
645 SIAtomicAddrSpace InstrAddrSpace)
const {
647 return std::tuple(SIAtomicScope::SYSTEM, SIAtomicAddrSpace::ATOMIC,
true);
648 if (SSID == MMI->getAgentSSID())
649 return std::tuple(SIAtomicScope::AGENT, SIAtomicAddrSpace::ATOMIC,
true);
650 if (SSID == MMI->getWorkgroupSSID())
651 return std::tuple(SIAtomicScope::WORKGROUP, SIAtomicAddrSpace::ATOMIC,
653 if (SSID == MMI->getWavefrontSSID())
654 return std::tuple(SIAtomicScope::WAVEFRONT, SIAtomicAddrSpace::ATOMIC,
657 return std::tuple(SIAtomicScope::SINGLETHREAD, SIAtomicAddrSpace::ATOMIC,
659 if (SSID == MMI->getSystemOneAddressSpaceSSID())
660 return std::tuple(SIAtomicScope::SYSTEM,
661 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace,
false);
662 if (SSID == MMI->getAgentOneAddressSpaceSSID())
663 return std::tuple(SIAtomicScope::AGENT,
664 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace,
false);
665 if (SSID == MMI->getWorkgroupOneAddressSpaceSSID())
666 return std::tuple(SIAtomicScope::WORKGROUP,
667 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace,
false);
668 if (SSID == MMI->getWavefrontOneAddressSpaceSSID())
669 return std::tuple(SIAtomicScope::WAVEFRONT,
670 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace,
false);
671 if (SSID == MMI->getSingleThreadOneAddressSpaceSSID())
672 return std::tuple(SIAtomicScope::SINGLETHREAD,
673 SIAtomicAddrSpace::ATOMIC & InstrAddrSpace,
false);
677SIAtomicAddrSpace SIMemOpAccess::toSIAtomicAddrSpace(
unsigned AS)
const {
679 return SIAtomicAddrSpace::FLAT;
681 return SIAtomicAddrSpace::GLOBAL;
683 return SIAtomicAddrSpace::LDS;
685 return SIAtomicAddrSpace::SCRATCH;
687 return SIAtomicAddrSpace::GDS;
689 return SIAtomicAddrSpace::OTHER;
696std::optional<SIMemOpInfo> SIMemOpAccess::constructFromMIWithMMO(
698 assert(
MI->getNumMemOperands() > 0);
703 SIAtomicAddrSpace InstrAddrSpace = SIAtomicAddrSpace::NONE;
704 bool IsNonTemporal =
true;
709 for (
const auto &MMO :
MI->memoperands()) {
710 IsNonTemporal &= MMO->isNonTemporal();
713 toSIAtomicAddrSpace(MMO->getPointerInfo().getAddrSpace());
715 if (OpOrdering != AtomicOrdering::NotAtomic) {
716 const auto &IsSyncScopeInclusion =
717 MMI->isSyncScopeInclusion(SSID, MMO->getSyncScopeID());
718 if (!IsSyncScopeInclusion) {
719 reportUnsupported(
MI,
720 "Unsupported non-inclusive atomic synchronization scope");
724 SSID = *IsSyncScopeInclusion ? SSID : MMO->getSyncScopeID();
726 assert(MMO->getFailureOrdering() != AtomicOrdering::Release &&
727 MMO->getFailureOrdering() != AtomicOrdering::AcquireRelease);
733 SIAtomicScope
Scope = SIAtomicScope::NONE;
734 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE;
735 bool IsCrossAddressSpaceOrdering =
false;
736 if (Ordering != AtomicOrdering::NotAtomic) {
737 auto ScopeOrNone = toSIAtomicScope(SSID, InstrAddrSpace);
739 reportUnsupported(
MI,
"Unsupported atomic synchronization scope");
742 std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) =
744 if ((OrderingAddrSpace == SIAtomicAddrSpace::NONE) ||
745 ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != OrderingAddrSpace) ||
746 ((InstrAddrSpace & SIAtomicAddrSpace::ATOMIC) == SIAtomicAddrSpace::NONE)) {
747 reportUnsupported(
MI,
"Unsupported atomic address space");
751 return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, InstrAddrSpace,
752 IsCrossAddressSpaceOrdering, FailureOrdering, IsVolatile,
756std::optional<SIMemOpInfo>
760 if (!(
MI->mayLoad() && !
MI->mayStore()))
764 if (
MI->getNumMemOperands() == 0)
765 return SIMemOpInfo();
767 return constructFromMIWithMMO(
MI);
770std::optional<SIMemOpInfo>
774 if (!(!
MI->mayLoad() &&
MI->mayStore()))
778 if (
MI->getNumMemOperands() == 0)
779 return SIMemOpInfo();
781 return constructFromMIWithMMO(
MI);
784std::optional<SIMemOpInfo>
788 if (
MI->getOpcode() != AMDGPU::ATOMIC_FENCE)
795 auto ScopeOrNone = toSIAtomicScope(SSID, SIAtomicAddrSpace::ATOMIC);
797 reportUnsupported(
MI,
"Unsupported atomic synchronization scope");
801 SIAtomicScope
Scope = SIAtomicScope::NONE;
802 SIAtomicAddrSpace OrderingAddrSpace = SIAtomicAddrSpace::NONE;
803 bool IsCrossAddressSpaceOrdering =
false;
804 std::tie(Scope, OrderingAddrSpace, IsCrossAddressSpaceOrdering) =
807 if ((OrderingAddrSpace == SIAtomicAddrSpace::NONE) ||
808 ((OrderingAddrSpace & SIAtomicAddrSpace::ATOMIC) != OrderingAddrSpace)) {
809 reportUnsupported(
MI,
"Unsupported atomic address space");
813 return SIMemOpInfo(Ordering, Scope, OrderingAddrSpace, SIAtomicAddrSpace::ATOMIC,
814 IsCrossAddressSpaceOrdering, AtomicOrdering::NotAtomic);
817std::optional<SIMemOpInfo> SIMemOpAccess::getAtomicCmpxchgOrRmwInfo(
821 if (!(
MI->mayLoad() &&
MI->mayStore()))
825 if (
MI->getNumMemOperands() == 0)
826 return SIMemOpInfo();
828 return constructFromMIWithMMO(
MI);
832 TII =
ST.getInstrInfo();
848std::unique_ptr<SICacheControl> SICacheControl::create(
const GCNSubtarget &ST) {
850 if (
ST.hasGFX940Insts())
851 return std::make_unique<SIGfx940CacheControl>(ST);
852 if (
ST.hasGFX90AInsts())
853 return std::make_unique<SIGfx90ACacheControl>(ST);
855 return std::make_unique<SIGfx6CacheControl>(ST);
857 return std::make_unique<SIGfx7CacheControl>(ST);
859 return std::make_unique<SIGfx10CacheControl>(ST);
860 return std::make_unique<SIGfx11CacheControl>(ST);
863bool SIGfx6CacheControl::enableLoadCacheBypass(
866 SIAtomicAddrSpace AddrSpace)
const {
868 bool Changed =
false;
870 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
872 case SIAtomicScope::SYSTEM:
873 case SIAtomicScope::AGENT:
876 Changed |= enableGLCBit(
MI);
878 case SIAtomicScope::WORKGROUP:
879 case SIAtomicScope::WAVEFRONT:
880 case SIAtomicScope::SINGLETHREAD:
898bool SIGfx6CacheControl::enableStoreCacheBypass(
901 SIAtomicAddrSpace AddrSpace)
const {
903 bool Changed =
false;
911bool SIGfx6CacheControl::enableRMWCacheBypass(
914 SIAtomicAddrSpace AddrSpace)
const {
916 bool Changed =
false;
926bool SIGfx6CacheControl::enableVolatileAndOrNonTemporal(
928 bool IsVolatile,
bool IsNonTemporal)
const {
938 assert(
Op == SIMemOp::LOAD ||
Op == SIMemOp::STORE);
940 bool Changed =
false;
946 if (
Op == SIMemOp::LOAD)
947 Changed |= enableGLCBit(
MI);
954 Changed |= insertWait(
MI, SIAtomicScope::SYSTEM, AddrSpace,
Op,
false,
963 Changed |= enableGLCBit(
MI);
964 Changed |= enableSLCBit(
MI);
973 SIAtomicAddrSpace AddrSpace,
975 bool IsCrossAddrSpaceOrdering,
976 Position Pos)
const {
977 bool Changed =
false;
982 if (Pos == Position::AFTER)
986 bool LGKMCnt =
false;
988 if ((AddrSpace & (SIAtomicAddrSpace::GLOBAL | SIAtomicAddrSpace::SCRATCH)) !=
989 SIAtomicAddrSpace::NONE) {
991 case SIAtomicScope::SYSTEM:
992 case SIAtomicScope::AGENT:
995 case SIAtomicScope::WORKGROUP:
996 case SIAtomicScope::WAVEFRONT:
997 case SIAtomicScope::SINGLETHREAD:
1006 if ((AddrSpace & SIAtomicAddrSpace::LDS) != SIAtomicAddrSpace::NONE) {
1008 case SIAtomicScope::SYSTEM:
1009 case SIAtomicScope::AGENT:
1010 case SIAtomicScope::WORKGROUP:
1017 LGKMCnt |= IsCrossAddrSpaceOrdering;
1019 case SIAtomicScope::WAVEFRONT:
1020 case SIAtomicScope::SINGLETHREAD:
1029 if ((AddrSpace & SIAtomicAddrSpace::GDS) != SIAtomicAddrSpace::NONE) {
1031 case SIAtomicScope::SYSTEM:
1032 case SIAtomicScope::AGENT:
1039 LGKMCnt |= IsCrossAddrSpaceOrdering;
1041 case SIAtomicScope::WORKGROUP:
1042 case SIAtomicScope::WAVEFRONT:
1043 case SIAtomicScope::SINGLETHREAD:
1052 if (VMCnt || LGKMCnt) {
1053 unsigned WaitCntImmediate =
1062 if (Pos == Position::AFTER)
1069 SIAtomicScope Scope,
1070 SIAtomicAddrSpace AddrSpace,
1071 Position Pos)
const {
1072 if (!InsertCacheInv)
1075 bool Changed =
false;
1080 if (Pos == Position::AFTER)
1083 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1085 case SIAtomicScope::SYSTEM:
1086 case SIAtomicScope::AGENT:
1090 case SIAtomicScope::WORKGROUP:
1091 case SIAtomicScope::WAVEFRONT:
1092 case SIAtomicScope::SINGLETHREAD:
1107 if (Pos == Position::AFTER)
1114 SIAtomicScope Scope,
1115 SIAtomicAddrSpace AddrSpace,
1116 bool IsCrossAddrSpaceOrdering,
1117 Position Pos)
const {
1118 return insertWait(
MI, Scope, AddrSpace, SIMemOp::LOAD | SIMemOp::STORE,
1119 IsCrossAddrSpaceOrdering, Pos);
1123 SIAtomicScope Scope,
1124 SIAtomicAddrSpace AddrSpace,
1125 Position Pos)
const {
1126 if (!InsertCacheInv)
1129 bool Changed =
false;
1137 ? AMDGPU::BUFFER_WBINVL1
1138 : AMDGPU::BUFFER_WBINVL1_VOL;
1140 if (Pos == Position::AFTER)
1143 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1145 case SIAtomicScope::SYSTEM:
1146 case SIAtomicScope::AGENT:
1150 case SIAtomicScope::WORKGROUP:
1151 case SIAtomicScope::WAVEFRONT:
1152 case SIAtomicScope::SINGLETHREAD:
1167 if (Pos == Position::AFTER)
1173bool SIGfx90ACacheControl::enableLoadCacheBypass(
1175 SIAtomicScope Scope,
1176 SIAtomicAddrSpace AddrSpace)
const {
1178 bool Changed =
false;
1180 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1182 case SIAtomicScope::SYSTEM:
1183 case SIAtomicScope::AGENT:
1186 Changed |= enableGLCBit(
MI);
1188 case SIAtomicScope::WORKGROUP:
1193 if (
ST.isTgSplitEnabled())
1194 Changed |= enableGLCBit(
MI);
1196 case SIAtomicScope::WAVEFRONT:
1197 case SIAtomicScope::SINGLETHREAD:
1215bool SIGfx90ACacheControl::enableStoreCacheBypass(
1217 SIAtomicScope Scope,
1218 SIAtomicAddrSpace AddrSpace)
const {
1220 bool Changed =
false;
1222 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1224 case SIAtomicScope::SYSTEM:
1225 case SIAtomicScope::AGENT:
1229 case SIAtomicScope::WORKGROUP:
1230 case SIAtomicScope::WAVEFRONT:
1231 case SIAtomicScope::SINGLETHREAD:
1250bool SIGfx90ACacheControl::enableRMWCacheBypass(
1252 SIAtomicScope Scope,
1253 SIAtomicAddrSpace AddrSpace)
const {
1255 bool Changed =
false;
1257 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1259 case SIAtomicScope::SYSTEM:
1260 case SIAtomicScope::AGENT:
1265 case SIAtomicScope::WORKGROUP:
1266 case SIAtomicScope::WAVEFRONT:
1267 case SIAtomicScope::SINGLETHREAD:
1278bool SIGfx90ACacheControl::enableVolatileAndOrNonTemporal(
1280 bool IsVolatile,
bool IsNonTemporal)
const {
1290 assert(
Op == SIMemOp::LOAD ||
Op == SIMemOp::STORE);
1292 bool Changed =
false;
1298 if (
Op == SIMemOp::LOAD)
1299 Changed |= enableGLCBit(
MI);
1306 Changed |= insertWait(
MI, SIAtomicScope::SYSTEM, AddrSpace,
Op,
false,
1312 if (IsNonTemporal) {
1315 Changed |= enableGLCBit(
MI);
1316 Changed |= enableSLCBit(
MI);
1324 SIAtomicScope Scope,
1325 SIAtomicAddrSpace AddrSpace,
1327 bool IsCrossAddrSpaceOrdering,
1328 Position Pos)
const {
1329 if (
ST.isTgSplitEnabled()) {
1337 if (((AddrSpace & (SIAtomicAddrSpace::GLOBAL | SIAtomicAddrSpace::SCRATCH |
1338 SIAtomicAddrSpace::GDS)) != SIAtomicAddrSpace::NONE) &&
1339 (Scope == SIAtomicScope::WORKGROUP)) {
1341 Scope = SIAtomicScope::AGENT;
1345 AddrSpace &= ~SIAtomicAddrSpace
::LDS;
1347 return SIGfx7CacheControl::insertWait(
MI, Scope, AddrSpace,
Op,
1348 IsCrossAddrSpaceOrdering, Pos);
1352 SIAtomicScope Scope,
1353 SIAtomicAddrSpace AddrSpace,
1354 Position Pos)
const {
1355 if (!InsertCacheInv)
1358 bool Changed =
false;
1363 if (Pos == Position::AFTER)
1366 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1368 case SIAtomicScope::SYSTEM:
1380 case SIAtomicScope::AGENT:
1383 case SIAtomicScope::WORKGROUP:
1388 if (
ST.isTgSplitEnabled()) {
1390 Scope = SIAtomicScope::AGENT;
1393 case SIAtomicScope::WAVEFRONT:
1394 case SIAtomicScope::SINGLETHREAD:
1409 if (Pos == Position::AFTER)
1412 Changed |= SIGfx7CacheControl::insertAcquire(
MI, Scope, AddrSpace, Pos);
1418 SIAtomicScope Scope,
1419 SIAtomicAddrSpace AddrSpace,
1420 bool IsCrossAddrSpaceOrdering,
1421 Position Pos)
const {
1422 bool Changed =
false;
1427 if (Pos == Position::AFTER)
1430 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1432 case SIAtomicScope::SYSTEM:
1446 case SIAtomicScope::AGENT:
1447 case SIAtomicScope::WORKGROUP:
1448 case SIAtomicScope::WAVEFRONT:
1449 case SIAtomicScope::SINGLETHREAD:
1457 if (Pos == Position::AFTER)
1461 SIGfx7CacheControl::insertRelease(
MI, Scope, AddrSpace,
1462 IsCrossAddrSpaceOrdering, Pos);
1467bool SIGfx940CacheControl::enableLoadCacheBypass(
1469 SIAtomicAddrSpace AddrSpace)
const {
1471 bool Changed =
false;
1473 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1475 case SIAtomicScope::SYSTEM:
1477 Changed |= enableSC0Bit(
MI);
1478 Changed |= enableSC1Bit(
MI);
1480 case SIAtomicScope::AGENT:
1482 Changed |= enableSC1Bit(
MI);
1484 case SIAtomicScope::WORKGROUP:
1490 Changed |= enableSC0Bit(
MI);
1492 case SIAtomicScope::WAVEFRONT:
1493 case SIAtomicScope::SINGLETHREAD:
1511bool SIGfx940CacheControl::enableStoreCacheBypass(
1513 SIAtomicScope Scope, SIAtomicAddrSpace AddrSpace)
const {
1515 bool Changed =
false;
1517 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1519 case SIAtomicScope::SYSTEM:
1521 Changed |= enableSC0Bit(
MI);
1522 Changed |= enableSC1Bit(
MI);
1524 case SIAtomicScope::AGENT:
1526 Changed |= enableSC1Bit(
MI);
1528 case SIAtomicScope::WORKGROUP:
1530 Changed |= enableSC0Bit(
MI);
1532 case SIAtomicScope::WAVEFRONT:
1533 case SIAtomicScope::SINGLETHREAD:
1551bool SIGfx940CacheControl::enableRMWCacheBypass(
1553 SIAtomicAddrSpace AddrSpace)
const {
1555 bool Changed =
false;
1557 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1559 case SIAtomicScope::SYSTEM:
1561 Changed |= enableSC1Bit(
MI);
1563 case SIAtomicScope::AGENT:
1564 case SIAtomicScope::WORKGROUP:
1565 case SIAtomicScope::WAVEFRONT:
1566 case SIAtomicScope::SINGLETHREAD:
1580bool SIGfx940CacheControl::enableVolatileAndOrNonTemporal(
1582 bool IsVolatile,
bool IsNonTemporal)
const {
1592 assert(
Op == SIMemOp::LOAD ||
Op == SIMemOp::STORE);
1594 bool Changed =
false;
1598 Changed |= enableSC0Bit(
MI);
1599 Changed |= enableSC1Bit(
MI);
1606 Changed |= insertWait(
MI, SIAtomicScope::SYSTEM, AddrSpace,
Op,
false,
1612 if (IsNonTemporal) {
1613 Changed |= enableNTBit(
MI);
1621 SIAtomicScope Scope,
1622 SIAtomicAddrSpace AddrSpace,
1623 Position Pos)
const {
1624 if (!InsertCacheInv)
1627 bool Changed =
false;
1632 if (Pos == Position::AFTER)
1635 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1637 case SIAtomicScope::SYSTEM:
1651 case SIAtomicScope::AGENT:
1664 case SIAtomicScope::WORKGROUP:
1669 if (
ST.isTgSplitEnabled()) {
1683 case SIAtomicScope::WAVEFRONT:
1684 case SIAtomicScope::SINGLETHREAD:
1700 if (Pos == Position::AFTER)
1707 SIAtomicScope Scope,
1708 SIAtomicAddrSpace AddrSpace,
1709 bool IsCrossAddrSpaceOrdering,
1710 Position Pos)
const {
1711 bool Changed =
false;
1716 if (Pos == Position::AFTER)
1719 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1721 case SIAtomicScope::SYSTEM:
1736 case SIAtomicScope::AGENT:
1746 case SIAtomicScope::WORKGROUP:
1747 case SIAtomicScope::WAVEFRONT:
1748 case SIAtomicScope::SINGLETHREAD:
1758 if (Pos == Position::AFTER)
1763 Changed |= insertWait(
MI, Scope, AddrSpace, SIMemOp::LOAD | SIMemOp::STORE,
1764 IsCrossAddrSpaceOrdering, Pos);
1769bool SIGfx10CacheControl::enableLoadCacheBypass(
1771 SIAtomicScope Scope,
1772 SIAtomicAddrSpace AddrSpace)
const {
1774 bool Changed =
false;
1776 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
1778 case SIAtomicScope::SYSTEM:
1779 case SIAtomicScope::AGENT:
1782 Changed |= enableGLCBit(
MI);
1783 Changed |= enableDLCBit(
MI);
1785 case SIAtomicScope::WORKGROUP:
1790 if (!
ST.isCuModeEnabled())
1791 Changed |= enableGLCBit(
MI);
1793 case SIAtomicScope::WAVEFRONT:
1794 case SIAtomicScope::SINGLETHREAD:
1812bool SIGfx10CacheControl::enableVolatileAndOrNonTemporal(
1814 bool IsVolatile,
bool IsNonTemporal)
const {
1825 assert(
Op == SIMemOp::LOAD ||
Op == SIMemOp::STORE);
1827 bool Changed =
false;
1833 if (
Op == SIMemOp::LOAD) {
1834 Changed |= enableGLCBit(
MI);
1835 Changed |= enableDLCBit(
MI);
1843 Changed |= insertWait(
MI, SIAtomicScope::SYSTEM, AddrSpace,
Op,
false,
1848 if (IsNonTemporal) {
1853 if (
Op == SIMemOp::STORE)
1854 Changed |= enableGLCBit(
MI);
1855 Changed |= enableSLCBit(
MI);
1864 SIAtomicScope Scope,
1865 SIAtomicAddrSpace AddrSpace,
1867 bool IsCrossAddrSpaceOrdering,
1868 Position Pos)
const {
1869 bool Changed =
false;
1874 if (Pos == Position::AFTER)
1879 bool LGKMCnt =
false;
1881 if ((AddrSpace & (SIAtomicAddrSpace::GLOBAL | SIAtomicAddrSpace::SCRATCH)) !=
1882 SIAtomicAddrSpace::NONE) {
1884 case SIAtomicScope::SYSTEM:
1885 case SIAtomicScope::AGENT:
1886 if ((
Op & SIMemOp::LOAD) != SIMemOp::NONE)
1888 if ((
Op & SIMemOp::STORE) != SIMemOp::NONE)
1891 case SIAtomicScope::WORKGROUP:
1897 if (!
ST.isCuModeEnabled()) {
1898 if ((
Op & SIMemOp::LOAD) != SIMemOp::NONE)
1900 if ((
Op & SIMemOp::STORE) != SIMemOp::NONE)
1904 case SIAtomicScope::WAVEFRONT:
1905 case SIAtomicScope::SINGLETHREAD:
1914 if ((AddrSpace & SIAtomicAddrSpace::LDS) != SIAtomicAddrSpace::NONE) {
1916 case SIAtomicScope::SYSTEM:
1917 case SIAtomicScope::AGENT:
1918 case SIAtomicScope::WORKGROUP:
1925 LGKMCnt |= IsCrossAddrSpaceOrdering;
1927 case SIAtomicScope::WAVEFRONT:
1928 case SIAtomicScope::SINGLETHREAD:
1937 if ((AddrSpace & SIAtomicAddrSpace::GDS) != SIAtomicAddrSpace::NONE) {
1939 case SIAtomicScope::SYSTEM:
1940 case SIAtomicScope::AGENT:
1947 LGKMCnt |= IsCrossAddrSpaceOrdering;
1949 case SIAtomicScope::WORKGROUP:
1950 case SIAtomicScope::WAVEFRONT:
1951 case SIAtomicScope::SINGLETHREAD:
1960 if (VMCnt || LGKMCnt) {
1961 unsigned WaitCntImmediate =
1977 if (Pos == Position::AFTER)
1984 SIAtomicScope Scope,
1985 SIAtomicAddrSpace AddrSpace,
1986 Position Pos)
const {
1987 if (!InsertCacheInv)
1990 bool Changed =
false;
1995 if (Pos == Position::AFTER)
1998 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
2000 case SIAtomicScope::SYSTEM:
2001 case SIAtomicScope::AGENT:
2006 case SIAtomicScope::WORKGROUP:
2011 if (!
ST.isCuModeEnabled()) {
2016 case SIAtomicScope::WAVEFRONT:
2017 case SIAtomicScope::SINGLETHREAD:
2032 if (Pos == Position::AFTER)
2038bool SIGfx11CacheControl::enableLoadCacheBypass(
2040 SIAtomicAddrSpace AddrSpace)
const {
2042 bool Changed =
false;
2044 if ((AddrSpace & SIAtomicAddrSpace::GLOBAL) != SIAtomicAddrSpace::NONE) {
2046 case SIAtomicScope::SYSTEM:
2047 case SIAtomicScope::AGENT:
2050 Changed |= enableGLCBit(
MI);
2052 case SIAtomicScope::WORKGROUP:
2057 if (!
ST.isCuModeEnabled())
2058 Changed |= enableGLCBit(
MI);
2060 case SIAtomicScope::WAVEFRONT:
2061 case SIAtomicScope::SINGLETHREAD:
2079bool SIGfx11CacheControl::enableVolatileAndOrNonTemporal(
2081 bool IsVolatile,
bool IsNonTemporal)
const {
2092 assert(
Op == SIMemOp::LOAD ||
Op == SIMemOp::STORE);
2094 bool Changed =
false;
2100 if (
Op == SIMemOp::LOAD)
2101 Changed |= enableGLCBit(
MI);
2104 Changed |= enableDLCBit(
MI);
2111 Changed |= insertWait(
MI, SIAtomicScope::SYSTEM, AddrSpace,
Op,
false,
2116 if (IsNonTemporal) {
2121 if (
Op == SIMemOp::STORE)
2122 Changed |= enableGLCBit(
MI);
2123 Changed |= enableSLCBit(
MI);
2126 Changed |= enableDLCBit(
MI);
2133bool SIMemoryLegalizer::removeAtomicPseudoMIs() {
2134 if (AtomicPseudoMIs.empty())
2137 for (
auto &
MI : AtomicPseudoMIs)
2138 MI->eraseFromParent();
2140 AtomicPseudoMIs.clear();
2144bool SIMemoryLegalizer::expandLoad(
const SIMemOpInfo &MOI,
2148 bool Changed =
false;
2150 if (MOI.isAtomic()) {
2151 if (MOI.getOrdering() == AtomicOrdering::Monotonic ||
2152 MOI.getOrdering() == AtomicOrdering::Acquire ||
2153 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
2154 Changed |=
CC->enableLoadCacheBypass(
MI, MOI.getScope(),
2155 MOI.getOrderingAddrSpace());
2158 if (MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
2159 Changed |=
CC->insertWait(
MI, MOI.getScope(),
2160 MOI.getOrderingAddrSpace(),
2161 SIMemOp::LOAD | SIMemOp::STORE,
2162 MOI.getIsCrossAddressSpaceOrdering(),
2165 if (MOI.getOrdering() == AtomicOrdering::Acquire ||
2166 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
2167 Changed |=
CC->insertWait(
MI, MOI.getScope(),
2168 MOI.getInstrAddrSpace(),
2170 MOI.getIsCrossAddressSpaceOrdering(),
2172 Changed |=
CC->insertAcquire(
MI, MOI.getScope(),
2173 MOI.getOrderingAddrSpace(),
2183 Changed |=
CC->enableVolatileAndOrNonTemporal(
MI, MOI.getInstrAddrSpace(),
2184 SIMemOp::LOAD, MOI.isVolatile(),
2185 MOI.isNonTemporal());
2189bool SIMemoryLegalizer::expandStore(
const SIMemOpInfo &MOI,
2193 bool Changed =
false;
2195 if (MOI.isAtomic()) {
2196 if (MOI.getOrdering() == AtomicOrdering::Monotonic ||
2197 MOI.getOrdering() == AtomicOrdering::Release ||
2198 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
2199 Changed |=
CC->enableStoreCacheBypass(
MI, MOI.getScope(),
2200 MOI.getOrderingAddrSpace());
2203 if (MOI.getOrdering() == AtomicOrdering::Release ||
2204 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
2205 Changed |=
CC->insertRelease(
MI, MOI.getScope(),
2206 MOI.getOrderingAddrSpace(),
2207 MOI.getIsCrossAddressSpaceOrdering(),
2216 Changed |=
CC->enableVolatileAndOrNonTemporal(
2217 MI, MOI.getInstrAddrSpace(), SIMemOp::STORE, MOI.isVolatile(),
2218 MOI.isNonTemporal());
2222bool SIMemoryLegalizer::expandAtomicFence(
const SIMemOpInfo &MOI,
2224 assert(
MI->getOpcode() == AMDGPU::ATOMIC_FENCE);
2226 AtomicPseudoMIs.push_back(
MI);
2227 bool Changed =
false;
2229 if (MOI.isAtomic()) {
2230 if (MOI.getOrdering() == AtomicOrdering::Acquire)
2231 Changed |=
CC->insertWait(
MI, MOI.getScope(), MOI.getOrderingAddrSpace(),
2232 SIMemOp::LOAD | SIMemOp::STORE,
2233 MOI.getIsCrossAddressSpaceOrdering(),
2236 if (MOI.getOrdering() == AtomicOrdering::Release ||
2237 MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
2238 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
2246 Changed |=
CC->insertRelease(
MI, MOI.getScope(),
2247 MOI.getOrderingAddrSpace(),
2248 MOI.getIsCrossAddressSpaceOrdering(),
2256 if (MOI.getOrdering() == AtomicOrdering::Acquire ||
2257 MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
2258 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent)
2259 Changed |=
CC->insertAcquire(
MI, MOI.getScope(),
2260 MOI.getOrderingAddrSpace(),
2269bool SIMemoryLegalizer::expandAtomicCmpxchgOrRmw(
const SIMemOpInfo &MOI,
2273 bool Changed =
false;
2275 if (MOI.isAtomic()) {
2276 if (MOI.getOrdering() == AtomicOrdering::Monotonic ||
2277 MOI.getOrdering() == AtomicOrdering::Acquire ||
2278 MOI.getOrdering() == AtomicOrdering::Release ||
2279 MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
2280 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent) {
2281 Changed |=
CC->enableRMWCacheBypass(
MI, MOI.getScope(),
2282 MOI.getInstrAddrSpace());
2285 if (MOI.getOrdering() == AtomicOrdering::Release ||
2286 MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
2287 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
2288 MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent)
2289 Changed |=
CC->insertRelease(
MI, MOI.getScope(),
2290 MOI.getOrderingAddrSpace(),
2291 MOI.getIsCrossAddressSpaceOrdering(),
2294 if (MOI.getOrdering() == AtomicOrdering::Acquire ||
2295 MOI.getOrdering() == AtomicOrdering::AcquireRelease ||
2296 MOI.getOrdering() == AtomicOrdering::SequentiallyConsistent ||
2297 MOI.getFailureOrdering() == AtomicOrdering::Acquire ||
2298 MOI.getFailureOrdering() == AtomicOrdering::SequentiallyConsistent) {
2299 Changed |=
CC->insertWait(
MI, MOI.getScope(),
2300 MOI.getInstrAddrSpace(),
2301 isAtomicRet(*
MI) ? SIMemOp::LOAD :
2303 MOI.getIsCrossAddressSpaceOrdering(),
2305 Changed |=
CC->insertAcquire(
MI, MOI.getScope(),
2306 MOI.getOrderingAddrSpace(),
2317 bool Changed =
false;
2319 SIMemOpAccess MOA(MF);
2322 for (
auto &
MBB : MF) {
2326 if (
MI->isBundle() &&
MI->mayLoadOrStore()) {
2329 I !=
E &&
I->isBundledWithPred(); ++
I) {
2330 I->unbundleFromPred();
2333 MO.setIsInternalRead(
false);
2336 MI->eraseFromParent();
2337 MI = II->getIterator();
2343 if (
const auto &MOI = MOA.getLoadInfo(
MI))
2344 Changed |= expandLoad(*MOI,
MI);
2345 else if (
const auto &MOI = MOA.getStoreInfo(
MI)) {
2346 Changed |= expandStore(*MOI,
MI);
2347 Changed |=
CC->tryForceStoreSC0SC1(*MOI,
MI);
2348 }
else if (
const auto &MOI = MOA.getAtomicFenceInfo(
MI))
2349 Changed |= expandAtomicFence(*MOI,
MI);
2350 else if (
const auto &MOI = MOA.getAtomicCmpxchgOrRmwInfo(
MI))
2351 Changed |= expandAtomicCmpxchgOrRmw(*MOI,
MI);
2355 Changed |= removeAtomicPseudoMIs();
2361char SIMemoryLegalizer::
ID = 0;
2365 return new SIMemoryLegalizer();
static std::optional< LoadInfo > getLoadInfo(const MachineInstr &MI)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Provides AMDGPU specific target descriptions.
AMDGPU Machine Module Info.
Atomic ordering constants.
#define LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE() pulls the operator overloads used by LLVM_MARK_AS_BITMASK_EN...
#define LLVM_MARK_AS_BITMASK_ENUM(LargestValue)
LLVM_MARK_AS_BITMASK_ENUM lets you opt in an individual enum type so you can perform bitwise operatio...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static cl::opt< bool > AmdgcnSkipCacheInvalidations("amdgcn-skip-cache-invalidations", cl::init(false), cl::Hidden, cl::desc("Use this to skip inserting cache invalidating instructions."))
static SPIRV::Scope::Scope getScope(SyncScope::ID Ord)
static const uint32_t IV[8]
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
Diagnostic information for unsupported feature in backend.
FunctionPass class - This class is used to implement most global optimizations.
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineModuleInfo & getMMI() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
Ty & getObjFileInfo()
Keep track of various per-module pieces of information for backends that would like to do so.
MachineOperand class - Representation of each machine instruction operand.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
static bool isAtomicRet(const MachineInstr &MI)
StringRef - Represent a constant reference to a string, i.e.
Iterator for intrusive lists based on ilist_node.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ REGION_ADDRESS
Address space for region memory. (GDS)
@ LOCAL_ADDRESS
Address space for local memory.
@ FLAT_ADDRESS
Address space for flat memory.
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
@ PRIVATE_ADDRESS
Address space for private memory.
constexpr char IsVolatile[]
Key for Kernel::Arg::Metadata::mIsVolatile.
IsaVersion getIsaVersion(StringRef GPU)
unsigned encodeWaitcnt(const IsaVersion &Version, unsigned Vmcnt, unsigned Expcnt, unsigned Lgkmcnt)
Encodes Vmcnt, Expcnt and Lgkmcnt into Waitcnt for given isa Version.
unsigned getVmcntBitMask(const IsaVersion &Version)
unsigned getLgkmcntBitMask(const IsaVersion &Version)
unsigned getExpcntBitMask(const IsaVersion &Version)
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ Undef
Value of the register doesn't matter.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
@ System
Synchronized with respect to all concurrently executing threads.
initializer< Ty > init(const Ty &Val)
Scope
Defines the scope in which this symbol should be visible: Default – Visible in the public interface o...
NodeAddr< FuncNode * > Func
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
char & SIMemoryLegalizerID
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
AtomicOrdering getMergedAtomicOrdering(AtomicOrdering AO, AtomicOrdering Other)
Return a single atomic ordering that is at least as strong as both the AO and Other orderings for an ...
AtomicOrdering
Atomic ordering for LLVM's memory model.
FunctionPass * createSIMemoryLegalizerPass()
Instruction set architecture version.