LLVM 24.0.0git
SILoadStoreOptimizer.cpp
Go to the documentation of this file.
1//===- SILoadStoreOptimizer.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass tries to fuse DS instructions with close by immediate offsets.
10// This will fuse operations such as
11// ds_read_b32 v0, v2 offset:16
12// ds_read_b32 v1, v2 offset:32
13// ==>
14// ds_read2_b32 v[0:1], v2, offset0:4 offset1:8
15//
16// The same is done for certain SMEM and VMEM opcodes, e.g.:
17// s_buffer_load_dword s4, s[0:3], 4
18// s_buffer_load_dword s5, s[0:3], 8
19// ==>
20// s_buffer_load_dwordx2 s[4:5], s[0:3], 4
21//
22// This pass also tries to promote constant offset to the immediate by
23// adjusting the base. It tries to use a base from the nearby instructions that
24// allows it to have a 13bit constant offset and then promotes the 13bit offset
25// to the immediate.
26// E.g.
27// s_movk_i32 s0, 0x1800
28// v_add_co_u32_e32 v0, vcc, s0, v2
29// v_addc_co_u32_e32 v1, vcc, 0, v6, vcc
30//
31// s_movk_i32 s0, 0x1000
32// v_add_co_u32_e32 v5, vcc, s0, v2
33// v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
34// global_load_dwordx2 v[5:6], v[5:6], off
35// global_load_dwordx2 v[0:1], v[0:1], off
36// =>
37// s_movk_i32 s0, 0x1000
38// v_add_co_u32_e32 v5, vcc, s0, v2
39// v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
40// global_load_dwordx2 v[5:6], v[5:6], off
41// global_load_dwordx2 v[0:1], v[5:6], off offset:2048
42//
43// Future improvements:
44//
45// - This is currently missing stores of constants because loading
46// the constant into the data register is placed between the stores, although
47// this is arguably a scheduling problem.
48//
49// - Live interval recomputing seems inefficient. This currently only matches
50// one pair, and recomputes live intervals and moves on to the next pair. It
51// would be better to compute a list of all merges that need to occur.
52//
53// - With a list of instructions to process, we can also merge more. If a
54// cluster of loads have offsets that are too large to fit in the 8-bit
55// offsets, but are close enough to fit in the 8 bits, we can add to the base
56// pointer and use the new reduced offsets.
57//
58//===----------------------------------------------------------------------===//
59
60#include "SILoadStoreOptimizer.h"
61#include "AMDGPU.h"
62#include "GCNSubtarget.h"
63#include "SIDefines.h"
64#include "llvm/Analysis/AliasAnalysis.h"
65#include "llvm/CodeGen/MachineFunctionPass.h"
66#include "llvm/InitializePasses.h"
67
68using namespace llvm;
69
70#define DEBUG_TYPE "si-load-store-opt"
71
72namespace {
73enum InstClassEnum {
74 UNKNOWN,
75 DS_READ,
76 DS_WRITE,
77 S_BUFFER_LOAD_IMM,
78 S_BUFFER_LOAD_SGPR_IMM,
79 S_LOAD_IMM,
80 BUFFER_LOAD,
81 BUFFER_STORE,
82 MIMG,
83 TBUFFER_LOAD,
84 TBUFFER_STORE,
85 GLOBAL_LOAD_SADDR,
86 GLOBAL_STORE_SADDR,
87 FLAT_LOAD,
88 FLAT_STORE,
89 FLAT_LOAD_SADDR,
90 FLAT_STORE_SADDR,
91 GLOBAL_LOAD, // GLOBAL_LOAD/GLOBAL_STORE are never used as the InstClass of
92 GLOBAL_STORE // any CombineInfo, they are only ever returned by
93 // getCommonInstClass.
94};
95
96struct AddressRegs {
97 unsigned char NumVAddrs = 0;
98 bool SBase = false;
99 bool SRsrc = false;
100 bool SOffset = false;
101 bool SAddr = false;
102 bool VAddr = false;
103 bool Addr = false;
104 bool SSamp = false;
105};
106
107// GFX10 image_sample instructions can have 12 vaddrs + srsrc + ssamp.
108const unsigned MaxAddressRegs = 12 + 1 + 1;
109
110class SILoadStoreOptimizer {
111 struct CombineInfo {
112 MachineBasicBlock::iterator I;
113 unsigned EltSize;
114 unsigned Offset;
115 unsigned Width;
116 unsigned Format;
117 unsigned BaseOff;
118 unsigned DMask;
119 InstClassEnum InstClass;
120 unsigned CPol = 0;
121 bool GDS = false;
122 const TargetRegisterClass *DataRC;
123 bool UseST64;
124 int AddrIdx[MaxAddressRegs];
125 const MachineOperand *AddrReg[MaxAddressRegs];
126 unsigned NumAddresses;
127 unsigned Order;
128
129 bool hasSameBaseAddress(const CombineInfo &CI) {
130 if (NumAddresses != CI.NumAddresses)
131 return false;
132
133 const MachineInstr &MI = *CI.I;
134 for (unsigned i = 0; i < NumAddresses; i++) {
135 const MachineOperand &AddrRegNext = MI.getOperand(AddrIdx[i]);
136
137 if (AddrReg[i]->isImm() || AddrRegNext.isImm()) {
138 if (AddrReg[i]->isImm() != AddrRegNext.isImm() ||
139 AddrReg[i]->getImm() != AddrRegNext.getImm()) {
140 return false;
141 }
142 continue;
143 }
144
145 // Check same base pointer. Be careful of subregisters, which can occur
146 // with vectors of pointers.
147 if (AddrReg[i]->getReg() != AddrRegNext.getReg() ||
148 AddrReg[i]->getSubReg() != AddrRegNext.getSubReg()) {
149 return false;
150 }
151 }
152 return true;
153 }
154
155 bool hasMergeableAddress(const MachineRegisterInfo &MRI) {
156 for (unsigned i = 0; i < NumAddresses; ++i) {
157 const MachineOperand *AddrOp = AddrReg[i];
158 // Immediates are always OK.
159 if (AddrOp->isImm())
160 continue;
161
162 // Don't try to merge addresses that aren't either immediates or registers.
163 // TODO: Should be possible to merge FrameIndexes and maybe some other
164 // non-register
165 if (!AddrOp->isReg())
166 return false;
167
168 // TODO: We should be able to merge instructions with other physical reg
169 // addresses too.
170 if (AddrOp->getReg().isPhysical() &&
171 AddrOp->getReg() != AMDGPU::SGPR_NULL)
172 return false;
173
174 // If an address has only one use then there will be no other
175 // instructions with the same address, so we can't merge this one.
176 if (MRI.hasOneNonDBGUse(AddrOp->getReg()))
177 return false;
178 }
179 return true;
180 }
181
182 void setMI(MachineBasicBlock::iterator MI, const SILoadStoreOptimizer &LSO);
183
184 // Compare by pointer order.
185 bool operator<(const CombineInfo& Other) const {
186 return (InstClass == MIMG) ? DMask < Other.DMask : Offset < Other.Offset;
187 }
188 };
189
190 struct BaseRegisters {
191 Register LoReg;
192 Register HiReg;
193
194 unsigned LoSubReg = 0;
195 unsigned HiSubReg = 0;
196 // True when using V_ADD_U64_e64 pattern
197 bool UseV64Pattern = false;
198 // True when using V_ADD_U32_e64 pattern (32-bit FLAT_SCRATCH vaddr)
199 bool UseScratchVGPRPattern = false;
200 };
201
202 struct MemAddress {
203 BaseRegisters Base;
204 int64_t Offset = 0;
205 };
206
207 using MemInfoMap = DenseMap<MachineInstr *, MemAddress>;
208
209private:
210 MachineFunction *MF = nullptr;
211 const GCNSubtarget *STM = nullptr;
212 const SIInstrInfo *TII = nullptr;
213 const SIRegisterInfo *TRI = nullptr;
214 MachineRegisterInfo *MRI = nullptr;
215 AliasAnalysis *AA = nullptr;
216 bool OptimizeAgain;
217
218 bool canSwapInstructions(const DenseSet<Register> &ARegDefs,
219 const DenseSet<Register> &ARegUses,
220 const MachineInstr &A, const MachineInstr &B) const;
221 static bool dmasksCanBeCombined(const CombineInfo &CI,
222 const SIInstrInfo &TII,
223 const CombineInfo &Paired);
224 static bool offsetsCanBeCombined(CombineInfo &CI, const GCNSubtarget &STI,
225 CombineInfo &Paired, bool Modify = false);
226 static bool widthsFit(const GCNSubtarget &STI, const CombineInfo &CI,
227 const CombineInfo &Paired);
228 unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired);
229 static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI,
230 const CombineInfo &Paired);
231 const TargetRegisterClass *
232 getTargetRegisterClass(const CombineInfo &CI,
233 const CombineInfo &Paired) const;
234 const TargetRegisterClass *getDataRegClass(const MachineInstr &MI) const;
235
236 CombineInfo *checkAndPrepareMerge(CombineInfo &CI, CombineInfo &Paired);
237
238 void copyToDestRegs(CombineInfo &CI, CombineInfo &Paired,
239 MachineBasicBlock::iterator InsertBefore,
240 const DebugLoc &DL, AMDGPU::OpName OpName,
241 Register DestReg) const;
242 Register copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
243 MachineBasicBlock::iterator InsertBefore,
244 const DebugLoc &DL, AMDGPU::OpName OpName) const;
245
246 unsigned read2Opcode(unsigned EltSize, bool GDS) const;
247 unsigned read2ST64Opcode(unsigned EltSize, bool GDS) const;
248 MachineBasicBlock::iterator
249 mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
250 MachineBasicBlock::iterator InsertBefore);
251
252 unsigned write2Opcode(unsigned EltSize, bool GDS) const;
253 unsigned write2ST64Opcode(unsigned EltSize, bool GDS) const;
254 unsigned getWrite2Opcode(const CombineInfo &CI) const;
255
256 MachineBasicBlock::iterator
257 mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired,
258 MachineBasicBlock::iterator InsertBefore);
259 MachineBasicBlock::iterator
260 mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
261 MachineBasicBlock::iterator InsertBefore);
262 MachineBasicBlock::iterator
263 mergeSMemLoadImmPair(CombineInfo &CI, CombineInfo &Paired,
264 MachineBasicBlock::iterator InsertBefore);
265 MachineBasicBlock::iterator
266 mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
267 MachineBasicBlock::iterator InsertBefore);
268 MachineBasicBlock::iterator
269 mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
270 MachineBasicBlock::iterator InsertBefore);
271 MachineBasicBlock::iterator
272 mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired,
273 MachineBasicBlock::iterator InsertBefore);
274 MachineBasicBlock::iterator
275 mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired,
276 MachineBasicBlock::iterator InsertBefore);
277 MachineBasicBlock::iterator
278 mergeFlatLoadPair(CombineInfo &CI, CombineInfo &Paired,
279 MachineBasicBlock::iterator InsertBefore);
280 MachineBasicBlock::iterator
281 mergeFlatStorePair(CombineInfo &CI, CombineInfo &Paired,
282 MachineBasicBlock::iterator InsertBefore);
283
284 void updateBaseAndOffset(MachineInstr &I, Register NewBase,
285 int32_t NewOffset) const;
286 void updateAsyncLDSAddress(MachineInstr &MI, int32_t OffsetDiff) const;
287 Register computeBase(MachineInstr &MI, const MemAddress &Addr) const;
288 MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
289 bool processBaseWithConstOffset64(MachineInstr *AddDef,
290 const MachineOperand &Base,
291 MemAddress &Addr) const;
292 bool processBaseWithConstOffset32(MachineInstr *AddDef,
293 MemAddress &Addr) const;
294 void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const;
295 /// Promotes constant offset to the immediate by adjusting the base. It
296 /// tries to use a base from the nearby instructions that allows it to have
297 /// a 13bit constant offset which gets promoted to the immediate.
298 bool promoteConstantOffsetToImm(MachineInstr &CI,
299 MemInfoMap &Visited,
300 SmallPtrSet<MachineInstr *, 4> &Promoted) const;
301 void addInstToMergeableList(const CombineInfo &CI,
302 std::list<std::list<CombineInfo> > &MergeableInsts) const;
303
304 std::pair<MachineBasicBlock::iterator, bool> collectMergeableInsts(
305 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End,
306 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList,
307 std::list<std::list<CombineInfo>> &MergeableInsts) const;
308
309 static MachineMemOperand *combineKnownAdjacentMMOs(const CombineInfo &CI,
310 const CombineInfo &Paired);
311
312 static InstClassEnum getCommonInstClass(const CombineInfo &CI,
313 const CombineInfo &Paired);
314
315 bool optimizeInstsWithSameBaseAddr(std::list<CombineInfo> &MergeList,
316 bool &OptimizeListAgain);
317 bool optimizeBlock(std::list<std::list<CombineInfo> > &MergeableInsts);
318
319public:
320 SILoadStoreOptimizer(AliasAnalysis *AA) : AA(AA) {}
321 bool run(MachineFunction &MF);
322};
323
324class SILoadStoreOptimizerLegacy : public MachineFunctionPass {
325public:
326 static char ID;
327
328 SILoadStoreOptimizerLegacy() : MachineFunctionPass(ID) {}
329
330 bool runOnMachineFunction(MachineFunction &MF) override;
331
332 StringRef getPassName() const override { return "SI Load Store Optimizer"; }
333
334 void getAnalysisUsage(AnalysisUsage &AU) const override {
335 AU.setPreservesCFG();
336 AU.addRequired<AAResultsWrapperPass>();
337
338 MachineFunctionPass::getAnalysisUsage(AU);
339 }
340
341 MachineFunctionProperties getRequiredProperties() const override {
342 return MachineFunctionProperties().setIsSSA();
343 }
344};
345
346static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
347 const unsigned Opc = MI.getOpcode();
348
349 if (TII.isMUBUF(Opc)) {
350 // FIXME: Handle d16 correctly
351 return AMDGPU::getMUBUFElements(Opc);
352 }
353 if (TII.isImage(MI)) {
354 uint64_t DMaskImm =
355 TII.getNamedOperand(MI, AMDGPU::OpName::dmask)->getImm();
356 return llvm::popcount(DMaskImm);
357 }
358 if (TII.isMTBUF(Opc)) {
359 return AMDGPU::getMTBUFElements(Opc);
360 }
361
362 switch (Opc) {
363 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
364 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
365 case AMDGPU::S_LOAD_DWORD_IMM:
366 case AMDGPU::GLOBAL_LOAD_DWORD:
367 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
368 case AMDGPU::GLOBAL_STORE_DWORD:
369 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
370 case AMDGPU::FLAT_LOAD_DWORD:
371 case AMDGPU::FLAT_STORE_DWORD:
372 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
373 case AMDGPU::FLAT_STORE_DWORD_SADDR:
374 return 1;
375 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
376 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
377 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
378 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
379 case AMDGPU::S_LOAD_DWORDX2_IMM:
380 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
381 case AMDGPU::GLOBAL_LOAD_DWORDX2:
382 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
383 case AMDGPU::GLOBAL_STORE_DWORDX2:
384 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
385 case AMDGPU::FLAT_LOAD_DWORDX2:
386 case AMDGPU::FLAT_STORE_DWORDX2:
387 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
388 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
389 return 2;
390 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
391 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
392 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
393 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
394 case AMDGPU::S_LOAD_DWORDX3_IMM:
395 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
396 case AMDGPU::GLOBAL_LOAD_DWORDX3:
397 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
398 case AMDGPU::GLOBAL_STORE_DWORDX3:
399 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
400 case AMDGPU::FLAT_LOAD_DWORDX3:
401 case AMDGPU::FLAT_STORE_DWORDX3:
402 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
403 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
404 return 3;
405 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
406 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
407 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
408 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
409 case AMDGPU::S_LOAD_DWORDX4_IMM:
410 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
411 case AMDGPU::GLOBAL_LOAD_DWORDX4:
412 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
413 case AMDGPU::GLOBAL_STORE_DWORDX4:
414 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
415 case AMDGPU::FLAT_LOAD_DWORDX4:
416 case AMDGPU::FLAT_STORE_DWORDX4:
417 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
418 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
419 return 4;
420 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
421 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
422 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
423 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
424 case AMDGPU::S_LOAD_DWORDX8_IMM:
425 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
426 return 8;
427 case AMDGPU::DS_READ_B32:
428 case AMDGPU::DS_READ_B32_gfx9:
429 case AMDGPU::DS_WRITE_B32:
430 case AMDGPU::DS_WRITE_B32_gfx9:
431 return 1;
432 case AMDGPU::DS_READ_B64:
433 case AMDGPU::DS_READ_B64_gfx9:
434 case AMDGPU::DS_WRITE_B64:
435 case AMDGPU::DS_WRITE_B64_gfx9:
436 return 2;
437 default:
438 return 0;
439 }
440}
441
442/// Maps instruction opcode to enum InstClassEnum.
443static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
444 switch (Opc) {
445 default:
446 if (TII.isMUBUF(Opc)) {
447 switch (AMDGPU::getMUBUFBaseOpcode(Opc)) {
448 default:
449 return UNKNOWN;
450 case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN:
451 case AMDGPU::BUFFER_LOAD_DWORD_BOTHEN_exact:
452 case AMDGPU::BUFFER_LOAD_DWORD_IDXEN:
453 case AMDGPU::BUFFER_LOAD_DWORD_IDXEN_exact:
454 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
455 case AMDGPU::BUFFER_LOAD_DWORD_OFFEN_exact:
456 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
457 case AMDGPU::BUFFER_LOAD_DWORD_OFFSET_exact:
458 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN:
459 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_BOTHEN_exact:
460 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN:
461 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_IDXEN_exact:
462 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN:
463 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFEN_exact:
464 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET:
465 case AMDGPU::BUFFER_LOAD_DWORD_VBUFFER_OFFSET_exact:
466 return BUFFER_LOAD;
467 case AMDGPU::BUFFER_STORE_DWORD_BOTHEN:
468 case AMDGPU::BUFFER_STORE_DWORD_BOTHEN_exact:
469 case AMDGPU::BUFFER_STORE_DWORD_IDXEN:
470 case AMDGPU::BUFFER_STORE_DWORD_IDXEN_exact:
471 case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
472 case AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact:
473 case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
474 case AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact:
475 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN:
476 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_BOTHEN_exact:
477 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN:
478 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_IDXEN_exact:
479 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN:
480 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFEN_exact:
481 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET:
482 case AMDGPU::BUFFER_STORE_DWORD_VBUFFER_OFFSET_exact:
483 return BUFFER_STORE;
484 }
485 }
486 if (TII.isImage(Opc)) {
487 // Ignore instructions encoded without vaddr.
488 if (!AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr) &&
489 !AMDGPU::hasNamedOperand(Opc, AMDGPU::OpName::vaddr0))
490 return UNKNOWN;
491 // Ignore BVH instructions
492 if (AMDGPU::getMIMGBaseOpcode(Opc)->BVH)
493 return UNKNOWN;
494 // TODO: Support IMAGE_GET_RESINFO and IMAGE_GET_LOD.
495 if (TII.get(Opc).mayStore() || !TII.get(Opc).mayLoad() ||
496 TII.isGather4(Opc))
497 return UNKNOWN;
498 return MIMG;
499 }
500 if (TII.isMTBUF(Opc)) {
501 switch (AMDGPU::getMTBUFBaseOpcode(Opc)) {
502 default:
503 return UNKNOWN;
504 case AMDGPU::TBUFFER_LOAD_FORMAT_X_BOTHEN:
505 case AMDGPU::TBUFFER_LOAD_FORMAT_X_BOTHEN_exact:
506 case AMDGPU::TBUFFER_LOAD_FORMAT_X_IDXEN:
507 case AMDGPU::TBUFFER_LOAD_FORMAT_X_IDXEN_exact:
508 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN:
509 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN_exact:
510 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET:
511 case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET_exact:
512 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_BOTHEN:
513 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_BOTHEN_exact:
514 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_IDXEN:
515 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_IDXEN_exact:
516 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFEN:
517 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFEN_exact:
518 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFSET:
519 case AMDGPU::TBUFFER_LOAD_FORMAT_X_VBUFFER_OFFSET_exact:
520 return TBUFFER_LOAD;
521 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN:
522 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN_exact:
523 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET:
524 case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET_exact:
525 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN:
526 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact:
527 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFSET:
528 case AMDGPU::TBUFFER_STORE_FORMAT_X_VBUFFER_OFFSET_exact:
529 return TBUFFER_STORE;
530 }
531 }
532 return UNKNOWN;
533 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
534 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
535 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
536 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
537 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
538 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
539 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
540 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
541 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
542 return S_BUFFER_LOAD_IMM;
543 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
544 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
545 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
546 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
547 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
548 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
549 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
550 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
551 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
552 return S_BUFFER_LOAD_SGPR_IMM;
553 case AMDGPU::S_LOAD_DWORD_IMM:
554 case AMDGPU::S_LOAD_DWORDX2_IMM:
555 case AMDGPU::S_LOAD_DWORDX3_IMM:
556 case AMDGPU::S_LOAD_DWORDX4_IMM:
557 case AMDGPU::S_LOAD_DWORDX8_IMM:
558 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
559 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
560 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
561 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
562 return S_LOAD_IMM;
563 case AMDGPU::DS_READ_B32:
564 case AMDGPU::DS_READ_B32_gfx9:
565 case AMDGPU::DS_READ_B64:
566 case AMDGPU::DS_READ_B64_gfx9:
567 return DS_READ;
568 case AMDGPU::DS_WRITE_B32:
569 case AMDGPU::DS_WRITE_B32_gfx9:
570 case AMDGPU::DS_WRITE_B64:
571 case AMDGPU::DS_WRITE_B64_gfx9:
572 return DS_WRITE;
573 case AMDGPU::GLOBAL_LOAD_DWORD:
574 case AMDGPU::GLOBAL_LOAD_DWORDX2:
575 case AMDGPU::GLOBAL_LOAD_DWORDX3:
576 case AMDGPU::GLOBAL_LOAD_DWORDX4:
577 case AMDGPU::FLAT_LOAD_DWORD:
578 case AMDGPU::FLAT_LOAD_DWORDX2:
579 case AMDGPU::FLAT_LOAD_DWORDX3:
580 case AMDGPU::FLAT_LOAD_DWORDX4:
581 return FLAT_LOAD;
582 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
583 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
584 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
585 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
586 return GLOBAL_LOAD_SADDR;
587 case AMDGPU::GLOBAL_STORE_DWORD:
588 case AMDGPU::GLOBAL_STORE_DWORDX2:
589 case AMDGPU::GLOBAL_STORE_DWORDX3:
590 case AMDGPU::GLOBAL_STORE_DWORDX4:
591 case AMDGPU::FLAT_STORE_DWORD:
592 case AMDGPU::FLAT_STORE_DWORDX2:
593 case AMDGPU::FLAT_STORE_DWORDX3:
594 case AMDGPU::FLAT_STORE_DWORDX4:
595 return FLAT_STORE;
596 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
597 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
598 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
599 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
600 return GLOBAL_STORE_SADDR;
601 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
602 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
603 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
604 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
605 return FLAT_LOAD_SADDR;
606 case AMDGPU::FLAT_STORE_DWORD_SADDR:
607 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
608 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
609 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
610 return FLAT_STORE_SADDR;
611 }
612}
613
614/// Determines instruction subclass from opcode. Only instructions
615/// of the same subclass can be merged together. The merged instruction may have
616/// a different subclass but must have the same class.
617static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) {
618 switch (Opc) {
619 default:
620 if (TII.isMUBUF(Opc))
621 return AMDGPU::getMUBUFBaseOpcode(Opc);
622 if (TII.isImage(Opc)) {
623 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
624 assert(Info);
625 return Info->BaseOpcode;
626 }
627 if (TII.isMTBUF(Opc))
628 return AMDGPU::getMTBUFBaseOpcode(Opc);
629 return -1;
630 case AMDGPU::DS_READ_B32:
631 case AMDGPU::DS_READ_B32_gfx9:
632 case AMDGPU::DS_READ_B64:
633 case AMDGPU::DS_READ_B64_gfx9:
634 case AMDGPU::DS_WRITE_B32:
635 case AMDGPU::DS_WRITE_B32_gfx9:
636 case AMDGPU::DS_WRITE_B64:
637 case AMDGPU::DS_WRITE_B64_gfx9:
638 return Opc;
639 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
640 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
641 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
642 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
643 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
644 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
645 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
646 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
647 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
648 return AMDGPU::S_BUFFER_LOAD_DWORD_IMM;
649 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
650 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
651 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
652 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
653 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
654 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
655 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
656 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
657 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
658 return AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM;
659 case AMDGPU::S_LOAD_DWORD_IMM:
660 case AMDGPU::S_LOAD_DWORDX2_IMM:
661 case AMDGPU::S_LOAD_DWORDX3_IMM:
662 case AMDGPU::S_LOAD_DWORDX4_IMM:
663 case AMDGPU::S_LOAD_DWORDX8_IMM:
664 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
665 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
666 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
667 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
668 return AMDGPU::S_LOAD_DWORD_IMM;
669 case AMDGPU::GLOBAL_LOAD_DWORD:
670 case AMDGPU::GLOBAL_LOAD_DWORDX2:
671 case AMDGPU::GLOBAL_LOAD_DWORDX3:
672 case AMDGPU::GLOBAL_LOAD_DWORDX4:
673 case AMDGPU::FLAT_LOAD_DWORD:
674 case AMDGPU::FLAT_LOAD_DWORDX2:
675 case AMDGPU::FLAT_LOAD_DWORDX3:
676 case AMDGPU::FLAT_LOAD_DWORDX4:
677 return AMDGPU::FLAT_LOAD_DWORD;
678 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
679 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
680 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
681 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
682 return AMDGPU::GLOBAL_LOAD_DWORD_SADDR;
683 case AMDGPU::GLOBAL_STORE_DWORD:
684 case AMDGPU::GLOBAL_STORE_DWORDX2:
685 case AMDGPU::GLOBAL_STORE_DWORDX3:
686 case AMDGPU::GLOBAL_STORE_DWORDX4:
687 case AMDGPU::FLAT_STORE_DWORD:
688 case AMDGPU::FLAT_STORE_DWORDX2:
689 case AMDGPU::FLAT_STORE_DWORDX3:
690 case AMDGPU::FLAT_STORE_DWORDX4:
691 return AMDGPU::FLAT_STORE_DWORD;
692 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
693 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
694 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
695 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
696 return AMDGPU::GLOBAL_STORE_DWORD_SADDR;
697 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
698 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
699 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
700 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
701 return AMDGPU::FLAT_LOAD_DWORD_SADDR;
702 case AMDGPU::FLAT_STORE_DWORD_SADDR:
703 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
704 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
705 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
706 return AMDGPU::FLAT_STORE_DWORD_SADDR;
707 }
708}
709
710// GLOBAL loads and stores are classified as FLAT initially. If both combined
711// instructions are FLAT GLOBAL adjust the class to GLOBAL_LOAD or GLOBAL_STORE.
712// If either or both instructions are non segment specific FLAT the resulting
713// combined operation will be FLAT, potentially promoting one of the GLOBAL
714// operations to FLAT.
715// For other instructions return the original unmodified class.
716InstClassEnum
717SILoadStoreOptimizer::getCommonInstClass(const CombineInfo &CI,
718 const CombineInfo &Paired) {
719 assert(CI.InstClass == Paired.InstClass);
720
721 if ((CI.InstClass == FLAT_LOAD || CI.InstClass == FLAT_STORE) &&
722 SIInstrInfo::isFLATGlobal(*CI.I) && SIInstrInfo::isFLATGlobal(*Paired.I))
723 return (CI.InstClass == FLAT_STORE) ? GLOBAL_STORE : GLOBAL_LOAD;
724
725 return CI.InstClass;
726}
727
728static AddressRegs getRegs(unsigned Opc, const SIInstrInfo &TII) {
729 AddressRegs Result;
730
731 if (TII.isMUBUF(Opc)) {
732 if (AMDGPU::getMUBUFHasVAddr(Opc))
733 Result.VAddr = true;
734 if (AMDGPU::getMUBUFHasSrsrc(Opc))
735 Result.SRsrc = true;
736 if (AMDGPU::getMUBUFHasSoffset(Opc))
737 Result.SOffset = true;
738
739 return Result;
740 }
741
742 if (TII.isImage(Opc)) {
743 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
744 if (VAddr0Idx >= 0) {
745 AMDGPU::OpName RsrcName =
746 TII.isMIMG(Opc) ? AMDGPU::OpName::srsrc : AMDGPU::OpName::rsrc;
747 int RsrcIdx = AMDGPU::getNamedOperandIdx(Opc, RsrcName);
748 Result.NumVAddrs = RsrcIdx - VAddr0Idx;
749 } else {
750 Result.VAddr = true;
751 }
752 Result.SRsrc = true;
753 const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
754 if (Info && AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode)->Sampler)
755 Result.SSamp = true;
756
757 return Result;
758 }
759 if (TII.isMTBUF(Opc)) {
760 if (AMDGPU::getMTBUFHasVAddr(Opc))
761 Result.VAddr = true;
762 if (AMDGPU::getMTBUFHasSrsrc(Opc))
763 Result.SRsrc = true;
764 if (AMDGPU::getMTBUFHasSoffset(Opc))
765 Result.SOffset = true;
766
767 return Result;
768 }
769
770 switch (Opc) {
771 default:
772 return Result;
773 case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR_IMM:
774 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM:
775 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM:
776 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM:
777 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM:
778 case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec:
779 case AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec:
780 case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec:
781 case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec:
782 Result.SOffset = true;
783 [[fallthrough]];
784 case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
785 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
786 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM:
787 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
788 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM:
789 case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec:
790 case AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec:
791 case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec:
792 case AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec:
793 case AMDGPU::S_LOAD_DWORD_IMM:
794 case AMDGPU::S_LOAD_DWORDX2_IMM:
795 case AMDGPU::S_LOAD_DWORDX3_IMM:
796 case AMDGPU::S_LOAD_DWORDX4_IMM:
797 case AMDGPU::S_LOAD_DWORDX8_IMM:
798 case AMDGPU::S_LOAD_DWORDX2_IMM_ec:
799 case AMDGPU::S_LOAD_DWORDX3_IMM_ec:
800 case AMDGPU::S_LOAD_DWORDX4_IMM_ec:
801 case AMDGPU::S_LOAD_DWORDX8_IMM_ec:
802 Result.SBase = true;
803 return Result;
804 case AMDGPU::DS_READ_B32:
805 case AMDGPU::DS_READ_B64:
806 case AMDGPU::DS_READ_B32_gfx9:
807 case AMDGPU::DS_READ_B64_gfx9:
808 case AMDGPU::DS_WRITE_B32:
809 case AMDGPU::DS_WRITE_B64:
810 case AMDGPU::DS_WRITE_B32_gfx9:
811 case AMDGPU::DS_WRITE_B64_gfx9:
812 Result.Addr = true;
813 return Result;
814 case AMDGPU::GLOBAL_LOAD_DWORD_SADDR:
815 case AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR:
816 case AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR:
817 case AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR:
818 case AMDGPU::GLOBAL_STORE_DWORD_SADDR:
819 case AMDGPU::GLOBAL_STORE_DWORDX2_SADDR:
820 case AMDGPU::GLOBAL_STORE_DWORDX3_SADDR:
821 case AMDGPU::GLOBAL_STORE_DWORDX4_SADDR:
822 case AMDGPU::FLAT_LOAD_DWORD_SADDR:
823 case AMDGPU::FLAT_LOAD_DWORDX2_SADDR:
824 case AMDGPU::FLAT_LOAD_DWORDX3_SADDR:
825 case AMDGPU::FLAT_LOAD_DWORDX4_SADDR:
826 case AMDGPU::FLAT_STORE_DWORD_SADDR:
827 case AMDGPU::FLAT_STORE_DWORDX2_SADDR:
828 case AMDGPU::FLAT_STORE_DWORDX3_SADDR:
829 case AMDGPU::FLAT_STORE_DWORDX4_SADDR:
830 Result.SAddr = true;
831 [[fallthrough]];
832 case AMDGPU::GLOBAL_LOAD_DWORD:
833 case AMDGPU::GLOBAL_LOAD_DWORDX2:
834 case AMDGPU::GLOBAL_LOAD_DWORDX3:
835 case AMDGPU::GLOBAL_LOAD_DWORDX4:
836 case AMDGPU::GLOBAL_STORE_DWORD:
837 case AMDGPU::GLOBAL_STORE_DWORDX2:
838 case AMDGPU::GLOBAL_STORE_DWORDX3:
839 case AMDGPU::GLOBAL_STORE_DWORDX4:
840 case AMDGPU::FLAT_LOAD_DWORD:
841 case AMDGPU::FLAT_LOAD_DWORDX2:
842 case AMDGPU::FLAT_LOAD_DWORDX3:
843 case AMDGPU::FLAT_LOAD_DWORDX4:
844 case AMDGPU::FLAT_STORE_DWORD:
845 case AMDGPU::FLAT_STORE_DWORDX2:
846 case AMDGPU::FLAT_STORE_DWORDX3:
847 case AMDGPU::FLAT_STORE_DWORDX4:
848 Result.VAddr = true;
849 return Result;
850 }
851}
852
853void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
854 const SILoadStoreOptimizer &LSO) {
855 I = MI;
856 unsigned Opc = MI->getOpcode();
857 InstClass = getInstClass(Opc, *LSO.TII);
858
859 if (InstClass == UNKNOWN)
860 return;
861
862 DataRC = LSO.getDataRegClass(*MI);
863
864 switch (InstClass) {
865 case DS_READ:
866 EltSize =
867 (Opc == AMDGPU::DS_READ_B64 || Opc == AMDGPU::DS_READ_B64_gfx9) ? 8
868 : 4;
869 break;
870 case DS_WRITE:
871 EltSize =
872 (Opc == AMDGPU::DS_WRITE_B64 || Opc == AMDGPU::DS_WRITE_B64_gfx9) ? 8
873 : 4;
874 break;
875 case S_BUFFER_LOAD_IMM:
876 case S_BUFFER_LOAD_SGPR_IMM:
877 case S_LOAD_IMM:
878 EltSize = AMDGPU::convertSMRDOffsetUnits(*LSO.STM, 4);
879 break;
880 default:
881 EltSize = 4;
882 break;
883 }
884
885 if (InstClass == MIMG) {
886 DMask = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::dmask)->getImm();
887 // Offset is not considered for MIMG instructions.
888 Offset = 0;
889 } else {
890 int OffsetIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::offset);
891 Offset = I->getOperand(OffsetIdx).getImm();
892 }
893
894 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
895 Format = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::format)->getImm();
896 const AMDGPU::GcnBufferFormatInfo *Info =
897 AMDGPU::getGcnBufferFormatInfo(Format, *LSO.STM);
898 EltSize = Info->BitsPerComp / 8;
899 }
900
901 Width = getOpcodeWidth(*I, *LSO.TII);
902
903 if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) {
904 Offset &= 0xffff;
905 GDS = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::gds)->getImm();
906 } else if (InstClass != MIMG) {
907 CPol = LSO.TII->getNamedOperand(*I, AMDGPU::OpName::cpol)->getImm();
908 }
909
910 AddressRegs Regs = getRegs(Opc, *LSO.TII);
911 bool isVIMAGEorVSAMPLE = LSO.TII->isVIMAGE(*I) || LSO.TII->isVSAMPLE(*I);
912
913 NumAddresses = 0;
914 for (unsigned J = 0; J < Regs.NumVAddrs; J++)
915 AddrIdx[NumAddresses++] =
916 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0) + J;
917 if (Regs.Addr)
918 AddrIdx[NumAddresses++] =
919 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::addr);
920 if (Regs.SBase)
921 AddrIdx[NumAddresses++] =
922 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sbase);
923 if (Regs.SRsrc)
924 AddrIdx[NumAddresses++] = AMDGPU::getNamedOperandIdx(
925 Opc, isVIMAGEorVSAMPLE ? AMDGPU::OpName::rsrc : AMDGPU::OpName::srsrc);
926 if (Regs.SOffset)
927 AddrIdx[NumAddresses++] =
928 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::soffset);
929 if (Regs.SAddr)
930 AddrIdx[NumAddresses++] =
931 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::saddr);
932 if (Regs.VAddr)
933 AddrIdx[NumAddresses++] =
934 AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr);
935 if (Regs.SSamp)
936 AddrIdx[NumAddresses++] = AMDGPU::getNamedOperandIdx(
937 Opc, isVIMAGEorVSAMPLE ? AMDGPU::OpName::samp : AMDGPU::OpName::ssamp);
938 assert(NumAddresses <= MaxAddressRegs);
939
940 for (unsigned J = 0; J < NumAddresses; J++)
941 AddrReg[J] = &I->getOperand(AddrIdx[J]);
942}
943
944} // end anonymous namespace.
945
946INITIALIZE_PASS_BEGIN(SILoadStoreOptimizerLegacy, DEBUG_TYPE,
947 "SI Load Store Optimizer", false, false)
948INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
949INITIALIZE_PASS_END(SILoadStoreOptimizerLegacy, DEBUG_TYPE,
950 "SI Load Store Optimizer", false, false)
951
952char SILoadStoreOptimizerLegacy::ID = 0;
953
954char &llvm::SILoadStoreOptimizerLegacyID = SILoadStoreOptimizerLegacy::ID;
955
956static void addDefsUsesToList(const MachineInstr &MI,
957 DenseSet<Register> &RegDefs,
958 DenseSet<Register> &RegUses) {
959 for (const auto &Op : MI.operands()) {
960 if (!Op.isReg())
961 continue;
962 if (Op.isDef())
963 RegDefs.insert(Op.getReg());
964 if (Op.readsReg())
965 RegUses.insert(Op.getReg());
966 }
967}
968
969bool SILoadStoreOptimizer::canSwapInstructions(
970 const DenseSet<Register> &ARegDefs, const DenseSet<Register> &ARegUses,
971 const MachineInstr &A, const MachineInstr &B) const {
972 if (A.mayLoadOrStore() && B.mayLoadOrStore() &&
973 (A.mayStore() || B.mayStore()) && A.mayAlias(AA, B, true))
974 return false;
975 for (const auto &BOp : B.operands()) {
976 if (!BOp.isReg())
977 continue;
978 if ((BOp.isDef() || BOp.readsReg()) && ARegDefs.contains(BOp.getReg()))
979 return false;
980 if (BOp.isDef() && ARegUses.contains(BOp.getReg()))
981 return false;
982 }
983 return true;
984}
985
986// Given that \p CI and \p Paired are adjacent memory operations produce a new
987// MMO for the combined operation with a new access size.
988MachineMemOperand *
989SILoadStoreOptimizer::combineKnownAdjacentMMOs(const CombineInfo &CI,
990 const CombineInfo &Paired) {
991 const MachineMemOperand *MMOa = *CI.I->memoperands_begin();
992 const MachineMemOperand *MMOb = *Paired.I->memoperands_begin();
993
994 unsigned Size = MMOa->getSize().getValue() + MMOb->getSize().getValue();
995
996 // A base pointer for the combined operation is the same as the leading
997 // operation's pointer.
998 if (Paired < CI)
999 std::swap(MMOa, MMOb);
1000
1001 MachinePointerInfo PtrInfo(MMOa->getPointerInfo());
1002 // If merging FLAT and GLOBAL set address space to FLAT.
1003 if (MMOb->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
1004 PtrInfo.AddrSpace = AMDGPUAS::FLAT_ADDRESS;
1005
1006 MachineFunction *MF = CI.I->getMF();
1007 return MF->getMachineMemOperand(MMOa, PtrInfo, Size);
1008}
1009
1010bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI,
1011 const SIInstrInfo &TII,
1012 const CombineInfo &Paired) {
1013 assert(CI.InstClass == MIMG);
1014
1015 // Ignore instructions with tfe/lwe set.
1016 const auto *TFEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::tfe);
1017 const auto *LWEOp = TII.getNamedOperand(*CI.I, AMDGPU::OpName::lwe);
1018
1019 if ((TFEOp && TFEOp->getImm()) || (LWEOp && LWEOp->getImm()))
1020 return false;
1021
1022 // Check other optional immediate operands for equality.
1023 AMDGPU::OpName OperandsToMatch[] = {
1024 AMDGPU::OpName::cpol, AMDGPU::OpName::d16, AMDGPU::OpName::unorm,
1025 AMDGPU::OpName::da, AMDGPU::OpName::r128, AMDGPU::OpName::a16,
1026 AMDGPU::OpName::dim};
1027
1028 for (AMDGPU::OpName op : OperandsToMatch) {
1029 int Idx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), op);
1030 if (AMDGPU::getNamedOperandIdx(Paired.I->getOpcode(), op) != Idx)
1031 return false;
1032 if (Idx != -1 &&
1033 CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm())
1034 return false;
1035 }
1036
1037 // Check DMask for overlaps.
1038 unsigned MaxMask = std::max(CI.DMask, Paired.DMask);
1039 unsigned MinMask = std::min(CI.DMask, Paired.DMask);
1040
1041 if (!MaxMask)
1042 return false;
1043
1044 unsigned AllowedBitsForMin = llvm::countr_zero(MaxMask);
1045 if ((1u << AllowedBitsForMin) <= MinMask)
1046 return false;
1047
1048 return true;
1049}
1050
1051static unsigned getBufferFormatWithCompCount(unsigned OldFormat,
1052 unsigned ComponentCount,
1053 const GCNSubtarget &STI) {
1054 if (ComponentCount > 4)
1055 return 0;
1056
1057 const llvm::AMDGPU::GcnBufferFormatInfo *OldFormatInfo =
1058 llvm::AMDGPU::getGcnBufferFormatInfo(OldFormat, STI);
1059 if (!OldFormatInfo)
1060 return 0;
1061
1062 const llvm::AMDGPU::GcnBufferFormatInfo *NewFormatInfo =
1063 llvm::AMDGPU::getGcnBufferFormatInfo(OldFormatInfo->BitsPerComp,
1064 ComponentCount,
1065 OldFormatInfo->NumFormat, STI);
1066
1067 if (!NewFormatInfo)
1068 return 0;
1069
1070 assert(NewFormatInfo->NumFormat == OldFormatInfo->NumFormat &&
1071 NewFormatInfo->BitsPerComp == OldFormatInfo->BitsPerComp);
1072
1073 return NewFormatInfo->Format;
1074}
1075
1076// Return the value in the inclusive range [Lo,Hi] that is aligned to the
1077// highest power of two. Note that the result is well defined for all inputs
1078// including corner cases like:
1079// - if Lo == Hi, return that value
1080// - if Lo == 0, return 0 (even though the "- 1" below underflows
1081// - if Lo > Hi, return 0 (as if the range wrapped around)
1085
1086bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI,
1087 const GCNSubtarget &STI,
1088 CombineInfo &Paired,
1089 bool Modify) {
1090 assert(CI.InstClass != MIMG);
1091
1092 // XXX - Would the same offset be OK? Is there any reason this would happen or
1093 // be useful?
1094 if (CI.Offset == Paired.Offset)
1095 return false;
1096
1097 if (CI.GDS != Paired.GDS)
1098 return false;
1099
1100 // This won't be valid if the offset isn't aligned.
1101 if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0))
1102 return false;
1103
1104 if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) {
1105
1106 const llvm::AMDGPU::GcnBufferFormatInfo *Info0 =
1107 llvm::AMDGPU::getGcnBufferFormatInfo(CI.Format, STI);
1108 const llvm::AMDGPU::GcnBufferFormatInfo *Info1 =
1109 llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI);
1110
1111 if (Info0->BitsPerComp != Info1->BitsPerComp ||
1112 Info0->NumFormat != Info1->NumFormat)
1113 return false;
1114
1115 // For 8-bit or 16-bit formats there is no 3-component variant.
1116 // If NumCombinedComponents is 3, try the 4-component format and use XYZ.
1117 // Example:
1118 // tbuffer_load_format_x + tbuffer_load_format_x + tbuffer_load_format_x
1119 // ==> tbuffer_load_format_xyz with format:[BUF_FMT_16_16_16_16_SNORM]
1120 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1121 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1122 NumCombinedComponents = 4;
1123
1124 if (getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, STI) ==
1125 0)
1126 return false;
1127
1128 // Merge only when the two access ranges are strictly back-to-back,
1129 // any gap or overlap can over-write data or leave holes.
1130 unsigned ElemIndex0 = CI.Offset / CI.EltSize;
1131 unsigned ElemIndex1 = Paired.Offset / Paired.EltSize;
1132 if (ElemIndex0 + CI.Width != ElemIndex1 &&
1133 ElemIndex1 + Paired.Width != ElemIndex0)
1134 return false;
1135
1136 // 1-byte formats require 1-byte alignment.
1137 // 2-byte formats require 2-byte alignment.
1138 // 4-byte and larger formats require 4-byte alignment.
1139 unsigned MergedBytes = CI.EltSize * NumCombinedComponents;
1140 Align RequiredAlign(std::min(MergedBytes, 4u));
1141
1142 // The merged access starts at the address of the leading access. The
1143 // leading access must already satisfy the rule above for its own size, so
1144 // its address is aligned to at least the component size.
1145 const CombineInfo &Leading = Paired < CI ? Paired : CI;
1146 Align KnownAlign =
1147 std::max(getSingleElement(Leading.I->memoperands())->getAlign(),
1148 Align(std::min(CI.EltSize, 4u)));
1149 return KnownAlign >= RequiredAlign;
1150 }
1151
1152 uint32_t EltOffset0 = CI.Offset / CI.EltSize;
1153 uint32_t EltOffset1 = Paired.Offset / CI.EltSize;
1154 CI.UseST64 = false;
1155 CI.BaseOff = 0;
1156
1157 // Handle all non-DS instructions.
1158 if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) {
1159 if (EltOffset0 + CI.Width != EltOffset1 &&
1160 EltOffset1 + Paired.Width != EltOffset0)
1161 return false;
1162 // Instructions with scale_offset modifier cannot be combined unless we
1163 // also generate a code to scale the offset and reset that bit.
1164 if (CI.CPol != Paired.CPol || (CI.CPol & AMDGPU::CPol::SCAL))
1165 return false;
1166 if (CI.InstClass == S_LOAD_IMM || CI.InstClass == S_BUFFER_LOAD_IMM ||
1167 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM) {
1168 // Reject cases like:
1169 // dword + dwordx2 -> dwordx3
1170 // dword + dwordx3 -> dwordx4
1171 // If we tried to combine these cases, we would fail to extract a subreg
1172 // for the result of the second load due to SGPR alignment requirements.
1173 if (CI.Width != Paired.Width &&
1174 (CI.Width < Paired.Width) == (CI.Offset < Paired.Offset))
1175 return false;
1176 }
1177 return true;
1178 }
1179
1180 // If the offset in elements doesn't fit in 8-bits, we might be able to use
1181 // the stride 64 versions.
1182 if ((EltOffset0 % 64 == 0) && (EltOffset1 % 64) == 0 &&
1183 isUInt<8>(EltOffset0 / 64) && isUInt<8>(EltOffset1 / 64)) {
1184 if (Modify) {
1185 CI.Offset = EltOffset0 / 64;
1186 Paired.Offset = EltOffset1 / 64;
1187 CI.UseST64 = true;
1188 }
1189 return true;
1190 }
1191
1192 // Check if the new offsets fit in the reduced 8-bit range.
1193 if (isUInt<8>(EltOffset0) && isUInt<8>(EltOffset1)) {
1194 if (Modify) {
1195 CI.Offset = EltOffset0;
1196 Paired.Offset = EltOffset1;
1197 }
1198 return true;
1199 }
1200
1201 // Try to shift base address to decrease offsets.
1202 uint32_t Min = std::min(EltOffset0, EltOffset1);
1203 uint32_t Max = std::max(EltOffset0, EltOffset1);
1204
1205 const uint32_t Mask = maskTrailingOnes<uint32_t>(8) * 64;
1206 if (((Max - Min) & ~Mask) == 0) {
1207 if (Modify) {
1208 // From the range of values we could use for BaseOff, choose the one that
1209 // is aligned to the highest power of two, to maximise the chance that
1210 // the same offset can be reused for other load/store pairs.
1211 uint32_t BaseOff = mostAlignedValueInRange(Max - 0xff * 64, Min);
1212 // Copy the low bits of the offsets, so that when we adjust them by
1213 // subtracting BaseOff they will be multiples of 64.
1214 BaseOff |= Min & maskTrailingOnes<uint32_t>(6);
1215 CI.BaseOff = BaseOff * CI.EltSize;
1216 CI.Offset = (EltOffset0 - BaseOff) / 64;
1217 Paired.Offset = (EltOffset1 - BaseOff) / 64;
1218 CI.UseST64 = true;
1219 }
1220 return true;
1221 }
1222
1223 if (isUInt<8>(Max - Min)) {
1224 if (Modify) {
1225 // From the range of values we could use for BaseOff, choose the one that
1226 // is aligned to the highest power of two, to maximise the chance that
1227 // the same offset can be reused for other load/store pairs.
1228 uint32_t BaseOff = mostAlignedValueInRange(Max - 0xff, Min);
1229 CI.BaseOff = BaseOff * CI.EltSize;
1230 CI.Offset = EltOffset0 - BaseOff;
1231 Paired.Offset = EltOffset1 - BaseOff;
1232 }
1233 return true;
1234 }
1235
1236 return false;
1237}
1238
1239bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM,
1240 const CombineInfo &CI,
1241 const CombineInfo &Paired) {
1242 const unsigned Width = (CI.Width + Paired.Width);
1243 switch (CI.InstClass) {
1244 default:
1245 return (Width <= 4) && (STM.hasDwordx3LoadStores() || (Width != 3));
1246 case S_BUFFER_LOAD_IMM:
1247 case S_BUFFER_LOAD_SGPR_IMM:
1248 case S_LOAD_IMM:
1249 switch (Width) {
1250 default:
1251 return false;
1252 case 2:
1253 case 4:
1254 case 8:
1255 return true;
1256 case 3:
1257 return STM.hasScalarDwordx3Loads();
1258 }
1259 }
1260}
1261
1262const TargetRegisterClass *
1263SILoadStoreOptimizer::getDataRegClass(const MachineInstr &MI) const {
1264 if (const auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst)) {
1265 return TRI->getRegClassForReg(*MRI, Dst->getReg());
1266 }
1267 if (const auto *Src = TII->getNamedOperand(MI, AMDGPU::OpName::vdata)) {
1268 return TRI->getRegClassForReg(*MRI, Src->getReg());
1269 }
1270 if (const auto *Src = TII->getNamedOperand(MI, AMDGPU::OpName::data0)) {
1271 return TRI->getRegClassForReg(*MRI, Src->getReg());
1272 }
1273 if (const auto *Dst = TII->getNamedOperand(MI, AMDGPU::OpName::sdst)) {
1274 return TRI->getRegClassForReg(*MRI, Dst->getReg());
1275 }
1276 if (const auto *Src = TII->getNamedOperand(MI, AMDGPU::OpName::sdata)) {
1277 return TRI->getRegClassForReg(*MRI, Src->getReg());
1278 }
1279 return nullptr;
1280}
1281
1282/// This function assumes that CI comes before Paired in a basic block. Return
1283/// an insertion point for the merged instruction or nullptr on failure.
1284SILoadStoreOptimizer::CombineInfo *
1285SILoadStoreOptimizer::checkAndPrepareMerge(CombineInfo &CI,
1286 CombineInfo &Paired) {
1287 // If another instruction has already been merged into CI, it may now be a
1288 // type that we can't do any further merging into.
1289 if (CI.InstClass == UNKNOWN || Paired.InstClass == UNKNOWN)
1290 return nullptr;
1291 assert(CI.InstClass == Paired.InstClass);
1292
1293 if (getInstSubclass(CI.I->getOpcode(), *TII) !=
1294 getInstSubclass(Paired.I->getOpcode(), *TII))
1295 return nullptr;
1296
1297 // Check both offsets (or masks for MIMG) can be combined and fit in the
1298 // reduced range.
1299 if (CI.InstClass == MIMG) {
1300 if (!dmasksCanBeCombined(CI, *TII, Paired))
1301 return nullptr;
1302 } else {
1303 if (!widthsFit(*STM, CI, Paired) || !offsetsCanBeCombined(CI, *STM, Paired))
1304 return nullptr;
1305 }
1306
1307 DenseSet<Register> RegDefs;
1308 DenseSet<Register> RegUses;
1309 CombineInfo *Where;
1310 if (CI.I->mayLoad()) {
1311 // Try to hoist Paired up to CI.
1312 addDefsUsesToList(*Paired.I, RegDefs, RegUses);
1313 for (MachineBasicBlock::iterator MBBI = Paired.I; --MBBI != CI.I;) {
1314 if (!canSwapInstructions(RegDefs, RegUses, *Paired.I, *MBBI))
1315 return nullptr;
1316 }
1317 Where = &CI;
1318 } else {
1319 // Try to sink CI down to Paired.
1320 addDefsUsesToList(*CI.I, RegDefs, RegUses);
1321 for (MachineBasicBlock::iterator MBBI = CI.I; ++MBBI != Paired.I;) {
1322 if (!canSwapInstructions(RegDefs, RegUses, *CI.I, *MBBI))
1323 return nullptr;
1324 }
1325 Where = &Paired;
1326 }
1327
1328 // Call offsetsCanBeCombined with modify = true so that the offsets are
1329 // correct for the new instruction. This should return true, because
1330 // this function should only be called on CombineInfo objects that
1331 // have already been confirmed to be mergeable.
1332 if (CI.InstClass == DS_READ || CI.InstClass == DS_WRITE) {
1333 if (STM->hasNeedsAligned2addrDS() &&
1334 (CI.I->memoperands_empty() ||
1335 (*CI.I->memoperands_begin())->getAlign().value() < CI.Width * 4))
1336 return nullptr;
1337 offsetsCanBeCombined(CI, *STM, Paired, true);
1338 }
1339
1340 if (CI.InstClass == DS_WRITE) {
1341 // Both data operands must be AGPR or VGPR, so the data registers needs to
1342 // be constrained to one or the other. We expect to only emit the VGPR form
1343 // here for now.
1344 //
1345 // FIXME: There is currently a hack in getRegClass to report that the write2
1346 // operands are VGPRs. In the future we should have separate agpr
1347 // instruction definitions.
1348 const MachineOperand *Data0 =
1349 TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0);
1350 const MachineOperand *Data1 =
1351 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0);
1352
1353 const MCInstrDesc &Write2Opc = TII->get(getWrite2Opcode(CI));
1354 int Data0Idx = AMDGPU::getNamedOperandIdx(Write2Opc.getOpcode(),
1355 AMDGPU::OpName::data0);
1356 int Data1Idx = AMDGPU::getNamedOperandIdx(Write2Opc.getOpcode(),
1357 AMDGPU::OpName::data1);
1358
1359 const TargetRegisterClass *DataRC0 = TII->getRegClass(Write2Opc, Data0Idx);
1360
1361 const TargetRegisterClass *DataRC1 = TII->getRegClass(Write2Opc, Data1Idx);
1362
1363 if (unsigned SubReg = Data0->getSubReg()) {
1364 DataRC0 = TRI->getMatchingSuperRegClass(MRI->getRegClass(Data0->getReg()),
1365 DataRC0, SubReg);
1366 }
1367
1368 if (unsigned SubReg = Data1->getSubReg()) {
1369 DataRC1 = TRI->getMatchingSuperRegClass(MRI->getRegClass(Data1->getReg()),
1370 DataRC1, SubReg);
1371 }
1372
1373 if (!MRI->constrainRegClass(Data0->getReg(), DataRC0) ||
1374 !MRI->constrainRegClass(Data1->getReg(), DataRC1))
1375 return nullptr;
1376
1377 // TODO: If one register can be constrained, and not the other, insert a
1378 // copy.
1379 }
1380
1381 return Where;
1382}
1383
1384// Copy the merged load result from DestReg to the original dest regs of CI and
1385// Paired.
1386void SILoadStoreOptimizer::copyToDestRegs(
1387 CombineInfo &CI, CombineInfo &Paired,
1388 MachineBasicBlock::iterator InsertBefore, const DebugLoc &DL,
1389 AMDGPU::OpName OpName, Register DestReg) const {
1390 MachineBasicBlock *MBB = CI.I->getParent();
1391
1392 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired);
1393
1394 // Copy to the old destination registers.
1395 const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY);
1396 auto *Dest0 = TII->getNamedOperand(*CI.I, OpName);
1397 auto *Dest1 = TII->getNamedOperand(*Paired.I, OpName);
1398
1399 // The constrained sload instructions in S_LOAD_IMM class will have
1400 // `early-clobber` flag in the dst operand. Remove the flag before using the
1401 // MOs in copies.
1402 Dest0->setIsEarlyClobber(false);
1403 Dest1->setIsEarlyClobber(false);
1404
1405 BuildMI(*MBB, InsertBefore, DL, CopyDesc)
1406 .add(*Dest0) // Copy to same destination including flags and sub reg.
1407 .addReg(DestReg, {}, SubRegIdx0);
1408 BuildMI(*MBB, InsertBefore, DL, CopyDesc)
1409 .add(*Dest1)
1410 .addReg(DestReg, {}, SubRegIdx1);
1411}
1412
1413// Return a register for the source of the merged store after copying the
1414// original source regs of CI and Paired into it.
1415Register
1416SILoadStoreOptimizer::copyFromSrcRegs(CombineInfo &CI, CombineInfo &Paired,
1417 MachineBasicBlock::iterator InsertBefore,
1418 const DebugLoc &DL,
1419 AMDGPU::OpName OpName) const {
1420 MachineBasicBlock *MBB = CI.I->getParent();
1421
1422 auto [SubRegIdx0, SubRegIdx1] = getSubRegIdxs(CI, Paired);
1423
1424 // Copy to the new source register.
1425 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1426 Register SrcReg = MRI->createVirtualRegister(SuperRC);
1427
1428 const auto *Src0 = TII->getNamedOperand(*CI.I, OpName);
1429 const auto *Src1 = TII->getNamedOperand(*Paired.I, OpName);
1430
1431 BuildMI(*MBB, InsertBefore, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg)
1432 .add(*Src0)
1433 .addImm(SubRegIdx0)
1434 .add(*Src1)
1435 .addImm(SubRegIdx1);
1436
1437 return SrcReg;
1438}
1439
1440unsigned SILoadStoreOptimizer::read2Opcode(unsigned EltSize, bool GDS) const {
1441 if (GDS || STM->ldsRequiresM0Init())
1442 return (EltSize == 4) ? AMDGPU::DS_READ2_B32 : AMDGPU::DS_READ2_B64;
1443 return (EltSize == 4) ? AMDGPU::DS_READ2_B32_gfx9 : AMDGPU::DS_READ2_B64_gfx9;
1444}
1445
1446unsigned SILoadStoreOptimizer::read2ST64Opcode(unsigned EltSize,
1447 bool GDS) const {
1448 if (GDS || STM->ldsRequiresM0Init())
1449 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32 : AMDGPU::DS_READ2ST64_B64;
1450
1451 return (EltSize == 4) ? AMDGPU::DS_READ2ST64_B32_gfx9
1452 : AMDGPU::DS_READ2ST64_B64_gfx9;
1453}
1454
1455MachineBasicBlock::iterator
1456SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired,
1457 MachineBasicBlock::iterator InsertBefore) {
1458 MachineBasicBlock *MBB = CI.I->getParent();
1459
1460 // Be careful, since the addresses could be subregisters themselves in weird
1461 // cases, like vectors of pointers.
1462 const auto *AddrReg = TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr);
1463
1464 unsigned NewOffset0 = std::min(CI.Offset, Paired.Offset);
1465 unsigned NewOffset1 = std::max(CI.Offset, Paired.Offset);
1466 unsigned Opc = CI.UseST64 ? read2ST64Opcode(CI.EltSize, CI.GDS)
1467 : read2Opcode(CI.EltSize, CI.GDS);
1468
1469 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
1470 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
1471
1472 const MCInstrDesc &Read2Desc = TII->get(Opc);
1473
1474 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1475 Register DestReg = MRI->createVirtualRegister(SuperRC);
1476
1477 DebugLoc DL =
1478 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1479
1480 Register BaseReg = AddrReg->getReg();
1481 unsigned BaseSubReg = AddrReg->getSubReg();
1482 if (CI.BaseOff) {
1483 Register ImmReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1484 BuildMI(*MBB, InsertBefore, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg)
1485 .addImm(CI.BaseOff);
1486
1487 BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1488
1489 TII->getAddNoCarry(*MBB, InsertBefore, DL, BaseReg)
1490 .addReg(ImmReg)
1491 .addReg(AddrReg->getReg(), {}, BaseSubReg)
1492 .addImm(0); // clamp bit
1493 BaseSubReg = 0;
1494 }
1495
1496 MachineInstrBuilder Read2 =
1497 BuildMI(*MBB, InsertBefore, DL, Read2Desc, DestReg)
1498 .addReg(BaseReg, {}, BaseSubReg) // addr
1499 .addImm(NewOffset0) // offset0
1500 .addImm(NewOffset1) // offset1
1501 .addImm(CI.GDS) // gds
1502 .cloneMergedMemRefs({&*CI.I, &*Paired.I});
1503
1504 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdst, DestReg);
1505
1506 CI.I->eraseFromParent();
1507 Paired.I->eraseFromParent();
1508
1509 LLVM_DEBUG(dbgs() << "Inserted read2: " << *Read2 << '\n');
1510 return Read2;
1511}
1512
1513unsigned SILoadStoreOptimizer::write2Opcode(unsigned EltSize, bool GDS) const {
1514 if (GDS || STM->ldsRequiresM0Init())
1515 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32 : AMDGPU::DS_WRITE2_B64;
1516 return (EltSize == 4) ? AMDGPU::DS_WRITE2_B32_gfx9
1517 : AMDGPU::DS_WRITE2_B64_gfx9;
1518}
1519
1520unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize,
1521 bool GDS) const {
1522 if (GDS || STM->ldsRequiresM0Init())
1523 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32
1524 : AMDGPU::DS_WRITE2ST64_B64;
1525
1526 return (EltSize == 4) ? AMDGPU::DS_WRITE2ST64_B32_gfx9
1527 : AMDGPU::DS_WRITE2ST64_B64_gfx9;
1528}
1529
1530unsigned SILoadStoreOptimizer::getWrite2Opcode(const CombineInfo &CI) const {
1531 return CI.UseST64 ? write2ST64Opcode(CI.EltSize, CI.GDS)
1532 : write2Opcode(CI.EltSize, CI.GDS);
1533}
1534
1535MachineBasicBlock::iterator SILoadStoreOptimizer::mergeWrite2Pair(
1536 CombineInfo &CI, CombineInfo &Paired,
1537 MachineBasicBlock::iterator InsertBefore) {
1538 MachineBasicBlock *MBB = CI.I->getParent();
1539
1540 // Be sure to use .addOperand(), and not .addReg() with these. We want to be
1541 // sure we preserve the subregister index and any register flags set on them.
1542 const MachineOperand *AddrReg =
1543 TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr);
1544 const MachineOperand *Data0 =
1545 TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0);
1546 const MachineOperand *Data1 =
1547 TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0);
1548
1549 unsigned NewOffset0 = CI.Offset;
1550 unsigned NewOffset1 = Paired.Offset;
1551 unsigned Opc = getWrite2Opcode(CI);
1552
1553 if (NewOffset0 > NewOffset1) {
1554 // Canonicalize the merged instruction so the smaller offset comes first.
1555 std::swap(NewOffset0, NewOffset1);
1556 std::swap(Data0, Data1);
1557 }
1558
1559 assert((isUInt<8>(NewOffset0) && isUInt<8>(NewOffset1)) &&
1560 (NewOffset0 != NewOffset1) && "Computed offset doesn't fit");
1561
1562 const MCInstrDesc &Write2Desc = TII->get(Opc);
1563 DebugLoc DL =
1564 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1565
1566 Register BaseReg = AddrReg->getReg();
1567 unsigned BaseSubReg = AddrReg->getSubReg();
1568 if (CI.BaseOff) {
1569 Register ImmReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1570 BuildMI(*MBB, InsertBefore, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg)
1571 .addImm(CI.BaseOff);
1572
1573 BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1574
1575 TII->getAddNoCarry(*MBB, InsertBefore, DL, BaseReg)
1576 .addReg(ImmReg)
1577 .addReg(AddrReg->getReg(), {}, BaseSubReg)
1578 .addImm(0); // clamp bit
1579 BaseSubReg = 0;
1580 }
1581
1582 MachineInstrBuilder Write2 = BuildMI(*MBB, InsertBefore, DL, Write2Desc)
1583 .addReg(BaseReg, {}, BaseSubReg) // addr
1584 .add(*Data0) // data0
1585 .add(*Data1) // data1
1586 .addImm(NewOffset0) // offset0
1587 .addImm(NewOffset1) // offset1
1588 .addImm(CI.GDS) // gds
1589 .cloneMergedMemRefs({&*CI.I, &*Paired.I});
1590
1591 CI.I->eraseFromParent();
1592 Paired.I->eraseFromParent();
1593
1594 LLVM_DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n');
1595 return Write2;
1596}
1597
1598MachineBasicBlock::iterator
1599SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired,
1600 MachineBasicBlock::iterator InsertBefore) {
1601 MachineBasicBlock *MBB = CI.I->getParent();
1602 DebugLoc DL =
1603 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1604
1605 const unsigned Opcode = getNewOpcode(CI, Paired);
1606
1607 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1608
1609 Register DestReg = MRI->createVirtualRegister(SuperRC);
1610 unsigned MergedDMask = CI.DMask | Paired.DMask;
1611 unsigned DMaskIdx =
1612 AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::dmask);
1613
1614 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg);
1615 for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) {
1616 if (I == DMaskIdx)
1617 MIB.addImm(MergedDMask);
1618 else
1619 MIB.add((*CI.I).getOperand(I));
1620 }
1621
1622 // It shouldn't be possible to get this far if the two instructions
1623 // don't have a single memoperand, because MachineInstr::mayAlias()
1624 // will return true if this is the case.
1625 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1626
1627 MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1628
1629 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata, DestReg);
1630
1631 CI.I->eraseFromParent();
1632 Paired.I->eraseFromParent();
1633 return New;
1634}
1635
1636MachineBasicBlock::iterator SILoadStoreOptimizer::mergeSMemLoadImmPair(
1637 CombineInfo &CI, CombineInfo &Paired,
1638 MachineBasicBlock::iterator InsertBefore) {
1639 MachineBasicBlock *MBB = CI.I->getParent();
1640 DebugLoc DL =
1641 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1642
1643 const unsigned Opcode = getNewOpcode(CI, Paired);
1644
1645 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1646
1647 Register DestReg = MRI->createVirtualRegister(SuperRC);
1648 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset);
1649
1650 // It shouldn't be possible to get this far if the two instructions
1651 // don't have a single memoperand, because MachineInstr::mayAlias()
1652 // will return true if this is the case.
1653 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1654
1655 MachineInstrBuilder New =
1656 BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg)
1657 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::sbase));
1658 if (CI.InstClass == S_BUFFER_LOAD_SGPR_IMM)
1659 New.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset));
1660 New.addImm(MergedOffset);
1661 New.addImm(CI.CPol).addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1662
1663 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::sdst, DestReg);
1664
1665 CI.I->eraseFromParent();
1666 Paired.I->eraseFromParent();
1667 return New;
1668}
1669
1670MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferLoadPair(
1671 CombineInfo &CI, CombineInfo &Paired,
1672 MachineBasicBlock::iterator InsertBefore) {
1673 MachineBasicBlock *MBB = CI.I->getParent();
1674
1675 DebugLoc DL =
1676 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1677
1678 const unsigned Opcode = getNewOpcode(CI, Paired);
1679
1680 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1681
1682 // Copy to the new source register.
1683 Register DestReg = MRI->createVirtualRegister(SuperRC);
1684 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset);
1685
1686 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg);
1687
1688 AddressRegs Regs = getRegs(Opcode, *TII);
1689
1690 if (Regs.VAddr)
1691 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
1692
1693 // It shouldn't be possible to get this far if the two instructions
1694 // don't have a single memoperand, because MachineInstr::mayAlias()
1695 // will return true if this is the case.
1696 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1697
1698 MachineInstr *New =
1699 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc))
1700 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset))
1701 .addImm(MergedOffset) // offset
1702 .addImm(CI.CPol) // cpol
1703 .addImm(0) // swz
1704 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1705
1706 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata, DestReg);
1707
1708 CI.I->eraseFromParent();
1709 Paired.I->eraseFromParent();
1710 return New;
1711}
1712
1713MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferLoadPair(
1714 CombineInfo &CI, CombineInfo &Paired,
1715 MachineBasicBlock::iterator InsertBefore) {
1716 MachineBasicBlock *MBB = CI.I->getParent();
1717
1718 DebugLoc DL =
1719 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1720
1721 const unsigned Opcode = getNewOpcode(CI, Paired);
1722
1723 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1724
1725 // Copy to the new source register.
1726 Register DestReg = MRI->createVirtualRegister(SuperRC);
1727 unsigned MergedOffset = std::min(CI.Offset, Paired.Offset);
1728
1729 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg);
1730
1731 AddressRegs Regs = getRegs(Opcode, *TII);
1732
1733 if (Regs.VAddr)
1734 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
1735
1736 // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
1737 // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
1738 // and use XYZ of XYZW to enable the merge.
1739 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1740 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1741 NumCombinedComponents = 4;
1742 unsigned JoinedFormat =
1743 getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, *STM);
1744
1745 // It shouldn't be possible to get this far if the two instructions
1746 // don't have a single memoperand, because MachineInstr::mayAlias()
1747 // will return true if this is the case.
1748 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1749
1750 MachineInstr *New =
1751 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc))
1752 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset))
1753 .addImm(MergedOffset) // offset
1754 .addImm(JoinedFormat) // format
1755 .addImm(CI.CPol) // cpol
1756 .addImm(0) // swz
1757 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1758
1759 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata, DestReg);
1760
1761 CI.I->eraseFromParent();
1762 Paired.I->eraseFromParent();
1763 return New;
1764}
1765
1766MachineBasicBlock::iterator SILoadStoreOptimizer::mergeTBufferStorePair(
1767 CombineInfo &CI, CombineInfo &Paired,
1768 MachineBasicBlock::iterator InsertBefore) {
1769 MachineBasicBlock *MBB = CI.I->getParent();
1770 DebugLoc DL =
1771 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1772
1773 const unsigned Opcode = getNewOpcode(CI, Paired);
1774
1775 Register SrcReg =
1776 copyFromSrcRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata);
1777
1778 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode)).addReg(SrcReg);
1779
1780 AddressRegs Regs = getRegs(Opcode, *TII);
1781
1782 if (Regs.VAddr)
1783 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
1784
1785 // For 8-bit or 16-bit tbuffer formats there is no 3-component encoding.
1786 // If the combined count is 3 (e.g. X+X+X or XY+X), promote to 4 components
1787 // and use XYZ of XYZW to enable the merge.
1788 unsigned NumCombinedComponents = CI.Width + Paired.Width;
1789 if (NumCombinedComponents == 3 && CI.EltSize <= 2)
1790 NumCombinedComponents = 4;
1791 unsigned JoinedFormat =
1792 getBufferFormatWithCompCount(CI.Format, NumCombinedComponents, *STM);
1793
1794 // It shouldn't be possible to get this far if the two instructions
1795 // don't have a single memoperand, because MachineInstr::mayAlias()
1796 // will return true if this is the case.
1797 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
1798
1799 MachineInstr *New =
1800 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc))
1801 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset))
1802 .addImm(std::min(CI.Offset, Paired.Offset)) // offset
1803 .addImm(JoinedFormat) // format
1804 .addImm(CI.CPol) // cpol
1805 .addImm(0) // swz
1806 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1807
1808 CI.I->eraseFromParent();
1809 Paired.I->eraseFromParent();
1810 return New;
1811}
1812
1813MachineBasicBlock::iterator SILoadStoreOptimizer::mergeFlatLoadPair(
1814 CombineInfo &CI, CombineInfo &Paired,
1815 MachineBasicBlock::iterator InsertBefore) {
1816 MachineBasicBlock *MBB = CI.I->getParent();
1817
1818 DebugLoc DL =
1819 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1820
1821 const unsigned Opcode = getNewOpcode(CI, Paired);
1822
1823 const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired);
1824 Register DestReg = MRI->createVirtualRegister(SuperRC);
1825
1826 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode), DestReg);
1827
1828 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr))
1829 MIB.add(*SAddr);
1830
1831 MachineInstr *New =
1832 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr))
1833 .addImm(std::min(CI.Offset, Paired.Offset))
1834 .addImm(CI.CPol)
1835 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1836
1837 copyToDestRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdst, DestReg);
1838
1839 CI.I->eraseFromParent();
1840 Paired.I->eraseFromParent();
1841 return New;
1842}
1843
1844MachineBasicBlock::iterator SILoadStoreOptimizer::mergeFlatStorePair(
1845 CombineInfo &CI, CombineInfo &Paired,
1846 MachineBasicBlock::iterator InsertBefore) {
1847 MachineBasicBlock *MBB = CI.I->getParent();
1848
1849 DebugLoc DL =
1850 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
1851
1852 const unsigned Opcode = getNewOpcode(CI, Paired);
1853
1854 Register SrcReg =
1855 copyFromSrcRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata);
1856
1857 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode))
1858 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr))
1859 .addReg(SrcReg);
1860
1861 if (auto *SAddr = TII->getNamedOperand(*CI.I, AMDGPU::OpName::saddr))
1862 MIB.add(*SAddr);
1863
1864 MachineInstr *New =
1865 MIB.addImm(std::min(CI.Offset, Paired.Offset))
1866 .addImm(CI.CPol)
1867 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
1868
1869 CI.I->eraseFromParent();
1870 Paired.I->eraseFromParent();
1871 return New;
1872}
1873
1874static bool needsConstrainedOpcode(const GCNSubtarget &STM,
1875 ArrayRef<MachineMemOperand *> MMOs,
1876 unsigned Width) {
1877 // Conservatively returns true if not found the MMO.
1878 return STM.isXNACKEnabled() &&
1879 (MMOs.size() != 1 || MMOs[0]->getAlign().value() < Width * 4);
1880}
1881
1882unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI,
1883 const CombineInfo &Paired) {
1884 const unsigned Width = CI.Width + Paired.Width;
1885 const CombineInfo &Leading = Paired < CI ? Paired : CI;
1886 // If XNACK is enabled, use the constrained opcodes when the first load is
1887 // under-aligned.
1888 const bool NeedsConstrainedOpc =
1889 needsConstrainedOpcode(*STM, Leading.I->memoperands(), Width);
1890
1891 switch (getCommonInstClass(CI, Paired)) {
1892 default:
1893 assert(CI.InstClass == BUFFER_LOAD || CI.InstClass == BUFFER_STORE);
1894 // FIXME: Handle d16 correctly
1895 return AMDGPU::getMUBUFOpcode(AMDGPU::getMUBUFBaseOpcode(CI.I->getOpcode()),
1896 Width);
1897 case TBUFFER_LOAD:
1898 case TBUFFER_STORE:
1899 return AMDGPU::getMTBUFOpcode(AMDGPU::getMTBUFBaseOpcode(CI.I->getOpcode()),
1900 Width);
1901
1902 case UNKNOWN:
1903 llvm_unreachable("Unknown instruction class");
1904 case S_BUFFER_LOAD_IMM: {
1905 switch (Width) {
1906 default:
1907 return 0;
1908 case 2:
1909 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM_ec
1910 : AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM;
1911 case 3:
1912 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM_ec
1913 : AMDGPU::S_BUFFER_LOAD_DWORDX3_IMM;
1914 case 4:
1915 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM_ec
1916 : AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM;
1917 case 8:
1918 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM_ec
1919 : AMDGPU::S_BUFFER_LOAD_DWORDX8_IMM;
1920 }
1921 }
1922 case S_BUFFER_LOAD_SGPR_IMM: {
1923 switch (Width) {
1924 default:
1925 return 0;
1926 case 2:
1927 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM_ec
1928 : AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR_IMM;
1929 case 3:
1930 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM_ec
1931 : AMDGPU::S_BUFFER_LOAD_DWORDX3_SGPR_IMM;
1932 case 4:
1933 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM_ec
1934 : AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR_IMM;
1935 case 8:
1936 return NeedsConstrainedOpc ? AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM_ec
1937 : AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR_IMM;
1938 }
1939 }
1940 case S_LOAD_IMM: {
1941 switch (Width) {
1942 default:
1943 return 0;
1944 case 2:
1945 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX2_IMM_ec
1946 : AMDGPU::S_LOAD_DWORDX2_IMM;
1947 case 3:
1948 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX3_IMM_ec
1949 : AMDGPU::S_LOAD_DWORDX3_IMM;
1950 case 4:
1951 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX4_IMM_ec
1952 : AMDGPU::S_LOAD_DWORDX4_IMM;
1953 case 8:
1954 return NeedsConstrainedOpc ? AMDGPU::S_LOAD_DWORDX8_IMM_ec
1955 : AMDGPU::S_LOAD_DWORDX8_IMM;
1956 }
1957 }
1958 case GLOBAL_LOAD:
1959 switch (Width) {
1960 default:
1961 return 0;
1962 case 2:
1963 return AMDGPU::GLOBAL_LOAD_DWORDX2;
1964 case 3:
1965 return AMDGPU::GLOBAL_LOAD_DWORDX3;
1966 case 4:
1967 return AMDGPU::GLOBAL_LOAD_DWORDX4;
1968 }
1969 case GLOBAL_LOAD_SADDR:
1970 switch (Width) {
1971 default:
1972 return 0;
1973 case 2:
1974 return AMDGPU::GLOBAL_LOAD_DWORDX2_SADDR;
1975 case 3:
1976 return AMDGPU::GLOBAL_LOAD_DWORDX3_SADDR;
1977 case 4:
1978 return AMDGPU::GLOBAL_LOAD_DWORDX4_SADDR;
1979 }
1980 case GLOBAL_STORE:
1981 switch (Width) {
1982 default:
1983 return 0;
1984 case 2:
1985 return AMDGPU::GLOBAL_STORE_DWORDX2;
1986 case 3:
1987 return AMDGPU::GLOBAL_STORE_DWORDX3;
1988 case 4:
1989 return AMDGPU::GLOBAL_STORE_DWORDX4;
1990 }
1991 case GLOBAL_STORE_SADDR:
1992 switch (Width) {
1993 default:
1994 return 0;
1995 case 2:
1996 return AMDGPU::GLOBAL_STORE_DWORDX2_SADDR;
1997 case 3:
1998 return AMDGPU::GLOBAL_STORE_DWORDX3_SADDR;
1999 case 4:
2000 return AMDGPU::GLOBAL_STORE_DWORDX4_SADDR;
2001 }
2002 case FLAT_LOAD:
2003 switch (Width) {
2004 default:
2005 return 0;
2006 case 2:
2007 return AMDGPU::FLAT_LOAD_DWORDX2;
2008 case 3:
2009 return AMDGPU::FLAT_LOAD_DWORDX3;
2010 case 4:
2011 return AMDGPU::FLAT_LOAD_DWORDX4;
2012 }
2013 case FLAT_STORE:
2014 switch (Width) {
2015 default:
2016 return 0;
2017 case 2:
2018 return AMDGPU::FLAT_STORE_DWORDX2;
2019 case 3:
2020 return AMDGPU::FLAT_STORE_DWORDX3;
2021 case 4:
2022 return AMDGPU::FLAT_STORE_DWORDX4;
2023 }
2024 case FLAT_LOAD_SADDR:
2025 switch (Width) {
2026 default:
2027 return 0;
2028 case 2:
2029 return AMDGPU::FLAT_LOAD_DWORDX2_SADDR;
2030 case 3:
2031 return AMDGPU::FLAT_LOAD_DWORDX3_SADDR;
2032 case 4:
2033 return AMDGPU::FLAT_LOAD_DWORDX4_SADDR;
2034 }
2035 case FLAT_STORE_SADDR:
2036 switch (Width) {
2037 default:
2038 return 0;
2039 case 2:
2040 return AMDGPU::FLAT_STORE_DWORDX2_SADDR;
2041 case 3:
2042 return AMDGPU::FLAT_STORE_DWORDX3_SADDR;
2043 case 4:
2044 return AMDGPU::FLAT_STORE_DWORDX4_SADDR;
2045 }
2046 case MIMG:
2047 assert(((unsigned)llvm::popcount(CI.DMask | Paired.DMask) == Width) &&
2048 "No overlaps");
2049 return AMDGPU::getMaskedMIMGOp(CI.I->getOpcode(), Width);
2050 }
2051}
2052
2053std::pair<unsigned, unsigned>
2054SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI,
2055 const CombineInfo &Paired) {
2056 assert((CI.InstClass != MIMG ||
2057 ((unsigned)llvm::popcount(CI.DMask | Paired.DMask) ==
2058 CI.Width + Paired.Width)) &&
2059 "No overlaps");
2060
2061 unsigned Idx0;
2062 unsigned Idx1;
2063
2064 static const unsigned Idxs[5][4] = {
2065 {AMDGPU::sub0, AMDGPU::sub0_sub1, AMDGPU::sub0_sub1_sub2, AMDGPU::sub0_sub1_sub2_sub3},
2066 {AMDGPU::sub1, AMDGPU::sub1_sub2, AMDGPU::sub1_sub2_sub3, AMDGPU::sub1_sub2_sub3_sub4},
2067 {AMDGPU::sub2, AMDGPU::sub2_sub3, AMDGPU::sub2_sub3_sub4, AMDGPU::sub2_sub3_sub4_sub5},
2068 {AMDGPU::sub3, AMDGPU::sub3_sub4, AMDGPU::sub3_sub4_sub5, AMDGPU::sub3_sub4_sub5_sub6},
2069 {AMDGPU::sub4, AMDGPU::sub4_sub5, AMDGPU::sub4_sub5_sub6, AMDGPU::sub4_sub5_sub6_sub7},
2070 };
2071
2072 assert(CI.Width >= 1 && CI.Width <= 4);
2073 assert(Paired.Width >= 1 && Paired.Width <= 4);
2074
2075 if (Paired < CI) {
2076 Idx1 = Idxs[0][Paired.Width - 1];
2077 Idx0 = Idxs[Paired.Width][CI.Width - 1];
2078 } else {
2079 Idx0 = Idxs[0][CI.Width - 1];
2080 Idx1 = Idxs[CI.Width][Paired.Width - 1];
2081 }
2082
2083 return {Idx0, Idx1};
2084}
2085
2086const TargetRegisterClass *
2087SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI,
2088 const CombineInfo &Paired) const {
2089 if (CI.InstClass == S_BUFFER_LOAD_IMM ||
2090 CI.InstClass == S_BUFFER_LOAD_SGPR_IMM || CI.InstClass == S_LOAD_IMM) {
2091 switch (CI.Width + Paired.Width) {
2092 default:
2093 return nullptr;
2094 case 2:
2095 return &AMDGPU::SReg_64_XEXECRegClass;
2096 case 3:
2097 return &AMDGPU::SGPR_96RegClass;
2098 case 4:
2099 return &AMDGPU::SGPR_128RegClass;
2100 case 8:
2101 return &AMDGPU::SGPR_256RegClass;
2102 case 16:
2103 return &AMDGPU::SGPR_512RegClass;
2104 }
2105 }
2106
2107 // FIXME: This should compute the instruction to use, and then use the result
2108 // of TII->getRegClass.
2109 unsigned BitWidth = 32 * (CI.Width + Paired.Width);
2110 return TRI->isAGPRClass(getDataRegClass(*CI.I))
2111 ? TRI->getAGPRClassForBitWidth(BitWidth)
2112 : TRI->getVGPRClassForBitWidth(BitWidth);
2113}
2114
2115MachineBasicBlock::iterator SILoadStoreOptimizer::mergeBufferStorePair(
2116 CombineInfo &CI, CombineInfo &Paired,
2117 MachineBasicBlock::iterator InsertBefore) {
2118 MachineBasicBlock *MBB = CI.I->getParent();
2119 DebugLoc DL =
2120 DebugLoc::getMergedLocation(CI.I->getDebugLoc(), Paired.I->getDebugLoc());
2121
2122 const unsigned Opcode = getNewOpcode(CI, Paired);
2123
2124 Register SrcReg =
2125 copyFromSrcRegs(CI, Paired, InsertBefore, DL, AMDGPU::OpName::vdata);
2126
2127 auto MIB = BuildMI(*MBB, InsertBefore, DL, TII->get(Opcode)).addReg(SrcReg);
2128
2129 AddressRegs Regs = getRegs(Opcode, *TII);
2130
2131 if (Regs.VAddr)
2132 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
2133
2134
2135 // It shouldn't be possible to get this far if the two instructions
2136 // don't have a single memoperand, because MachineInstr::mayAlias()
2137 // will return true if this is the case.
2138 assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand());
2139
2140 MachineInstr *New =
2141 MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc))
2142 .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset))
2143 .addImm(std::min(CI.Offset, Paired.Offset)) // offset
2144 .addImm(CI.CPol) // cpol
2145 .addImm(0) // swz
2146 .addMemOperand(combineKnownAdjacentMMOs(CI, Paired));
2147
2148 CI.I->eraseFromParent();
2149 Paired.I->eraseFromParent();
2150 return New;
2151}
2152
2153MachineOperand
2154SILoadStoreOptimizer::createRegOrImm(int32_t Val, MachineInstr &MI) const {
2155 APInt V(32, Val, true);
2156 if (TII->isInlineConstant(V))
2157 return MachineOperand::CreateImm(Val);
2158
2159 Register Reg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
2160 MachineInstr *Mov =
2161 BuildMI(*MI.getParent(), MI.getIterator(), MI.getDebugLoc(),
2162 TII->get(AMDGPU::S_MOV_B32), Reg)
2163 .addImm(Val);
2164 (void)Mov;
2165 LLVM_DEBUG(dbgs() << " "; Mov->dump());
2166 return MachineOperand::CreateReg(Reg, false);
2167}
2168
2169// Compute base address using Addr and return the final register.
2170Register SILoadStoreOptimizer::computeBase(MachineInstr &MI,
2171 const MemAddress &Addr) const {
2172 MachineBasicBlock *MBB = MI.getParent();
2173 MachineBasicBlock::iterator MBBI = MI.getIterator();
2174 const DebugLoc &DL = MI.getDebugLoc();
2175
2176 LLVM_DEBUG(dbgs() << " Re-Computed Anchor-Base:\n");
2177
2178 // Use V_ADD_U64_e64 when the original pattern used it (gfx1250+)
2179 if (Addr.Base.UseV64Pattern) {
2180 Register FullDestReg = MRI->createVirtualRegister(
2181 TII->getRegClass(TII->get(AMDGPU::V_ADD_U64_e64), 0));
2182
2183 // Load the 64-bit offset into an SGPR pair if needed
2184 Register OffsetReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
2185 MachineInstr *MovOffset =
2186 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::S_MOV_B64_IMM_PSEUDO),
2187 OffsetReg)
2188 .addImm(Addr.Offset);
2189 MachineInstr *Add64 =
2190 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_U64_e64), FullDestReg)
2191 .addReg(Addr.Base.LoReg)
2192 .addReg(OffsetReg)
2193 .addImm(0);
2194 (void)MovOffset;
2195 (void)Add64;
2196 LLVM_DEBUG(dbgs() << " " << *MovOffset << "\n";
2197 dbgs() << " " << *Add64 << "\n\n";);
2198
2199 return FullDestReg;
2200 }
2201
2202 // Use V_ADD_U32_e64 for the 32-bit FLAT_SCRATCH vaddr base.
2203 if (Addr.Base.UseScratchVGPRPattern) {
2204 MachineOperand OffsetOp =
2205 createRegOrImm(static_cast<int32_t>(Addr.Offset), MI);
2206
2207 Register FullDestReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2208 MachineInstr *Add32 =
2209 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_U32_e64), FullDestReg)
2210 .addReg(Addr.Base.LoReg, {}, Addr.Base.LoSubReg)
2211 .add(OffsetOp)
2212 .addImm(0); // clamp bit
2213 (void)Add32;
2214 LLVM_DEBUG(dbgs() << " " << *Add32 << "\n\n";);
2215
2216 return FullDestReg;
2217 }
2218
2219 // Original carry-chain pattern (V_ADD_CO_U32 + V_ADDC_U32)
2220 assert((TRI->getRegSizeInBits(Addr.Base.LoReg, *MRI) == 32 ||
2221 Addr.Base.LoSubReg) &&
2222 "Expected 32-bit Base-Register-Low!!");
2223
2224 assert((TRI->getRegSizeInBits(Addr.Base.HiReg, *MRI) == 32 ||
2225 Addr.Base.HiSubReg) &&
2226 "Expected 32-bit Base-Register-Hi!!");
2227
2228 MachineOperand OffsetLo = createRegOrImm(static_cast<int32_t>(Addr.Offset), MI);
2229 MachineOperand OffsetHi =
2230 createRegOrImm(static_cast<int32_t>(Addr.Offset >> 32), MI);
2231
2232 const auto *CarryRC = TRI->getWaveMaskRegClass();
2233 Register CarryReg = MRI->createVirtualRegister(CarryRC);
2234 Register DeadCarryReg = MRI->createVirtualRegister(CarryRC);
2235
2236 Register DestSub0 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2237 Register DestSub1 = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
2238 MachineInstr *LoHalf =
2239 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADD_CO_U32_e64), DestSub0)
2240 .addReg(CarryReg, RegState::Define)
2241 .addReg(Addr.Base.LoReg, {}, Addr.Base.LoSubReg)
2242 .add(OffsetLo)
2243 .addImm(0); // clamp bit
2244
2245 MachineInstr *HiHalf =
2246 BuildMI(*MBB, MBBI, DL, TII->get(AMDGPU::V_ADDC_U32_e64), DestSub1)
2247 .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
2248 .addReg(Addr.Base.HiReg, {}, Addr.Base.HiSubReg)
2249 .add(OffsetHi)
2250 .addReg(CarryReg)
2251 .addImm(0); // clamp bit
2252
2253 Register FullDestReg = MRI->createVirtualRegister(TRI->getVGPR64Class());
2254 MachineInstr *FullBase =
2255 BuildMI(*MBB, MBBI, DL, TII->get(TargetOpcode::REG_SEQUENCE), FullDestReg)
2256 .addReg(DestSub0)
2257 .addImm(AMDGPU::sub0)
2258 .addReg(DestSub1)
2259 .addImm(AMDGPU::sub1);
2260
2261 (void)LoHalf;
2262 (void)HiHalf;
2263 (void)FullBase;
2264 LLVM_DEBUG(dbgs() << " " << *LoHalf << "\n";
2265 dbgs() << " " << *HiHalf << "\n";
2266 dbgs() << " " << *FullBase << "\n\n";);
2267
2268 return FullDestReg;
2269}
2270
2271// Update base and offset with the NewBase and NewOffset in MI.
2272void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI,
2273 Register NewBase,
2274 int32_t NewOffset) const {
2275 auto *Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
2276 Base->setReg(NewBase);
2277 Base->setIsKill(false);
2278 TII->getNamedOperand(MI, AMDGPU::OpName::offset)->setImm(NewOffset);
2279}
2280
2281// Helper to extract a 64-bit constant offset from a V_ADD_U64_e64 instruction.
2282// Returns true if successful, populating Addr with base register info and
2283// offset.
2284bool SILoadStoreOptimizer::processBaseWithConstOffset64(
2285 MachineInstr *AddDef, const MachineOperand &Base, MemAddress &Addr) const {
2286 if (!Base.isReg())
2287 return false;
2288
2289 MachineOperand *Src0 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src0);
2290 MachineOperand *Src1 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src1);
2291
2292 const MachineOperand *BaseOp = nullptr;
2293
2294 auto Offset = TII->getImmOrMaterializedImm(*MRI, *Src1);
2295
2296 if (Offset) {
2297 BaseOp = Src0;
2298 Addr.Offset = *Offset;
2299 } else {
2300 // Both or neither are constants - can't handle this pattern
2301 return false;
2302 }
2303
2304 // Now extract the base register (which should be a 64-bit VGPR).
2305 Addr.Base.LoReg = BaseOp->getReg();
2306 Addr.Base.UseV64Pattern = true;
2307 return true;
2308}
2309
2310// Helper to extract a 32-bit constant offset from a V_ADD_U32_e64 instruction.
2311// This is the pattern used for the 32-bit vaddr of FLAT_SCRATCH instructions.
2312// Returns true if successful, populating Addr with base register info and
2313// offset.
2314bool SILoadStoreOptimizer::processBaseWithConstOffset32(
2315 MachineInstr *AddDef, MemAddress &Addr) const {
2316 // A clamped add saturates, so its constant cannot be split off.
2317 if (TII->getNamedOperand(*AddDef, AMDGPU::OpName::clamp)->getImm())
2318 return false;
2319
2320 MachineOperand *Src0 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src0);
2321 MachineOperand *Src1 = TII->getNamedOperand(*AddDef, AMDGPU::OpName::src1);
2322
2323 const MachineOperand *BaseOp = nullptr;
2324 std::optional<int64_t> Offset;
2325
2326 if ((Offset = TII->getImmOrMaterializedImm(*MRI, *Src1))) {
2327 BaseOp = Src0;
2328 } else if ((Offset = TII->getImmOrMaterializedImm(*MRI, *Src0))) {
2329 BaseOp = Src1;
2330 } else {
2331 // Both or neither are constants - can't handle this pattern
2332 return false;
2333 }
2334
2335 // An SGPR base could make the rebuilt add read two SGPRs.
2336 if (!BaseOp->isReg() || !TRI->isVGPR(*MRI, BaseOp->getReg()))
2337 return false;
2338
2339 Addr.Base.LoReg = BaseOp->getReg();
2340 Addr.Base.LoSubReg = BaseOp->getSubReg();
2341 Addr.Base.UseScratchVGPRPattern = true;
2342 Addr.Offset = *Offset;
2343 return true;
2344}
2345
2346// Analyze Base and extracts:
2347// - 32bit base registers, subregisters
2348// - 64bit constant offset
2349// Expecting base computation as:
2350// %OFFSET0:sgpr_32 = S_MOV_B32 8000
2351// %LO:vgpr_32, %c:sreg_64_xexec =
2352// V_ADD_CO_U32_e64 %BASE_LO:vgpr_32, %103:sgpr_32,
2353// %HI:vgpr_32, = V_ADDC_U32_e64 %BASE_HI:vgpr_32, 0, killed %c:sreg_64_xexec
2354// %Base:vreg_64 =
2355// REG_SEQUENCE %LO:vgpr_32, %subreg.sub0, %HI:vgpr_32, %subreg.sub1
2356//
2357// Also handles V_ADD_U64_e64 pattern (gfx1250+):
2358// %OFFSET:sreg_64 = S_MOV_B64_IMM_PSEUDO 256
2359// %Base:vreg_64 = V_ADD_U64_e64 %BASE:vreg_64, %OFFSET:sreg_64, 0
2360//
2361// Also handles the 32-bit VGPR pattern used for FLAT_SCRATCH vaddr:
2362// %OFFSET:sreg_32 = S_MOV_B32 4096
2363// %Base:vgpr_32 = V_ADD_U32_e64 %BASE:vgpr_32, %OFFSET:sreg_32, 0
2364void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base,
2365 MemAddress &Addr) const {
2366 if (!Base.isReg())
2367 return;
2368
2369 MachineInstr *Def = MRI->getVRegDef(Base.getReg());
2370 if (!Def)
2371 return;
2372
2373 // Try V_ADD_U64_e64 pattern first (simpler, used on gfx1250+)
2374 if (Def->getOpcode() == AMDGPU::V_ADD_U64_e64) {
2375 if (processBaseWithConstOffset64(Def, Base, Addr))
2376 return;
2377 }
2378
2379 // Try V_ADD_U32_e64 pattern (32-bit base, used for FLAT_SCRATCH vaddr)
2380 // Not GLOBAL saddr: it zero-extends vaddr, so a 32-bit wrap cannot be split.
2381 if (Def->getOpcode() == AMDGPU::V_ADD_U32_e64 &&
2382 SIInstrInfo::isFLATScratch(*Base.getParent())) {
2383 if (processBaseWithConstOffset32(Def, Addr))
2384 return;
2385 }
2386
2387 // Fall through to REG_SEQUENCE + V_ADD_CO_U32 + V_ADDC_U32 pattern
2388 if (Def->getOpcode() != AMDGPU::REG_SEQUENCE || Def->getNumOperands() != 5)
2389 return;
2390
2391 MachineOperand BaseLo = Def->getOperand(1);
2392 MachineOperand BaseHi = Def->getOperand(3);
2393 if (!BaseLo.isReg() || !BaseHi.isReg())
2394 return;
2395
2396 MachineInstr *BaseLoDef = MRI->getVRegDef(BaseLo.getReg());
2397 MachineInstr *BaseHiDef = MRI->getVRegDef(BaseHi.getReg());
2398
2399 if (!BaseLoDef || BaseLoDef->getOpcode() != AMDGPU::V_ADD_CO_U32_e64 ||
2400 !BaseHiDef || BaseHiDef->getOpcode() != AMDGPU::V_ADDC_U32_e64)
2401 return;
2402
2403 MachineOperand *Src0 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src0);
2404 MachineOperand *Src1 = TII->getNamedOperand(*BaseLoDef, AMDGPU::OpName::src1);
2405
2406 auto Offset0P = TII->getImmOrMaterializedImm(*MRI, *Src0);
2407 if (Offset0P)
2408 BaseLo = *Src1;
2409 else {
2410 if (!(Offset0P = TII->getImmOrMaterializedImm(*MRI, *Src1)))
2411 return;
2412 BaseLo = *Src0;
2413 }
2414
2415 if (!BaseLo.isReg())
2416 return;
2417
2418 Src0 = TII->getNamedOperand(*BaseHiDef, AMDGPU::OpName::src0);
2419 Src1 = TII->getNamedOperand(*BaseHiDef, AMDGPU::OpName::src1);
2420
2421 if (Src0->isImm())
2422 std::swap(Src0, Src1);
2423
2424 if (!Src1->isImm() || Src0->isImm())
2425 return;
2426
2427 uint64_t Offset1 = Src1->getImm();
2428 BaseHi = *Src0;
2429
2430 if (!BaseHi.isReg())
2431 return;
2432
2433 Addr.Base.LoReg = BaseLo.getReg();
2434 Addr.Base.HiReg = BaseHi.getReg();
2435 Addr.Base.LoSubReg = BaseLo.getSubReg();
2436 Addr.Base.HiSubReg = BaseHi.getSubReg();
2437 Addr.Offset = (*Offset0P & 0x00000000ffffffff) | (Offset1 << 32);
2438}
2439
2440// Maintain the correct LDS address for async loads and stores.
2441// It becomes incorrect when promoteConstantOffsetToImm adds an offset only
2442// meant for the global address operand. For async loads the LDS address is in
2443// vdst. For async stores, the LDS address is in vdata.
2444void SILoadStoreOptimizer::updateAsyncLDSAddress(MachineInstr &MI,
2445 int32_t OffsetDiff) const {
2446 if (!TII->usesASYNC_CNT(MI) || OffsetDiff == 0)
2447 return;
2448
2449 MachineOperand *LDSAddr = TII->getNamedOperand(MI, AMDGPU::OpName::vdst);
2450 if (!LDSAddr)
2451 LDSAddr = TII->getNamedOperand(MI, AMDGPU::OpName::vdata);
2452 assert(LDSAddr);
2453
2454 Register OldReg = LDSAddr->getReg();
2455 Register NewReg = MRI->createVirtualRegister(MRI->getRegClass(OldReg));
2456 MachineBasicBlock &MBB = *MI.getParent();
2457 const DebugLoc &DL = MI.getDebugLoc();
2458 BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_ADD_U32_e64), NewReg)
2459 .addReg(OldReg)
2460 .addImm(-OffsetDiff)
2461 .addImm(0);
2462
2463 LDSAddr->setReg(NewReg);
2464}
2465
2466bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
2467 MachineInstr &MI,
2468 MemInfoMap &Visited,
2469 SmallPtrSet<MachineInstr *, 4> &AnchorList) const {
2470
2471 if (!STM->hasFlatInstOffsets() || !SIInstrInfo::isFLAT(MI))
2472 return false;
2473
2474 if (SIInstrInfo::isFLATScratch(MI) &&
2475 !TII->getNamedOperand(MI, AMDGPU::OpName::vaddr))
2476 return false;
2477
2478 // Moving part of vaddr into the offset can trigger the SVS swizzle bug.
2479 if (STM->hasFlatScratchSVSSwizzleBug() && SIInstrInfo::isFLATScratch(MI) &&
2480 TII->getNamedOperand(MI, AMDGPU::OpName::saddr))
2481 return false;
2482
2483 unsigned AS = SIInstrInfo::isFLATGlobal(MI) ? AMDGPUAS::GLOBAL_ADDRESS
2484 : SIInstrInfo::isFLATScratch(MI) ? AMDGPUAS::PRIVATE_ADDRESS
2485 : AMDGPUAS::FLAT_ADDRESS;
2486
2487 AMDGPU::FlatAddrSpace FlatVariant =
2488 AS == AMDGPUAS::GLOBAL_ADDRESS ? AMDGPU::FlatAddrSpace::FlatGlobal
2489 : AS == AMDGPUAS::PRIVATE_ADDRESS ? AMDGPU::FlatAddrSpace::FlatScratch
2490 : AMDGPU::FlatAddrSpace::FLAT;
2491 bool AllowNegativeOffset =
2492 TII->allowNegativeFlatOffset(FlatVariant) && !TII->usesASYNC_CNT(MI);
2493 // The async global instructions use i24 offset for global address but u16
2494 // offset for LDS address. In this case, we just only promote when the offset
2495 // is u16.
2496 bool IsOffsetU16 = TII->usesASYNC_CNT(MI);
2497
2498 if (AnchorList.count(&MI))
2499 return false;
2500
2501 LLVM_DEBUG(dbgs() << "\nTryToPromoteConstantOffsetToImmFor "; MI.dump());
2502
2503 if (TII->getNamedOperand(MI, AMDGPU::OpName::offset)->getImm()) {
2504 LLVM_DEBUG(dbgs() << " Const-offset is already promoted.\n";);
2505 return false;
2506 }
2507
2508 // Step1: Find the base-registers and a 64bit constant offset.
2509 MachineOperand &Base = *TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
2510 auto [It, Inserted] = Visited.try_emplace(&MI);
2511 MemAddress MAddr;
2512 if (Inserted) {
2513 processBaseWithConstOffset(Base, MAddr);
2514 It->second = MAddr;
2515 } else
2516 MAddr = It->second;
2517
2518 if (MAddr.Offset == 0) {
2519 LLVM_DEBUG(dbgs() << " Failed to extract constant-offset or there are no"
2520 " constant offsets that can be promoted.\n";);
2521 return false;
2522 }
2523
2524 LLVM_DEBUG(dbgs() << " BASE: {" << printReg(MAddr.Base.HiReg, TRI) << ", "
2525 << printReg(MAddr.Base.LoReg, TRI)
2526 << "} Offset: " << MAddr.Offset << "\n\n";);
2527
2528 // Step2: Traverse through MI's basic block and find an anchor(that has the
2529 // same base-registers) with the highest 13bit distance from MI's offset.
2530 // E.g. (64bit loads)
2531 // bb:
2532 // addr1 = &a + 4096; load1 = load(addr1, 0)
2533 // addr2 = &a + 6144; load2 = load(addr2, 0)
2534 // addr3 = &a + 8192; load3 = load(addr3, 0)
2535 // addr4 = &a + 10240; load4 = load(addr4, 0)
2536 // addr5 = &a + 12288; load5 = load(addr5, 0)
2537 //
2538 // Starting from the first load, the optimization will try to find a new base
2539 // from which (&a + 4096) has 13 bit distance. Both &a + 6144 and &a + 8192
2540 // has 13bit distance from &a + 4096. The heuristic considers &a + 8192
2541 // as the new-base(anchor) because of the maximum distance which can
2542 // accommodate more intermediate bases presumably.
2543 //
2544 // Step3: move (&a + 8192) above load1. Compute and promote offsets from
2545 // (&a + 8192) for load1, load2, load4.
2546 // addr = &a + 8192
2547 // load1 = load(addr, -4096)
2548 // load2 = load(addr, -2048)
2549 // load3 = load(addr, 0)
2550 // load4 = load(addr, 2048)
2551 // addr5 = &a + 12288; load5 = load(addr5, 0)
2552 //
2553 MachineInstr *AnchorInst = nullptr;
2554 MemAddress AnchorAddr;
2555 uint32_t MaxDist = std::numeric_limits<uint32_t>::min();
2556 SmallVector<std::pair<MachineInstr *, int64_t>, 4> InstsWCommonBase;
2557 bool MIIsAnchor = false;
2558
2559 MachineBasicBlock *MBB = MI.getParent();
2560 MachineBasicBlock::iterator E = MBB->end();
2561 MachineBasicBlock::iterator MBBI = MI.getIterator();
2562 ++MBBI;
2563 const SITargetLowering *TLI = STM->getTargetLowering();
2564
2565 for ( ; MBBI != E; ++MBBI) {
2566 MachineInstr &MINext = *MBBI;
2567 // TODO: Support finding an anchor(with same base) from store addresses or
2568 // any other load addresses where the opcodes are different.
2569 if (MINext.getOpcode() != MI.getOpcode() ||
2570 TII->getNamedOperand(MINext, AMDGPU::OpName::offset)->getImm())
2571 continue;
2572
2573 const MachineOperand &BaseNext =
2574 *TII->getNamedOperand(MINext, AMDGPU::OpName::vaddr);
2575 MemAddress MAddrNext;
2576 auto [It, Inserted] = Visited.try_emplace(&MINext);
2577 if (Inserted) {
2578 processBaseWithConstOffset(BaseNext, MAddrNext);
2579 It->second = MAddrNext;
2580 } else
2581 MAddrNext = It->second;
2582
2583 if (MAddrNext.Base.LoReg != MAddr.Base.LoReg ||
2584 MAddrNext.Base.HiReg != MAddr.Base.HiReg ||
2585 MAddrNext.Base.LoSubReg != MAddr.Base.LoSubReg ||
2586 MAddrNext.Base.HiSubReg != MAddr.Base.HiSubReg)
2587 continue;
2588
2589 InstsWCommonBase.emplace_back(&MINext, MAddrNext.Offset);
2590
2591 if (AllowNegativeOffset) {
2592 int64_t Dist = MAddr.Offset - MAddrNext.Offset;
2593 TargetLoweringBase::AddrMode AM;
2594 AM.HasBaseReg = true;
2595 AM.BaseOffs = Dist;
2596 if (TLI->isLegalFlatAddressingMode(AM, AS) &&
2597 (uint32_t)std::abs(Dist) > MaxDist) {
2598 MaxDist = std::abs(Dist);
2599
2600 AnchorAddr = MAddrNext;
2601 AnchorInst = &MINext;
2602 }
2603 }
2604 }
2605
2606 // When negative offsets are not allowed, pick the candidate with the smallest
2607 // offset as anchor so all promoted offsets are non-negative. If MI itself has
2608 // the smallest offset, MI becomes the reference point (MIIsAnchor).
2609 if (!AllowNegativeOffset && !InstsWCommonBase.empty()) {
2610 for (auto &[Inst, Offset] : InstsWCommonBase) {
2611 int64_t Dist = MAddr.Offset - Offset;
2612 TargetLoweringBase::AddrMode AM;
2613 AM.HasBaseReg = true;
2614 AM.BaseOffs = Dist;
2615 if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AS) &&
2616 (!IsOffsetU16 || isUInt<16>(Dist)) &&
2617 (!AnchorInst || Offset < AnchorAddr.Offset)) {
2618 AnchorAddr = Visited[Inst];
2619 AnchorInst = Inst;
2620 }
2621 }
2622 if (!AnchorInst)
2623 MIIsAnchor = true;
2624 }
2625
2626 if (AnchorInst) {
2627 LLVM_DEBUG(dbgs() << " Anchor-Inst(with max-distance from Offset): ";
2628 AnchorInst->dump());
2629 LLVM_DEBUG(dbgs() << " Anchor-Offset from BASE: "
2630 << AnchorAddr.Offset << "\n\n");
2631
2632 // Instead of moving up, just re-compute anchor-instruction's base address.
2633 Register Base = computeBase(MI, AnchorAddr);
2634
2635 int32_t OffsetDiff = MAddr.Offset - AnchorAddr.Offset;
2636 updateBaseAndOffset(MI, Base, OffsetDiff);
2637 updateAsyncLDSAddress(MI, OffsetDiff);
2638 LLVM_DEBUG(dbgs() << " After promotion: "; MI.dump(););
2639
2640 for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
2641 TargetLoweringBase::AddrMode AM;
2642 AM.HasBaseReg = true;
2643 AM.BaseOffs = OtherOffset - AnchorAddr.Offset;
2644
2645 if (TLI->isLegalFlatAddressingMode(AM, AS) &&
2646 (AllowNegativeOffset || AM.BaseOffs >= 0) &&
2647 (!IsOffsetU16 || isUInt<16>(AM.BaseOffs))) {
2648 LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset; dbgs() << ")";
2649 OtherMI->dump());
2650 int32_t OtherOffsetDiff = OtherOffset - AnchorAddr.Offset;
2651 updateBaseAndOffset(*OtherMI, Base, OtherOffsetDiff);
2652 updateAsyncLDSAddress(*OtherMI, OtherOffsetDiff);
2653 LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
2654 }
2655 }
2656 AnchorList.insert(AnchorInst);
2657 return true;
2658 }
2659
2660 if (MIIsAnchor) {
2661 LLVM_DEBUG(dbgs() << " MI is anchor (smallest offset); promoting "
2662 "candidates relative to MI's base.\n");
2663
2664 Register Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr)->getReg();
2665 bool AnyPromoted = false;
2666
2667 for (auto [OtherMI, OtherOffset] : InstsWCommonBase) {
2668 int64_t Dist = OtherOffset - MAddr.Offset;
2669 TargetLoweringBase::AddrMode AM;
2670 AM.HasBaseReg = true;
2671 AM.BaseOffs = Dist;
2672 if (Dist >= 0 && TLI->isLegalFlatAddressingMode(AM, AS) &&
2673 (!IsOffsetU16 || isUInt<16>(Dist))) {
2674 LLVM_DEBUG(dbgs() << " Promote Offset(" << OtherOffset << ")";
2675 OtherMI->dump());
2676 updateBaseAndOffset(*OtherMI, Base, Dist);
2677 updateAsyncLDSAddress(*OtherMI, Dist);
2678 LLVM_DEBUG(dbgs() << " After promotion: "; OtherMI->dump());
2679 AnyPromoted = true;
2680 }
2681 }
2682
2683 if (AnyPromoted) {
2684 TII->getNamedOperand(MI, AMDGPU::OpName::vaddr)->setIsKill(false);
2685 AnchorList.insert(&MI);
2686 return true;
2687 }
2688 }
2689
2690 return false;
2691}
2692
2693void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI,
2694 std::list<std::list<CombineInfo> > &MergeableInsts) const {
2695 for (std::list<CombineInfo> &AddrList : MergeableInsts) {
2696 if (AddrList.front().InstClass == CI.InstClass &&
2697 AddrList.front().hasSameBaseAddress(CI)) {
2698 AddrList.emplace_back(CI);
2699 return;
2700 }
2701 }
2702
2703 // Base address not found, so add a new list.
2704 MergeableInsts.emplace_back(1, CI);
2705}
2706
2707std::pair<MachineBasicBlock::iterator, bool>
2708SILoadStoreOptimizer::collectMergeableInsts(
2709 MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End,
2710 MemInfoMap &Visited, SmallPtrSet<MachineInstr *, 4> &AnchorList,
2711 std::list<std::list<CombineInfo>> &MergeableInsts) const {
2712 bool Modified = false;
2713
2714 // Sort potential mergeable instructions into lists. One list per base address.
2715 unsigned Order = 0;
2716 MachineBasicBlock::iterator BlockI = Begin;
2717 for (; BlockI != End; ++BlockI) {
2718 MachineInstr &MI = *BlockI;
2719
2720 // We run this before checking if an address is mergeable, because it can produce
2721 // better code even if the instructions aren't mergeable.
2722 if (promoteConstantOffsetToImm(MI, Visited, AnchorList))
2723 Modified = true;
2724
2725 // Treat volatile accesses, ordered accesses and unmodeled side effects as
2726 // barriers. We can look after this barrier for separate merges.
2727 if (MI.hasOrderedMemoryRef() || MI.hasUnmodeledSideEffects()) {
2728 LLVM_DEBUG(dbgs() << "Breaking search on barrier: " << MI);
2729
2730 // Search will resume after this instruction in a separate merge list.
2731 ++BlockI;
2732 break;
2733 }
2734
2735 const InstClassEnum InstClass = getInstClass(MI.getOpcode(), *TII);
2736 if (InstClass == UNKNOWN)
2737 continue;
2738
2739 // Do not merge VMEM buffer instructions with "swizzled" bit set.
2740 int Swizzled =
2741 AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::swz);
2742 if (Swizzled != -1 && MI.getOperand(Swizzled).getImm())
2743 continue;
2744
2745 if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) {
2746 if (!STM->hasRelaxedTBufferOOBMode()) {
2747 LLVM_DEBUG(
2748 dbgs() << "Skip tbuffer combine: relaxed OOB mode not enabled\n");
2749 continue;
2750 }
2751
2752 const MachineOperand *Fmt =
2753 TII->getNamedOperand(MI, AMDGPU::OpName::format);
2754 if (!AMDGPU::getGcnBufferFormatInfo(Fmt->getImm(), *STM)) {
2755 LLVM_DEBUG(dbgs() << "Skip tbuffer with unknown format: " << MI);
2756 continue;
2757 }
2758 } else if (InstClass == MIMG) {
2759 // Do not merge MIMG instructions with tfe or lwe enabled.
2760 // TFE/LWE add a status result that the image merge path does not model.
2761 const auto *TFEOp = TII->getNamedOperand(MI, AMDGPU::OpName::tfe);
2762 if (TFEOp && TFEOp->getImm())
2763 continue;
2764
2765 const auto *LWEOp = TII->getNamedOperand(MI, AMDGPU::OpName::lwe);
2766 if (LWEOp && LWEOp->getImm())
2767 continue;
2768 }
2769
2770 CombineInfo CI;
2771 CI.setMI(MI, *this);
2772 CI.Order = Order++;
2773
2774 if (!CI.hasMergeableAddress(*MRI))
2775 continue;
2776
2777 LLVM_DEBUG(dbgs() << "Mergeable: " << MI);
2778
2779 addInstToMergeableList(CI, MergeableInsts);
2780 }
2781
2782 // At this point we have lists of Mergeable instructions.
2783 //
2784 // Part 2: Sort lists by offset and then for each CombineInfo object in the
2785 // list try to find an instruction that can be merged with I. If an instruction
2786 // is found, it is stored in the Paired field. If no instructions are found, then
2787 // the CombineInfo object is deleted from the list.
2788
2789 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(),
2790 E = MergeableInsts.end(); I != E;) {
2791
2792 std::list<CombineInfo> &MergeList = *I;
2793 if (MergeList.size() <= 1) {
2794 // This means we have found only one instruction with a given address
2795 // that can be merged, and we need at least 2 instructions to do a merge,
2796 // so this list can be discarded.
2797 I = MergeableInsts.erase(I);
2798 continue;
2799 }
2800
2801 // Sort the lists by offsets, this way mergeable instructions will be
2802 // adjacent to each other in the list, which will make it easier to find
2803 // matches.
2804 MergeList.sort(
2805 [] (const CombineInfo &A, const CombineInfo &B) {
2806 return A.Offset < B.Offset;
2807 });
2808 ++I;
2809 }
2810
2811 return {BlockI, Modified};
2812}
2813
2814// Scan through looking for adjacent LDS operations with constant offsets from
2815// the same base register. We rely on the scheduler to do the hard work of
2816// clustering nearby loads, and assume these are all adjacent.
2817bool SILoadStoreOptimizer::optimizeBlock(
2818 std::list<std::list<CombineInfo> > &MergeableInsts) {
2819 bool Modified = false;
2820
2821 for (std::list<std::list<CombineInfo>>::iterator I = MergeableInsts.begin(),
2822 E = MergeableInsts.end(); I != E;) {
2823 std::list<CombineInfo> &MergeList = *I;
2824
2825 bool OptimizeListAgain = false;
2826 if (!optimizeInstsWithSameBaseAddr(MergeList, OptimizeListAgain)) {
2827 // We weren't able to make any changes, so delete the list so we don't
2828 // process the same instructions the next time we try to optimize this
2829 // block.
2830 I = MergeableInsts.erase(I);
2831 continue;
2832 }
2833
2834 Modified = true;
2835
2836 // We made changes, but also determined that there were no more optimization
2837 // opportunities, so we don't need to reprocess the list
2838 if (!OptimizeListAgain) {
2839 I = MergeableInsts.erase(I);
2840 continue;
2841 }
2842 OptimizeAgain = true;
2843 }
2844 return Modified;
2845}
2846
2847bool
2848SILoadStoreOptimizer::optimizeInstsWithSameBaseAddr(
2849 std::list<CombineInfo> &MergeList,
2850 bool &OptimizeListAgain) {
2851 if (MergeList.empty())
2852 return false;
2853
2854 bool Modified = false;
2855
2856 for (auto I = MergeList.begin(), Next = std::next(I); Next != MergeList.end();
2857 Next = std::next(I)) {
2858
2859 auto First = I;
2860 auto Second = Next;
2861
2862 if ((*First).Order > (*Second).Order)
2863 std::swap(First, Second);
2864 CombineInfo &CI = *First;
2865 CombineInfo &Paired = *Second;
2866
2867 CombineInfo *Where = checkAndPrepareMerge(CI, Paired);
2868 if (!Where) {
2869 ++I;
2870 continue;
2871 }
2872
2873 Modified = true;
2874
2875 LLVM_DEBUG(dbgs() << "Merging: " << *CI.I << " with: " << *Paired.I);
2876
2877 MachineBasicBlock::iterator NewMI;
2878 switch (CI.InstClass) {
2879 default:
2880 llvm_unreachable("unknown InstClass");
2881 break;
2882 case DS_READ:
2883 NewMI = mergeRead2Pair(CI, Paired, Where->I);
2884 break;
2885 case DS_WRITE:
2886 NewMI = mergeWrite2Pair(CI, Paired, Where->I);
2887 break;
2888 case S_BUFFER_LOAD_IMM:
2889 case S_BUFFER_LOAD_SGPR_IMM:
2890 case S_LOAD_IMM:
2891 NewMI = mergeSMemLoadImmPair(CI, Paired, Where->I);
2892 OptimizeListAgain |= CI.Width + Paired.Width < 8;
2893 break;
2894 case BUFFER_LOAD:
2895 NewMI = mergeBufferLoadPair(CI, Paired, Where->I);
2896 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2897 break;
2898 case BUFFER_STORE:
2899 NewMI = mergeBufferStorePair(CI, Paired, Where->I);
2900 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2901 break;
2902 case MIMG:
2903 NewMI = mergeImagePair(CI, Paired, Where->I);
2904 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2905 break;
2906 case TBUFFER_LOAD:
2907 NewMI = mergeTBufferLoadPair(CI, Paired, Where->I);
2908 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2909 break;
2910 case TBUFFER_STORE:
2911 NewMI = mergeTBufferStorePair(CI, Paired, Where->I);
2912 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2913 break;
2914 case FLAT_LOAD:
2915 case FLAT_LOAD_SADDR:
2916 case GLOBAL_LOAD:
2917 case GLOBAL_LOAD_SADDR:
2918 NewMI = mergeFlatLoadPair(CI, Paired, Where->I);
2919 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2920 break;
2921 case FLAT_STORE:
2922 case FLAT_STORE_SADDR:
2923 case GLOBAL_STORE:
2924 case GLOBAL_STORE_SADDR:
2925 NewMI = mergeFlatStorePair(CI, Paired, Where->I);
2926 OptimizeListAgain |= CI.Width + Paired.Width < 4;
2927 break;
2928 }
2929 CI.setMI(NewMI, *this);
2930 CI.Order = Where->Order;
2931 if (I == Second)
2932 I = Next;
2933
2934 MergeList.erase(Second);
2935 }
2936
2937 return Modified;
2938}
2939
2940bool SILoadStoreOptimizerLegacy::runOnMachineFunction(MachineFunction &MF) {
2941 if (skipFunction(MF.getFunction()))
2942 return false;
2943 return SILoadStoreOptimizer(
2944 &getAnalysis<AAResultsWrapperPass>().getAAResults())
2945 .run(MF);
2946}
2947
2948bool SILoadStoreOptimizer::run(MachineFunction &MF) {
2949 this->MF = &MF;
2950 STM = &MF.getSubtarget<GCNSubtarget>();
2951 if (!STM->loadStoreOptEnabled())
2952 return false;
2953
2954 TII = STM->getInstrInfo();
2955 TRI = &TII->getRegisterInfo();
2956
2957 MRI = &MF.getRegInfo();
2958
2959 LLVM_DEBUG(dbgs() << "Running SILoadStoreOptimizer\n");
2960
2961 bool Modified = false;
2962
2963 // Contains the list of instructions for which constant offsets are being
2964 // promoted to the IMM. This is tracked for an entire block at time.
2965 SmallPtrSet<MachineInstr *, 4> AnchorList;
2966 MemInfoMap Visited;
2967
2968 for (MachineBasicBlock &MBB : MF) {
2969 MachineBasicBlock::iterator SectionEnd;
2970 for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E;
2971 I = SectionEnd) {
2972 bool CollectModified;
2973 std::list<std::list<CombineInfo>> MergeableInsts;
2974
2975 // First pass: Collect list of all instructions we know how to merge in a
2976 // subset of the block.
2977 std::tie(SectionEnd, CollectModified) =
2978 collectMergeableInsts(I, E, Visited, AnchorList, MergeableInsts);
2979
2980 Modified |= CollectModified;
2981
2982 do {
2983 OptimizeAgain = false;
2984 Modified |= optimizeBlock(MergeableInsts);
2985 } while (OptimizeAgain);
2986 }
2987
2988 Visited.clear();
2989 AnchorList.clear();
2990 }
2991
2992 return Modified;
2993}
2994
2995PreservedAnalyses
2996SILoadStoreOptimizerPass::run(MachineFunction &MF,
2997 MachineFunctionAnalysisManager &MFAM) {
2998 MFPropsModifier _(*this, MF);
2999
3000 if (MF.getFunction().hasOptNone())
3001 return PreservedAnalyses::all();
3002
3003 auto &FAM = MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF)
3004 .getManager();
3005 AAResults &AA = FAM.getResult<AAManager>(MF.getFunction());
3006
3007 bool Changed = SILoadStoreOptimizer(&AA).run(MF);
3008 if (!Changed)
3009 return PreservedAnalyses::all();
3010
3011 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
3012 PA.preserveSet<CFGAnalyses>();
3013 return PA;
3014}
TLI
const TargetLowering & TLI
Definition AArch64ISelLowering.cpp:28558
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const
aarch64 promote const
Definition AArch64PromoteConstant.cpp:223
uint64_t
uint64_t uint64_t
Definition AMDGPUDisassembler.cpp:553
addInstToMergeableList
INITIALIZE_PASS(AMDGPUImageIntrinsicOptimizer, DEBUG_TYPE, "AMDGPU Image Intrinsic Optimizer", false, false) char AMDGPUImageIntrinsicOptimizer void addInstToMergeableList(IntrinsicInst *II, SmallVector< SmallVector< IntrinsicInst *, 4 > > &MergeableInsts, const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr)
Definition AMDGPUImageIntrinsicOptimizer.cpp:98
collectMergeableInsts
BasicBlock::iterator collectMergeableInsts(BasicBlock::iterator I, BasicBlock::iterator E, SmallVector< SmallVector< IntrinsicInst *, 4 > > &MergeableInsts)
Definition AMDGPUImageIntrinsicOptimizer.cpp:142
MBB
MachineBasicBlock & MBB
Definition ARMSLSHardening.cpp:71
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition ARMSLSHardening.cpp:72
A
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
GCNSubtarget.h
AMD GCN specific subclass of TargetSubtarget.
DEBUG_TYPE
#define DEBUG_TYPE
Definition GenericCycleImpl.h:34
op
#define op(i)
TII
const HexagonInstrInfo * TII
Definition HexagonCopyToCombine.cpp:118
_
#define _
Definition HexagonMCCodeEmitter.cpp:46
getAlign
static MaybeAlign getAlign(Value *Ptr)
Definition IRBuilder.cpp:544
MI
IRTranslator LLVM IR MI
Definition IRTranslator.cpp:876
OffsetOp
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
Definition InstCombineCompares.cpp:6256
I
#define I(x, y, z)
Definition MD5.cpp:57
Reg
Register Reg
Definition MachineSink.cpp:2076
TRI
Register const TargetRegisterInfo * TRI
Definition MachineSink.cpp:2077
Register
Promote Memory to Register
Definition Mem2Reg.cpp:110
getReg
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Definition MipsDisassembler.cpp:106
FAM
FunctionAnalysisManager FAM
Definition PassBuilderBindings.cpp:61
INITIALIZE_PASS_DEPENDENCY
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition PassSupport.h:42
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition PassSupport.h:44
INITIALIZE_PASS_BEGIN
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
Definition PassSupport.h:39
MRI
MachineRegisterInfo & MRI
Definition RISCVLateBranchOpt.cpp:60
Lo
MachineInstr *& Lo
Definition RISCVMergeBaseOffset.cpp:77
mostAlignedValueInRange
static uint32_t mostAlignedValueInRange(uint32_t Lo, uint32_t Hi)
Definition SILoadStoreOptimizer.cpp:1082
needsConstrainedOpcode
static bool needsConstrainedOpcode(const GCNSubtarget &STM, ArrayRef< MachineMemOperand * > MMOs, unsigned Width)
Definition SILoadStoreOptimizer.cpp:1874
addDefsUsesToList
static void addDefsUsesToList(const MachineInstr &MI, DenseSet< Register > &RegDefs, DenseSet< Register > &RegUses)
Definition SILoadStoreOptimizer.cpp:956
getBufferFormatWithCompCount
static unsigned getBufferFormatWithCompCount(unsigned OldFormat, unsigned ComponentCount, const GCNSubtarget &STI)
Definition SILoadStoreOptimizer.cpp:1051
optimizeBlock
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
Definition ScalarizeMaskedMemIntrin.cpp:1232
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:119
llvm::AAManager
A manager for alias analyses.
Definition AliasAnalysis.h:967
llvm::AAResultsWrapperPass
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Definition AliasAnalysis.h:1015
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition PassManager.h:436
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition PassAnalysisSupport.h:48
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition PassAnalysisSupport.h:76
llvm::AnalysisUsage::setPreservesCFG
LLVM_ABI void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition Pass.cpp:280
llvm::ArrayRef
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
llvm::ArrayRef::size
size_t size() const
Get the array size.
Definition ArrayRef.h:141
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
llvm::DebugLoc
A debug info location.
Definition DebugLoc.h:126
llvm::DebugLoc::getMergedLocation
static LLVM_ABI DebugLoc getMergedLocation(DebugLoc LocA, DebugLoc LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
Definition DebugLoc.cpp:186
llvm::DenseSet
Implements a dense probed hash-table based set.
Definition DenseSet.h:283
llvm::Function::hasOptNone
bool hasOptNone() const
Do not optimize this function (-O0).
Definition Function.h:686
llvm::GCNSubtarget::loadStoreOptEnabled
bool loadStoreOptEnabled() const
Definition GCNSubtarget.h:575
llvm::GCNSubtarget::getInstrInfo
const SIInstrInfo * getInstrInfo() const override
Definition GCNSubtarget.h:135
llvm::GCNSubtarget::hasDwordx3LoadStores
bool hasDwordx3LoadStores() const
Definition GCNSubtarget.h:586
llvm::GCNSubtarget::getTargetLowering
const SITargetLowering * getTargetLowering() const override
Definition GCNSubtarget.h:141
llvm::GCNSubtarget::hasRelaxedTBufferOOBMode
bool hasRelaxedTBufferOOBMode() const
Definition GCNSubtarget.h:349
llvm::GCNSubtarget::ldsRequiresM0Init
bool ldsRequiresM0Init() const
Return if most LDS instructions have an m0 use that require m0 to be initialized.
Definition GCNSubtarget.h:436
llvm::GCNSubtarget::hasFlatScratchSVSSwizzleBug
bool hasFlatScratchSVSSwizzleBug() const
Definition GCNSubtarget.h:689
llvm::GCNSubtarget::isXNACKEnabled
bool isXNACKEnabled() const
Definition GCNSubtarget.h:346
llvm::HexagonInstrInfo::getRegisterInfo
const HexagonRegisterInfo & getRegisterInfo() const
Definition HexagonInstrInfo.h:53
llvm::LocationSize::getValue
TypeSize getValue() const
Definition MemoryLocation.h:158
llvm::MCInstrDesc::getOpcode
unsigned getOpcode() const
Return the opcode number for this descriptor.
Definition MCInstrDesc.h:233
llvm::MFPropsModifier
An RAII based helper class to modify MachineFunctionProperties when running pass.
Definition MachinePassManager.h:40
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition MachineBasicBlock.h:322
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition MachineBasicBlock.h:340
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition MachineFunctionPass.h:31
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition MachineFunctionPass.cpp:148
llvm::MachineFunctionProperties
Properties which a MachineFunction may have at a given point in time.
Definition MachineFunction.h:138
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition MachineFunction.h:778
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition MachineFunction.h:788
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition MachineFunction.h:739
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition MachineFunction.cpp:570
llvm::MachineInstrBuilder::cloneMergedMemRefs
const MachineInstrBuilder & cloneMergedMemRefs(ArrayRef< const MachineInstr * > OtherMIs) const
Definition MachineInstrBuilder.h:321
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
Definition MachineInstrBuilder.h:199
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition MachineInstrBuilder.h:233
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition MachineInstrBuilder.h:326
llvm::MachineInstr
Representation of each machine instruction.
Definition MachineInstr.h:74
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition MachineInstr.h:609
llvm::MachineInstr::dump
LLVM_ABI void dump() const
Definition MachineInstr.cpp:1782
llvm::MachineMemOperand
A description of a memory reference used in the backend.
Definition MachineMemOperand.h:141
llvm::MachineMemOperand::getSize
LocationSize getSize() const
Return the size in bytes of the memory reference.
Definition MachineMemOperand.h:255
llvm::MachineMemOperand::getAddrSpace
unsigned getAddrSpace() const
Definition MachineMemOperand.h:248
llvm::MachineMemOperand::getPointerInfo
const MachinePointerInfo & getPointerInfo() const
Definition MachineMemOperand.h:219
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition MachineOperand.h:49
llvm::MachineOperand::getSubReg
unsigned getSubReg() const
Definition MachineOperand.h:380
llvm::MachineOperand::getImm
int64_t getImm() const
Definition MachineOperand.h:563
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition MachineOperand.h:334
llvm::MachineOperand::setReg
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
Definition MachineOperand.cpp:60
llvm::MachineOperand::isImm
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Definition MachineOperand.h:336
llvm::MachineOperand::CreateImm
static MachineOperand CreateImm(int64_t Val)
Definition MachineOperand.h:836
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition MachineOperand.h:375
llvm::MachineOperand::CreateReg
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
Definition MachineOperand.h:854
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition MachineRegisterInfo.h:55
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition MachineRegisterInfo.h:684
llvm::MachineRegisterInfo::getVRegDef
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition MachineRegisterInfo.cpp:406
llvm::MachineRegisterInfo::createVirtualRegister
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition MachineRegisterInfo.cpp:156
llvm::MachineRegisterInfo::constrainRegClass
LLVM_ABI const TargetRegisterClass * constrainRegClass(Register Reg, const TargetRegisterClass *RC, unsigned MinNumRegs=0)
constrainRegClass - Constrain the register class of the specified virtual register to be a common sub...
Definition MachineRegisterInfo.cpp:86
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
llvm::PreservedAnalyses::preserveSet
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
llvm::Register
Wrapper class representing virtual and physical registers.
Definition Register.h:20
llvm::Register::isPhysical
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
llvm::SIInstrInfo::isFLATScratch
static bool isFLATScratch(const MachineInstr &MI)
Definition SIInstrInfo.h:761
llvm::SIInstrInfo::isVIMAGE
static bool isVIMAGE(const MachineInstr &MI)
Definition SIInstrInfo.h:715
llvm::SIInstrInfo::isFLATGlobal
static bool isFLATGlobal(const MachineInstr &MI)
Definition SIInstrInfo.h:753
llvm::SIInstrInfo::isVSAMPLE
static bool isVSAMPLE(const MachineInstr &MI)
Definition SIInstrInfo.h:723
llvm::SIInstrInfo::isFLAT
static bool isFLAT(const MachineInstr &MI)
Definition SIInstrInfo.h:739
llvm::SIInstrInfo::getNamedOperand
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, AMDGPU::OpName OperandName) const
Returns the operand named Op.
Definition SIInstrInfo.cpp:10312
llvm::SILoadStoreOptimizerPass::run
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Definition SILoadStoreOptimizer.cpp:2996
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition SmallPtrSet.h:519
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition SmallVector.h:995
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::detail::DenseSetImpl::contains
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition DenseSet.h:183
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
uint32_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AA
Abstract Attribute helper functions.
Definition Attributor.h:165
llvm::AMDGPUAS::FLAT_ADDRESS
@ FLAT_ADDRESS
Address space for flat memory.
Definition AMDGPUAddrSpace.h:31
llvm::AMDGPUAS::GLOBAL_ADDRESS
@ GLOBAL_ADDRESS
Address space for global memory (RAT0, VTX0).
Definition AMDGPUAddrSpace.h:32
llvm::AMDGPUAS::PRIVATE_ADDRESS
@ PRIVATE_ADDRESS
Address space for private memory.
Definition AMDGPUAddrSpace.h:37
llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
Definition AMDGPUMetadata.h:183
llvm::AMDGPU::getMIMGInfo
LLVM_READONLY const MIMGInfo * getMIMGInfo(unsigned Opc)
llvm::AMDGPU::convertSMRDOffsetUnits
uint64_t convertSMRDOffsetUnits(const MCSubtargetInfo &ST, uint64_t ByteOffset)
Convert ByteOffset to dwords if the subtarget uses dword SMRD immediate offsets.
Definition AMDGPUBaseInfo.cpp:3289
llvm::AMDGPU::getMTBUFHasSrsrc
bool getMTBUFHasSrsrc(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:511
llvm::AMDGPU::getMTBUFElements
int getMTBUFElements(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:501
llvm::AMDGPU::getMTBUFHasSoffset
bool getMTBUFHasSoffset(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:516
llvm::AMDGPU::getMUBUFOpcode
int getMUBUFOpcode(unsigned BaseOpc, unsigned Elements)
Definition AMDGPUBaseInfo.cpp:526
llvm::AMDGPU::getMUBUFBaseOpcode
int getMUBUFBaseOpcode(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:521
llvm::AMDGPU::hasNamedOperand
LLVM_READONLY bool hasNamedOperand(uint32_t Opcode, OpName NamedIdx)
Definition AMDGPUBaseInfo.h:349
llvm::AMDGPU::getMTBUFBaseOpcode
int getMTBUFBaseOpcode(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:490
llvm::AMDGPU::getMUBUFHasVAddr
bool getMUBUFHasVAddr(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:537
llvm::AMDGPU::getMTBUFOpcode
int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements)
Definition AMDGPUBaseInfo.cpp:495
llvm::AMDGPU::getMUBUFHasSoffset
bool getMUBUFHasSoffset(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:547
llvm::AMDGPU::getMIMGBaseOpcode
const MIMGBaseOpcodeInfo * getMIMGBaseOpcode(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:329
llvm::AMDGPU::getMIMGBaseOpcodeInfo
LLVM_READONLY const MIMGBaseOpcodeInfo * getMIMGBaseOpcodeInfo(unsigned BaseOpcode)
llvm::AMDGPU::getMaskedMIMGOp
int getMaskedMIMGOp(unsigned Opc, unsigned NewChannels)
Definition AMDGPUBaseInfo.cpp:334
llvm::AMDGPU::getMTBUFHasVAddr
bool getMTBUFHasVAddr(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:506
llvm::AMDGPU::getMUBUFElements
int getMUBUFElements(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:532
llvm::AMDGPU::getGcnBufferFormatInfo
const GcnBufferFormatInfo * getGcnBufferFormatInfo(uint8_t BitsPerComp, uint8_t NumComponents, uint8_t NumFormat, const MCSubtargetInfo &STI)
Definition AMDGPUBaseInfo.cpp:3419
llvm::AMDGPU::getMUBUFHasSrsrc
bool getMUBUFHasSrsrc(unsigned Opc)
Definition AMDGPUBaseInfo.cpp:542
llvm::ARM_AM::add
@ add
Definition ARMAddressingModes.h:39
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition BitmaskEnum.h:126
llvm::dwarf_linker::DebugSectionKind::DebugLoc
@ DebugLoc
Definition DWARFLinkerBase.h:35
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
Definition MicrosoftDemangleNodes.h:127
llvm::rdf::Def
NodeAddr< DefNode * > Def
Definition RDFGraph.h:384
llvm::sframe::BaseReg
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition DefaultTargetInfo.h:21
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::operator<
bool operator<(int64_t V1, const APSInt &V2)
Definition APSInt.h:360
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition MachineInstrBuilder.h:449
llvm::maskLeadingOnes
constexpr T maskLeadingOnes(unsigned N)
Create a bitmask with the N left-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:89
llvm::getSingleElement
decltype(auto) getSingleElement(ContainerTy &&C)
Asserts that the given container has a single element and returns that element.
Definition STLExtras.h:309
llvm::MachineFunctionAnalysisManager
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
Definition MachineFunctionAnalysisManager.h:24
llvm::SILoadStoreOptimizerLegacyID
char & SILoadStoreOptimizerLegacyID
Definition SILoadStoreOptimizer.cpp:954
llvm::popcount
constexpr int popcount(T Value) noexcept
Count the number of set bits in a value.
Definition bit.h:156
llvm::countr_zero
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
llvm::getMachineFunctionPassPreservedAnalyses
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
Definition MachinePassManager.cpp:162
llvm::countl_zero
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::isUInt
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
llvm::SmallVector
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
Definition SmallVector.h:1187
llvm::IRMemLocation::Other
@ Other
Any other memory.
Definition ModRef.h:68
llvm::IRMemLocation::First
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:25
llvm::AnchorList
std::vector< std::pair< LineLocation, FunctionId > > AnchorList
Definition SampleProfileMatcher.h:27
llvm::BitWidth
constexpr unsigned BitWidth
Definition BitmaskEnum.h:219
llvm::Next
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
llvm::maskTrailingOnes
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
llvm::MachineVerifierMode::End
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
llvm::AliasAnalysis
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
Definition AliasAnalysis.h:692
llvm::printReg
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Definition TargetRegisterInfo.cpp:106
llvm::TargetRegisterClass
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
Generated on for LLVM by doxygen 1.14.0