LLVM 24.0.0git
AArch64LoadStoreOptimizer.cpp
Go to the documentation of this file.
1//===- AArch64LoadStoreOptimizer.cpp - AArch64 load/store opt. pass -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that performs load / store related peephole
10// optimizations. This pass should be run after register allocation.
11//
12// The pass runs after the PrologEpilogInserter where we emit the CFI
13// instructions. In order to preserve the correctness of the unwind information,
14// the pass should not change the order of any two instructions, one of which
15// has the FrameSetup/FrameDestroy flag or, alternatively, apply an add-hoc fix
16// to unwind information.
17//
18//===----------------------------------------------------------------------===//
19
20#include "AArch64InstrInfo.h"
21#include "AArch64MachineFunctionInfo.h"
22#include "AArch64Subtarget.h"
23#include "MCTargetDesc/AArch64AddressingModes.h"
24#include "llvm/ADT/SetVector.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/iterator_range.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineFunction.h"
32#include "llvm/CodeGen/MachineFunctionPass.h"
33#include "llvm/CodeGen/MachineInstr.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineOperand.h"
36#include "llvm/CodeGen/MachineRegisterInfo.h"
37#include "llvm/CodeGen/TargetRegisterInfo.h"
38#include "llvm/IR/DebugLoc.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCDwarf.h"
41#include "llvm/Pass.h"
42#include "llvm/Support/Debug.h"
43#include "llvm/Support/DebugCounter.h"
44#include "llvm/Support/ErrorHandling.h"
45#include <cassert>
46#include <cstdint>
47#include <functional>
48#include <iterator>
49#include <limits>
50#include <optional>
51
52using namespace llvm;
53
54#define DEBUG_TYPE "aarch64-ldst-opt"
55
56STATISTIC(NumPairCreated, "Number of load/store pair instructions generated");
57STATISTIC(NumPostFolded, "Number of post-index updates folded");
58STATISTIC(NumPreFolded, "Number of pre-index updates folded");
59STATISTIC(NumUnscaledPairCreated,
60 "Number of load/store from unscaled generated");
61STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted");
62STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted");
63STATISTIC(NumFailedAlignmentCheck, "Number of load/store pair transformation "
64 "not passed the alignment check");
65STATISTIC(NumConstOffsetFolded,
66 "Number of const offset of index address folded");
67STATISTIC(NumUMOVFoldedToFPRStore,
68 "Number of UMOV + GPR stores folded to FPR stores");
69
70DEBUG_COUNTER(RegRenamingCounter, DEBUG_TYPE "-reg-renaming",
71 "Controls which pairs are considered for renaming");
72
73#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"
74
75namespace {
76
77using LdStPairFlags = struct LdStPairFlags {
78 // If a matching instruction is found, MergeForward is set to true if the
79 // merge is to remove the first instruction and replace the second with
80 // a pair-wise insn, and false if the reverse is true.
81 bool MergeForward = false;
82
83 // SExtIdx gives the index of the result of the load pair that must be
84 // extended. The value of SExtIdx assumes that the paired load produces the
85 // value in this order: (I, returned iterator), i.e., -1 means no value has
86 // to be extended, 0 means I, and 1 means the returned iterator.
87 int SExtIdx = -1;
88
89 // If not none, RenameReg can be used to rename the result register of the
90 // first store in a pair. Currently this only works when merging stores
91 // forward.
92 std::optional<MCPhysReg> RenameReg;
93
94 LdStPairFlags() = default;
95
96 void setMergeForward(bool V = true) { MergeForward = V; }
97 bool getMergeForward() const { return MergeForward; }
98
99 void setSExtIdx(int V) { SExtIdx = V; }
100 int getSExtIdx() const { return SExtIdx; }
101
102 void setRenameReg(MCPhysReg R) { RenameReg = R; }
103 void clearRenameReg() { RenameReg = std::nullopt; }
104 std::optional<MCPhysReg> getRenameReg() const { return RenameReg; }
105};
106
107struct AArch64LoadStoreOpt {
108 AliasAnalysis *AA;
109 const AArch64InstrInfo *TII;
110 const TargetRegisterInfo *TRI;
111 const AArch64Subtarget *Subtarget;
112
113 // Track which register units have been modified and used.
114 LiveRegUnits ModifiedRegUnits, UsedRegUnits;
115 LiveRegUnits DefinedInBB;
116
117 // Scan the instructions looking for a load/store that can be combined
118 // with the current instruction into a load/store pair.
119 // Return the matching instruction if one is found, else MBB->end().
120 MachineBasicBlock::iterator findMatchingInsn(MachineBasicBlock::iterator I,
121 LdStPairFlags &Flags,
122 unsigned Limit,
123 bool FindNarrowMerge);
124
125 // Scan the instructions looking for a store that writes to the address from
126 // which the current load instruction reads. Return true if one is found.
127 bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit,
128 MachineBasicBlock::iterator &StoreI);
129
130 // Merge the two instructions indicated into a wider narrow store instruction.
131 MachineBasicBlock::iterator
132 mergeNarrowZeroStores(MachineBasicBlock::iterator I,
133 MachineBasicBlock::iterator MergeMI,
134 const LdStPairFlags &Flags);
135
136 // Merge the two instructions indicated into a single pair-wise instruction.
137 MachineBasicBlock::iterator
138 mergePairedInsns(MachineBasicBlock::iterator I,
139 MachineBasicBlock::iterator Paired,
140 const LdStPairFlags &Flags);
141
142 // Promote the load that reads directly from the address stored to.
143 MachineBasicBlock::iterator
144 promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
145 MachineBasicBlock::iterator StoreI);
146
147 // Scan the instruction list to find a base register update that can
148 // be combined with the current instruction (a load or store) using
149 // pre or post indexed addressing with writeback. Scan forwards.
150 MachineBasicBlock::iterator
151 findMatchingUpdateInsnForward(MachineBasicBlock::iterator I,
152 int UnscaledOffset, unsigned Limit);
153
154 // Scan the instruction list to find a register assigned with a const
155 // value that can be combined with the current instruction (a load or store)
156 // using base addressing with writeback. Scan backwards.
157 MachineBasicBlock::iterator
158 findMatchingConstOffsetBackward(MachineBasicBlock::iterator I, unsigned Limit,
159 unsigned &Offset);
160
161 // Scan the instruction list to find a base register update that can
162 // be combined with the current instruction (a load or store) using
163 // pre or post indexed addressing with writeback. Scan backwards.
164 // `MergeEither` is set to true if the combined instruction may be placed
165 // either at the location of the load/store instruction or at the location of
166 // the update instruction.
167 MachineBasicBlock::iterator
168 findMatchingUpdateInsnBackward(MachineBasicBlock::iterator I, unsigned Limit,
169 bool &MergeEither);
170
171 // Find an instruction that updates the base register of the ld/st
172 // instruction.
173 bool isMatchingUpdateInsn(MachineInstr &MemMI, MachineInstr &MI,
174 unsigned BaseReg, int Offset);
175
176 bool isMatchingMovConstInsn(MachineInstr &MemMI, MachineInstr &MI,
177 unsigned IndexReg, unsigned &Offset);
178
179 // Merge a pre- or post-index base register update into a ld/st instruction.
180 std::optional<MachineBasicBlock::iterator>
181 mergeUpdateInsn(MachineBasicBlock::iterator I,
182 MachineBasicBlock::iterator Update, bool IsForward,
183 bool IsPreIdx, bool MergeEither);
184
185 MachineBasicBlock::iterator
186 mergeConstOffsetInsn(MachineBasicBlock::iterator I,
187 MachineBasicBlock::iterator Update, unsigned Offset,
188 int Scale);
189
190 // Find and merge zero store instructions.
191 bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI);
192
193 // Find and pair ldr/str instructions.
194 bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI);
195
196 // Find and promote load instructions which read directly from store.
197 bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI);
198
199 // Find and merge a base register updates before or after a ld/st instruction.
200 bool tryToMergeLdStUpdate(MachineBasicBlock::iterator &MBBI);
201
202 // Find and merge an index ldr/st instruction into a base ld/st instruction.
203 bool tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI, int Scale);
204
205 // Replace a UMOV (lane 0) + GPR store with a direct FPR sub-register store.
206 bool tryToReplaceUMOVStore(MachineBasicBlock::iterator &MBBI);
207
208 bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt);
209
210 bool runOnMachineFunction(MachineFunction &MF);
211};
212
213struct AArch64LoadStoreOptLegacy : public MachineFunctionPass {
214 static char ID;
215
216 AArch64LoadStoreOptLegacy() : MachineFunctionPass(ID) {}
217
218 bool runOnMachineFunction(MachineFunction &Fn) override;
219
220 void getAnalysisUsage(AnalysisUsage &AU) const override {
221 AU.addRequired<AAResultsWrapperPass>();
222 MachineFunctionPass::getAnalysisUsage(AU);
223 }
224
225 MachineFunctionProperties getRequiredProperties() const override {
226 return MachineFunctionProperties().setNoVRegs();
227 }
228
229 StringRef getPassName() const override { return AARCH64_LOAD_STORE_OPT_NAME; }
230};
231
232char AArch64LoadStoreOptLegacy::ID = 0;
233
234} // end anonymous namespace
235
236INITIALIZE_PASS(AArch64LoadStoreOptLegacy, "aarch64-ldst-opt",
237 AARCH64_LOAD_STORE_OPT_NAME, false, false)
238
239static bool isNarrowStore(unsigned Opc) {
240 switch (Opc) {
241 default:
242 return false;
243 case AArch64::STRBBui:
244 case AArch64::STURBBi:
245 case AArch64::STRHHui:
246 case AArch64::STURHHi:
247 return true;
248 }
249}
250
251// These instruction set memory tag and either keep memory contents unchanged or
252// set it to zero, ignoring the address part of the source register.
253static bool isTagStore(const MachineInstr &MI) {
254 switch (MI.getOpcode()) {
255 default:
256 return false;
257 case AArch64::STGi:
258 case AArch64::STZGi:
259 case AArch64::ST2Gi:
260 case AArch64::STZ2Gi:
261 return true;
262 }
263}
264
265static unsigned getMatchingNonSExtOpcode(unsigned Opc,
266 bool *IsValidLdStrOpc = nullptr) {
267 if (IsValidLdStrOpc)
268 *IsValidLdStrOpc = true;
269 switch (Opc) {
270 default:
271 if (IsValidLdStrOpc)
272 *IsValidLdStrOpc = false;
273 return std::numeric_limits<unsigned>::max();
274 case AArch64::STRDui:
275 case AArch64::STURDi:
276 case AArch64::STRDpre:
277 case AArch64::STRQui:
278 case AArch64::STURQi:
279 case AArch64::STRQpre:
280 case AArch64::STRBBui:
281 case AArch64::STURBBi:
282 case AArch64::STRHHui:
283 case AArch64::STURHHi:
284 case AArch64::STRWui:
285 case AArch64::STRWpre:
286 case AArch64::STURWi:
287 case AArch64::STRXui:
288 case AArch64::STRXpre:
289 case AArch64::STURXi:
290 case AArch64::STR_ZXI:
291 case AArch64::LDRDui:
292 case AArch64::LDURDi:
293 case AArch64::LDRDpre:
294 case AArch64::LDRQui:
295 case AArch64::LDURQi:
296 case AArch64::LDRQpre:
297 case AArch64::LDRWui:
298 case AArch64::LDURWi:
299 case AArch64::LDRWpre:
300 case AArch64::LDRXui:
301 case AArch64::LDURXi:
302 case AArch64::LDRXpre:
303 case AArch64::STRSui:
304 case AArch64::STURSi:
305 case AArch64::STRSpre:
306 case AArch64::LDRSui:
307 case AArch64::LDURSi:
308 case AArch64::LDRSpre:
309 case AArch64::LDR_ZXI:
310 return Opc;
311 case AArch64::LDRSWui:
312 return AArch64::LDRWui;
313 case AArch64::LDURSWi:
314 return AArch64::LDURWi;
315 case AArch64::LDRSWpre:
316 return AArch64::LDRWpre;
317 }
318}
319
320static unsigned getMatchingWideOpcode(unsigned Opc) {
321 switch (Opc) {
322 default:
323 llvm_unreachable("Opcode has no wide equivalent!");
324 case AArch64::STRBBui:
325 return AArch64::STRHHui;
326 case AArch64::STRHHui:
327 return AArch64::STRWui;
328 case AArch64::STURBBi:
329 return AArch64::STURHHi;
330 case AArch64::STURHHi:
331 return AArch64::STURWi;
332 case AArch64::STURWi:
333 return AArch64::STURXi;
334 case AArch64::STRWui:
335 return AArch64::STRXui;
336 }
337}
338
339static unsigned getMatchingPairOpcode(unsigned Opc) {
340 switch (Opc) {
341 default:
342 llvm_unreachable("Opcode has no pairwise equivalent!");
343 case AArch64::STRSui:
344 case AArch64::STURSi:
345 return AArch64::STPSi;
346 case AArch64::STRSpre:
347 return AArch64::STPSpre;
348 case AArch64::STRDui:
349 case AArch64::STURDi:
350 return AArch64::STPDi;
351 case AArch64::STRDpre:
352 return AArch64::STPDpre;
353 case AArch64::STRQui:
354 case AArch64::STURQi:
355 case AArch64::STR_ZXI:
356 return AArch64::STPQi;
357 case AArch64::STRQpre:
358 return AArch64::STPQpre;
359 case AArch64::STRWui:
360 case AArch64::STURWi:
361 return AArch64::STPWi;
362 case AArch64::STRWpre:
363 return AArch64::STPWpre;
364 case AArch64::STRXui:
365 case AArch64::STURXi:
366 return AArch64::STPXi;
367 case AArch64::STRXpre:
368 return AArch64::STPXpre;
369 case AArch64::LDRSui:
370 case AArch64::LDURSi:
371 return AArch64::LDPSi;
372 case AArch64::LDRSpre:
373 return AArch64::LDPSpre;
374 case AArch64::LDRDui:
375 case AArch64::LDURDi:
376 return AArch64::LDPDi;
377 case AArch64::LDRDpre:
378 return AArch64::LDPDpre;
379 case AArch64::LDRQui:
380 case AArch64::LDURQi:
381 case AArch64::LDR_ZXI:
382 return AArch64::LDPQi;
383 case AArch64::LDRQpre:
384 return AArch64::LDPQpre;
385 case AArch64::LDRWui:
386 case AArch64::LDURWi:
387 return AArch64::LDPWi;
388 case AArch64::LDRWpre:
389 return AArch64::LDPWpre;
390 case AArch64::LDRXui:
391 case AArch64::LDURXi:
392 return AArch64::LDPXi;
393 case AArch64::LDRXpre:
394 return AArch64::LDPXpre;
395 case AArch64::LDRSWui:
396 case AArch64::LDURSWi:
397 return AArch64::LDPSWi;
398 case AArch64::LDRSWpre:
399 return AArch64::LDPSWpre;
400 }
401}
402
403static unsigned isMatchingStore(MachineInstr &LoadInst,
404 MachineInstr &StoreInst) {
405 unsigned LdOpc = LoadInst.getOpcode();
406 unsigned StOpc = StoreInst.getOpcode();
407 switch (LdOpc) {
408 default:
409 llvm_unreachable("Unsupported load instruction!");
410 case AArch64::LDRBBui:
411 return StOpc == AArch64::STRBBui || StOpc == AArch64::STRHHui ||
412 StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
413 case AArch64::LDURBBi:
414 return StOpc == AArch64::STURBBi || StOpc == AArch64::STURHHi ||
415 StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
416 case AArch64::LDRHHui:
417 return StOpc == AArch64::STRHHui || StOpc == AArch64::STRWui ||
418 StOpc == AArch64::STRXui;
419 case AArch64::LDURHHi:
420 return StOpc == AArch64::STURHHi || StOpc == AArch64::STURWi ||
421 StOpc == AArch64::STURXi;
422 case AArch64::LDRWui:
423 return StOpc == AArch64::STRWui || StOpc == AArch64::STRXui;
424 case AArch64::LDURWi:
425 return StOpc == AArch64::STURWi || StOpc == AArch64::STURXi;
426 case AArch64::LDRXui:
427 return StOpc == AArch64::STRXui;
428 case AArch64::LDURXi:
429 return StOpc == AArch64::STURXi;
430 }
431}
432
433static unsigned getPreIndexedOpcode(unsigned Opc) {
434 // FIXME: We don't currently support creating pre-indexed loads/stores when
435 // the load or store is the unscaled version. If we decide to perform such an
436 // optimization in the future the cases for the unscaled loads/stores will
437 // need to be added here.
438 switch (Opc) {
439 default:
440 llvm_unreachable("Opcode has no pre-indexed equivalent!");
441 case AArch64::STRBui:
442 return AArch64::STRBpre;
443 case AArch64::STRHui:
444 return AArch64::STRHpre;
445 case AArch64::STRSui:
446 return AArch64::STRSpre;
447 case AArch64::STRDui:
448 return AArch64::STRDpre;
449 case AArch64::STRQui:
450 return AArch64::STRQpre;
451 case AArch64::STRBBui:
452 return AArch64::STRBBpre;
453 case AArch64::STRHHui:
454 return AArch64::STRHHpre;
455 case AArch64::STRWui:
456 return AArch64::STRWpre;
457 case AArch64::STRXui:
458 return AArch64::STRXpre;
459 case AArch64::LDRBui:
460 return AArch64::LDRBpre;
461 case AArch64::LDRHui:
462 return AArch64::LDRHpre;
463 case AArch64::LDRSui:
464 return AArch64::LDRSpre;
465 case AArch64::LDRDui:
466 return AArch64::LDRDpre;
467 case AArch64::LDRQui:
468 return AArch64::LDRQpre;
469 case AArch64::LDRBBui:
470 return AArch64::LDRBBpre;
471 case AArch64::LDRHHui:
472 return AArch64::LDRHHpre;
473 case AArch64::LDRWui:
474 return AArch64::LDRWpre;
475 case AArch64::LDRXui:
476 return AArch64::LDRXpre;
477 case AArch64::LDRSWui:
478 return AArch64::LDRSWpre;
479 case AArch64::LDPSi:
480 return AArch64::LDPSpre;
481 case AArch64::LDPSWi:
482 return AArch64::LDPSWpre;
483 case AArch64::LDPDi:
484 return AArch64::LDPDpre;
485 case AArch64::LDPQi:
486 return AArch64::LDPQpre;
487 case AArch64::LDPWi:
488 return AArch64::LDPWpre;
489 case AArch64::LDPXi:
490 return AArch64::LDPXpre;
491 case AArch64::STPSi:
492 return AArch64::STPSpre;
493 case AArch64::STPDi:
494 return AArch64::STPDpre;
495 case AArch64::STPQi:
496 return AArch64::STPQpre;
497 case AArch64::STPWi:
498 return AArch64::STPWpre;
499 case AArch64::STPXi:
500 return AArch64::STPXpre;
501 case AArch64::STGi:
502 return AArch64::STGPreIndex;
503 case AArch64::STZGi:
504 return AArch64::STZGPreIndex;
505 case AArch64::ST2Gi:
506 return AArch64::ST2GPreIndex;
507 case AArch64::STZ2Gi:
508 return AArch64::STZ2GPreIndex;
509 case AArch64::STGPi:
510 return AArch64::STGPpre;
511 }
512}
513
514static unsigned getBaseAddressOpcode(unsigned Opc) {
515 // TODO: Add more index address stores.
516 switch (Opc) {
517 default:
518 llvm_unreachable("Opcode has no base address equivalent!");
519 case AArch64::LDRBroX:
520 return AArch64::LDRBui;
521 case AArch64::LDRBBroX:
522 return AArch64::LDRBBui;
523 case AArch64::LDRSBXroX:
524 return AArch64::LDRSBXui;
525 case AArch64::LDRSBWroX:
526 return AArch64::LDRSBWui;
527 case AArch64::LDRHroX:
528 return AArch64::LDRHui;
529 case AArch64::LDRHHroX:
530 return AArch64::LDRHHui;
531 case AArch64::LDRSHXroX:
532 return AArch64::LDRSHXui;
533 case AArch64::LDRSHWroX:
534 return AArch64::LDRSHWui;
535 case AArch64::LDRWroX:
536 return AArch64::LDRWui;
537 case AArch64::LDRSroX:
538 return AArch64::LDRSui;
539 case AArch64::LDRSWroX:
540 return AArch64::LDRSWui;
541 case AArch64::LDRDroX:
542 return AArch64::LDRDui;
543 case AArch64::LDRXroX:
544 return AArch64::LDRXui;
545 case AArch64::LDRQroX:
546 return AArch64::LDRQui;
547 }
548}
549
550static unsigned getPostIndexedOpcode(unsigned Opc) {
551 switch (Opc) {
552 default:
553 llvm_unreachable("Opcode has no post-indexed wise equivalent!");
554 case AArch64::STRBui:
555 return AArch64::STRBpost;
556 case AArch64::STRHui:
557 return AArch64::STRHpost;
558 case AArch64::STRSui:
559 case AArch64::STURSi:
560 return AArch64::STRSpost;
561 case AArch64::STRDui:
562 case AArch64::STURDi:
563 return AArch64::STRDpost;
564 case AArch64::STRQui:
565 case AArch64::STURQi:
566 return AArch64::STRQpost;
567 case AArch64::STRBBui:
568 return AArch64::STRBBpost;
569 case AArch64::STRHHui:
570 return AArch64::STRHHpost;
571 case AArch64::STRWui:
572 case AArch64::STURWi:
573 return AArch64::STRWpost;
574 case AArch64::STRXui:
575 case AArch64::STURXi:
576 return AArch64::STRXpost;
577 case AArch64::LDRBui:
578 return AArch64::LDRBpost;
579 case AArch64::LDRHui:
580 return AArch64::LDRHpost;
581 case AArch64::LDRSui:
582 case AArch64::LDURSi:
583 return AArch64::LDRSpost;
584 case AArch64::LDRDui:
585 case AArch64::LDURDi:
586 return AArch64::LDRDpost;
587 case AArch64::LDRQui:
588 case AArch64::LDURQi:
589 return AArch64::LDRQpost;
590 case AArch64::LDRBBui:
591 return AArch64::LDRBBpost;
592 case AArch64::LDRHHui:
593 return AArch64::LDRHHpost;
594 case AArch64::LDRWui:
595 case AArch64::LDURWi:
596 return AArch64::LDRWpost;
597 case AArch64::LDRXui:
598 case AArch64::LDURXi:
599 return AArch64::LDRXpost;
600 case AArch64::LDRSWui:
601 return AArch64::LDRSWpost;
602 case AArch64::LDPSi:
603 return AArch64::LDPSpost;
604 case AArch64::LDPSWi:
605 return AArch64::LDPSWpost;
606 case AArch64::LDPDi:
607 return AArch64::LDPDpost;
608 case AArch64::LDPQi:
609 return AArch64::LDPQpost;
610 case AArch64::LDPWi:
611 return AArch64::LDPWpost;
612 case AArch64::LDPXi:
613 return AArch64::LDPXpost;
614 case AArch64::STPSi:
615 return AArch64::STPSpost;
616 case AArch64::STPDi:
617 return AArch64::STPDpost;
618 case AArch64::STPQi:
619 return AArch64::STPQpost;
620 case AArch64::STPWi:
621 return AArch64::STPWpost;
622 case AArch64::STPXi:
623 return AArch64::STPXpost;
624 case AArch64::STGi:
625 return AArch64::STGPostIndex;
626 case AArch64::STZGi:
627 return AArch64::STZGPostIndex;
628 case AArch64::ST2Gi:
629 return AArch64::ST2GPostIndex;
630 case AArch64::STZ2Gi:
631 return AArch64::STZ2GPostIndex;
632 case AArch64::STGPi:
633 return AArch64::STGPpost;
634 }
635}
636
637static bool isPreLdStPairCandidate(MachineInstr &FirstMI, MachineInstr &MI) {
638
639 unsigned OpcA = FirstMI.getOpcode();
640 unsigned OpcB = MI.getOpcode();
641
642 switch (OpcA) {
643 default:
644 return false;
645 case AArch64::STRSpre:
646 return (OpcB == AArch64::STRSui) || (OpcB == AArch64::STURSi);
647 case AArch64::STRDpre:
648 return (OpcB == AArch64::STRDui) || (OpcB == AArch64::STURDi);
649 case AArch64::STRQpre:
650 return (OpcB == AArch64::STRQui) || (OpcB == AArch64::STURQi);
651 case AArch64::STRWpre:
652 return (OpcB == AArch64::STRWui) || (OpcB == AArch64::STURWi);
653 case AArch64::STRXpre:
654 return (OpcB == AArch64::STRXui) || (OpcB == AArch64::STURXi);
655 case AArch64::LDRSpre:
656 return (OpcB == AArch64::LDRSui) || (OpcB == AArch64::LDURSi);
657 case AArch64::LDRDpre:
658 return (OpcB == AArch64::LDRDui) || (OpcB == AArch64::LDURDi);
659 case AArch64::LDRQpre:
660 return (OpcB == AArch64::LDRQui) || (OpcB == AArch64::LDURQi);
661 case AArch64::LDRWpre:
662 return (OpcB == AArch64::LDRWui) || (OpcB == AArch64::LDURWi);
663 case AArch64::LDRXpre:
664 return (OpcB == AArch64::LDRXui) || (OpcB == AArch64::LDURXi);
665 case AArch64::LDRSWpre:
666 return (OpcB == AArch64::LDRSWui) || (OpcB == AArch64::LDURSWi);
667 }
668}
669
670// Returns the scale and offset range of pre/post indexed variants of MI.
671static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale,
672 int &MinOffset, int &MaxOffset) {
673 bool IsPaired = AArch64InstrInfo::isPairedLdSt(MI);
674 bool IsTagStore = isTagStore(MI);
675 // ST*G and all paired ldst have the same scale in pre/post-indexed variants
676 // as in the "unsigned offset" variant.
677 // All other pre/post indexed ldst instructions are unscaled.
678 Scale = (IsTagStore || IsPaired) ? AArch64InstrInfo::getMemScale(MI) : 1;
679
680 if (IsPaired) {
681 MinOffset = -64;
682 MaxOffset = 63;
683 } else {
684 MinOffset = -256;
685 MaxOffset = 255;
686 }
687}
688
689static MachineOperand &getLdStRegOp(MachineInstr &MI,
690 unsigned PairedRegOp = 0) {
691 assert(PairedRegOp < 2 && "Unexpected register operand idx.");
692 bool IsPreLdSt = AArch64InstrInfo::isPreLdSt(MI);
693 if (IsPreLdSt)
694 PairedRegOp += 1;
695 unsigned Idx =
696 AArch64InstrInfo::isPairedLdSt(MI) || IsPreLdSt ? PairedRegOp : 0;
697 return MI.getOperand(Idx);
698}
699
700static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst,
701 MachineInstr &StoreInst,
702 const AArch64InstrInfo *TII) {
703 assert(isMatchingStore(LoadInst, StoreInst) && "Expect only matched ld/st.");
704 int LoadSize = TII->getMemScale(LoadInst);
705 int StoreSize = TII->getMemScale(StoreInst);
706 int UnscaledStOffset =
707 TII->hasUnscaledLdStOffset(StoreInst)
708 ? AArch64InstrInfo::getLdStOffsetOp(StoreInst).getImm()
709 : AArch64InstrInfo::getLdStOffsetOp(StoreInst).getImm() * StoreSize;
710 int UnscaledLdOffset =
711 TII->hasUnscaledLdStOffset(LoadInst)
712 ? AArch64InstrInfo::getLdStOffsetOp(LoadInst).getImm()
713 : AArch64InstrInfo::getLdStOffsetOp(LoadInst).getImm() * LoadSize;
714 return (UnscaledStOffset <= UnscaledLdOffset) &&
715 (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize));
716}
717
718static bool isPromotableZeroStoreInst(MachineInstr &MI) {
719 unsigned Opc = MI.getOpcode();
720 return (Opc == AArch64::STRWui || Opc == AArch64::STURWi ||
721 isNarrowStore(Opc)) &&
722 getLdStRegOp(MI).getReg() == AArch64::WZR;
723}
724
725static bool isPromotableLoadFromStore(MachineInstr &MI) {
726 switch (MI.getOpcode()) {
727 default:
728 return false;
729 // Scaled instructions.
730 case AArch64::LDRBBui:
731 case AArch64::LDRHHui:
732 case AArch64::LDRWui:
733 case AArch64::LDRXui:
734 // Unscaled instructions.
735 case AArch64::LDURBBi:
736 case AArch64::LDURHHi:
737 case AArch64::LDURWi:
738 case AArch64::LDURXi:
739 return true;
740 }
741}
742
743static bool isMergeableLdStUpdate(MachineInstr &MI, AArch64FunctionInfo &AFI) {
744 unsigned Opc = MI.getOpcode();
745 switch (Opc) {
746 default:
747 return false;
748 // Scaled instructions.
749 case AArch64::STRBui:
750 case AArch64::STRHui:
751 case AArch64::STRSui:
752 case AArch64::STRDui:
753 case AArch64::STRQui:
754 case AArch64::STRXui:
755 case AArch64::STRWui:
756 case AArch64::STRHHui:
757 case AArch64::STRBBui:
758 case AArch64::LDRBui:
759 case AArch64::LDRHui:
760 case AArch64::LDRSui:
761 case AArch64::LDRDui:
762 case AArch64::LDRQui:
763 case AArch64::LDRXui:
764 case AArch64::LDRWui:
765 case AArch64::LDRHHui:
766 case AArch64::LDRBBui:
767 case AArch64::STGi:
768 case AArch64::STZGi:
769 case AArch64::ST2Gi:
770 case AArch64::STZ2Gi:
771 case AArch64::STGPi:
772 // Unscaled instructions.
773 case AArch64::STURSi:
774 case AArch64::STURDi:
775 case AArch64::STURQi:
776 case AArch64::STURWi:
777 case AArch64::STURXi:
778 case AArch64::LDURSi:
779 case AArch64::LDURDi:
780 case AArch64::LDURQi:
781 case AArch64::LDURWi:
782 case AArch64::LDURXi:
783 // Paired instructions.
784 case AArch64::LDPSi:
785 case AArch64::LDPSWi:
786 case AArch64::LDPDi:
787 case AArch64::LDPQi:
788 case AArch64::LDPWi:
789 case AArch64::LDPXi:
790 case AArch64::STPSi:
791 case AArch64::STPDi:
792 case AArch64::STPQi:
793 case AArch64::STPWi:
794 case AArch64::STPXi:
795 // Make sure this is a reg+imm (as opposed to an address reloc).
796 if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
797 return false;
798
799 // When using stack tagging, simple sp+imm loads and stores are not
800 // tag-checked, but pre- and post-indexed versions of them are, so we can't
801 // replace the former with the latter. This transformation would be valid
802 // if the load/store accesses an untagged stack slot, but we don't have
803 // that information available after frame indices have been eliminated.
804 if (AFI.isMTETagged() &&
805 AArch64InstrInfo::getLdStBaseOp(MI).getReg() == AArch64::SP)
806 return false;
807
808 return true;
809 }
810}
811
812// Make sure this is a reg+reg Ld/St
813static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale) {
814 unsigned Opc = MI.getOpcode();
815 switch (Opc) {
816 default:
817 return false;
818 // Scaled instructions.
819 // TODO: Add more index address stores.
820 case AArch64::LDRBroX:
821 case AArch64::LDRBBroX:
822 case AArch64::LDRSBXroX:
823 case AArch64::LDRSBWroX:
824 Scale = 1;
825 return true;
826 case AArch64::LDRHroX:
827 case AArch64::LDRHHroX:
828 case AArch64::LDRSHXroX:
829 case AArch64::LDRSHWroX:
830 Scale = 2;
831 return true;
832 case AArch64::LDRWroX:
833 case AArch64::LDRSroX:
834 case AArch64::LDRSWroX:
835 Scale = 4;
836 return true;
837 case AArch64::LDRDroX:
838 case AArch64::LDRXroX:
839 Scale = 8;
840 return true;
841 case AArch64::LDRQroX:
842 Scale = 16;
843 return true;
844 }
845}
846
847static bool isRewritableImplicitDef(const MachineInstr &MI,
848 const MachineOperand &MO) {
849 switch (MI.getOpcode()) {
850 default:
851 return MO.isRenamable();
852 case AArch64::ORRWrs:
853 case AArch64::ADDWri:
854 return true;
855 }
856}
857
858MachineBasicBlock::iterator
859AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I,
860 MachineBasicBlock::iterator MergeMI,
861 const LdStPairFlags &Flags) {
862 assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) &&
863 "Expected promotable zero stores.");
864
865 MachineBasicBlock::iterator E = I->getParent()->end();
866 MachineBasicBlock::iterator NextI = next_nodbg(I, E);
867 // If NextI is the second of the two instructions to be merged, we need
868 // to skip one further. Either way we merge will invalidate the iterator,
869 // and we don't need to scan the new instruction, as it's a pairwise
870 // instruction, which we're not considering for further action anyway.
871 if (NextI == MergeMI)
872 NextI = next_nodbg(NextI, E);
873
874 unsigned Opc = I->getOpcode();
875 unsigned MergeMIOpc = MergeMI->getOpcode();
876 bool IsScaled = !TII->hasUnscaledLdStOffset(Opc);
877 bool IsMergedMIScaled = !TII->hasUnscaledLdStOffset(MergeMIOpc);
878 int OffsetStride = IsScaled ? TII->getMemScale(*I) : 1;
879 int MergeMIOffsetStride = IsMergedMIScaled ? TII->getMemScale(*MergeMI) : 1;
880
881 bool MergeForward = Flags.getMergeForward();
882 // Insert our new paired instruction after whichever of the paired
883 // instructions MergeForward indicates.
884 MachineBasicBlock::iterator InsertionPoint = MergeForward ? MergeMI : I;
885 // Also based on MergeForward is from where we copy the base register operand
886 // so we get the flags compatible with the input code.
887 const MachineOperand &BaseRegOp =
888 MergeForward ? AArch64InstrInfo::getLdStBaseOp(*MergeMI)
889 : AArch64InstrInfo::getLdStBaseOp(*I);
890
891 // Which register is Rt and which is Rt2 depends on the offset order.
892 int64_t IOffsetInBytes =
893 AArch64InstrInfo::getLdStOffsetOp(*I).getImm() * OffsetStride;
894 int64_t MIOffsetInBytes =
895 AArch64InstrInfo::getLdStOffsetOp(*MergeMI).getImm() *
896 MergeMIOffsetStride;
897 // Select final offset based on the offset order.
898 int64_t OffsetImm;
899 if (IOffsetInBytes > MIOffsetInBytes)
900 OffsetImm = MIOffsetInBytes;
901 else
902 OffsetImm = IOffsetInBytes;
903
904 int NewOpcode = getMatchingWideOpcode(Opc);
905 // Adjust final offset on scaled stores because the new instruction
906 // has a different scale.
907 if (!TII->hasUnscaledLdStOffset(NewOpcode)) {
908 int NewOffsetStride = TII->getMemScale(NewOpcode);
909 assert(((OffsetImm % NewOffsetStride) == 0) &&
910 "Offset should be a multiple of the store memory scale");
911 OffsetImm = OffsetImm / NewOffsetStride;
912 }
913
914 // Construct the new instruction.
915 DebugLoc DL = I->getDebugLoc();
916 MachineBasicBlock *MBB = I->getParent();
917 MachineInstrBuilder MIB;
918 MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(NewOpcode))
919 .addReg(isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR)
920 .add(BaseRegOp)
921 .addImm(OffsetImm)
922 .cloneMergedMemRefs({&*I, &*MergeMI})
923 .setMIFlags(I->mergeFlagsWith(*MergeMI));
924 (void)MIB;
925
926 LLVM_DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n ");
927 LLVM_DEBUG(I->print(dbgs()));
928 LLVM_DEBUG(dbgs() << " ");
929 LLVM_DEBUG(MergeMI->print(dbgs()));
930 LLVM_DEBUG(dbgs() << " with instruction:\n ");
931 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
932 LLVM_DEBUG(dbgs() << "\n");
933
934 // Erase the old instructions.
935 I->eraseFromParent();
936 MergeMI->eraseFromParent();
937 return NextI;
938}
939
940// Apply Fn to all instructions between MI and the beginning of the block, until
941// a def for DefReg is reached. Returns true, iff Fn returns true for all
942// visited instructions. Stop after visiting Limit iterations.
943static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg,
944 const TargetRegisterInfo *TRI, unsigned Limit,
945 std::function<bool(MachineInstr &, bool)> &Fn) {
946 auto MBB = MI.getParent();
947 for (MachineInstr &I :
948 instructionsWithoutDebug(MI.getReverseIterator(), MBB->instr_rend())) {
949 if (!Limit)
950 return false;
951 --Limit;
952
953 bool isDef = any_of(I.operands(), [DefReg, TRI](MachineOperand &MOP) {
954 return MOP.isReg() && MOP.isDef() && !MOP.isDebug() && MOP.getReg() &&
955 TRI->regsOverlap(MOP.getReg(), DefReg);
956 });
957 if (!Fn(I, isDef))
958 return false;
959 if (isDef)
960 break;
961 }
962 return true;
963}
964
965static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units,
966 const TargetRegisterInfo *TRI) {
967
968 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
969 if (MOP.isReg() && MOP.isKill())
970 Units.removeReg(MOP.getReg());
971
972 for (const MachineOperand &MOP : phys_regs_and_masks(MI))
973 if (MOP.isReg() && !MOP.isKill())
974 Units.addReg(MOP.getReg());
975}
976
977/// This function will add a new entry into the debugValueSubstitutions table
978/// when two instruction have been merged into a new one represented by \p
979/// MergedInstr.
980static void addDebugSubstitutionsToTable(MachineFunction *MF,
981 unsigned InstrNumToSet,
982 MachineInstr &OriginalInstr,
983 MachineInstr &MergedInstr) {
984
985 // Figure out the Operand Index of the destination register of the
986 // OriginalInstr in the new MergedInstr.
987 auto Reg = OriginalInstr.getOperand(0).getReg();
988 unsigned OperandNo = 0;
989 bool RegFound = false;
990 for (const auto Op : MergedInstr.operands()) {
991 if (Op.getReg() == Reg) {
992 RegFound = true;
993 break;
994 }
995 OperandNo++;
996 }
997
998 if (RegFound)
999 MF->makeDebugValueSubstitution({OriginalInstr.peekDebugInstrNum(), 0},
1000 {InstrNumToSet, OperandNo});
1001}
1002
1003MachineBasicBlock::iterator
1004AArch64LoadStoreOpt::mergePairedInsns(MachineBasicBlock::iterator I,
1005 MachineBasicBlock::iterator Paired,
1006 const LdStPairFlags &Flags) {
1007 MachineBasicBlock::iterator E = I->getParent()->end();
1008 MachineBasicBlock::iterator NextI = next_nodbg(I, E);
1009 // If NextI is the second of the two instructions to be merged, we need
1010 // to skip one further. Either way we merge will invalidate the iterator,
1011 // and we don't need to scan the new instruction, as it's a pairwise
1012 // instruction, which we're not considering for further action anyway.
1013 if (NextI == Paired)
1014 NextI = next_nodbg(NextI, E);
1015
1016 int SExtIdx = Flags.getSExtIdx();
1017 unsigned Opc =
1018 SExtIdx == -1 ? I->getOpcode() : getMatchingNonSExtOpcode(I->getOpcode());
1019 bool IsUnscaled = TII->hasUnscaledLdStOffset(Opc);
1020 int OffsetStride = IsUnscaled ? TII->getMemScale(*I) : 1;
1021
1022 bool MergeForward = Flags.getMergeForward();
1023
1024 std::optional<MCPhysReg> RenameReg = Flags.getRenameReg();
1025 if (RenameReg) {
1026 MCRegister RegToRename = getLdStRegOp(*I).getReg();
1027 DefinedInBB.addReg(*RenameReg);
1028
1029 // Return the sub/super register for RenameReg, matching the size of
1030 // OriginalReg.
1031 auto GetMatchingSubReg =
1032 [this, RenameReg](const TargetRegisterClass *C) -> MCPhysReg {
1033 for (MCPhysReg SubOrSuper :
1034 TRI->sub_and_superregs_inclusive(*RenameReg)) {
1035 if (C->contains(SubOrSuper))
1036 return SubOrSuper;
1037 }
1038 llvm_unreachable("Should have found matching sub or super register!");
1039 };
1040
1041 std::function<bool(MachineInstr &, bool)> UpdateMIs =
1042 [this, RegToRename, GetMatchingSubReg, MergeForward](MachineInstr &MI,
1043 bool IsDef) {
1044 if (IsDef) {
1045 bool SeenDef = false;
1046 for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
1047 MachineOperand &MOP = MI.getOperand(OpIdx);
1048 // Rename the first explicit definition and all implicit
1049 // definitions matching RegToRename.
1050 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1051 (!MergeForward || !SeenDef ||
1052 (MOP.isDef() && MOP.isImplicit())) &&
1053 TRI->regsOverlap(MOP.getReg(), RegToRename)) {
1054 assert((MOP.isImplicit() ||
1055 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
1056 "Need renamable operands");
1057 Register MatchingReg;
1058 if (const TargetRegisterClass *RC =
1059 MI.getRegClassConstraint(OpIdx, TII, TRI))
1060 MatchingReg = GetMatchingSubReg(RC);
1061 else {
1062 if (!isRewritableImplicitDef(MI, MOP))
1063 continue;
1064 MatchingReg = GetMatchingSubReg(
1065 TRI->getMinimalPhysRegClass(MOP.getReg()));
1066 }
1067 MOP.setReg(MatchingReg);
1068 SeenDef = true;
1069 }
1070 }
1071 } else {
1072 for (unsigned OpIdx = 0; OpIdx < MI.getNumOperands(); ++OpIdx) {
1073 MachineOperand &MOP = MI.getOperand(OpIdx);
1074 if (MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1075 TRI->regsOverlap(MOP.getReg(), RegToRename)) {
1076 assert((MOP.isImplicit() ||
1077 (MOP.isRenamable() && !MOP.isEarlyClobber())) &&
1078 "Need renamable operands");
1079 Register MatchingReg;
1080 if (const TargetRegisterClass *RC =
1081 MI.getRegClassConstraint(OpIdx, TII, TRI))
1082 MatchingReg = GetMatchingSubReg(RC);
1083 else
1084 MatchingReg = GetMatchingSubReg(
1085 TRI->getMinimalPhysRegClass(MOP.getReg()));
1086 assert(MatchingReg.isValid() &&
1087 "Cannot find matching regs for renaming");
1088 MOP.setReg(MatchingReg);
1089 }
1090 }
1091 }
1092 LLVM_DEBUG(dbgs() << "Renamed " << MI);
1093 return true;
1094 };
1095 forAllMIsUntilDef(MergeForward ? *I : *Paired->getPrevNode(), RegToRename,
1096 TRI, UINT32_MAX, UpdateMIs);
1097
1098#if !defined(NDEBUG)
1099 // For forward merging store:
1100 // Make sure the register used for renaming is not used between the
1101 // paired instructions. That would trash the content before the new
1102 // paired instruction.
1103 MCPhysReg RegToCheck = *RenameReg;
1104 // For backward merging load:
1105 // Make sure the register being renamed is not used between the
1106 // paired instructions. That would trash the content after the new
1107 // paired instruction.
1108 if (!MergeForward)
1109 RegToCheck = RegToRename;
1110 for (auto &MI :
1111 iterator_range<MachineInstrBundleIterator<llvm::MachineInstr>>(
1112 MergeForward ? std::next(I) : I,
1113 MergeForward ? std::next(Paired) : Paired))
1114 assert(all_of(MI.operands(),
1115 [this, RegToCheck](const MachineOperand &MOP) {
1116 return !MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1117 MOP.isUndef() ||
1118 !TRI->regsOverlap(MOP.getReg(), RegToCheck);
1119 }) &&
1120 "Rename register used between paired instruction, trashing the "
1121 "content");
1122#endif
1123 }
1124
1125 // Insert our new paired instruction after whichever of the paired
1126 // instructions MergeForward indicates.
1127 MachineBasicBlock::iterator InsertionPoint = MergeForward ? Paired : I;
1128 // Also based on MergeForward is from where we copy the base register operand
1129 // so we get the flags compatible with the input code.
1130 const MachineOperand &BaseRegOp =
1131 MergeForward ? AArch64InstrInfo::getLdStBaseOp(*Paired)
1132 : AArch64InstrInfo::getLdStBaseOp(*I);
1133
1134 int Offset = AArch64InstrInfo::getLdStOffsetOp(*I).getImm();
1135 int PairedOffset = AArch64InstrInfo::getLdStOffsetOp(*Paired).getImm();
1136 bool PairedIsUnscaled = TII->hasUnscaledLdStOffset(Paired->getOpcode());
1137 if (IsUnscaled != PairedIsUnscaled) {
1138 // We're trying to pair instructions that differ in how they are scaled. If
1139 // I is scaled then scale the offset of Paired accordingly. Otherwise, do
1140 // the opposite (i.e., make Paired's offset unscaled).
1141 int MemSize = TII->getMemScale(*Paired);
1142 if (PairedIsUnscaled) {
1143 // If the unscaled offset isn't a multiple of the MemSize, we can't
1144 // pair the operations together.
1145 assert(!(PairedOffset % TII->getMemScale(*Paired)) &&
1146 "Offset should be a multiple of the stride!");
1147 PairedOffset /= MemSize;
1148 } else {
1149 PairedOffset *= MemSize;
1150 }
1151 }
1152
1153 // Which register is Rt and which is Rt2 depends on the offset order.
1154 // However, for pre load/stores the Rt should be the one of the pre
1155 // load/store.
1156 MachineInstr *RtMI, *Rt2MI;
1157 if (Offset == PairedOffset + OffsetStride &&
1158 !AArch64InstrInfo::isPreLdSt(*I)) {
1159 RtMI = &*Paired;
1160 Rt2MI = &*I;
1161 // Here we swapped the assumption made for SExtIdx.
1162 // I.e., we turn ldp I, Paired into ldp Paired, I.
1163 // Update the index accordingly.
1164 if (SExtIdx != -1)
1165 SExtIdx = (SExtIdx + 1) % 2;
1166 } else {
1167 RtMI = &*I;
1168 Rt2MI = &*Paired;
1169 }
1170 int OffsetImm = AArch64InstrInfo::getLdStOffsetOp(*RtMI).getImm();
1171 // Scale the immediate offset, if necessary.
1172 if (TII->hasUnscaledLdStOffset(RtMI->getOpcode())) {
1173 assert(!(OffsetImm % TII->getMemScale(*RtMI)) &&
1174 "Unscaled offset cannot be scaled.");
1175 OffsetImm /= TII->getMemScale(*RtMI);
1176 }
1177
1178 // Construct the new instruction.
1179 MachineInstrBuilder MIB;
1180 DebugLoc DL = I->getDebugLoc();
1181 MachineBasicBlock *MBB = I->getParent();
1182 MachineOperand RegOp0 = getLdStRegOp(*RtMI);
1183 MachineOperand RegOp1 = getLdStRegOp(*Rt2MI);
1184 MachineOperand &PairedRegOp = RtMI == &*Paired ? RegOp0 : RegOp1;
1185 // Kill flags may become invalid when moving stores for pairing.
1186 if (RegOp0.isUse()) {
1187 if (!MergeForward) {
1188 // Clear kill flags on store if moving upwards. Example:
1189 // STRWui kill %w0, ...
1190 // USE %w1
1191 // STRWui kill %w1 ; need to clear kill flag when moving STRWui upwards
1192 // We are about to move the store of w1, so its kill flag may become
1193 // invalid; not the case for w0.
1194 // Since w1 is used between the stores, the kill flag on w1 is cleared
1195 // after merging.
1196 // STPWi kill %w0, %w1, ...
1197 // USE %w1
1198 for (auto It = std::next(I); It != Paired && PairedRegOp.isKill(); ++It)
1199 if (It->readsRegister(PairedRegOp.getReg(), TRI))
1200 PairedRegOp.setIsKill(false);
1201 } else {
1202 // Clear kill flags of the first stores register. Example:
1203 // STRWui %w1, ...
1204 // USE kill %w1 ; need to clear kill flag when moving STRWui downwards
1205 // STRW %w0
1206 Register Reg = getLdStRegOp(*I).getReg();
1207 for (MachineInstr &MI :
1208 make_range(std::next(I->getIterator()), Paired->getIterator()))
1209 MI.clearRegisterKills(Reg, TRI);
1210 }
1211 }
1212
1213 unsigned int MatchPairOpcode = getMatchingPairOpcode(Opc);
1214 MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(MatchPairOpcode));
1215
1216 // Adds the pre-index operand for pre-indexed ld/st pairs.
1217 if (AArch64InstrInfo::isPreLdSt(*RtMI))
1218 MIB.addReg(BaseRegOp.getReg(), RegState::Define);
1219
1220 MIB.add(RegOp0)
1221 .add(RegOp1)
1222 .add(BaseRegOp)
1223 .addImm(OffsetImm)
1224 .cloneMergedMemRefs({&*I, &*Paired})
1225 .setMIFlags(I->mergeFlagsWith(*Paired));
1226
1227 (void)MIB;
1228
1229 LLVM_DEBUG(
1230 dbgs() << "Creating pair load/store. Replacing instructions:\n ");
1231 LLVM_DEBUG(I->print(dbgs()));
1232 LLVM_DEBUG(dbgs() << " ");
1233 LLVM_DEBUG(Paired->print(dbgs()));
1234 LLVM_DEBUG(dbgs() << " with instruction:\n ");
1235 if (SExtIdx != -1) {
1236 // Generate the sign extension for the proper result of the ldp.
1237 // I.e., with X1, that would be:
1238 // %w1 = KILL %w1, implicit-def %x1
1239 // %x1 = SBFMXri killed %x1, 0, 31
1240 MachineOperand &DstMO = MIB->getOperand(SExtIdx);
1241 // Right now, DstMO has the extended register, since it comes from an
1242 // extended opcode.
1243 Register DstRegX = DstMO.getReg();
1244 // Get the W variant of that register.
1245 Register DstRegW = TRI->getSubReg(DstRegX, AArch64::sub_32);
1246 // Update the result of LDP to use the W instead of the X variant.
1247 DstMO.setReg(DstRegW);
1248 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1249 LLVM_DEBUG(dbgs() << "\n");
1250 // Make the machine verifier happy by providing a definition for
1251 // the X register.
1252 // Insert this definition right after the generated LDP, i.e., before
1253 // InsertionPoint.
1254 MachineInstrBuilder MIBKill =
1255 BuildMI(*MBB, InsertionPoint, DL, TII->get(TargetOpcode::KILL), DstRegW)
1256 .addReg(DstRegW)
1257 .addReg(DstRegX, RegState::Define);
1258 MIBKill->getOperand(2).setImplicit();
1259 // Create the sign extension.
1260 MachineInstrBuilder MIBSXTW =
1261 BuildMI(*MBB, InsertionPoint, DL, TII->get(AArch64::SBFMXri), DstRegX)
1262 .addReg(DstRegX)
1263 .addImm(0)
1264 .addImm(31);
1265 (void)MIBSXTW;
1266
1267 // In the case of a sign-extend, where we have something like:
1268 // debugValueSubstitutions:[]
1269 // $w1 = LDRWui $x0, 1, debug-instr-number 1
1270 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1271 // $x0 = LDRSWui $x0, 0, debug-instr-number 2
1272 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1273
1274 // It will be converted to:
1275 // debugValueSubstitutions:[]
1276 // $w0, $w1 = LDPWi $x0, 0
1277 // $w0 = KILL $w0, implicit-def $x0
1278 // $x0 = SBFMXri $x0, 0, 31
1279 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1280 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1281
1282 // We want the final result to look like:
1283 // debugValueSubstitutions:
1284 // - { srcinst: 1, srcop: 0, dstinst: 4, dstop: 1, subreg: 0 }
1285 // - { srcinst: 2, srcop: 0, dstinst: 3, dstop: 0, subreg: 0 }
1286 // $w0, $w1 = LDPWi $x0, 0, debug-instr-number 4
1287 // $w0 = KILL $w0, implicit-def $x0
1288 // $x0 = SBFMXri $x0, 0, 31, debug-instr-number 3
1289 // DBG_INSTR_REF !7, dbg-instr-ref(1, 0), debug-location !9
1290 // DBG_INSTR_REF !8, dbg-instr-ref(2, 0), debug-location !9
1291
1292 // $x0 is where the final value is stored, so the sign extend (SBFMXri)
1293 // instruction contains the final value we care about we give it a new
1294 // debug-instr-number 3. Whereas, $w1 contains the final value that we care
1295 // about, therefore the LDP instruction is also given a new
1296 // debug-instr-number 4. We have to add these substitutions to the
1297 // debugValueSubstitutions table. However, we also have to ensure that the
1298 // OpIndex that pointed to debug-instr-number 1 gets updated to 1, because
1299 // $w1 is the second operand of the LDP instruction.
1300
1301 if (I->peekDebugInstrNum()) {
1302 // If I is the instruction which got sign extended and has a
1303 // debug-instr-number, give the SBFMXri instruction a new
1304 // debug-instr-number, and update the debugValueSubstitutions table with
1305 // the new debug-instr-number and OpIndex pair. Otherwise, give the Merged
1306 // instruction a new debug-instr-number, and update the
1307 // debugValueSubstitutions table with the new debug-instr-number and
1308 // OpIndex pair.
1309 unsigned NewInstrNum;
1310 if (DstRegX == I->getOperand(0).getReg()) {
1311 NewInstrNum = MIBSXTW->getDebugInstrNum();
1312 addDebugSubstitutionsToTable(MBB->getParent(), NewInstrNum, *I,
1313 *MIBSXTW);
1314 } else {
1315 NewInstrNum = MIB->getDebugInstrNum();
1316 addDebugSubstitutionsToTable(MBB->getParent(), NewInstrNum, *I, *MIB);
1317 }
1318 }
1319 if (Paired->peekDebugInstrNum()) {
1320 // If Paired is the instruction which got sign extended and has a
1321 // debug-instr-number, give the SBFMXri instruction a new
1322 // debug-instr-number, and update the debugValueSubstitutions table with
1323 // the new debug-instr-number and OpIndex pair. Otherwise, give the Merged
1324 // instruction a new debug-instr-number, and update the
1325 // debugValueSubstitutions table with the new debug-instr-number and
1326 // OpIndex pair.
1327 unsigned NewInstrNum;
1328 if (DstRegX == Paired->getOperand(0).getReg()) {
1329 NewInstrNum = MIBSXTW->getDebugInstrNum();
1330 addDebugSubstitutionsToTable(MBB->getParent(), NewInstrNum, *Paired,
1331 *MIBSXTW);
1332 } else {
1333 NewInstrNum = MIB->getDebugInstrNum();
1334 addDebugSubstitutionsToTable(MBB->getParent(), NewInstrNum, *Paired,
1335 *MIB);
1336 }
1337 }
1338
1339 LLVM_DEBUG(dbgs() << " Extend operand:\n ");
1340 LLVM_DEBUG(((MachineInstr *)MIBSXTW)->print(dbgs()));
1341 } else if (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI) {
1342 // We are combining SVE fill/spill to LDP/STP, so we need to use the Q
1343 // variant of the registers.
1344 MachineOperand &MOp0 = MIB->getOperand(0);
1345 MachineOperand &MOp1 = MIB->getOperand(1);
1346 assert(AArch64::ZPRRegClass.contains(MOp0.getReg()) &&
1347 AArch64::ZPRRegClass.contains(MOp1.getReg()) && "Invalid register.");
1348 MOp0.setReg(AArch64::Q0 + (MOp0.getReg() - AArch64::Z0));
1349 MOp1.setReg(AArch64::Q0 + (MOp1.getReg() - AArch64::Z0));
1350 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1351 } else {
1352
1353 // In the case that the merge doesn't result in a sign-extend, if we have
1354 // something like:
1355 // debugValueSubstitutions:[]
1356 // $x1 = LDRXui $x0, 1, debug-instr-number 1
1357 // DBG_INSTR_REF !13, dbg-instr-ref(1, 0), debug-location !11
1358 // $x0 = LDRXui killed $x0, 0, debug-instr-number 2
1359 // DBG_INSTR_REF !14, dbg-instr-ref(2, 0), debug-location !11
1360
1361 // It will be converted to:
1362 // debugValueSubstitutions: []
1363 // $x0, $x1 = LDPXi $x0, 0
1364 // DBG_INSTR_REF !12, dbg-instr-ref(1, 0), debug-location !14
1365 // DBG_INSTR_REF !13, dbg-instr-ref(2, 0), debug-location !14
1366
1367 // We want the final result to look like:
1368 // debugValueSubstitutions:
1369 // - { srcinst: 1, srcop: 0, dstinst: 3, dstop: 1, subreg: 0 }
1370 // - { srcinst: 2, srcop: 0, dstinst: 3, dstop: 0, subreg: 0 }
1371 // $x0, $x1 = LDPXi $x0, 0, debug-instr-number 3
1372 // DBG_INSTR_REF !12, dbg-instr-ref(1, 0), debug-location !14
1373 // DBG_INSTR_REF !12, dbg-instr-ref(2, 0), debug-location !14
1374
1375 // Here all that needs to be done is, that the LDP instruction needs to be
1376 // updated with a new debug-instr-number, we then need to add entries into
1377 // the debugSubstitutions table to map the old instr-refs to the new ones.
1378
1379 // Assign new DebugInstrNum to the Paired instruction.
1380 if (I->peekDebugInstrNum()) {
1381 unsigned NewDebugInstrNum = MIB->getDebugInstrNum();
1382 addDebugSubstitutionsToTable(MBB->getParent(), NewDebugInstrNum, *I,
1383 *MIB);
1384 }
1385 if (Paired->peekDebugInstrNum()) {
1386 unsigned NewDebugInstrNum = MIB->getDebugInstrNum();
1387 addDebugSubstitutionsToTable(MBB->getParent(), NewDebugInstrNum, *Paired,
1388 *MIB);
1389 }
1390
1391 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
1392 }
1393 LLVM_DEBUG(dbgs() << "\n");
1394
1395 if (MergeForward)
1396 for (const MachineOperand &MOP : phys_regs_and_masks(*I))
1397 if (MOP.isReg() && MOP.isKill())
1398 DefinedInBB.addReg(MOP.getReg());
1399
1400 // Copy over any implicit-def operands. This is like MI.copyImplicitOps, but
1401 // only copies implicit defs and makes sure that each operand is only added
1402 // once in case of duplicates.
1403 auto CopyImplicitOps = [&](MachineBasicBlock::iterator MI1,
1404 MachineBasicBlock::iterator MI2) {
1405 SmallSetVector<Register, 4> Ops;
1406 for (const MachineOperand &MO :
1407 llvm::drop_begin(MI1->operands(), MI1->getDesc().getNumOperands()))
1408 if (MO.isReg() && MO.isImplicit() && MO.isDef())
1409 Ops.insert(MO.getReg());
1410 for (const MachineOperand &MO :
1411 llvm::drop_begin(MI2->operands(), MI2->getDesc().getNumOperands()))
1412 if (MO.isReg() && MO.isImplicit() && MO.isDef())
1413 Ops.insert(MO.getReg());
1414 for (auto Op : Ops)
1415 MIB.addDef(Op, RegState::Implicit);
1416 };
1417 CopyImplicitOps(I, Paired);
1418
1419 // Erase the old instructions.
1420 I->eraseFromParent();
1421 Paired->eraseFromParent();
1422
1423 return NextI;
1424}
1425
1426MachineBasicBlock::iterator
1427AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI,
1428 MachineBasicBlock::iterator StoreI) {
1429 MachineBasicBlock::iterator NextI =
1430 next_nodbg(LoadI, LoadI->getParent()->end());
1431
1432 int LoadSize = TII->getMemScale(*LoadI);
1433 int StoreSize = TII->getMemScale(*StoreI);
1434 Register LdRt = getLdStRegOp(*LoadI).getReg();
1435 const MachineOperand &StMO = getLdStRegOp(*StoreI);
1436 Register StRt = getLdStRegOp(*StoreI).getReg();
1437 bool IsStoreXReg = TRI->getRegClass(AArch64::GPR64RegClassID)->contains(StRt);
1438
1439 assert((IsStoreXReg ||
1440 TRI->getRegClass(AArch64::GPR32RegClassID)->contains(StRt)) &&
1441 "Unexpected RegClass");
1442
1443 MachineInstr *BitExtMI;
1444 if (LoadSize == StoreSize && (LoadSize == 4 || LoadSize == 8)) {
1445 // Remove the load, if the destination register of the loads is the same
1446 // register for stored value.
1447 if (StRt == LdRt && LoadSize == 8) {
1448 for (MachineInstr &MI : make_range(StoreI->getIterator(),
1449 LoadI->getIterator())) {
1450 if (MI.killsRegister(StRt, TRI)) {
1451 MI.clearRegisterKills(StRt, TRI);
1452 break;
1453 }
1454 }
1455 LLVM_DEBUG(dbgs() << "Remove load instruction:\n ");
1456 LLVM_DEBUG(LoadI->print(dbgs()));
1457 LLVM_DEBUG(dbgs() << "\n");
1458 LoadI->eraseFromParent();
1459 return NextI;
1460 }
1461 // Replace the load with a mov if the load and store are in the same size.
1462 BitExtMI =
1463 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1464 TII->get(IsStoreXReg ? AArch64::ORRXrs : AArch64::ORRWrs), LdRt)
1465 .addReg(IsStoreXReg ? AArch64::XZR : AArch64::WZR)
1466 .add(StMO)
1467 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0))
1468 .setMIFlags(LoadI->getFlags());
1469 } else {
1470 // FIXME: Currently we disable this transformation in big-endian targets as
1471 // performance and correctness are verified only in little-endian.
1472 if (!Subtarget->isLittleEndian())
1473 return NextI;
1474 bool IsUnscaled = TII->hasUnscaledLdStOffset(*LoadI);
1475 assert(IsUnscaled == TII->hasUnscaledLdStOffset(*StoreI) &&
1476 "Unsupported ld/st match");
1477 assert(LoadSize <= StoreSize && "Invalid load size");
1478 int UnscaledLdOffset =
1479 IsUnscaled
1480 ? AArch64InstrInfo::getLdStOffsetOp(*LoadI).getImm()
1481 : AArch64InstrInfo::getLdStOffsetOp(*LoadI).getImm() * LoadSize;
1482 int UnscaledStOffset =
1483 IsUnscaled
1484 ? AArch64InstrInfo::getLdStOffsetOp(*StoreI).getImm()
1485 : AArch64InstrInfo::getLdStOffsetOp(*StoreI).getImm() * StoreSize;
1486 int Width = LoadSize * 8;
1487 Register DestReg =
1488 IsStoreXReg ? Register(TRI->getMatchingSuperReg(
1489 LdRt, AArch64::sub_32, &AArch64::GPR64RegClass))
1490 : LdRt;
1491
1492 assert((UnscaledLdOffset >= UnscaledStOffset &&
1493 (UnscaledLdOffset + LoadSize) <= UnscaledStOffset + StoreSize) &&
1494 "Invalid offset");
1495
1496 int Immr = 8 * (UnscaledLdOffset - UnscaledStOffset);
1497 int Imms = Immr + Width - 1;
1498 if (UnscaledLdOffset == UnscaledStOffset) {
1499 uint32_t AndMaskEncoded = ((IsStoreXReg ? 1 : 0) << 12) // N
1500 | ((Immr) << 6) // immr
1501 | ((Imms) << 0) // imms
1502 ;
1503
1504 BitExtMI =
1505 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1506 TII->get(IsStoreXReg ? AArch64::ANDXri : AArch64::ANDWri),
1507 DestReg)
1508 .add(StMO)
1509 .addImm(AndMaskEncoded)
1510 .setMIFlags(LoadI->getFlags());
1511 } else if (IsStoreXReg && Imms == 31) {
1512 // Use the 32 bit variant of UBFM if it's the LSR alias of the
1513 // instruction.
1514 assert(Immr <= Imms && "Expected LSR alias of UBFM");
1515 BitExtMI = BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1516 TII->get(AArch64::UBFMWri),
1517 TRI->getSubReg(DestReg, AArch64::sub_32))
1518 .addReg(TRI->getSubReg(StRt, AArch64::sub_32))
1519 .addImm(Immr)
1520 .addImm(Imms)
1521 .setMIFlags(LoadI->getFlags());
1522 } else {
1523 BitExtMI =
1524 BuildMI(*LoadI->getParent(), LoadI, LoadI->getDebugLoc(),
1525 TII->get(IsStoreXReg ? AArch64::UBFMXri : AArch64::UBFMWri),
1526 DestReg)
1527 .add(StMO)
1528 .addImm(Immr)
1529 .addImm(Imms)
1530 .setMIFlags(LoadI->getFlags());
1531 }
1532 }
1533
1534 // Clear kill flags between store and load.
1535 for (MachineInstr &MI : make_range(StoreI->getIterator(),
1536 BitExtMI->getIterator()))
1537 if (MI.killsRegister(StRt, TRI)) {
1538 MI.clearRegisterKills(StRt, TRI);
1539 break;
1540 }
1541
1542 LLVM_DEBUG(dbgs() << "Promoting load by replacing :\n ");
1543 LLVM_DEBUG(StoreI->print(dbgs()));
1544 LLVM_DEBUG(dbgs() << " ");
1545 LLVM_DEBUG(LoadI->print(dbgs()));
1546 LLVM_DEBUG(dbgs() << " with instructions:\n ");
1547 LLVM_DEBUG(StoreI->print(dbgs()));
1548 LLVM_DEBUG(dbgs() << " ");
1549 LLVM_DEBUG((BitExtMI)->print(dbgs()));
1550 LLVM_DEBUG(dbgs() << "\n");
1551
1552 // Erase the old instructions.
1553 LoadI->eraseFromParent();
1554 return NextI;
1555}
1556
1557static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) {
1558 // Convert the byte-offset used by unscaled into an "element" offset used
1559 // by the scaled pair load/store instructions.
1560 if (IsUnscaled) {
1561 // If the byte-offset isn't a multiple of the stride, there's no point
1562 // trying to match it.
1563 if (Offset % OffsetStride)
1564 return false;
1565 Offset /= OffsetStride;
1566 }
1567 return Offset <= 63 && Offset >= -64;
1568}
1569
1570// Do alignment, specialized to power of 2 and for signed ints,
1571// avoiding having to do a C-style cast from uint_64t to int when
1572// using alignTo from include/llvm/Support/MathExtras.h.
1573// FIXME: Move this function to include/MathExtras.h?
1574static int alignTo(int Num, int PowOf2) {
1575 return (Num + PowOf2 - 1) & ~(PowOf2 - 1);
1576}
1577
1578static bool mayAlias(MachineInstr &MIa,
1579 SmallVectorImpl<MachineInstr *> &MemInsns,
1580 AliasAnalysis *AA) {
1581 for (MachineInstr *MIb : MemInsns) {
1582 if (MIa.mayAlias(AA, *MIb, /*UseTBAA*/ false)) {
1583 LLVM_DEBUG(dbgs() << "Aliasing with: "; MIb->dump());
1584 return true;
1585 }
1586 }
1587
1588 LLVM_DEBUG(dbgs() << "No aliases found\n");
1589 return false;
1590}
1591
1592bool AArch64LoadStoreOpt::findMatchingStore(
1593 MachineBasicBlock::iterator I, unsigned Limit,
1594 MachineBasicBlock::iterator &StoreI) {
1595 MachineBasicBlock::iterator B = I->getParent()->begin();
1596 MachineBasicBlock::iterator MBBI = I;
1597 MachineInstr &LoadMI = *I;
1598 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(LoadMI).getReg();
1599
1600 // If the load is the first instruction in the block, there's obviously
1601 // not any matching store.
1602 if (MBBI == B)
1603 return false;
1604
1605 // Track which register units have been modified and used between the first
1606 // insn and the second insn.
1607 ModifiedRegUnits.clear();
1608 UsedRegUnits.clear();
1609
1610 unsigned Count = 0;
1611 do {
1612 MBBI = prev_nodbg(MBBI, B);
1613 MachineInstr &MI = *MBBI;
1614
1615 // Don't count transient instructions towards the search limit since there
1616 // may be different numbers of them if e.g. debug information is present.
1617 if (!MI.isTransient())
1618 ++Count;
1619
1620 // If the load instruction reads directly from the address to which the
1621 // store instruction writes and the stored value is not modified, we can
1622 // promote the load. Since we do not handle stores with pre-/post-index,
1623 // it's unnecessary to check if BaseReg is modified by the store itself.
1624 // Also we can't handle stores without an immediate offset operand,
1625 // while the operand might be the address for a global variable.
1626 if (MI.mayStore() && isMatchingStore(LoadMI, MI) &&
1627 BaseReg == AArch64InstrInfo::getLdStBaseOp(MI).getReg() &&
1628 AArch64InstrInfo::getLdStOffsetOp(MI).isImm() &&
1629 isLdOffsetInRangeOfSt(LoadMI, MI, TII) &&
1630 ModifiedRegUnits.available(getLdStRegOp(MI).getReg())) {
1631 StoreI = MBBI;
1632 return true;
1633 }
1634
1635 if (MI.isCall())
1636 return false;
1637
1638 // Update modified / uses register units.
1639 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
1640
1641 // Otherwise, if the base register is modified, we have no match, so
1642 // return early.
1643 if (!ModifiedRegUnits.available(BaseReg))
1644 return false;
1645
1646 // If we encounter a store aliased with the load, return early.
1647 if (MI.mayStore() && LoadMI.mayAlias(AA, MI, /*UseTBAA*/ false))
1648 return false;
1649 } while (MBBI != B && Count < Limit);
1650 return false;
1651}
1652
1653// Returns true if FirstMI and MI are candidates for merging or pairing.
1654// Otherwise, returns false.
1655static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI,
1656 LdStPairFlags &Flags,
1657 const AArch64InstrInfo *TII) {
1658 // If this is volatile or if pairing is suppressed, not a candidate.
1659 if (MI.hasOrderedMemoryRef() || TII->isLdStPairSuppressed(MI))
1660 return false;
1661
1662 // We should have already checked FirstMI for pair suppression and volatility.
1663 assert(!FirstMI.hasOrderedMemoryRef() &&
1664 !TII->isLdStPairSuppressed(FirstMI) &&
1665 "FirstMI shouldn't get here if either of these checks are true.");
1666
1667 if ((MI.getFlag(MachineInstr::FrameSetup) ||
1668 MI.getFlag(MachineInstr::FrameDestroy)) &&
1669 MI.getMF()->needsWinCFI())
1670 return false;
1671
1672 unsigned OpcA = FirstMI.getOpcode();
1673 unsigned OpcB = MI.getOpcode();
1674
1675 // Opcodes match: If the opcodes are pre ld/st there is nothing more to check.
1676 if (OpcA == OpcB)
1677 return !AArch64InstrInfo::isPreLdSt(FirstMI);
1678
1679 // Bail out if one of the opcodes is SVE fill/spill, as we currently don't
1680 // allow pairing them with other instructions.
1681 if (OpcA == AArch64::LDR_ZXI || OpcA == AArch64::STR_ZXI ||
1682 OpcB == AArch64::LDR_ZXI || OpcB == AArch64::STR_ZXI)
1683 return false;
1684
1685 // Two pre ld/st of different opcodes cannot be merged either
1686 if (AArch64InstrInfo::isPreLdSt(FirstMI) && AArch64InstrInfo::isPreLdSt(MI))
1687 return false;
1688
1689 // Try to match a sign-extended load/store with a zero-extended load/store.
1690 bool IsValidLdStrOpc, PairIsValidLdStrOpc;
1691 unsigned NonSExtOpc = getMatchingNonSExtOpcode(OpcA, &IsValidLdStrOpc);
1692 assert(IsValidLdStrOpc &&
1693 "Given Opc should be a Load or Store with an immediate");
1694 // OpcA will be the first instruction in the pair.
1695 if (NonSExtOpc == getMatchingNonSExtOpcode(OpcB, &PairIsValidLdStrOpc)) {
1696 Flags.setSExtIdx(NonSExtOpc == OpcA ? 1 : 0);
1697 return true;
1698 }
1699
1700 // If the second instruction isn't even a mergable/pairable load/store, bail
1701 // out.
1702 if (!PairIsValidLdStrOpc)
1703 return false;
1704
1705 // Narrow stores do not have a matching pair opcodes, so constrain their
1706 // merging to zero stores.
1707 if (isNarrowStore(OpcA) || isNarrowStore(OpcB))
1708 return getLdStRegOp(FirstMI).getReg() == AArch64::WZR &&
1709 getLdStRegOp(MI).getReg() == AArch64::WZR &&
1710 TII->getMemScale(FirstMI) == TII->getMemScale(MI);
1711
1712 // The STR<S,D,Q,W,X>pre - STR<S,D,Q,W,X>ui and
1713 // LDR<S,D,Q,W,X,SW>pre-LDR<S,D,Q,W,X,SW>ui
1714 // are candidate pairs that can be merged.
1715 if (isPreLdStPairCandidate(FirstMI, MI))
1716 return true;
1717
1718 // Try to match an unscaled load/store with a scaled load/store.
1719 return TII->hasUnscaledLdStOffset(OpcA) != TII->hasUnscaledLdStOffset(OpcB) &&
1720 getMatchingPairOpcode(OpcA) == getMatchingPairOpcode(OpcB);
1721
1722 // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
1723}
1724
1725static bool canRenameMOP(const MachineInstr &MI, const MachineOperand &MOP,
1726 const TargetRegisterInfo *TRI) {
1727 if (MOP.isReg()) {
1728 auto *RegClass = TRI->getMinimalPhysRegClass(MOP.getReg());
1729 // Renaming registers with multiple disjunct sub-registers (e.g. the
1730 // result of a LD3) means that all sub-registers are renamed, potentially
1731 // impacting other instructions we did not check. Bail out.
1732 // Note that this relies on the structure of the AArch64 register file. In
1733 // particular, a subregister cannot be written without overwriting the
1734 // whole register.
1735 if (RegClass->HasDisjunctSubRegs && RegClass->CoveredBySubRegs &&
1736 (TRI->getSubRegisterClass(RegClass, AArch64::dsub0) ||
1737 TRI->getSubRegisterClass(RegClass, AArch64::qsub0) ||
1738 TRI->getSubRegisterClass(RegClass, AArch64::zsub0))) {
1739 LLVM_DEBUG(
1740 dbgs()
1741 << " Cannot rename operands with multiple disjunct subregisters ("
1742 << MOP << ")\n");
1743 return false;
1744 }
1745
1746 // We cannot rename arbitrary implicit-defs, the specific rule to rewrite
1747 // them must be known. For example, in ORRWrs the implicit-def
1748 // corresponds to the result register.
1749 if (MOP.isImplicit() && MOP.isDef()) {
1750 if (!isRewritableImplicitDef(MI, MOP))
1751 return false;
1752 return TRI->isSuperOrSubRegisterEq(MI.getOperand(0).getReg(),
1753 MOP.getReg());
1754 }
1755 }
1756 return MOP.isImplicit() ||
1757 (MOP.isRenamable() && !MOP.isEarlyClobber() && !MOP.isTied());
1758}
1759
1760static bool
1761canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween,
1762 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1763 const TargetRegisterInfo *TRI) {
1764 if (!FirstMI.mayStore())
1765 return false;
1766
1767 // Check if we can find an unused register which we can use to rename
1768 // the register used by the first load/store.
1769
1770 auto RegToRename = getLdStRegOp(FirstMI).getReg();
1771 // For now, we only rename if the store operand gets killed at the store.
1772 if (!getLdStRegOp(FirstMI).isKill() &&
1773 !any_of(FirstMI.operands(),
1774 [TRI, RegToRename](const MachineOperand &MOP) {
1775 return MOP.isReg() && !MOP.isDebug() && MOP.getReg() &&
1776 MOP.isImplicit() && MOP.isKill() &&
1777 TRI->regsOverlap(RegToRename, MOP.getReg());
1778 })) {
1779 LLVM_DEBUG(dbgs() << " Operand not killed at " << FirstMI);
1780 return false;
1781 }
1782
1783 bool FoundDef = false;
1784
1785 // For each instruction between FirstMI and the previous def for RegToRename,
1786 // we
1787 // * check if we can rename RegToRename in this instruction
1788 // * collect the registers used and required register classes for RegToRename.
1789 std::function<bool(MachineInstr &, bool)> CheckMIs = [&](MachineInstr &MI,
1790 bool IsDef) {
1791 LLVM_DEBUG(dbgs() << "Checking " << MI);
1792 // Currently we do not try to rename across frame-setup instructions.
1793 if (MI.getFlag(MachineInstr::FrameSetup)) {
1794 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions "
1795 << "currently\n");
1796 return false;
1797 }
1798
1799 UsedInBetween.accumulate(MI);
1800
1801 // For a definition, check that we can rename the definition and exit the
1802 // loop.
1803 FoundDef = IsDef;
1804
1805 // For defs, check if we can rename the first def of RegToRename.
1806 if (FoundDef) {
1807 // For some pseudo instructions, we might not generate code in the end
1808 // (e.g. KILL) and we would end up without a correct def for the rename
1809 // register.
1810 // TODO: This might be overly conservative and we could handle those cases
1811 // in multiple ways:
1812 // 1. Insert an extra copy, to materialize the def.
1813 // 2. Skip pseudo-defs until we find an non-pseudo def.
1814 if (MI.isPseudo()) {
1815 LLVM_DEBUG(dbgs() << " Cannot rename pseudo/bundle instruction\n");
1816 return false;
1817 }
1818
1819 for (auto &MOP : MI.operands()) {
1820 if (!MOP.isReg() || !MOP.isDef() || MOP.isDebug() || !MOP.getReg() ||
1821 !TRI->regsOverlap(MOP.getReg(), RegToRename))
1822 continue;
1823 if (!canRenameMOP(MI, MOP, TRI)) {
1824 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1825 return false;
1826 }
1827 RequiredClasses.insert(TRI->getMinimalPhysRegClass(MOP.getReg()));
1828 }
1829 return true;
1830 } else {
1831 for (auto &MOP : MI.operands()) {
1832 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1833 !TRI->regsOverlap(MOP.getReg(), RegToRename))
1834 continue;
1835
1836 if (!canRenameMOP(MI, MOP, TRI)) {
1837 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1838 return false;
1839 }
1840 RequiredClasses.insert(TRI->getMinimalPhysRegClass(MOP.getReg()));
1841 }
1842 }
1843 return true;
1844 };
1845
1846 const AArch64Options &CLOpts =
1847 FirstMI.getMF()->getSubtarget<AArch64Subtarget>().getCLOpts();
1848 if (!forAllMIsUntilDef(FirstMI, RegToRename, TRI,
1849 CLOpts.load_store_scan_limit, CheckMIs))
1850 return false;
1851
1852 if (!FoundDef) {
1853 LLVM_DEBUG(dbgs() << " Did not find definition for register in BB\n");
1854 return false;
1855 }
1856 return true;
1857}
1858
1859// We want to merge the second load into the first by rewriting the usages of
1860// the same reg between first (incl.) and second (excl.). We don't need to care
1861// about any insns before FirstLoad or after SecondLoad.
1862// 1. The second load writes new value into the same reg.
1863// - The renaming is impossible to impact later use of the reg.
1864// - The second load always trash the value written by the first load which
1865// means the reg must be killed before the second load.
1866// 2. The first load must be a def for the same reg so we don't need to look
1867// into anything before it.
1868static bool canRenameUntilSecondLoad(
1869 MachineInstr &FirstLoad, MachineInstr &SecondLoad,
1870 LiveRegUnits &UsedInBetween,
1871 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1872 const TargetRegisterInfo *TRI) {
1873 if (FirstLoad.isPseudo())
1874 return false;
1875
1876 UsedInBetween.accumulate(FirstLoad);
1877 auto RegToRename = getLdStRegOp(FirstLoad).getReg();
1878 bool Success = std::all_of(
1879 FirstLoad.getIterator(), SecondLoad.getIterator(),
1880 [&](MachineInstr &MI) {
1881 LLVM_DEBUG(dbgs() << "Checking " << MI);
1882 // Currently we do not try to rename across frame-setup instructions.
1883 if (MI.getFlag(MachineInstr::FrameSetup)) {
1884 LLVM_DEBUG(dbgs() << " Cannot rename framesetup instructions "
1885 << "currently\n");
1886 return false;
1887 }
1888
1889 for (auto &MOP : MI.operands()) {
1890 if (!MOP.isReg() || MOP.isDebug() || !MOP.getReg() ||
1891 !TRI->regsOverlap(MOP.getReg(), RegToRename))
1892 continue;
1893 if (!canRenameMOP(MI, MOP, TRI)) {
1894 LLVM_DEBUG(dbgs() << " Cannot rename " << MOP << " in " << MI);
1895 return false;
1896 }
1897 RequiredClasses.insert(TRI->getMinimalPhysRegClass(MOP.getReg()));
1898 }
1899
1900 return true;
1901 });
1902 return Success;
1903}
1904
1905// Check if we can find a physical register for renaming \p Reg. This register
1906// must:
1907// * not be defined already in \p DefinedInBB; DefinedInBB must contain all
1908// defined registers up to the point where the renamed register will be used,
1909// * not used in \p UsedInBetween; UsedInBetween must contain all accessed
1910// registers in the range the rename register will be used,
1911// * is available in all used register classes (checked using RequiredClasses).
1912static std::optional<MCPhysReg> tryToFindRegisterToRename(
1913 const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB,
1914 LiveRegUnits &UsedInBetween,
1915 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1916 const TargetRegisterInfo *TRI) {
1917 const MachineRegisterInfo &RegInfo = MF.getRegInfo();
1918
1919 // Checks if any sub- or super-register of PR is callee saved.
1920 auto AnySubOrSuperRegCalleePreserved = [&MF, TRI](MCPhysReg PR) {
1921 return any_of(TRI->sub_and_superregs_inclusive(PR),
1922 [&MF, TRI](MCPhysReg SubOrSuper) {
1923 return TRI->isCalleeSavedPhysReg(SubOrSuper, MF);
1924 });
1925 };
1926
1927 // Check if PR or one of its sub- or super-registers can be used for all
1928 // required register classes.
1929 auto CanBeUsedForAllClasses = [&RequiredClasses, TRI](MCPhysReg PR) {
1930 return all_of(RequiredClasses, [PR, TRI](const TargetRegisterClass *C) {
1931 return any_of(
1932 TRI->sub_and_superregs_inclusive(PR),
1933 [C](MCPhysReg SubOrSuper) { return C->contains(SubOrSuper); });
1934 });
1935 };
1936
1937 auto *RegClass = TRI->getMinimalPhysRegClass(Reg);
1938 for (const MCPhysReg &PR : *RegClass) {
1939 if (DefinedInBB.available(PR) && UsedInBetween.available(PR) &&
1940 !RegInfo.isReserved(PR) && !AnySubOrSuperRegCalleePreserved(PR) &&
1941 CanBeUsedForAllClasses(PR)) {
1942 DefinedInBB.addReg(PR);
1943 LLVM_DEBUG(dbgs() << "Found rename register " << printReg(PR, TRI)
1944 << "\n");
1945 return {PR};
1946 }
1947 }
1948 LLVM_DEBUG(dbgs() << "No rename register found from "
1949 << TRI->getRegClassName(RegClass) << "\n");
1950 return std::nullopt;
1951}
1952
1953// For store pairs: returns a register from FirstMI to the beginning of the
1954// block that can be renamed.
1955// For load pairs: returns a register from FirstMI to MI that can be renamed.
1956static std::optional<MCPhysReg> findRenameRegForSameLdStRegPair(
1957 std::optional<bool> MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI,
1958 Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween,
1959 SmallPtrSetImpl<const TargetRegisterClass *> &RequiredClasses,
1960 const TargetRegisterInfo *TRI) {
1961 std::optional<MCPhysReg> RenameReg;
1962 if (!DebugCounter::shouldExecute(RegRenamingCounter))
1963 return RenameReg;
1964
1965 auto *RegClass = TRI->getMinimalPhysRegClass(getLdStRegOp(FirstMI).getReg());
1966 MachineFunction &MF = *FirstMI.getParent()->getParent();
1967 if (!RegClass || !MF.getRegInfo().tracksLiveness())
1968 return RenameReg;
1969
1970 const bool IsLoad = FirstMI.mayLoad();
1971
1972 if (!MaybeCanRename) {
1973 if (IsLoad)
1974 MaybeCanRename = {canRenameUntilSecondLoad(FirstMI, MI, UsedInBetween,
1975 RequiredClasses, TRI)};
1976 else
1977 MaybeCanRename = {
1978 canRenameUpToDef(FirstMI, UsedInBetween, RequiredClasses, TRI)};
1979 }
1980
1981 if (*MaybeCanRename) {
1982 RenameReg = tryToFindRegisterToRename(MF, Reg, DefinedInBB, UsedInBetween,
1983 RequiredClasses, TRI);
1984 }
1985 return RenameReg;
1986}
1987
1988/// Scan the instructions looking for a load/store that can be combined with the
1989/// current instruction into a wider equivalent or a load/store pair.
1990MachineBasicBlock::iterator
1991AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I,
1992 LdStPairFlags &Flags, unsigned Limit,
1993 bool FindNarrowMerge) {
1994 MachineBasicBlock::iterator E = I->getParent()->end();
1995 MachineBasicBlock::iterator MBBI = I;
1996 MachineInstr &FirstMI = *I;
1997 MBBI = next_nodbg(MBBI, E);
1998
1999 bool MayLoad = FirstMI.mayLoad();
2000 bool IsUnscaled = TII->hasUnscaledLdStOffset(FirstMI);
2001 Register Reg = getLdStRegOp(FirstMI).getReg();
2002 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(FirstMI).getReg();
2003 int Offset = AArch64InstrInfo::getLdStOffsetOp(FirstMI).getImm();
2004 int OffsetStride = IsUnscaled ? TII->getMemScale(FirstMI) : 1;
2005 bool IsPromotableZeroStore = isPromotableZeroStoreInst(FirstMI);
2006
2007 std::optional<bool> MaybeCanRename;
2008 if (!Subtarget->getCLOpts().load_store_renaming)
2009 MaybeCanRename = {false};
2010
2011 SmallPtrSet<const TargetRegisterClass *, 5> RequiredClasses;
2012 LiveRegUnits UsedInBetween;
2013 UsedInBetween.init(*TRI);
2014
2015 Flags.clearRenameReg();
2016
2017 // Track which register units have been modified and used between the first
2018 // insn (inclusive) and the second insn.
2019 ModifiedRegUnits.clear();
2020 UsedRegUnits.clear();
2021
2022 // Remember any instructions that read/write memory between FirstMI and MI.
2023 SmallVector<MachineInstr *, 4> MemInsns;
2024
2025 LLVM_DEBUG(dbgs() << "Find match for: "; FirstMI.dump());
2026 for (unsigned Count = 0; MBBI != E && Count < Limit;
2027 MBBI = next_nodbg(MBBI, E)) {
2028 MachineInstr &MI = *MBBI;
2029 LLVM_DEBUG(dbgs() << "Analysing 2nd insn: "; MI.dump());
2030
2031 UsedInBetween.accumulate(MI);
2032
2033 // Don't count transient instructions towards the search limit since there
2034 // may be different numbers of them if e.g. debug information is present.
2035 if (!MI.isTransient())
2036 ++Count;
2037
2038 Flags.setSExtIdx(-1);
2039 if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) &&
2040 AArch64InstrInfo::getLdStOffsetOp(MI).isImm()) {
2041 assert(MI.mayLoadOrStore() && "Expected memory operation.");
2042 // If we've found another instruction with the same opcode, check to see
2043 // if the base and offset are compatible with our starting instruction.
2044 // These instructions all have scaled immediate operands, so we just
2045 // check for +1/-1. Make sure to check the new instruction offset is
2046 // actually an immediate and not a symbolic reference destined for
2047 // a relocation.
2048 Register MIBaseReg = AArch64InstrInfo::getLdStBaseOp(MI).getReg();
2049 int MIOffset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
2050 bool MIIsUnscaled = TII->hasUnscaledLdStOffset(MI);
2051 if (IsUnscaled != MIIsUnscaled) {
2052 // We're trying to pair instructions that differ in how they are scaled.
2053 // If FirstMI is scaled then scale the offset of MI accordingly.
2054 // Otherwise, do the opposite (i.e., make MI's offset unscaled).
2055 int MemSize = TII->getMemScale(MI);
2056 if (MIIsUnscaled) {
2057 // If the unscaled offset isn't a multiple of the MemSize, we can't
2058 // pair the operations together: bail and keep looking.
2059 if (MIOffset % MemSize) {
2060 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2061 UsedRegUnits, TRI);
2062 MemInsns.push_back(&MI);
2063 continue;
2064 }
2065 MIOffset /= MemSize;
2066 } else {
2067 MIOffset *= MemSize;
2068 }
2069 }
2070
2071 bool IsPreLdSt = isPreLdStPairCandidate(FirstMI, MI);
2072
2073 if (BaseReg == MIBaseReg) {
2074 // If the offset of the second ld/st is not equal to the size of the
2075 // destination register it can’t be paired with a pre-index ld/st
2076 // pair. Additionally if the base reg is used or modified the operations
2077 // can't be paired: bail and keep looking.
2078 if (IsPreLdSt) {
2079 bool IsOutOfBounds = MIOffset != TII->getMemScale(MI);
2080 bool IsBaseRegUsed = !UsedRegUnits.available(
2081 AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2082 bool IsBaseRegModified = !ModifiedRegUnits.available(
2083 AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2084 // If the stored value and the address of the second instruction is
2085 // the same, it needs to be using the updated register and therefore
2086 // it must not be folded.
2087 bool IsMIRegTheSame =
2088 TRI->regsOverlap(getLdStRegOp(MI).getReg(),
2089 AArch64InstrInfo::getLdStBaseOp(MI).getReg());
2090 if (IsOutOfBounds || IsBaseRegUsed || IsBaseRegModified ||
2091 IsMIRegTheSame) {
2092 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2093 UsedRegUnits, TRI);
2094 MemInsns.push_back(&MI);
2095 continue;
2096 }
2097 } else {
2098 if ((Offset != MIOffset + OffsetStride) &&
2099 (Offset + OffsetStride != MIOffset)) {
2100 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2101 UsedRegUnits, TRI);
2102 MemInsns.push_back(&MI);
2103 continue;
2104 }
2105 }
2106
2107 int MinOffset = Offset < MIOffset ? Offset : MIOffset;
2108 if (FindNarrowMerge) {
2109 // If the alignment requirements of the scaled wide load/store
2110 // instruction can't express the offset of the scaled narrow input,
2111 // bail and keep looking. For promotable zero stores, allow only when
2112 // the stored value is the same (i.e., WZR).
2113 if ((!IsUnscaled && alignTo(MinOffset, 2) != MinOffset) ||
2114 (IsPromotableZeroStore && Reg != getLdStRegOp(MI).getReg())) {
2115 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2116 UsedRegUnits, TRI);
2117 MemInsns.push_back(&MI);
2118 continue;
2119 }
2120 } else {
2121 // Pairwise instructions have a 7-bit signed offset field. Single
2122 // insns have a 12-bit unsigned offset field. If the resultant
2123 // immediate offset of merging these instructions is out of range for
2124 // a pairwise instruction, bail and keep looking.
2125 if (!inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) {
2126 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2127 UsedRegUnits, TRI);
2128 MemInsns.push_back(&MI);
2129 LLVM_DEBUG(dbgs() << "Offset doesn't fit in immediate, "
2130 << "keep looking.\n");
2131 continue;
2132 }
2133 // If the alignment requirements of the paired (scaled) instruction
2134 // can't express the offset of the unscaled input, bail and keep
2135 // looking.
2136 if (IsUnscaled && (alignTo(MinOffset, OffsetStride) != MinOffset)) {
2137 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2138 UsedRegUnits, TRI);
2139 MemInsns.push_back(&MI);
2140 LLVM_DEBUG(dbgs()
2141 << "Offset doesn't fit due to alignment requirements, "
2142 << "keep looking.\n");
2143 continue;
2144 }
2145 }
2146
2147 // If the BaseReg has been modified, then we cannot do the optimization.
2148 // For example, in the following pattern
2149 // ldr x1 [x2]
2150 // ldr x2 [x3]
2151 // ldr x4 [x2, #8],
2152 // the first and third ldr cannot be converted to ldp x1, x4, [x2]
2153 if (!ModifiedRegUnits.available(BaseReg))
2154 return E;
2155
2156 const bool SameLoadReg = MayLoad && TRI->isSuperOrSubRegisterEq(
2157 Reg, getLdStRegOp(MI).getReg());
2158
2159 // If the Rt of the second instruction (destination register of the
2160 // load) was not modified or used between the two instructions and none
2161 // of the instructions between the second and first alias with the
2162 // second, we can combine the second into the first.
2163 bool RtNotModified =
2164 ModifiedRegUnits.available(getLdStRegOp(MI).getReg());
2165 bool RtNotUsed = !(MI.mayLoad() && !SameLoadReg &&
2166 !UsedRegUnits.available(getLdStRegOp(MI).getReg()));
2167
2168 LLVM_DEBUG(dbgs() << "Checking, can combine 2nd into 1st insn:\n"
2169 << "Reg '" << getLdStRegOp(MI) << "' not modified: "
2170 << (RtNotModified ? "true" : "false") << "\n"
2171 << "Reg '" << getLdStRegOp(MI) << "' not used: "
2172 << (RtNotUsed ? "true" : "false") << "\n");
2173
2174 if (RtNotModified && RtNotUsed && !mayAlias(MI, MemInsns, AA)) {
2175 // For pairs loading into the same reg, try to find a renaming
2176 // opportunity to allow the renaming of Reg between FirstMI and MI
2177 // and combine MI into FirstMI; otherwise bail and keep looking.
2178 if (SameLoadReg) {
2179 std::optional<MCPhysReg> RenameReg =
2180 findRenameRegForSameLdStRegPair(MaybeCanRename, FirstMI, MI,
2181 Reg, DefinedInBB, UsedInBetween,
2182 RequiredClasses, TRI);
2183 if (!RenameReg) {
2184 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits,
2185 UsedRegUnits, TRI);
2186 MemInsns.push_back(&MI);
2187 LLVM_DEBUG(dbgs() << "Can't find reg for renaming, "
2188 << "keep looking.\n");
2189 continue;
2190 }
2191 Flags.setRenameReg(*RenameReg);
2192 }
2193
2194 Flags.setMergeForward(false);
2195 if (!SameLoadReg)
2196 Flags.clearRenameReg();
2197 return MBBI;
2198 }
2199
2200 // Likewise, if the Rt of the first instruction is not modified or used
2201 // between the two instructions and none of the instructions between the
2202 // first and the second alias with the first, we can combine the first
2203 // into the second.
2204 RtNotModified = !(
2205 MayLoad && !UsedRegUnits.available(getLdStRegOp(FirstMI).getReg()));
2206
2207 LLVM_DEBUG(dbgs() << "Checking, can combine 1st into 2nd insn:\n"
2208 << "Reg '" << getLdStRegOp(FirstMI)
2209 << "' not modified: "
2210 << (RtNotModified ? "true" : "false") << "\n");
2211
2212 if (RtNotModified && !mayAlias(FirstMI, MemInsns, AA)) {
2213 if (ModifiedRegUnits.available(getLdStRegOp(FirstMI).getReg())) {
2214 Flags.setMergeForward(true);
2215 Flags.clearRenameReg();
2216 return MBBI;
2217 }
2218
2219 std::optional<MCPhysReg> RenameReg = findRenameRegForSameLdStRegPair(
2220 MaybeCanRename, FirstMI, MI, Reg, DefinedInBB, UsedInBetween,
2221 RequiredClasses, TRI);
2222 if (RenameReg) {
2223 Flags.setMergeForward(true);
2224 Flags.setRenameReg(*RenameReg);
2225 return MBBI;
2226 }
2227 }
2228 LLVM_DEBUG(dbgs() << "Unable to combine these instructions due to "
2229 << "interference in between, keep looking.\n");
2230 }
2231 }
2232
2233 // If the instruction wasn't a matching load or store. Stop searching if we
2234 // encounter a call instruction that might modify memory.
2235 if (MI.isCall()) {
2236 LLVM_DEBUG(dbgs() << "Found a call, stop looking.\n");
2237 return E;
2238 }
2239
2240 // Update modified / uses register units.
2241 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2242
2243 // Otherwise, if the base register is modified, we have no match, so
2244 // return early.
2245 if (!ModifiedRegUnits.available(BaseReg)) {
2246 LLVM_DEBUG(dbgs() << "Base reg is modified, stop looking.\n");
2247 return E;
2248 }
2249
2250 // Update list of instructions that read/write memory.
2251 if (MI.mayLoadOrStore())
2252 MemInsns.push_back(&MI);
2253 }
2254 return E;
2255}
2256
2257static MachineBasicBlock::iterator
2258maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI) {
2259 assert((MI.getOpcode() == AArch64::SUBXri ||
2260 MI.getOpcode() == AArch64::ADDXri) &&
2261 "Expected a register update instruction");
2262 auto End = MI.getParent()->end();
2263 if (MaybeCFI == End ||
2264 MaybeCFI->getOpcode() != TargetOpcode::CFI_INSTRUCTION ||
2265 !(MI.getFlag(MachineInstr::FrameSetup) ||
2266 MI.getFlag(MachineInstr::FrameDestroy)) ||
2267 MI.getOperand(0).getReg() != AArch64::SP)
2268 return End;
2269
2270 const MachineFunction &MF = *MI.getParent()->getParent();
2271 unsigned CFIIndex = MaybeCFI->getOperand(0).getCFIIndex();
2272 const MCCFIInstruction &CFI = MF.getFrameInstructions()[CFIIndex];
2273 switch (CFI.getOperation()) {
2274 case MCCFIInstruction::OpDefCfa:
2275 case MCCFIInstruction::OpDefCfaOffset:
2276 return MaybeCFI;
2277 default:
2278 return End;
2279 }
2280}
2281
2282std::optional<MachineBasicBlock::iterator> AArch64LoadStoreOpt::mergeUpdateInsn(
2283 MachineBasicBlock::iterator I, MachineBasicBlock::iterator Update,
2284 bool IsForward, bool IsPreIdx, bool MergeEither) {
2285 assert((Update->getOpcode() == AArch64::ADDXri ||
2286 Update->getOpcode() == AArch64::SUBXri) &&
2287 "Unexpected base register update instruction to merge!");
2288 MachineBasicBlock::iterator E = I->getParent()->end();
2289 MachineBasicBlock::iterator NextI = next_nodbg(I, E);
2290
2291 // If updating the SP and the following instruction is CFA offset related CFI,
2292 // make sure the CFI follows the SP update either by merging at the location
2293 // of the update or by moving the CFI after the merged instruction. If unable
2294 // to do so, bail.
2295 MachineBasicBlock::iterator InsertPt = I;
2296 if (IsForward) {
2297 assert(IsPreIdx);
2298 if (auto CFI = maybeMoveCFI(*Update, next_nodbg(Update, E)); CFI != E) {
2299 if (MergeEither) {
2300 InsertPt = Update;
2301 } else {
2302 // Take care not to reorder CFIs.
2303 if (std::any_of(std::next(CFI), I, [](const auto &Insn) {
2304 return Insn.getOpcode() == TargetOpcode::CFI_INSTRUCTION;
2305 }))
2306 return std::nullopt;
2307
2308 MachineBasicBlock *MBB = InsertPt->getParent();
2309 MBB->splice(std::next(InsertPt), MBB, CFI);
2310 }
2311 }
2312 }
2313
2314 // Return the instruction following the merged instruction, which is
2315 // the instruction following our unmerged load. Unless that's the add/sub
2316 // instruction we're merging, in which case it's the one after that.
2317 if (NextI == Update)
2318 NextI = next_nodbg(NextI, E);
2319
2320 int Value = Update->getOperand(2).getImm();
2321 assert(AArch64_AM::getShiftValue(Update->getOperand(3).getImm()) == 0 &&
2322 "Can't merge 1 << 12 offset into pre-/post-indexed load / store");
2323 if (Update->getOpcode() == AArch64::SUBXri)
2324 Value = -Value;
2325
2326 unsigned NewOpc = IsPreIdx ? getPreIndexedOpcode(I->getOpcode())
2327 : getPostIndexedOpcode(I->getOpcode());
2328 MachineInstrBuilder MIB;
2329 int Scale, MinOffset, MaxOffset;
2330 getPrePostIndexedMemOpInfo(*I, Scale, MinOffset, MaxOffset);
2331 if (!AArch64InstrInfo::isPairedLdSt(*I)) {
2332 // Non-paired instruction.
2333 MIB = BuildMI(*InsertPt->getParent(), InsertPt, InsertPt->getDebugLoc(),
2334 TII->get(NewOpc))
2335 .add(Update->getOperand(0))
2336 .add(getLdStRegOp(*I))
2337 .add(AArch64InstrInfo::getLdStBaseOp(*I))
2338 .addImm(Value / Scale)
2339 .setMemRefs(I->memoperands())
2340 .setMIFlags(I->mergeFlagsWith(*Update));
2341 } else {
2342 // Paired instruction.
2343 MIB = BuildMI(*InsertPt->getParent(), InsertPt, InsertPt->getDebugLoc(),
2344 TII->get(NewOpc))
2345 .add(Update->getOperand(0))
2346 .add(getLdStRegOp(*I, 0))
2347 .add(getLdStRegOp(*I, 1))
2348 .add(AArch64InstrInfo::getLdStBaseOp(*I))
2349 .addImm(Value / Scale)
2350 .setMemRefs(I->memoperands())
2351 .setMIFlags(I->mergeFlagsWith(*Update));
2352 }
2353
2354 if (IsPreIdx) {
2355 ++NumPreFolded;
2356 LLVM_DEBUG(dbgs() << "Creating pre-indexed load/store.");
2357 } else {
2358 ++NumPostFolded;
2359 LLVM_DEBUG(dbgs() << "Creating post-indexed load/store.");
2360 }
2361 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
2362 LLVM_DEBUG(I->print(dbgs()));
2363 LLVM_DEBUG(dbgs() << " ");
2364 LLVM_DEBUG(Update->print(dbgs()));
2365 LLVM_DEBUG(dbgs() << " with instruction:\n ");
2366 LLVM_DEBUG(((MachineInstr *)MIB)->print(dbgs()));
2367 LLVM_DEBUG(dbgs() << "\n");
2368
2369 // Erase the old instructions for the block.
2370 I->eraseFromParent();
2371 Update->eraseFromParent();
2372
2373 return NextI;
2374}
2375
2376MachineBasicBlock::iterator
2377AArch64LoadStoreOpt::mergeConstOffsetInsn(MachineBasicBlock::iterator I,
2378 MachineBasicBlock::iterator Update,
2379 unsigned Offset, int Scale) {
2380 assert((Update->getOpcode() == AArch64::MOVKWi) &&
2381 "Unexpected const mov instruction to merge!");
2382 MachineBasicBlock::iterator E = I->getParent()->end();
2383 MachineBasicBlock::iterator NextI = next_nodbg(I, E);
2384 MachineBasicBlock::iterator PrevI = prev_nodbg(Update, E);
2385 MachineInstr &MemMI = *I;
2386 unsigned Mask = (1 << 12) * Scale - 1;
2387 unsigned Low = Offset & Mask;
2388 unsigned High = Offset - Low;
2389 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MemMI).getReg();
2390 Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MemMI).getReg();
2391 MachineInstrBuilder AddMIB, MemMIB;
2392
2393 // Add IndexReg, BaseReg, High (the BaseReg may be SP)
2394 AddMIB =
2395 BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(AArch64::ADDXri))
2396 .addDef(IndexReg)
2397 .addUse(BaseReg)
2398 .addImm(High >> 12) // shifted value
2399 .addImm(12); // shift 12
2400 (void)AddMIB;
2401 // Ld/St DestReg, IndexReg, Imm12
2402 unsigned NewOpc = getBaseAddressOpcode(I->getOpcode());
2403 MemMIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
2404 .add(getLdStRegOp(MemMI))
2405 .add(AArch64InstrInfo::getLdStOffsetOp(MemMI))
2406 .addImm(Low / Scale)
2407 .setMemRefs(I->memoperands())
2408 .setMIFlags(I->mergeFlagsWith(*Update));
2409 (void)MemMIB;
2410
2411 ++NumConstOffsetFolded;
2412 LLVM_DEBUG(dbgs() << "Creating base address load/store.\n");
2413 LLVM_DEBUG(dbgs() << " Replacing instructions:\n ");
2414 LLVM_DEBUG(PrevI->print(dbgs()));
2415 LLVM_DEBUG(dbgs() << " ");
2416 LLVM_DEBUG(Update->print(dbgs()));
2417 LLVM_DEBUG(dbgs() << " ");
2418 LLVM_DEBUG(I->print(dbgs()));
2419 LLVM_DEBUG(dbgs() << " with instruction:\n ");
2420 LLVM_DEBUG(((MachineInstr *)AddMIB)->print(dbgs()));
2421 LLVM_DEBUG(dbgs() << " ");
2422 LLVM_DEBUG(((MachineInstr *)MemMIB)->print(dbgs()));
2423 LLVM_DEBUG(dbgs() << "\n");
2424
2425 // Erase the old instructions for the block.
2426 I->eraseFromParent();
2427 PrevI->eraseFromParent();
2428 Update->eraseFromParent();
2429
2430 return NextI;
2431}
2432
2433bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
2434 MachineInstr &MI,
2435 unsigned BaseReg, int Offset) {
2436 switch (MI.getOpcode()) {
2437 default:
2438 break;
2439 case AArch64::SUBXri:
2440 case AArch64::ADDXri:
2441 // Make sure it's a vanilla immediate operand, not a relocation or
2442 // anything else we can't handle.
2443 if (!MI.getOperand(2).isImm())
2444 break;
2445 // Watch out for 1 << 12 shifted value.
2446 if (AArch64_AM::getShiftValue(MI.getOperand(3).getImm()))
2447 break;
2448
2449 // The update instruction source and destination register must be the
2450 // same as the load/store base register.
2451 if (MI.getOperand(0).getReg() != BaseReg ||
2452 MI.getOperand(1).getReg() != BaseReg)
2453 break;
2454
2455 int UpdateOffset = MI.getOperand(2).getImm();
2456 if (MI.getOpcode() == AArch64::SUBXri)
2457 UpdateOffset = -UpdateOffset;
2458
2459 // The immediate must be a multiple of the scaling factor of the pre/post
2460 // indexed instruction.
2461 int Scale, MinOffset, MaxOffset;
2462 getPrePostIndexedMemOpInfo(MemMI, Scale, MinOffset, MaxOffset);
2463 if (UpdateOffset % Scale != 0)
2464 break;
2465
2466 // Scaled offset must fit in the instruction immediate.
2467 int ScaledOffset = UpdateOffset / Scale;
2468 if (ScaledOffset > MaxOffset || ScaledOffset < MinOffset)
2469 break;
2470
2471 // If we have a non-zero Offset, we check that it matches the amount
2472 // we're adding to the register.
2473 if (!Offset || Offset == UpdateOffset)
2474 return true;
2475 break;
2476 }
2477 return false;
2478}
2479
2480bool AArch64LoadStoreOpt::isMatchingMovConstInsn(MachineInstr &MemMI,
2481 MachineInstr &MI,
2482 unsigned IndexReg,
2483 unsigned &Offset) {
2484 // The update instruction source and destination register must be the
2485 // same as the load/store index register.
2486 if (MI.getOpcode() == AArch64::MOVKWi &&
2487 TRI->isSuperOrSubRegisterEq(IndexReg, MI.getOperand(1).getReg())) {
2488
2489 // movz + movk hold a large offset of a Ld/St instruction.
2490 MachineBasicBlock::iterator B = MI.getParent()->begin();
2491 MachineBasicBlock::iterator MBBI = &MI;
2492 // Skip the scene when the MI is the first instruction of a block.
2493 if (MBBI == B)
2494 return false;
2495 MBBI = prev_nodbg(MBBI, B);
2496 MachineInstr &MovzMI = *MBBI;
2497 // Make sure the MOVKWi and MOVZWi set the same register.
2498 if (MovzMI.getOpcode() == AArch64::MOVZWi &&
2499 MovzMI.getOperand(0).getReg() == MI.getOperand(0).getReg()) {
2500 unsigned Low = MovzMI.getOperand(1).getImm();
2501 unsigned High = MI.getOperand(2).getImm() << MI.getOperand(3).getImm();
2502 Offset = High + Low;
2503 // 12-bit optionally shifted immediates are legal for adds.
2504 return Offset >> 24 == 0;
2505 }
2506 }
2507 return false;
2508}
2509
2510MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
2511 MachineBasicBlock::iterator I, int UnscaledOffset, unsigned Limit) {
2512 MachineBasicBlock::iterator E = I->getParent()->end();
2513 MachineInstr &MemMI = *I;
2514 MachineBasicBlock::iterator MBBI = I;
2515
2516 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MemMI).getReg();
2517 int MIUnscaledOffset = AArch64InstrInfo::getLdStOffsetOp(MemMI).getImm() *
2518 TII->getMemScale(MemMI);
2519
2520 // Scan forward looking for post-index opportunities. Updating instructions
2521 // can't be formed if the memory instruction doesn't have the offset we're
2522 // looking for.
2523 if (MIUnscaledOffset != UnscaledOffset)
2524 return E;
2525
2526 // If the base register overlaps a source/destination register, we can't
2527 // merge the update. This does not apply to tag store instructions which
2528 // ignore the address part of the source register.
2529 // This does not apply to STGPi as well, which does not have unpredictable
2530 // behavior in this case unlike normal stores, and always performs writeback
2531 // after reading the source register value.
2532 if (!isTagStore(MemMI) && MemMI.getOpcode() != AArch64::STGPi) {
2533 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MemMI);
2534 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
2535 Register DestReg = getLdStRegOp(MemMI, i).getReg();
2536 if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
2537 return E;
2538 }
2539 }
2540
2541 // Track which register units have been modified and used between the first
2542 // insn (inclusive) and the second insn.
2543 ModifiedRegUnits.clear();
2544 UsedRegUnits.clear();
2545 MBBI = next_nodbg(MBBI, E);
2546
2547 // We can't post-increment the stack pointer if any instruction between
2548 // the memory access (I) and the increment (MBBI) can access the memory
2549 // region defined by [SP, MBBI].
2550 const bool BaseRegSP = BaseReg == AArch64::SP;
2551 if (BaseRegSP && I->getMF()->needsWinCFI()) {
2552 // FIXME: For now, we always block the optimization over SP in windows
2553 // targets as it requires to adjust the unwind/debug info, messing up
2554 // the unwind info can actually cause a miscompile.
2555 return E;
2556 }
2557
2558 unsigned Count = 0;
2559 MachineBasicBlock *CurMBB = I->getParent();
2560 // choice of next block to visit is liveins-based
2561 bool VisitSucc = CurMBB->getParent()->getRegInfo().tracksLiveness();
2562
2563 while (true) {
2564 for (MachineBasicBlock::iterator CurEnd = CurMBB->end();
2565 MBBI != CurEnd && Count < Limit; MBBI = next_nodbg(MBBI, CurEnd)) {
2566 MachineInstr &MI = *MBBI;
2567
2568 // Don't count transient instructions towards the search limit since there
2569 // may be different numbers of them if e.g. debug information is present.
2570 if (!MI.isTransient())
2571 ++Count;
2572
2573 // If we found a match, return it.
2574 if (isMatchingUpdateInsn(*I, MI, BaseReg, UnscaledOffset))
2575 return MBBI;
2576
2577 // Update the status of what the instruction clobbered and used.
2578 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits,
2579 TRI);
2580
2581 // Otherwise, if the base register is used or modified, we have no match,
2582 // so return early. If we are optimizing SP, do not allow instructions
2583 // that may load or store in between the load and the optimized value
2584 // update.
2585 if (!ModifiedRegUnits.available(BaseReg) ||
2586 !UsedRegUnits.available(BaseReg) ||
2587 (BaseRegSP && MBBI->mayLoadOrStore()))
2588 return E;
2589 }
2590
2591 if (!VisitSucc || Limit <= Count)
2592 break;
2593
2594 // Try to go downward to successors along a CF path w/o side enters
2595 // such that BaseReg is alive along it but not at its exits
2596 MachineBasicBlock *SuccToVisit = nullptr;
2597 unsigned LiveSuccCount = 0;
2598 for (MachineBasicBlock *Succ : CurMBB->successors()) {
2599 for (MCRegAliasIterator AI(BaseReg, TRI, true); AI.isValid(); ++AI) {
2600 if (Succ->isLiveIn(*AI)) {
2601 if (LiveSuccCount++)
2602 return E;
2603 if (Succ->pred_size() == 1)
2604 SuccToVisit = Succ;
2605 break;
2606 }
2607 }
2608 }
2609 if (!SuccToVisit)
2610 break;
2611 CurMBB = SuccToVisit;
2612 MBBI = CurMBB->begin();
2613 }
2614
2615 return E;
2616}
2617
2618MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
2619 MachineBasicBlock::iterator I, unsigned Limit, bool &MergeEither) {
2620 MachineBasicBlock::iterator B = I->getParent()->begin();
2621 MachineBasicBlock::iterator E = I->getParent()->end();
2622 MachineInstr &MemMI = *I;
2623 MachineBasicBlock::iterator MBBI = I;
2624 MachineFunction &MF = *MemMI.getMF();
2625
2626 Register BaseReg = AArch64InstrInfo::getLdStBaseOp(MemMI).getReg();
2627 int Offset = AArch64InstrInfo::getLdStOffsetOp(MemMI).getImm();
2628
2629 bool IsPairedInsn = AArch64InstrInfo::isPairedLdSt(MemMI);
2630 Register DestReg[] = {getLdStRegOp(MemMI, 0).getReg(),
2631 IsPairedInsn ? getLdStRegOp(MemMI, 1).getReg()
2632 : Register()};
2633
2634 // If the load/store is the first instruction in the block, there's obviously
2635 // not any matching update. Ditto if the memory offset isn't zero.
2636 if (MBBI == B || Offset != 0)
2637 return E;
2638 // If the base register overlaps a destination register, we can't
2639 // merge the update.
2640 if (!isTagStore(MemMI)) {
2641 for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i)
2642 if (DestReg[i] == BaseReg || TRI->isSubRegister(BaseReg, DestReg[i]))
2643 return E;
2644 }
2645
2646 const bool BaseRegSP = BaseReg == AArch64::SP;
2647 if (BaseRegSP && I->getMF()->needsWinCFI()) {
2648 // FIXME: For now, we always block the optimization over SP in windows
2649 // targets as it requires to adjust the unwind/debug info, messing up
2650 // the unwind info can actually cause a miscompile.
2651 return E;
2652 }
2653
2654 const AArch64Subtarget &Subtarget = MF.getSubtarget<AArch64Subtarget>();
2655 unsigned RedZoneSize =
2656 Subtarget.getTargetLowering()->getRedZoneSize(MF.getFunction());
2657
2658 // Track which register units have been modified and used between the first
2659 // insn (inclusive) and the second insn.
2660 ModifiedRegUnits.clear();
2661 UsedRegUnits.clear();
2662 unsigned Count = 0;
2663 bool MemAccessBeforeSPPreInc = false;
2664 MergeEither = true;
2665 do {
2666 MBBI = prev_nodbg(MBBI, B);
2667 MachineInstr &MI = *MBBI;
2668
2669 // Don't count transient instructions towards the search limit since there
2670 // may be different numbers of them if e.g. debug information is present.
2671 if (!MI.isTransient())
2672 ++Count;
2673
2674 // If we found a match, return it.
2675 if (isMatchingUpdateInsn(*I, MI, BaseReg, Offset)) {
2676 // Check that the update value is within our red zone limit (which may be
2677 // zero).
2678 if (MemAccessBeforeSPPreInc && MBBI->getOperand(2).getImm() > RedZoneSize)
2679 return E;
2680 return MBBI;
2681 }
2682
2683 // Update the status of what the instruction clobbered and used.
2684 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2685
2686 // Otherwise, if the base register is used or modified, we have no match, so
2687 // return early.
2688 if (!ModifiedRegUnits.available(BaseReg) ||
2689 !UsedRegUnits.available(BaseReg))
2690 return E;
2691
2692 // If we have a destination register (i.e. a load instruction) and a
2693 // destination register is used or modified, then we can only merge forward,
2694 // i.e. the combined instruction is put in the place of the memory
2695 // instruction. Same applies if we see a memory access or side effects.
2696 if (MI.mayLoadOrStore() || MI.hasUnmodeledSideEffects() ||
2697 (DestReg[0].isValid() && !(ModifiedRegUnits.available(DestReg[0]) &&
2698 UsedRegUnits.available(DestReg[0]))) ||
2699 (DestReg[1].isValid() && !(ModifiedRegUnits.available(DestReg[1]) &&
2700 UsedRegUnits.available(DestReg[1]))))
2701 MergeEither = false;
2702
2703 // Keep track if we have a memory access before an SP pre-increment, in this
2704 // case we need to validate later that the update amount respects the red
2705 // zone.
2706 if (BaseRegSP && MBBI->mayLoadOrStore())
2707 MemAccessBeforeSPPreInc = true;
2708 } while (MBBI != B && Count < Limit);
2709 return E;
2710}
2711
2712MachineBasicBlock::iterator
2713AArch64LoadStoreOpt::findMatchingConstOffsetBackward(
2714 MachineBasicBlock::iterator I, unsigned Limit, unsigned &Offset) {
2715 MachineBasicBlock::iterator B = I->getParent()->begin();
2716 MachineBasicBlock::iterator E = I->getParent()->end();
2717 MachineInstr &MemMI = *I;
2718 MachineBasicBlock::iterator MBBI = I;
2719
2720 // If the load is the first instruction in the block, there's obviously
2721 // not any matching load or store.
2722 if (MBBI == B)
2723 return E;
2724
2725 // Make sure the IndexReg is killed and the shift amount is zero.
2726 // TODO: Relex this restriction to extend, simplify processing now.
2727 if (!AArch64InstrInfo::getLdStOffsetOp(MemMI).isKill() ||
2728 !AArch64InstrInfo::getLdStAmountOp(MemMI).isImm() ||
2729 (AArch64InstrInfo::getLdStAmountOp(MemMI).getImm() != 0))
2730 return E;
2731
2732 Register IndexReg = AArch64InstrInfo::getLdStOffsetOp(MemMI).getReg();
2733
2734 // Track which register units have been modified and used between the first
2735 // insn (inclusive) and the second insn.
2736 ModifiedRegUnits.clear();
2737 UsedRegUnits.clear();
2738 unsigned Count = 0;
2739 do {
2740 MBBI = prev_nodbg(MBBI, B);
2741 MachineInstr &MI = *MBBI;
2742
2743 // Don't count transient instructions towards the search limit since there
2744 // may be different numbers of them if e.g. debug information is present.
2745 if (!MI.isTransient())
2746 ++Count;
2747
2748 // If we found a match, return it.
2749 if (isMatchingMovConstInsn(*I, MI, IndexReg, Offset)) {
2750 return MBBI;
2751 }
2752
2753 // Update the status of what the instruction clobbered and used.
2754 LiveRegUnits::accumulateUsedDefed(MI, ModifiedRegUnits, UsedRegUnits, TRI);
2755
2756 // Otherwise, if the index register is used or modified, we have no match,
2757 // so return early.
2758 if (!ModifiedRegUnits.available(IndexReg) ||
2759 !UsedRegUnits.available(IndexReg))
2760 return E;
2761
2762 } while (MBBI != B && Count < Limit);
2763 return E;
2764}
2765
2766bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore(
2767 MachineBasicBlock::iterator &MBBI) {
2768 MachineInstr &MI = *MBBI;
2769 // If this is a volatile load, don't mess with it.
2770 if (MI.hasOrderedMemoryRef())
2771 return false;
2772
2773 if (MI.getFlag(MachineInstr::FrameDestroy) && MI.getMF()->needsWinCFI())
2774 return false;
2775
2776 // Make sure this is a reg+imm.
2777 // FIXME: It is possible to extend it to handle reg+reg cases.
2778 if (!AArch64InstrInfo::getLdStOffsetOp(MI).isImm())
2779 return false;
2780
2781 // Look backward up to the scan limit.
2782 MachineBasicBlock::iterator StoreI;
2783 if (findMatchingStore(MBBI, Subtarget->getCLOpts().load_store_scan_limit,
2784 StoreI)) {
2785 ++NumLoadsFromStoresPromoted;
2786 // Promote the load. Keeping the iterator straight is a
2787 // pain, so we let the merge routine tell us what the next instruction
2788 // is after it's done mucking about.
2789 MBBI = promoteLoadFromStore(MBBI, StoreI);
2790 return true;
2791 }
2792 return false;
2793}
2794
2795// Merge adjacent zero stores into a wider store.
2796bool AArch64LoadStoreOpt::tryToMergeZeroStInst(
2797 MachineBasicBlock::iterator &MBBI) {
2798 assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow store.");
2799 MachineInstr &MI = *MBBI;
2800 MachineBasicBlock::iterator E = MI.getParent()->end();
2801
2802 if (!TII->isCandidateToMergeOrPair(MI))
2803 return false;
2804
2805 // Look ahead up to the scan limit for a mergeable instruction.
2806 LdStPairFlags Flags;
2807 MachineBasicBlock::iterator MergeMI = findMatchingInsn(
2808 MBBI, Flags, Subtarget->getCLOpts().load_store_scan_limit,
2809 /*FindNarrowMerge=*/true);
2810 if (MergeMI != E) {
2811 ++NumZeroStoresPromoted;
2812
2813 // Keeping the iterator straight is a pain, so we let the merge routine tell
2814 // us what the next instruction is after it's done mucking about.
2815 MBBI = mergeNarrowZeroStores(MBBI, MergeMI, Flags);
2816 return true;
2817 }
2818 return false;
2819}
2820
2821// Find loads and stores that can be merged into a single load or store pair
2822// instruction.
2823bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) {
2824 MachineInstr &MI = *MBBI;
2825 MachineBasicBlock::iterator E = MI.getParent()->end();
2826
2827 if (!TII->isCandidateToMergeOrPair(MI))
2828 return false;
2829
2830 // If disable-ldp feature is opted, do not emit ldp.
2831 if (MI.mayLoad() && Subtarget->hasDisableLdp())
2832 return false;
2833
2834 // If disable-stp feature is opted, do not emit stp.
2835 if (MI.mayStore() && Subtarget->hasDisableStp())
2836 return false;
2837
2838 // Early exit if the offset is not possible to match. (6 bits of positive
2839 // range, plus allow an extra one in case we find a later insn that matches
2840 // with Offset-1)
2841 bool IsUnscaled = TII->hasUnscaledLdStOffset(MI);
2842 int Offset = AArch64InstrInfo::getLdStOffsetOp(MI).getImm();
2843 int OffsetStride = IsUnscaled ? TII->getMemScale(MI) : 1;
2844 // Allow one more for offset.
2845 if (Offset > 0)
2846 Offset -= OffsetStride;
2847 if (!inBoundsForPair(IsUnscaled, Offset, OffsetStride))
2848 return false;
2849
2850 // Look ahead up to the scan limit for a pairable instruction.
2851 LdStPairFlags Flags;
2852 MachineBasicBlock::iterator Paired = findMatchingInsn(
2853 MBBI, Flags, Subtarget->getCLOpts().load_store_scan_limit,
2854 /*FindNarrowMerge=*/false);
2855
2856 if (Paired == E)
2857 return false;
2858
2859 // Keeping the iterator straight is a pain, so we let the merge routine tell
2860 // us what the next instruction is after it's done mucking about.
2861 auto Prev = std::prev(MBBI);
2862
2863 // Fetch the memoperand of the load/store that is a candidate for combination.
2864 MachineMemOperand *MemOp =
2865 MI.memoperands_empty() ? nullptr : MI.memoperands().front();
2866
2867 // If a load/store arrives and ldp/stp-aligned-only feature is opted, check
2868 // that the alignment of the source pointer is at least double the alignment
2869 // of the type.
2870 if ((MI.mayLoad() && Subtarget->hasLdpAlignedOnly()) ||
2871 (MI.mayStore() && Subtarget->hasStpAlignedOnly())) {
2872 // If there is no size/align information, cancel the transformation.
2873 if (!MemOp || !MemOp->getMemoryType().isValid()) {
2874 NumFailedAlignmentCheck++;
2875 return false;
2876 }
2877
2878 // Get the needed alignments to check them if
2879 // ldp-aligned-only/stp-aligned-only features are opted.
2880 uint64_t MemAlignment = MemOp->getAlign().value();
2881 uint64_t TypeAlignment =
2882 Align(MemOp->getSize().getValue().getKnownMinValue()).value();
2883
2884 if (MemAlignment < 2 * TypeAlignment) {
2885 NumFailedAlignmentCheck++;
2886 return false;
2887 }
2888 }
2889
2890 ++NumPairCreated;
2891 if (TII->hasUnscaledLdStOffset(MI))
2892 ++NumUnscaledPairCreated;
2893
2894 MBBI = mergePairedInsns(MBBI, Paired, Flags);
2895 // Collect liveness info for instructions between Prev and the new position
2896 // MBBI.
2897 for (auto I = std::next(Prev); I != MBBI; I++)
2898 updateDefinedRegisters(*I, DefinedInBB, TRI);
2899
2900 return true;
2901}
2902
2903bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
2904 (MachineBasicBlock::iterator &MBBI) {
2905 MachineInstr &MI = *MBBI;
2906 MachineBasicBlock::iterator E = MI.getParent()->end();
2907 MachineBasicBlock::iterator Update;
2908
2909 // Do not form post-inc addressing mode for volatile accesses. Instructions
2910 // performing register writeback do not set a valid instruction syndrome,
2911 // making it impossible to handle MMIO in protected hypervisors.
2912 // Exclude accesses based on the stack pointer, as these can't be MMIO.
2913 // Also exclude MTE tag store instructions.
2914 if (MBBI->hasOrderedMemoryRef() &&
2915 AArch64InstrInfo::getLdStBaseOp(MI).getReg() != AArch64::SP &&
2916 !isTagStore(MI) && MI.getOpcode() != AArch64::STGPi)
2917 return false;
2918
2919 // Look forward to try to form a post-index instruction. For example,
2920 // ldr x0, [x20]
2921 // add x20, x20, #32
2922 // merged into:
2923 // ldr x0, [x20], #32
2924 Update = findMatchingUpdateInsnForward(
2925 MBBI, 0, Subtarget->getCLOpts().update_scan_limit);
2926 if (Update != E) {
2927 // Merge the update into the ld/st.
2928 if (auto NextI = mergeUpdateInsn(MBBI, Update, /*IsForward=*/false,
2929 /*IsPreIdx=*/false,
2930 /*MergeEither=*/false)) {
2931 MBBI = *NextI;
2932 return true;
2933 }
2934 }
2935
2936 // Don't know how to handle unscaled pre/post-index versions below, so bail.
2937 if (TII->hasUnscaledLdStOffset(MI.getOpcode()))
2938 return false;
2939
2940 // Look back to try to find a pre-index instruction. For example,
2941 // add x0, x0, #8
2942 // ldr x1, [x0]
2943 // merged into:
2944 // ldr x1, [x0, #8]!
2945 bool MergeEither;
2946 Update = findMatchingUpdateInsnBackward(
2947 MBBI, Subtarget->getCLOpts().update_scan_limit, MergeEither);
2948 if (Update != E) {
2949 // Merge the update into the ld/st.
2950 if (auto NextI = mergeUpdateInsn(MBBI, Update, /*IsForward=*/true,
2951 /*IsPreIdx=*/true, MergeEither)) {
2952 MBBI = *NextI;
2953 return true;
2954 }
2955 }
2956
2957 // The immediate in the load/store is scaled by the size of the memory
2958 // operation. The immediate in the add we're looking for,
2959 // however, is not, so adjust here.
2960 int UnscaledOffset =
2961 AArch64InstrInfo::getLdStOffsetOp(MI).getImm() * TII->getMemScale(MI);
2962
2963 // Look forward to try to find a pre-index instruction. For example,
2964 // ldr x1, [x0, #64]
2965 // add x0, x0, #64
2966 // merged into:
2967 // ldr x1, [x0, #64]!
2968 Update = findMatchingUpdateInsnForward(
2969 MBBI, UnscaledOffset, Subtarget->getCLOpts().update_scan_limit);
2970 if (Update != E) {
2971 // Merge the update into the ld/st.
2972 if (auto NextI = mergeUpdateInsn(MBBI, Update, /*IsForward=*/false,
2973 /*IsPreIdx=*/true,
2974 /*MergeEither=*/false)) {
2975 MBBI = *NextI;
2976 return true;
2977 }
2978 }
2979
2980 return false;
2981}
2982
2983bool AArch64LoadStoreOpt::tryToMergeIndexLdSt(MachineBasicBlock::iterator &MBBI,
2984 int Scale) {
2985 MachineInstr &MI = *MBBI;
2986 MachineBasicBlock::iterator E = MI.getParent()->end();
2987 MachineBasicBlock::iterator Update;
2988
2989 // Don't know how to handle unscaled pre/post-index versions below, so bail.
2990 if (TII->hasUnscaledLdStOffset(MI.getOpcode()))
2991 return false;
2992
2993 // Look back to try to find a const offset for index LdSt instruction. For
2994 // example,
2995 // mov x8, #LargeImm ; = a * (1<<12) + imm12
2996 // ldr x1, [x0, x8]
2997 // merged into:
2998 // add x8, x0, a * (1<<12)
2999 // ldr x1, [x8, imm12]
3000 unsigned Offset;
3001 Update = findMatchingConstOffsetBackward(
3002 MBBI, Subtarget->getCLOpts().load_store_const_scan_limit, Offset);
3003 if (Update != E && (Offset & (Scale - 1)) == 0) {
3004 // Merge the imm12 into the ld/st.
3005 MBBI = mergeConstOffsetInsn(MBBI, Update, Offset, Scale);
3006 return true;
3007 }
3008
3009 return false;
3010}
3011
3012// Map a GPR store opcode to its FPR equivalent at the same data width.
3013// Returns 0 if no mapping exists.
3014static unsigned getGPRToFPRStoreOpcode(unsigned GPRStoreOpc) {
3015 switch (GPRStoreOpc) {
3016 // Unsigned immediate.
3017 case AArch64::STRBBui:
3018 return AArch64::STRBui;
3019 case AArch64::STRHHui:
3020 return AArch64::STRHui;
3021 case AArch64::STRWui:
3022 return AArch64::STRSui;
3023 case AArch64::STRXui:
3024 return AArch64::STRDui;
3025 // Unscaled immediate.
3026 case AArch64::STURBBi:
3027 return AArch64::STURBi;
3028 case AArch64::STURHHi:
3029 return AArch64::STURHi;
3030 case AArch64::STURWi:
3031 return AArch64::STURSi;
3032 case AArch64::STURXi:
3033 return AArch64::STURDi;
3034 // Register offset.
3035 case AArch64::STRBBroW:
3036 return AArch64::STRBroW;
3037 case AArch64::STRBBroX:
3038 return AArch64::STRBroX;
3039 case AArch64::STRHHroW:
3040 return AArch64::STRHroW;
3041 case AArch64::STRHHroX:
3042 return AArch64::STRHroX;
3043 case AArch64::STRWroW:
3044 return AArch64::STRSroW;
3045 case AArch64::STRWroX:
3046 return AArch64::STRSroX;
3047 case AArch64::STRXroW:
3048 return AArch64::STRDroW;
3049 case AArch64::STRXroX:
3050 return AArch64::STRDroX;
3051 default:
3052 return 0;
3053 }
3054}
3055
3056// Given a UMOV-lane-0 opcode, return the sub-register index to extract from
3057// the vector register, or 0 if the opcode is not a supported UMOV.
3058static unsigned getUMOVSubRegIdx(unsigned UMOVOpc) {
3059 switch (UMOVOpc) {
3060 case AArch64::UMOVvi8_idx0:
3061 return AArch64::bsub;
3062 case AArch64::UMOVvi16_idx0:
3063 return AArch64::hsub;
3064 case AArch64::UMOVvi32_idx0:
3065 return AArch64::ssub;
3066 case AArch64::UMOVvi64_idx0:
3067 return AArch64::dsub;
3068 default:
3069 return 0;
3070 }
3071}
3072
3073bool AArch64LoadStoreOpt::tryToReplaceUMOVStore(
3074 MachineBasicBlock::iterator &MBBI) {
3075 MachineInstr &StoreMI = *MBBI;
3076
3077 unsigned FPRStoreOpc = getGPRToFPRStoreOpcode(StoreMI.getOpcode());
3078 if (!FPRStoreOpc)
3079 return false;
3080
3081 if (StoreMI.hasOrderedMemoryRef() || StoreMI.memoperands().size() != 1)
3082 return false;
3083
3084 MachineBasicBlock *MBB = StoreMI.getParent();
3085 MCPhysReg StoreValReg = StoreMI.getOperand(0).getReg();
3086
3087 if (!StoreMI.getOperand(0).isKill())
3088 return false;
3089
3090 // Bail out if the store uses the value register elsewhere (e.g., as the base
3091 // address in `str w8, [x8, #0]`).
3092 for (unsigned I = 1, E = StoreMI.getNumExplicitOperands(); I < E; ++I)
3093 if (StoreMI.getOperand(I).isReg() &&
3094 TRI->regsOverlap(StoreMI.getOperand(I).getReg(), StoreValReg))
3095 return false;
3096
3097 // Scan backward to find the UMOV that defines the store's value register.
3098 MachineInstr *UMOVMI = nullptr;
3099 MachineBasicBlock::iterator B = MBB->begin();
3100 unsigned SubRegIdx = 0;
3101 unsigned Count = 0;
3102 for (auto It = MBBI; It != B;) {
3103 MachineInstr &MI = *--It;
3104 if (MI.isDebugInstr())
3105 continue;
3106 if (++Count > Subtarget->getCLOpts().umov_fold_scan_limit)
3107 return false;
3108 if (MI.readsRegister(StoreValReg, TRI))
3109 return false;
3110 if (MI.modifiesRegister(StoreValReg, TRI)) {
3111 SubRegIdx = getUMOVSubRegIdx(MI.getOpcode());
3112 if (!SubRegIdx)
3113 return false;
3114 UMOVMI = &MI;
3115 break;
3116 }
3117 }
3118 if (!UMOVMI)
3119 return false;
3120 MCPhysReg VecReg = UMOVMI->getOperand(1).getReg();
3121 MCPhysReg FPRReg = TRI->getSubReg(VecReg, SubRegIdx);
3122 if ((*StoreMI.memoperands_begin())->getSizeInBits() !=
3123 TRI->getRegSizeInBits(*TRI->getMinimalPhysRegClass(FPRReg)))
3124 return false;
3125
3126 // Check that no instruction between UMOV and store clobbers the vector
3127 // register. Also track whether VecReg is killed anywhere from the UMOV
3128 // (inclusive) through the intervening instructions -- we need this to decide
3129 // whether the FPR sub-register can be marked killed on the new store.
3130 bool VecRegKilled = UMOVMI->killsRegister(VecReg, TRI);
3131 for (auto It = std::next(UMOVMI->getIterator()); It != MBBI; ++It) {
3132 if (It->modifiesRegister(VecReg, TRI))
3133 return false;
3134 if (!VecRegKilled && It->killsRegister(VecReg, TRI))
3135 VecRegKilled = true;
3136 }
3137
3138 // Safe to proceed. Clear kill flags on the vector register between UMOV and
3139 // the new store so the FPR sub-register stays live.
3140 UMOVMI->clearRegisterKills(VecReg, TRI);
3141 for (auto It = std::next(UMOVMI->getIterator()); It != MBBI; ++It)
3142 It->clearRegisterKills(VecReg, TRI);
3143
3144 LLVM_DEBUG(dbgs() << "Folding UMOV + store: " << *UMOVMI << " + "
3145 << StoreMI);
3146
3147 auto MIB = BuildMI(*MBB, MBBI, StoreMI.getDebugLoc(), TII->get(FPRStoreOpc))
3148 .addReg(FPRReg, getKillRegState(VecRegKilled));
3149 for (unsigned I = 1, E = StoreMI.getNumExplicitOperands(); I < E; ++I)
3150 MIB.add(StoreMI.getOperand(I));
3151 MIB.setMemRefs(StoreMI.memoperands());
3152
3153 MBBI = MBB->erase(MBBI);
3154 UMOVMI->eraseFromParent();
3155
3156 ++NumUMOVFoldedToFPRStore;
3157 return true;
3158}
3159
3160bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB,
3161 bool EnableNarrowZeroStOpt) {
3162 AArch64FunctionInfo &AFI = *MBB.getParent()->getInfo<AArch64FunctionInfo>();
3163
3164 bool Modified = false;
3165 // Six transformations to do here:
3166 // 1) Find loads that directly read from stores and promote them by
3167 // replacing with mov instructions. If the store is wider than the load,
3168 // the load will be replaced with a bitfield extract.
3169 // e.g.,
3170 // str w1, [x0, #4]
3171 // ldrh w2, [x0, #6]
3172 // ; becomes
3173 // str w1, [x0, #4]
3174 // lsr w2, w1, #16
3175 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3176 MBBI != E;) {
3177 if (isPromotableLoadFromStore(*MBBI) && tryToPromoteLoadFromStore(MBBI))
3178 Modified = true;
3179 else
3180 ++MBBI;
3181 }
3182 // 2) Merge adjacent zero stores into a wider store.
3183 // e.g.,
3184 // strh wzr, [x0]
3185 // strh wzr, [x0, #2]
3186 // ; becomes
3187 // str wzr, [x0]
3188 // e.g.,
3189 // str wzr, [x0]
3190 // str wzr, [x0, #4]
3191 // ; becomes
3192 // str xzr, [x0]
3193 if (EnableNarrowZeroStOpt)
3194 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3195 MBBI != E;) {
3196 if (isPromotableZeroStoreInst(*MBBI) && tryToMergeZeroStInst(MBBI))
3197 Modified = true;
3198 else
3199 ++MBBI;
3200 }
3201 // 3) Find loads and stores that can be merged into a single load or store
3202 // pair instruction.
3203 // When compiling for SVE 128, also try to combine SVE fill/spill
3204 // instructions into LDP/STP.
3205 // e.g.,
3206 // ldr x0, [x2]
3207 // ldr x1, [x2, #8]
3208 // ; becomes
3209 // ldp x0, x1, [x2]
3210 // e.g.,
3211 // ldr z0, [x2]
3212 // ldr z1, [x2, #1, mul vl]
3213 // ; becomes
3214 // ldp q0, q1, [x2]
3215
3216 if (MBB.getParent()->getRegInfo().tracksLiveness()) {
3217 DefinedInBB.clear();
3218 DefinedInBB.addLiveIns(MBB);
3219 }
3220
3221 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3222 MBBI != E;) {
3223 // Track currently live registers up to this point, to help with
3224 // searching for a rename register on demand.
3225 updateDefinedRegisters(*MBBI, DefinedInBB, TRI);
3226 if (TII->isPairableLdStInst(*MBBI) && tryToPairLdStInst(MBBI))
3227 Modified = true;
3228 else
3229 ++MBBI;
3230 }
3231 // 4) Find base register updates that can be merged into the load or store
3232 // as a base-reg writeback.
3233 // e.g.,
3234 // ldr x0, [x2]
3235 // add x2, x2, #4
3236 // ; becomes
3237 // ldr x0, [x2], #4
3238 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3239 MBBI != E;) {
3240 if (isMergeableLdStUpdate(*MBBI, AFI) && tryToMergeLdStUpdate(MBBI))
3241 Modified = true;
3242 else
3243 ++MBBI;
3244 }
3245
3246 // 5) Find a register assigned with a const value that can be combined with
3247 // into the load or store. e.g.,
3248 // mov x8, #LargeImm ; = a * (1<<12) + imm12
3249 // ldr x1, [x0, x8]
3250 // ; becomes
3251 // add x8, x0, a * (1<<12)
3252 // ldr x1, [x8, imm12]
3253 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3254 MBBI != E;) {
3255 int Scale;
3256 if (isMergeableIndexLdSt(*MBBI, Scale) && tryToMergeIndexLdSt(MBBI, Scale))
3257 Modified = true;
3258 else
3259 ++MBBI;
3260 }
3261
3262 // 6) Replace UMOV (lane 0) + GPR store with a direct FPR sub-register store.
3263 // e.g.,
3264 // umov w8, v0.h[0]
3265 // strh w8, [x0]
3266 // ; becomes
3267 // str h0, [x0]
3268 for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
3269 MBBI != E;) {
3270 if (tryToReplaceUMOVStore(MBBI))
3271 Modified = true;
3272 else
3273 ++MBBI;
3274 }
3275
3276 return Modified;
3277}
3278
3279bool AArch64LoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
3280 Subtarget = &Fn.getSubtarget<AArch64Subtarget>();
3281 TII = Subtarget->getInstrInfo();
3282 TRI = Subtarget->getRegisterInfo();
3283
3284 // Resize the modified and used register unit trackers. We do this once
3285 // per function and then clear the register units each time we optimize a load
3286 // or store.
3287 ModifiedRegUnits.init(*TRI);
3288 UsedRegUnits.init(*TRI);
3289 DefinedInBB.init(*TRI);
3290
3291 bool Modified = false;
3292 bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign();
3293 for (auto &MBB : Fn) {
3294 auto M = optimizeBlock(MBB, enableNarrowZeroStOpt);
3295 Modified |= M;
3296 }
3297
3298 return Modified;
3299}
3300
3301// FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep loads and
3302// stores near one another? Note: The pre-RA instruction scheduler already has
3303// hooks to try and schedule pairable loads/stores together to improve pairing
3304// opportunities. Thus, pre-RA pairing pass may not be worth the effort.
3305
3306// FIXME: When pairing store instructions it's very possible for this pass to
3307// hoist a store with a KILL marker above another use (without a KILL marker).
3308// The resulting IR is invalid, but nothing uses the KILL markers after this
3309// pass, so it's never caused a problem in practice.
3310
3311bool AArch64LoadStoreOptLegacy::runOnMachineFunction(MachineFunction &MF) {
3312 if (skipFunction(MF.getFunction()))
3313 return false;
3314 AArch64LoadStoreOpt Impl;
3315 Impl.AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
3316 return Impl.runOnMachineFunction(MF);
3317}
3318
3319/// createAArch64LoadStoreOptimizationPass - returns an instance of the
3320/// load / store optimization pass.
3321FunctionPass *llvm::createAArch64LoadStoreOptLegacyPass() {
3322 return new AArch64LoadStoreOptLegacy();
3323}
3324
3325PreservedAnalyses
3326AArch64LoadStoreOptPass::run(MachineFunction &MF,
3327 MachineFunctionAnalysisManager &MFAM) {
3328 AArch64LoadStoreOpt Impl;
3329 Impl.AA = &MFAM.getResult<FunctionAnalysisManagerMachineFunctionProxy>(MF)
3330 .getManager()
3331 .getResult<AAManager>(MF.getFunction());
3332 bool Changed = Impl.runOnMachineFunction(MF);
3333 if (!Changed)
3334 return PreservedAnalyses::all();
3335 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
3336 PA.preserveSet<CFGAnalyses>();
3337 return PA;
3338}
Success
#define Success
Definition AArch64Disassembler.cpp:42
for
for(const MachineOperand &MO :llvm::drop_begin(OldMI.operands(), Desc.getNumOperands()))
Definition AArch64ExpandPseudoInsts.cpp:128
getLdStRegOp
static MachineOperand & getLdStRegOp(MachineInstr &MI, unsigned PairedRegOp=0)
Definition AArch64LoadStoreOptimizer.cpp:689
isPromotableLoadFromStore
static bool isPromotableLoadFromStore(MachineInstr &MI)
Definition AArch64LoadStoreOptimizer.cpp:725
getPrePostIndexedMemOpInfo
static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale, int &MinOffset, int &MaxOffset)
Definition AArch64LoadStoreOptimizer.cpp:671
inBoundsForPair
static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride)
Definition AArch64LoadStoreOptimizer.cpp:1557
getMatchingPairOpcode
static unsigned getMatchingPairOpcode(unsigned Opc)
Definition AArch64LoadStoreOptimizer.cpp:339
getGPRToFPRStoreOpcode
static unsigned getGPRToFPRStoreOpcode(unsigned GPRStoreOpc)
Definition AArch64LoadStoreOptimizer.cpp:3014
areCandidatesToMergeOrPair
static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI, LdStPairFlags &Flags, const AArch64InstrInfo *TII)
Definition AArch64LoadStoreOptimizer.cpp:1655
tryToFindRegisterToRename
static std::optional< MCPhysReg > tryToFindRegisterToRename(const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
Definition AArch64LoadStoreOptimizer.cpp:1912
canRenameUntilSecondLoad
static bool canRenameUntilSecondLoad(MachineInstr &FirstLoad, MachineInstr &SecondLoad, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
Definition AArch64LoadStoreOptimizer.cpp:1868
findRenameRegForSameLdStRegPair
static std::optional< MCPhysReg > findRenameRegForSameLdStRegPair(std::optional< bool > MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
Definition AArch64LoadStoreOptimizer.cpp:1956
mayAlias
static bool mayAlias(MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
Definition AArch64LoadStoreOptimizer.cpp:1578
getPreIndexedOpcode
static unsigned getPreIndexedOpcode(unsigned Opc)
Definition AArch64LoadStoreOptimizer.cpp:433
AARCH64_LOAD_STORE_OPT_NAME
#define AARCH64_LOAD_STORE_OPT_NAME
Definition AArch64LoadStoreOptimizer.cpp:73
addDebugSubstitutionsToTable
static void addDebugSubstitutionsToTable(MachineFunction *MF, unsigned InstrNumToSet, MachineInstr &OriginalInstr, MachineInstr &MergedInstr)
This function will add a new entry into the debugValueSubstitutions table when two instruction have b...
Definition AArch64LoadStoreOptimizer.cpp:980
isPromotableZeroStoreInst
static bool isPromotableZeroStoreInst(MachineInstr &MI)
Definition AArch64LoadStoreOptimizer.cpp:718
getMatchingWideOpcode
static unsigned getMatchingWideOpcode(unsigned Opc)
Definition AArch64LoadStoreOptimizer.cpp:320
getMatchingNonSExtOpcode
static unsigned getMatchingNonSExtOpcode(unsigned Opc, bool *IsValidLdStrOpc=nullptr)
Definition AArch64LoadStoreOptimizer.cpp:265
maybeMoveCFI
static MachineBasicBlock::iterator maybeMoveCFI(MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI)
Definition AArch64LoadStoreOptimizer.cpp:2258
isTagStore
static bool isTagStore(const MachineInstr &MI)
Definition AArch64LoadStoreOptimizer.cpp:253
isMatchingStore
static unsigned isMatchingStore(MachineInstr &LoadInst, MachineInstr &StoreInst)
Definition AArch64LoadStoreOptimizer.cpp:403
forAllMIsUntilDef
static bool forAllMIsUntilDef(MachineInstr &MI, MCPhysReg DefReg, const TargetRegisterInfo *TRI, unsigned Limit, std::function< bool(MachineInstr &, bool)> &Fn)
Definition AArch64LoadStoreOptimizer.cpp:943
isRewritableImplicitDef
static bool isRewritableImplicitDef(const MachineInstr &MI, const MachineOperand &MO)
Definition AArch64LoadStoreOptimizer.cpp:847
getPostIndexedOpcode
static unsigned getPostIndexedOpcode(unsigned Opc)
Definition AArch64LoadStoreOptimizer.cpp:550
getUMOVSubRegIdx
static unsigned getUMOVSubRegIdx(unsigned UMOVOpc)
Definition AArch64LoadStoreOptimizer.cpp:3058
isMergeableLdStUpdate
static bool isMergeableLdStUpdate(MachineInstr &MI, AArch64FunctionInfo &AFI)
Definition AArch64LoadStoreOptimizer.cpp:743
canRenameMOP
static bool canRenameMOP(const MachineInstr &MI, const MachineOperand &MOP, const TargetRegisterInfo *TRI)
Definition AArch64LoadStoreOptimizer.cpp:1725
isLdOffsetInRangeOfSt
static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst, MachineInstr &StoreInst, const AArch64InstrInfo *TII)
Definition AArch64LoadStoreOptimizer.cpp:700
isPreLdStPairCandidate
static bool isPreLdStPairCandidate(MachineInstr &FirstMI, MachineInstr &MI)
Definition AArch64LoadStoreOptimizer.cpp:637
isMergeableIndexLdSt
static bool isMergeableIndexLdSt(MachineInstr &MI, int &Scale)
Definition AArch64LoadStoreOptimizer.cpp:813
updateDefinedRegisters
static void updateDefinedRegisters(MachineInstr &MI, LiveRegUnits &Units, const TargetRegisterInfo *TRI)
Definition AArch64LoadStoreOptimizer.cpp:965
canRenameUpToDef
static bool canRenameUpToDef(MachineInstr &FirstMI, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
Definition AArch64LoadStoreOptimizer.cpp:1761
getBaseAddressOpcode
static unsigned getBaseAddressOpcode(unsigned Opc)
Definition AArch64LoadStoreOptimizer.cpp:514
assert
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t
unsigned uint64_t
Definition AMDGPUDisassembler.cpp:540
MBB
MachineBasicBlock & MBB
Definition ARMSLSHardening.cpp:71
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition ARMSLSHardening.cpp:73
MBBI
MachineBasicBlock MachineBasicBlock::iterator MBBI
Definition ARMSLSHardening.cpp:72
print
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
Definition ArchiveWriter.cpp:213
C
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
DebugCounter.h
This file provides an implementation of debug counters.
DEBUG_COUNTER
#define DEBUG_COUNTER(VARNAME, COUNTERNAME, DESC)
Definition DebugCounter.h:189
DEBUG_TYPE
#define DEBUG_TYPE
Definition GenericCycleImpl.h:34
TII
const HexagonInstrInfo * TII
Definition HexagonCopyToCombine.cpp:118
MI
IRTranslator LLVM IR MI
Definition IRTranslator.cpp:863
Ops
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
Definition ItaniumDemangle.h:3392
I
#define I(x, y, z)
Definition MD5.cpp:57
Reg
Register Reg
Definition MachineSink.cpp:2127
TRI
Register const TargetRegisterInfo * TRI
Definition MachineSink.cpp:2128
Register
Promote Memory to Register
Definition Mem2Reg.cpp:110
getReg
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Definition MipsDisassembler.cpp:106
High
uint64_t High
Definition NVVMIntrRange.cpp:46
INITIALIZE_PASS
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
contains
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:484
optimizeBlock
static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT, const TargetTransformInfo &TTI, const DataLayout &DL, bool HasBranchDivergence, DomTreeUpdater *DTU)
Definition ScalarizeMaskedMemIntrin.cpp:1228
SetVector.h
This file implements a set that has insertion order iteration characteristics.
SmallVector.h
This file defines the SmallVector class.
Statistic.h
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
STATISTIC
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
LLVM_DEBUG
#define LLVM_DEBUG(...)
Definition Debug.h:119
llvm::AAManager
A manager for alias analyses.
Definition AliasAnalysis.h:964
llvm::AAResultsWrapperPass
A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.
Definition AliasAnalysis.h:1012
llvm::AArch64FunctionInfo
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
Definition AArch64MachineFunctionInfo.h:51
llvm::AArch64InstrInfo::getLdStOffsetOp
static const MachineOperand & getLdStOffsetOp(const MachineInstr &MI)
Returns the immediate offset operator of a load/store.
Definition AArch64InstrInfo.cpp:5487
llvm::AArch64InstrInfo::getLdStAmountOp
static const MachineOperand & getLdStAmountOp(const MachineInstr &MI)
Returns the shift amount operator of a load/store.
Definition AArch64InstrInfo.cpp:5496
llvm::AArch64InstrInfo::isPreLdSt
static bool isPreLdSt(const MachineInstr &MI)
Returns whether the instruction is a pre-indexed load/store.
Definition AArch64InstrInfo.cpp:5406
llvm::AArch64InstrInfo::isPairedLdSt
static bool isPairedLdSt(const MachineInstr &MI)
Returns whether the instruction is a paired load/store.
Definition AArch64InstrInfo.cpp:5458
llvm::AArch64InstrInfo::getMemScale
static int getMemScale(unsigned Opc)
Scaling factor for (scaled or unscaled) load or store.
Definition AArch64InstrInfo.cpp:5302
llvm::AArch64InstrInfo::getLdStBaseOp
static const MachineOperand & getLdStBaseOp(const MachineInstr &MI)
Returns the base register operator of a load/store.
Definition AArch64InstrInfo.cpp:5478
llvm::AArch64LoadStoreOptPass::run
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Definition AArch64LoadStoreOptimizer.cpp:3326
llvm::AArch64Subtarget::getRegisterInfo
const AArch64RegisterInfo * getRegisterInfo() const override
Definition AArch64Subtarget.h:173
llvm::AArch64Subtarget::getInstrInfo
const AArch64InstrInfo * getInstrInfo() const override
Definition AArch64Subtarget.h:172
llvm::AArch64Subtarget::getCLOpts
const AArch64Options & getCLOpts() const
Definition AArch64Subtarget.h:156
llvm::AArch64Subtarget::getTargetLowering
const AArch64TargetLowering * getTargetLowering() const override
Definition AArch64Subtarget.h:169
llvm::AArch64TargetLowering::getRedZoneSize
unsigned getRedZoneSize(const Function &F) const
Definition AArch64ISelLowering.h:561
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition PassManager.h:436
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition PassAnalysisSupport.h:48
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition PassAnalysisSupport.h:76
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition Analysis.h:73
llvm::DebugCounter::shouldExecute
static bool shouldExecute(CounterInfo &Counter)
Definition DebugCounter.h:110
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:314
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition Instruction.h:348
llvm::LiveRegUnits
A set of register units used to track register liveness.
Definition LiveRegUnits.h:31
llvm::LiveRegUnits::accumulateUsedDefed
static void accumulateUsedDefed(const MachineInstr &MI, LiveRegUnits &ModifiedRegUnits, LiveRegUnits &UsedRegUnits, const TargetRegisterInfo *TRI)
For a machine instruction MI, adds all register units used in UsedRegUnits and defined or clobbered i...
Definition LiveRegUnits.h:48
llvm::LiveRegUnits::available
bool available(MCRegister Reg) const
Returns true if no part of physical register Reg is live.
Definition LiveRegUnits.h:117
llvm::LiveRegUnits::init
void init(const TargetRegisterInfo &TRI)
Initialize and clear the set.
Definition LiveRegUnits.h:74
llvm::LiveRegUnits::addReg
void addReg(MCRegister Reg)
Adds register units covered by physical register Reg.
Definition LiveRegUnits.h:87
llvm::LiveRegUnits::removeReg
void removeReg(MCRegister Reg)
Removes all register units covered by physical register Reg.
Definition LiveRegUnits.h:103
llvm::LiveRegUnits::addLiveIns
LLVM_ABI void addLiveIns(const MachineBasicBlock &MBB)
Adds registers living into block MBB.
Definition LiveRegUnits.cpp:159
llvm::LiveRegUnits::clear
void clear()
Clears the set.
Definition LiveRegUnits.h:81
llvm::LiveRegUnits::accumulate
LLVM_ABI void accumulate(const MachineInstr &MI)
Adds all register units used, defined or clobbered in MI.
Definition LiveRegUnits.cpp:73
llvm::LoadInst
An instruction for reading from memory.
Definition Instructions.h:197
llvm::MCCFIInstruction::getOperation
OpType getOperation() const
Definition MCDwarf.h:833
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition MachineBasicBlock.h:322
llvm::MachineBasicBlock::erase
LLVM_ABI instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
Definition MachineBasicBlock.cpp:1488
llvm::MachineBasicBlock::successors
iterator_range< succ_iterator > successors()
Definition MachineBasicBlock.h:465
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition MachineBasicBlock.h:1160
llvm::MachineBasicBlock::iterator
MachineInstrBundleIterator< MachineInstr > iterator
Definition MachineBasicBlock.h:340
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition MachineFunctionPass.h:31
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition MachineFunctionPass.cpp:148
llvm::MachineFunctionProperties
Properties which a MachineFunction may have at a given point in time.
Definition MachineFunction.h:138
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition MachineFunction.h:778
llvm::MachineFunction::getFrameInstructions
const std::vector< MCCFIInstruction > & getFrameInstructions() const
Returns a reference to a list of cfi instructions in the function's prologue.
Definition MachineFunction.h:1260
llvm::MachineFunction::makeDebugValueSubstitution
void makeDebugValueSubstitution(DebugInstrOperandPair, DebugInstrOperandPair, unsigned SubReg=0)
Create a substitution between one <instr,operand> value to a different, new value.
Definition MachineFunction.cpp:1117
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition MachineFunction.h:788
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition MachineFunction.h:739
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition MachineFunction.h:890
llvm::MachineInstrBuilder::cloneMergedMemRefs
const MachineInstrBuilder & cloneMergedMemRefs(ArrayRef< const MachineInstr * > OtherMIs) const
Definition MachineInstrBuilder.h:321
llvm::MachineInstrBuilder::setMemRefs
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
Definition MachineInstrBuilder.h:310
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
Definition MachineInstrBuilder.h:225
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
Definition MachineInstrBuilder.h:199
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition MachineInstrBuilder.h:233
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition MachineInstrBuilder.h:326
llvm::MachineInstrBuilder::addDef
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
Definition MachineInstrBuilder.h:218
llvm::MachineInstrBuilder::setMIFlags
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
Definition MachineInstrBuilder.h:379
llvm::MachineInstr
Representation of each machine instruction.
Definition MachineInstr.h:74
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition MachineInstr.h:609
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition MachineInstr.h:381
llvm::MachineInstr::mayAlias
LLVM_ABI bool mayAlias(BatchAAResults *AA, const MachineInstr &Other, bool UseTBAA) const
Returns true if this instruction's memory access aliases the memory access of Other.
Definition MachineInstr.cpp:1558
llvm::MachineInstr::peekDebugInstrNum
unsigned peekDebugInstrNum() const
Examine the instruction number of this MachineInstr.
Definition MachineInstr.h:580
llvm::MachineInstr::getNumExplicitOperands
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
Definition MachineInstr.cpp:857
llvm::MachineInstr::setMemRefs
LLVM_ABI void setMemRefs(MachineFunction &MF, ArrayRef< MachineMemOperand * > MemRefs)
Assign this MachineInstr's memory reference descriptor list.
Definition MachineInstr.cpp:369
llvm::MachineInstr::mayLoad
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
Definition MachineInstr.h:1166
llvm::MachineInstr::operands
mop_range operands()
Definition MachineInstr.h:715
llvm::MachineInstr::killsRegister
bool killsRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr kills the specified register.
Definition MachineInstr.h:1552
llvm::MachineInstr::memoperands_begin
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition MachineInstr.h:820
llvm::MachineInstr::hasOrderedMemoryRef
LLVM_ABI bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
Definition MachineInstr.cpp:1619
llvm::MachineInstr::getMF
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
Definition MachineInstr.cpp:792
llvm::MachineInstr::memoperands
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
Definition MachineInstr.h:802
llvm::MachineInstr::mayStore
bool mayStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly modify memory.
Definition MachineInstr.h:1179
llvm::MachineInstr::getDebugLoc
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition MachineInstr.h:533
llvm::MachineInstr::isPseudo
bool isPseudo(QueryType Type=IgnoreBundle) const
Return true if this is a pseudo instruction that doesn't correspond to a real machine instruction.
Definition MachineInstr.h:958
llvm::MachineInstr::dump
LLVM_ABI void dump() const
Definition MachineInstr.cpp:1781
llvm::MachineInstr::getDebugInstrNum
LLVM_ABI unsigned getDebugInstrNum()
Fetch the instruction number of this MachineInstr.
Definition MachineInstr.cpp:2662
llvm::MachineInstr::clearRegisterKills
LLVM_ABI void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo)
Clear all kill flags affecting Reg.
Definition MachineInstr.cpp:2232
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition MachineInstr.h:617
llvm::MachineInstr::eraseFromParent
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Definition MachineInstr.cpp:806
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition MachineOperand.h:49
llvm::MachineOperand::setImplicit
void setImplicit(bool Val=true)
Definition MachineOperand.h:521
llvm::MachineOperand::getImm
int64_t getImm() const
Definition MachineOperand.h:563
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition MachineOperand.h:334
llvm::MachineOperand::setReg
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
Definition MachineOperand.cpp:60
llvm::MachineOperand::setIsKill
void setIsKill(bool Val=true)
Definition MachineOperand.h:526
llvm::MachineOperand::isRenamable
LLVM_ABI bool isRenamable() const
isRenamable - Returns true if this register may be renamed, i.e.
Definition MachineOperand.cpp:123
llvm::MachineOperand::isEarlyClobber
bool isEarlyClobber() const
Definition MachineOperand.h:451
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition MachineOperand.h:375
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition MachineRegisterInfo.h:55
llvm::MachineRegisterInfo::tracksLiveness
bool tracksLiveness() const
tracksLiveness - Returns true when tracking register liveness accurately.
Definition MachineRegisterInfo.h:229
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition Analysis.h:118
llvm::PreservedAnalyses::preserveSet
PreservedAnalyses & preserveSet()
Mark an analysis set as preserved.
Definition Analysis.h:151
llvm::Register
Wrapper class representing virtual and physical registers.
Definition Register.h:20
llvm::Register::isValid
constexpr bool isValid() const
Definition Register.h:112
llvm::SmallPtrSetImpl
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition SmallPtrSet.h:352
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition SmallPtrSet.h:373
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition SmallVector.h:592
llvm::SmallVectorTemplateBase::push_back
void push_back(const T &Elt)
Definition SmallVector.h:423
llvm::StoreInst
An instruction for storing to memory.
Definition Instructions.h:338
llvm::StringRef
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition TargetRegisterInfo.h:74
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition ilist_node.h:123
Changed
Changed
Definition ObjCARCOpts.cpp:2468
iterator_range.h
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition ErrorHandling.h:164
llvm::AA
Abstract Attribute helper functions.
Definition Attributor.h:165
llvm::AArch64_AM::getShiftValue
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
Definition AArch64AddressingModes.h:98
llvm::AArch64_AM::LSL
@ LSL
Definition AArch64AddressingModes.h:34
llvm::AArch64_AM::getShifterImm
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Definition AArch64AddressingModes.h:111
llvm::AMDGPU::HSAMD::Kernel::Arg::Key::Align
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
Definition AMDGPUMetadata.h:183
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition BitmaskEnum.h:126
llvm::dwarf_linker::DebugSectionKind::DebugLoc
@ DebugLoc
Definition DWARFLinkerBase.h:35
llvm::numbers::e
constexpr double e
Definition STLForwardCompat.h:64
llvm::sframe::BaseReg
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
llvm::vputils::getOpcode
unsigned getOpcode(const VPValue *V)
Return the instruction opcode for the recipe defining V or 0 for unsupported recipes and VPValues not...
Definition VPlanUtils.cpp:398
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition DefaultTargetInfo.h:21
llvm::next_nodbg
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
Definition MachineBasicBlock.h:1507
llvm::drop_begin
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
llvm::ThreadPriority::Low
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
llvm::Offset
@ Offset
Definition DWP.cpp:577
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition MachineInstrBuilder.h:449
llvm::getKillRegState
constexpr RegState getKillRegState(bool B)
Definition MachineInstrBuilder.h:90
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition iterator_range.h:67
llvm::phys_regs_and_masks
iterator_range< filter_iterator< ConstMIBundleOperands, bool(*)(const MachineOperand &)> > phys_regs_and_masks(const MachineInstr &MI)
Returns an iterator range over all physical register and mask operands for MI and bundled instruction...
Definition LiveRegUnits.h:167
llvm::MachineFunctionAnalysisManager
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
Definition MachineFunctionAnalysisManager.h:24
llvm::createAArch64LoadStoreOptLegacyPass
FunctionPass * createAArch64LoadStoreOptLegacyPass()
createAArch64LoadStoreOptimizationPass - returns an instance of the load / store optimization pass.
Definition AArch64LoadStoreOptimizer.cpp:3321
llvm::Value
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
llvm::getMachineFunctionPassPreservedAnalyses
LLVM_ABI PreservedAnalyses getMachineFunctionPassPreservedAnalyses()
Returns the minimum set of Analyses that all machine function passes must preserve.
Definition MachinePassManager.cpp:162
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
llvm::getImm
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
Definition SPIRVUtils.cpp:1170
llvm::dbgs
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
llvm::alignTo
constexpr uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition Alignment.h:149
llvm::instructionsWithoutDebug
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
Definition MachineBasicBlock.h:1521
llvm::iterator_range
iterator_range(Container &&) -> iterator_range< llvm::detail::IterOfRange< Container > >
llvm::MCPhysReg
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
llvm::Count
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Count
Definition InstrProf.h:145
llvm::Op
DWARFExpression::Operation Op
Definition DWARFExpressionPrinter.cpp:24
llvm::MachineVerifierMode::End
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:267
llvm::AliasAnalysis
AAResults AliasAnalysis
Temporary typedef for legacy code that uses a generic AliasAnalysis pointer or reference.
Definition AliasAnalysis.h:689
llvm::prev_nodbg
IterT prev_nodbg(IterT It, IterT Begin, bool SkipPseudoOp=true)
Decrement It, then continue decrementing it while it points to a debug instruction.
Definition MachineBasicBlock.h:1514
llvm::printReg
LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Definition TargetRegisterInfo.cpp:106
llvm::TargetRegisterClass
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
Generated on for LLVM by doxygen 1.14.0