LLVM  16.0.0git
SIFixSGPRCopies.cpp
Go to the documentation of this file.
1 //===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Copies from VGPR to SGPR registers are illegal and the register coalescer
11 /// will sometimes generate these illegal copies in situations like this:
12 ///
13 /// Register Class <vsrc> is the union of <vgpr> and <sgpr>
14 ///
15 /// BB0:
16 /// %0 <sgpr> = SCALAR_INST
17 /// %1 <vsrc> = COPY %0 <sgpr>
18 /// ...
19 /// BRANCH %cond BB1, BB2
20 /// BB1:
21 /// %2 <vgpr> = VECTOR_INST
22 /// %3 <vsrc> = COPY %2 <vgpr>
23 /// BB2:
24 /// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
25 /// %5 <vgpr> = VECTOR_INST %4 <vsrc>
26 ///
27 ///
28 /// The coalescer will begin at BB0 and eliminate its copy, then the resulting
29 /// code will look like this:
30 ///
31 /// BB0:
32 /// %0 <sgpr> = SCALAR_INST
33 /// ...
34 /// BRANCH %cond BB1, BB2
35 /// BB1:
36 /// %2 <vgpr> = VECTOR_INST
37 /// %3 <vsrc> = COPY %2 <vgpr>
38 /// BB2:
39 /// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
40 /// %5 <vgpr> = VECTOR_INST %4 <sgpr>
41 ///
42 /// Now that the result of the PHI instruction is an SGPR, the register
43 /// allocator is now forced to constrain the register class of %3 to
44 /// <sgpr> so we end up with final code like this:
45 ///
46 /// BB0:
47 /// %0 <sgpr> = SCALAR_INST
48 /// ...
49 /// BRANCH %cond BB1, BB2
50 /// BB1:
51 /// %2 <vgpr> = VECTOR_INST
52 /// %3 <sgpr> = COPY %2 <vgpr>
53 /// BB2:
54 /// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
55 /// %5 <vgpr> = VECTOR_INST %4 <sgpr>
56 ///
57 /// Now this code contains an illegal copy from a VGPR to an SGPR.
58 ///
59 /// In order to avoid this problem, this pass searches for PHI instructions
60 /// which define a <vsrc> register and constrains its definition class to
61 /// <vgpr> if the user of the PHI's definition register is a vector instruction.
62 /// If the PHI's definition class is constrained to <vgpr> then the coalescer
63 /// will be unable to perform the COPY removal from the above example which
64 /// ultimately led to the creation of an illegal COPY.
65 //===----------------------------------------------------------------------===//
66 
67 #include "AMDGPU.h"
68 #include "GCNSubtarget.h"
70 #include "llvm/ADT/SetOperations.h"
72 #include "llvm/InitializePasses.h"
74 
75 using namespace llvm;
76 
77 #define DEBUG_TYPE "si-fix-sgpr-copies"
78 
80  "amdgpu-enable-merge-m0",
81  cl::desc("Merge and hoist M0 initializations"),
82  cl::init(true));
83 
84 namespace {
85 
86 class V2SCopyInfo {
87 public:
88  // VGPR to SGPR copy being processed
89  MachineInstr *Copy;
90  // All SALU instructions reachable from this copy in SSA graph
92  // Number of SGPR to VGPR copies that are used to put the SALU computation
93  // results back to VALU.
94  unsigned NumSVCopies;
95 
96  unsigned Score;
97  // Actual count of v_readfirstlane_b32
98  // which need to be inserted to keep SChain SALU
99  unsigned NumReadfirstlanes;
100  // Current score state. To speedup selection V2SCopyInfos for processing
101  bool NeedToBeConvertedToVALU = false;
102  // Unique ID. Used as a key for mapping to keep permanent order.
103  unsigned ID;
104 
105  // Count of another VGPR to SGPR copies that contribute to the
106  // current copy SChain
107  unsigned SiblingPenalty = 0;
108  SetVector<unsigned> Siblings;
109  V2SCopyInfo() : Copy(nullptr), ID(0){};
110  V2SCopyInfo(unsigned Id, MachineInstr *C, unsigned Width)
111  : Copy(C), NumSVCopies(0), NumReadfirstlanes(Width / 32), ID(Id){};
112 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
113  void dump() {
114  dbgs() << ID << " : " << *Copy << "\n\tS:" << SChain.size()
115  << "\n\tSV:" << NumSVCopies << "\n\tSP: " << SiblingPenalty
116  << "\nScore: " << Score << "\n";
117  }
118 #endif
119 };
120 
121 class SIFixSGPRCopies : public MachineFunctionPass {
124  SmallVector<MachineInstr*, 4> RegSequences;
127  unsigned NextVGPRToSGPRCopyID;
130 
131 public:
132  static char ID;
133 
135  const SIRegisterInfo *TRI;
136  const SIInstrInfo *TII;
137 
138  SIFixSGPRCopies() : MachineFunctionPass(ID), NextVGPRToSGPRCopyID(0) {}
139 
140  bool runOnMachineFunction(MachineFunction &MF) override;
141  void fixSCCCopies(MachineFunction &MF);
142  void prepareRegSequenceAndPHIs(MachineFunction &MF);
143  unsigned getNextVGPRToSGPRCopyId() { return ++NextVGPRToSGPRCopyID; }
144  bool needToBeConvertedToVALU(V2SCopyInfo *I);
145  void analyzeVGPRToSGPRCopy(MachineInstr *MI);
146  void lowerVGPR2SGPRCopies(MachineFunction &MF);
147  // Handles copies which source register is:
148  // 1. Physical register
149  // 2. AGPR
150  // 3. Defined by the instruction the merely moves the immediate
151  bool lowerSpecialCase(MachineInstr &MI);
152 
153  void processPHINode(MachineInstr &MI);
154 
155  StringRef getPassName() const override { return "SI Fix SGPR copies"; }
156 
157  void getAnalysisUsage(AnalysisUsage &AU) const override {
160  AU.setPreservesCFG();
162  }
163 };
164 
165 } // end anonymous namespace
166 
167 INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE,
168  "SI Fix SGPR copies", false, false)
171  "SI Fix SGPR copies", false, false)
172 
173 char SIFixSGPRCopies::ID = 0;
174 
175 char &llvm::SIFixSGPRCopiesID = SIFixSGPRCopies::ID;
176 
178  return new SIFixSGPRCopies();
179 }
180 
181 static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
183  const SIRegisterInfo &TRI,
184  const MachineRegisterInfo &MRI) {
185  Register DstReg = Copy.getOperand(0).getReg();
186  Register SrcReg = Copy.getOperand(1).getReg();
187 
188  const TargetRegisterClass *SrcRC = SrcReg.isVirtual()
189  ? MRI.getRegClass(SrcReg)
190  : TRI.getPhysRegClass(SrcReg);
191 
192  // We don't really care about the subregister here.
193  // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
194 
195  const TargetRegisterClass *DstRC = DstReg.isVirtual()
196  ? MRI.getRegClass(DstReg)
197  : TRI.getPhysRegClass(DstReg);
198 
199  return std::make_pair(SrcRC, DstRC);
200 }
201 
202 static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
203  const TargetRegisterClass *DstRC,
204  const SIRegisterInfo &TRI) {
205  return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(DstRC) &&
206  TRI.hasVectorRegisters(SrcRC);
207 }
208 
209 static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
210  const TargetRegisterClass *DstRC,
211  const SIRegisterInfo &TRI) {
212  return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(SrcRC) &&
213  TRI.hasVectorRegisters(DstRC);
214 }
215 
217  const SIRegisterInfo *TRI,
218  const SIInstrInfo *TII) {
219  MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
220  auto &Src = MI.getOperand(1);
221  Register DstReg = MI.getOperand(0).getReg();
222  Register SrcReg = Src.getReg();
223  if (!SrcReg.isVirtual() || !DstReg.isVirtual())
224  return false;
225 
226  for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) {
227  const auto *UseMI = MO.getParent();
228  if (UseMI == &MI)
229  continue;
230  if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
231  UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END)
232  return false;
233 
234  unsigned OpIdx = UseMI->getOperandNo(&MO);
235  if (OpIdx >= UseMI->getDesc().getNumOperands() ||
236  !TII->isOperandLegal(*UseMI, OpIdx, &Src))
237  return false;
238  }
239  // Change VGPR to SGPR destination.
240  MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg)));
241  return true;
242 }
243 
244 // Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
245 //
246 // SGPRx = ...
247 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
248 // VGPRz = COPY SGPRy
249 //
250 // ==>
251 //
252 // VGPRx = COPY SGPRx
253 // VGPRz = REG_SEQUENCE VGPRx, sub0
254 //
255 // This exposes immediate folding opportunities when materializing 64-bit
256 // immediates.
258  const SIRegisterInfo *TRI,
259  const SIInstrInfo *TII,
261  assert(MI.isRegSequence());
262 
263  Register DstReg = MI.getOperand(0).getReg();
264  if (!TRI->isSGPRClass(MRI.getRegClass(DstReg)))
265  return false;
266 
267  if (!MRI.hasOneUse(DstReg))
268  return false;
269 
270  MachineInstr &CopyUse = *MRI.use_instr_begin(DstReg);
271  if (!CopyUse.isCopy())
272  return false;
273 
274  // It is illegal to have vreg inputs to a physreg defining reg_sequence.
275  if (CopyUse.getOperand(0).getReg().isPhysical())
276  return false;
277 
278  const TargetRegisterClass *SrcRC, *DstRC;
279  std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI);
280 
281  if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
282  return false;
283 
284  if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII))
285  return true;
286 
287  // TODO: Could have multiple extracts?
288  unsigned SubReg = CopyUse.getOperand(1).getSubReg();
289  if (SubReg != AMDGPU::NoSubRegister)
290  return false;
291 
292  MRI.setRegClass(DstReg, DstRC);
293 
294  // SGPRx = ...
295  // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
296  // VGPRz = COPY SGPRy
297 
298  // =>
299  // VGPRx = COPY SGPRx
300  // VGPRz = REG_SEQUENCE VGPRx, sub0
301 
302  MI.getOperand(0).setReg(CopyUse.getOperand(0).getReg());
303  bool IsAGPR = TRI->isAGPRClass(DstRC);
304 
305  for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
306  const TargetRegisterClass *SrcRC =
307  TRI->getRegClassForOperandReg(MRI, MI.getOperand(I));
308  assert(TRI->isSGPRClass(SrcRC) &&
309  "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
310  const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SrcRC);
311 
312  Register TmpReg = MRI.createVirtualRegister(NewSrcRC);
313 
314  BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY),
315  TmpReg)
316  .add(MI.getOperand(I));
317 
318  if (IsAGPR) {
319  const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SrcRC);
320  Register TmpAReg = MRI.createVirtualRegister(NewSrcRC);
321  unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
322  AMDGPU::V_ACCVGPR_WRITE_B32_e64 : AMDGPU::COPY;
323  BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(Opc),
324  TmpAReg)
325  .addReg(TmpReg, RegState::Kill);
326  TmpReg = TmpAReg;
327  }
328 
329  MI.getOperand(I).setReg(TmpReg);
330  }
331 
332  CopyUse.eraseFromParent();
333  return true;
334 }
335 
336 static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
337  const MachineInstr *MoveImm,
338  const SIInstrInfo *TII,
339  unsigned &SMovOp,
340  int64_t &Imm) {
341  if (Copy->getOpcode() != AMDGPU::COPY)
342  return false;
343 
344  if (!MoveImm->isMoveImmediate())
345  return false;
346 
347  const MachineOperand *ImmOp =
348  TII->getNamedOperand(*MoveImm, AMDGPU::OpName::src0);
349  if (!ImmOp->isImm())
350  return false;
351 
352  // FIXME: Handle copies with sub-regs.
353  if (Copy->getOperand(0).getSubReg())
354  return false;
355 
356  switch (MoveImm->getOpcode()) {
357  default:
358  return false;
359  case AMDGPU::V_MOV_B32_e32:
360  SMovOp = AMDGPU::S_MOV_B32;
361  break;
362  case AMDGPU::V_MOV_B64_PSEUDO:
363  SMovOp = AMDGPU::S_MOV_B64;
364  break;
365  }
366  Imm = ImmOp->getImm();
367  return true;
368 }
369 
370 template <class UnaryPredicate>
372  const MachineBasicBlock *CutOff,
373  UnaryPredicate Predicate) {
374  if (MBB == CutOff)
375  return false;
376 
379 
380  while (!Worklist.empty()) {
381  MachineBasicBlock *MBB = Worklist.pop_back_val();
382 
383  if (!Visited.insert(MBB).second)
384  continue;
385  if (MBB == CutOff)
386  continue;
387  if (Predicate(MBB))
388  return true;
389 
390  Worklist.append(MBB->pred_begin(), MBB->pred_end());
391  }
392 
393  return false;
394 }
395 
396 // Checks if there is potential path From instruction To instruction.
397 // If CutOff is specified and it sits in between of that path we ignore
398 // a higher portion of the path and report it is not reachable.
399 static bool isReachable(const MachineInstr *From,
400  const MachineInstr *To,
401  const MachineBasicBlock *CutOff,
402  MachineDominatorTree &MDT) {
403  if (MDT.dominates(From, To))
404  return true;
405 
406  const MachineBasicBlock *MBBFrom = From->getParent();
407  const MachineBasicBlock *MBBTo = To->getParent();
408 
409  // Do predecessor search.
410  // We should almost never get here since we do not usually produce M0 stores
411  // other than -1.
412  return searchPredecessors(MBBTo, CutOff, [MBBFrom]
413  (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
414 }
415 
416 // Return the first non-prologue instruction in the block.
420  while (I != MBB->end() && TII->isBasicBlockPrologue(*I))
421  ++I;
422 
423  return I;
424 }
425 
426 // Hoist and merge identical SGPR initializations into a common predecessor.
427 // This is intended to combine M0 initializations, but can work with any
428 // SGPR. A VGPR cannot be processed since we cannot guarantee vector
429 // executioon.
430 static bool hoistAndMergeSGPRInits(unsigned Reg,
431  const MachineRegisterInfo &MRI,
432  const TargetRegisterInfo *TRI,
434  const TargetInstrInfo *TII) {
435  // List of inits by immediate value.
436  using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
437  InitListMap Inits;
438  // List of clobbering instructions.
440  // List of instructions marked for deletion.
441  SmallSet<MachineInstr*, 8> MergedInstrs;
442 
443  bool Changed = false;
444 
445  for (auto &MI : MRI.def_instructions(Reg)) {
446  MachineOperand *Imm = nullptr;
447  for (auto &MO : MI.operands()) {
448  if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
449  (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
450  Imm = nullptr;
451  break;
452  } else if (MO.isImm())
453  Imm = &MO;
454  }
455  if (Imm)
456  Inits[Imm->getImm()].push_front(&MI);
457  else
458  Clobbers.push_back(&MI);
459  }
460 
461  for (auto &Init : Inits) {
462  auto &Defs = Init.second;
463 
464  for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
465  MachineInstr *MI1 = *I1;
466 
467  for (auto I2 = std::next(I1); I2 != E; ) {
468  MachineInstr *MI2 = *I2;
469 
470  // Check any possible interference
471  auto interferes = [&](MachineBasicBlock::iterator From,
472  MachineBasicBlock::iterator To) -> bool {
473 
474  assert(MDT.dominates(&*To, &*From));
475 
476  auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
477  const MachineBasicBlock *MBBFrom = From->getParent();
478  const MachineBasicBlock *MBBTo = To->getParent();
479  bool MayClobberFrom = isReachable(Clobber, &*From, MBBTo, MDT);
480  bool MayClobberTo = isReachable(Clobber, &*To, MBBTo, MDT);
481  if (!MayClobberFrom && !MayClobberTo)
482  return false;
483  if ((MayClobberFrom && !MayClobberTo) ||
484  (!MayClobberFrom && MayClobberTo))
485  return true;
486  // Both can clobber, this is not an interference only if both are
487  // dominated by Clobber and belong to the same block or if Clobber
488  // properly dominates To, given that To >> From, so it dominates
489  // both and located in a common dominator.
490  return !((MBBFrom == MBBTo &&
491  MDT.dominates(Clobber, &*From) &&
492  MDT.dominates(Clobber, &*To)) ||
493  MDT.properlyDominates(Clobber->getParent(), MBBTo));
494  };
495 
496  return (llvm::any_of(Clobbers, interferes)) ||
497  (llvm::any_of(Inits, [&](InitListMap::value_type &C) {
498  return C.first != Init.first &&
499  llvm::any_of(C.second, interferes);
500  }));
501  };
502 
503  if (MDT.dominates(MI1, MI2)) {
504  if (!interferes(MI2, MI1)) {
505  LLVM_DEBUG(dbgs()
506  << "Erasing from "
507  << printMBBReference(*MI2->getParent()) << " " << *MI2);
508  MergedInstrs.insert(MI2);
509  Changed = true;
510  ++I2;
511  continue;
512  }
513  } else if (MDT.dominates(MI2, MI1)) {
514  if (!interferes(MI1, MI2)) {
515  LLVM_DEBUG(dbgs()
516  << "Erasing from "
517  << printMBBReference(*MI1->getParent()) << " " << *MI1);
518  MergedInstrs.insert(MI1);
519  Changed = true;
520  ++I1;
521  break;
522  }
523  } else {
524  auto *MBB = MDT.findNearestCommonDominator(MI1->getParent(),
525  MI2->getParent());
526  if (!MBB) {
527  ++I2;
528  continue;
529  }
530 
532  if (!interferes(MI1, I) && !interferes(MI2, I)) {
533  LLVM_DEBUG(dbgs()
534  << "Erasing from "
535  << printMBBReference(*MI1->getParent()) << " " << *MI1
536  << "and moving from "
537  << printMBBReference(*MI2->getParent()) << " to "
538  << printMBBReference(*I->getParent()) << " " << *MI2);
539  I->getParent()->splice(I, MI2->getParent(), MI2);
540  MergedInstrs.insert(MI1);
541  Changed = true;
542  ++I1;
543  break;
544  }
545  }
546  ++I2;
547  }
548  ++I1;
549  }
550  }
551 
552  // Remove initializations that were merged into another.
553  for (auto &Init : Inits) {
554  auto &Defs = Init.second;
555  auto I = Defs.begin();
556  while (I != Defs.end()) {
557  if (MergedInstrs.count(*I)) {
558  (*I)->eraseFromParent();
559  I = Defs.erase(I);
560  } else
561  ++I;
562  }
563  }
564 
565  // Try to schedule SGPR initializations as early as possible in the MBB.
566  for (auto &Init : Inits) {
567  auto &Defs = Init.second;
568  for (auto *MI : Defs) {
569  auto MBB = MI->getParent();
570  MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
572  // Check if B should actually be a boundary. If not set the previous
573  // instruction as the boundary instead.
574  if (!TII->isBasicBlockPrologue(*B))
575  B++;
576 
577  auto R = std::next(MI->getReverseIterator());
578  const unsigned Threshold = 50;
579  // Search until B or Threshold for a place to insert the initialization.
580  for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
581  if (R->readsRegister(Reg, TRI) || R->definesRegister(Reg, TRI) ||
583  break;
584 
585  // Move to directly after R.
586  if (&*--R != MI)
587  MBB->splice(*R, MBB, MI);
588  }
589  }
590 
591  if (Changed)
593 
594  return Changed;
595 }
596 
597 bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) {
598  // Only need to run this in SelectionDAG path.
599  if (MF.getProperties().hasProperty(
601  return false;
602 
603  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
604  MRI = &MF.getRegInfo();
605  TRI = ST.getRegisterInfo();
606  TII = ST.getInstrInfo();
607  MDT = &getAnalysis<MachineDominatorTree>();
608 
609 
610  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
611  BI != BE; ++BI) {
612  MachineBasicBlock *MBB = &*BI;
613  for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
614  ++I) {
615  MachineInstr &MI = *I;
616 
617  switch (MI.getOpcode()) {
618  default:
619  continue;
620  case AMDGPU::COPY:
621  case AMDGPU::WQM:
622  case AMDGPU::STRICT_WQM:
623  case AMDGPU::SOFT_WQM:
624  case AMDGPU::STRICT_WWM: {
625  const TargetRegisterClass *SrcRC, *DstRC;
626  std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, *MRI);
627 
628  if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
629  // Since VGPR to SGPR copies affect VGPR to SGPR copy
630  // score and, hence the lowering decision, let's try to get rid of
631  // them as early as possible
633  continue;
634 
635  // Collect those not changed to try them after VGPR to SGPR copies
636  // lowering as there will be more opportunities.
637  S2VCopies.push_back(&MI);
638  }
639  if (!isVGPRToSGPRCopy(SrcRC, DstRC, *TRI))
640  continue;
641  if (lowerSpecialCase(MI))
642  continue;
643 
644  analyzeVGPRToSGPRCopy(&MI);
645 
646  break;
647  }
648  case AMDGPU::INSERT_SUBREG:
649  case AMDGPU::PHI:
650  case AMDGPU::REG_SEQUENCE: {
651  if (TRI->isSGPRClass(TII->getOpRegClass(MI, 0))) {
652  for (MachineOperand &MO : MI.operands()) {
653  if (!MO.isReg() || !MO.getReg().isVirtual())
654  continue;
655  const TargetRegisterClass *SrcRC = MRI->getRegClass(MO.getReg());
656  if (TRI->hasVectorRegisters(SrcRC)) {
657  const TargetRegisterClass *DestRC =
658  TRI->getEquivalentSGPRClass(SrcRC);
659  Register NewDst = MRI->createVirtualRegister(DestRC);
660  MachineBasicBlock *BlockToInsertCopy =
661  MI.isPHI() ? MI.getOperand(MI.getOperandNo(&MO) + 1).getMBB()
662  : MBB;
663  MachineBasicBlock::iterator PointToInsertCopy =
664  MI.isPHI() ? BlockToInsertCopy->getFirstInstrTerminator() : I;
665  MachineInstr *NewCopy =
666  BuildMI(*BlockToInsertCopy, PointToInsertCopy,
667  PointToInsertCopy->getDebugLoc(),
668  TII->get(AMDGPU::COPY), NewDst)
669  .addReg(MO.getReg());
670  MO.setReg(NewDst);
671  analyzeVGPRToSGPRCopy(NewCopy);
672  }
673  }
674  }
675 
676  if (MI.isPHI())
677  PHINodes.push_back(&MI);
678  else if (MI.isRegSequence())
679  RegSequences.push_back(&MI);
680 
681  break;
682  }
683  case AMDGPU::V_WRITELANE_B32: {
684  // Some architectures allow more than one constant bus access without
685  // SGPR restriction
686  if (ST.getConstantBusLimit(MI.getOpcode()) != 1)
687  break;
688 
689  // Writelane is special in that it can use SGPR and M0 (which would
690  // normally count as using the constant bus twice - but in this case it
691  // is allowed since the lane selector doesn't count as a use of the
692  // constant bus). However, it is still required to abide by the 1 SGPR
693  // rule. Apply a fix here as we might have multiple SGPRs after
694  // legalizing VGPRs to SGPRs
695  int Src0Idx =
696  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
697  int Src1Idx =
698  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
699  MachineOperand &Src0 = MI.getOperand(Src0Idx);
700  MachineOperand &Src1 = MI.getOperand(Src1Idx);
701 
702  // Check to see if the instruction violates the 1 SGPR rule
703  if ((Src0.isReg() && TRI->isSGPRReg(*MRI, Src0.getReg()) &&
704  Src0.getReg() != AMDGPU::M0) &&
705  (Src1.isReg() && TRI->isSGPRReg(*MRI, Src1.getReg()) &&
706  Src1.getReg() != AMDGPU::M0)) {
707 
708  // Check for trivially easy constant prop into one of the operands
709  // If this is the case then perform the operation now to resolve SGPR
710  // issue. If we don't do that here we will always insert a mov to m0
711  // that can't be resolved in later operand folding pass
712  bool Resolved = false;
713  for (MachineOperand *MO : {&Src0, &Src1}) {
714  if (MO->getReg().isVirtual()) {
715  MachineInstr *DefMI = MRI->getVRegDef(MO->getReg());
716  if (DefMI && TII->isFoldableCopy(*DefMI)) {
717  const MachineOperand &Def = DefMI->getOperand(0);
718  if (Def.isReg() &&
719  MO->getReg() == Def.getReg() &&
720  MO->getSubReg() == Def.getSubReg()) {
721  const MachineOperand &Copied = DefMI->getOperand(1);
722  if (Copied.isImm() &&
723  TII->isInlineConstant(APInt(64, Copied.getImm(), true))) {
724  MO->ChangeToImmediate(Copied.getImm());
725  Resolved = true;
726  break;
727  }
728  }
729  }
730  }
731  }
732 
733  if (!Resolved) {
734  // Haven't managed to resolve by replacing an SGPR with an immediate
735  // Move src1 to be in M0
736  BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
737  TII->get(AMDGPU::COPY), AMDGPU::M0)
738  .add(Src1);
739  Src1.ChangeToRegister(AMDGPU::M0, false);
740  }
741  }
742  break;
743  }
744  }
745  }
746  }
747 
748  lowerVGPR2SGPRCopies(MF);
749  // Postprocessing
750  fixSCCCopies(MF);
751  for (auto MI : S2VCopies) {
752  // Check if it is still valid
753  if (MI->isCopy()) {
754  const TargetRegisterClass *SrcRC, *DstRC;
755  std::tie(SrcRC, DstRC) = getCopyRegClasses(*MI, *TRI, *MRI);
756  if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
758  }
759  }
760  for (auto MI : RegSequences) {
761  // Check if it is still valid
762  if (MI->isRegSequence())
764  }
765  for (auto MI : PHINodes) {
766  processPHINode(*MI);
767  }
770 
771  SiblingPenalty.clear();
772  V2SCopies.clear();
773  SCCCopies.clear();
774  RegSequences.clear();
775  PHINodes.clear();
776  S2VCopies.clear();
777 
778  return true;
779 }
780 
781 void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
782  bool AllAGPRUses = true;
785  SetVector<MachineInstr *> PHIOperands;
786  worklist.insert(&MI);
787  Visited.insert(&MI);
788  // HACK to make MIR tests with no uses happy
789  bool HasUses = false;
790  while (!worklist.empty()) {
791  const MachineInstr *Instr = worklist.pop_back_val();
792  Register Reg = Instr->getOperand(0).getReg();
793  for (const auto &Use : MRI->use_operands(Reg)) {
794  HasUses = true;
795  const MachineInstr *UseMI = Use.getParent();
796  AllAGPRUses &= (UseMI->isCopy() &&
797  TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg())) ||
798  TRI->isAGPR(*MRI, Use.getReg());
799  if (UseMI->isCopy() || UseMI->isRegSequence()) {
800  if (Visited.insert(UseMI).second)
801  worklist.insert(UseMI);
802 
803  continue;
804  }
805  }
806  }
807 
808  Register PHIRes = MI.getOperand(0).getReg();
809  const TargetRegisterClass *RC0 = MRI->getRegClass(PHIRes);
810  if (HasUses && AllAGPRUses && !TRI->isAGPRClass(RC0)) {
811  LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
812  MRI->setRegClass(PHIRes, TRI->getEquivalentAGPRClass(RC0));
813  for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
814  MachineInstr *DefMI = MRI->getVRegDef(MI.getOperand(I).getReg());
815  if (DefMI && DefMI->isPHI())
816  PHIOperands.insert(DefMI);
817  }
818  }
819 
820  if (TRI->isVectorRegister(*MRI, PHIRes) ||
821  RC0 == &AMDGPU::VReg_1RegClass) {
822  LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI);
823  TII->legalizeOperands(MI, MDT);
824  }
825 
826  // Propagate register class back to PHI operands which are PHI themselves.
827  while (!PHIOperands.empty()) {
828  processPHINode(*PHIOperands.pop_back_val());
829  }
830 }
831 
832 bool SIFixSGPRCopies::lowerSpecialCase(MachineInstr &MI) {
833  Register DstReg = MI.getOperand(0).getReg();
834  Register SrcReg = MI.getOperand(1).getReg();
835  if (!DstReg.isVirtual()) {
836  // If the destination register is a physical register there isn't
837  // really much we can do to fix this.
838  // Some special instructions use M0 as an input. Some even only use
839  // the first lane. Insert a readfirstlane and hope for the best.
840  if (DstReg == AMDGPU::M0 &&
841  TRI->hasVectorRegisters(MRI->getRegClass(SrcReg))) {
842  Register TmpReg =
843  MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
844  BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
845  TII->get(AMDGPU::V_READFIRSTLANE_B32), TmpReg)
846  .add(MI.getOperand(1));
847  MI.getOperand(1).setReg(TmpReg);
848  }
849  return true;
850  }
851  if (!SrcReg.isVirtual() || TRI->isAGPR(*MRI, SrcReg)) {
852  TII->moveToVALU(MI, MDT);
853  return true;
854  }
855 
856  unsigned SMovOp;
857  int64_t Imm;
858  // If we are just copying an immediate, we can replace the copy with
859  // s_mov_b32.
860  if (isSafeToFoldImmIntoCopy(&MI, MRI->getVRegDef(SrcReg), TII, SMovOp, Imm)) {
861  MI.getOperand(1).ChangeToImmediate(Imm);
862  MI.addImplicitDefUseOperands(*MI.getParent()->getParent());
863  MI.setDesc(TII->get(SMovOp));
864  return true;
865  }
866  return false;
867 }
868 
869 void SIFixSGPRCopies::analyzeVGPRToSGPRCopy(MachineInstr* MI) {
870  Register DstReg = MI->getOperand(0).getReg();
871  const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg);
872 
873  V2SCopyInfo Info(getNextVGPRToSGPRCopyId(), MI,
874  TRI->getRegSizeInBits(*DstRC));
875  SmallVector<MachineInstr *, 8> AnalysisWorklist;
876  // Needed because the SSA is not a tree but a graph and may have
877  // forks and joins. We should not then go same way twice.
878  DenseSet<MachineInstr *> Visited;
879  AnalysisWorklist.push_back(Info.Copy);
880  while (!AnalysisWorklist.empty()) {
881 
882  MachineInstr *Inst = AnalysisWorklist.pop_back_val();
883 
884  if (!Visited.insert(Inst).second)
885  continue;
886 
887  // Copies and REG_SEQUENCE do not contribute to the final assembly
888  // So, skip them but take care of the SGPR to VGPR copies bookkeeping.
889  if (Inst->isCopy() || Inst->isRegSequence()) {
890  if (TRI->isVGPR(*MRI, Inst->getOperand(0).getReg())) {
891  if (!Inst->isCopy() ||
892  !tryChangeVGPRtoSGPRinCopy(*Inst, TRI, TII)) {
893  Info.NumSVCopies++;
894  continue;
895  }
896  }
897  }
898 
899  SiblingPenalty[Inst].insert(Info.ID);
900 
902  if ((TII->isSALU(*Inst) && Inst->isCompare()) ||
903  (Inst->isCopy() && Inst->getOperand(0).getReg() == AMDGPU::SCC)) {
904  auto I = Inst->getIterator();
905  auto E = Inst->getParent()->end();
906  while (++I != E && !I->findRegisterDefOperand(AMDGPU::SCC)) {
907  if (I->readsRegister(AMDGPU::SCC))
908  Users.push_back(&*I);
909  }
910  } else if (Inst->getNumExplicitDefs() != 0) {
911  Register Reg = Inst->getOperand(0).getReg();
912  if (TRI->isSGPRReg(*MRI, Reg) && !TII->isVALU(*Inst))
913  for (auto &U : MRI->use_instructions(Reg))
914  Users.push_back(&U);
915  }
916  for (auto U : Users) {
917  if (TII->isSALU(*U))
918  Info.SChain.insert(U);
919  AnalysisWorklist.push_back(U);
920  }
921  }
922  V2SCopies[Info.ID] = Info;
923 }
924 
925 // The main function that computes the VGPR to SGPR copy score
926 // and determines copy further lowering way: v_readfirstlane_b32 or moveToVALU
927 bool SIFixSGPRCopies::needToBeConvertedToVALU(V2SCopyInfo *Info) {
928  if (Info->SChain.empty()) {
929  Info->Score = 0;
930  return true;
931  }
932  Info->Siblings = SiblingPenalty[*std::max_element(
933  Info->SChain.begin(), Info->SChain.end(),
934  [&](MachineInstr *A, MachineInstr *B) -> bool {
935  return SiblingPenalty[A].size() < SiblingPenalty[B].size();
936  })];
937  Info->Siblings.remove_if([&](unsigned ID) { return ID == Info->ID; });
938  // The loop below computes the number of another VGPR to SGPR V2SCopies
939  // which contribute to the current copy SALU chain. We assume that all the
940  // V2SCopies with the same source virtual register will be squashed to one
941  // by regalloc. Also we take care of the V2SCopies of the differnt subregs
942  // of the same register.
944  for (auto J : Info->Siblings) {
945  auto InfoIt = V2SCopies.find(J);
946  if (InfoIt != V2SCopies.end()) {
947  MachineInstr *SiblingCopy = InfoIt->getSecond().Copy;
948  if (SiblingCopy->isImplicitDef())
949  // the COPY has already been MoveToVALUed
950  continue;
951 
952  SrcRegs.insert(std::make_pair(SiblingCopy->getOperand(1).getReg(),
953  SiblingCopy->getOperand(1).getSubReg()));
954  }
955  }
956  Info->SiblingPenalty = SrcRegs.size();
957 
958  unsigned Penalty =
959  Info->NumSVCopies + Info->SiblingPenalty + Info->NumReadfirstlanes;
960  unsigned Profit = Info->SChain.size();
961  Info->Score = Penalty > Profit ? 0 : Profit - Penalty;
962  Info->NeedToBeConvertedToVALU = Info->Score < 3;
963  return Info->NeedToBeConvertedToVALU;
964 }
965 
966 void SIFixSGPRCopies::lowerVGPR2SGPRCopies(MachineFunction &MF) {
967 
968  SmallVector<unsigned, 8> LoweringWorklist;
969  for (auto &C : V2SCopies) {
970  if (needToBeConvertedToVALU(&C.second))
971  LoweringWorklist.push_back(C.second.ID);
972  }
973 
974  while (!LoweringWorklist.empty()) {
975  unsigned CurID = LoweringWorklist.pop_back_val();
976  auto CurInfoIt = V2SCopies.find(CurID);
977  if (CurInfoIt != V2SCopies.end()) {
978  V2SCopyInfo C = CurInfoIt->getSecond();
979  LLVM_DEBUG(dbgs() << "Processing ...\n"; C.dump());
980  for (auto S : C.Siblings) {
981  auto SibInfoIt = V2SCopies.find(S);
982  if (SibInfoIt != V2SCopies.end()) {
983  V2SCopyInfo &SI = SibInfoIt->getSecond();
984  LLVM_DEBUG(dbgs() << "Sibling:\n"; SI.dump());
985  if (!SI.NeedToBeConvertedToVALU) {
986  set_subtract(SI.SChain, C.SChain);
987  if (needToBeConvertedToVALU(&SI))
988  LoweringWorklist.push_back(SI.ID);
989  }
990  SI.Siblings.remove_if([&](unsigned ID) { return ID == C.ID; });
991  }
992  }
993  LLVM_DEBUG(dbgs() << "V2S copy " << *C.Copy
994  << " is being turned to VALU\n");
995  V2SCopies.erase(C.ID);
996  TII->moveToVALU(*C.Copy, MDT);
997  }
998  }
999 
1000  // Now do actual lowering
1001  for (auto C : V2SCopies) {
1002  MachineInstr *MI = C.second.Copy;
1003  MachineBasicBlock *MBB = MI->getParent();
1004  // We decide to turn V2S copy to v_readfirstlane_b32
1005  // remove it from the V2SCopies and remove it from all its siblings
1006  LLVM_DEBUG(dbgs() << "V2S copy " << *MI
1007  << " is being turned to v_readfirstlane_b32"
1008  << " Score: " << C.second.Score << "\n");
1009  Register DstReg = MI->getOperand(0).getReg();
1010  Register SrcReg = MI->getOperand(1).getReg();
1011  unsigned SubReg = MI->getOperand(1).getSubReg();
1012  const TargetRegisterClass *SrcRC =
1013  TRI->getRegClassForOperandReg(*MRI, MI->getOperand(1));
1014  size_t SrcSize = TRI->getRegSizeInBits(*SrcRC);
1015  if (SrcSize == 16) {
1016  // HACK to handle possible 16bit VGPR source
1017  auto MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
1018  TII->get(AMDGPU::V_READFIRSTLANE_B32), DstReg);
1019  MIB.addReg(SrcReg, 0, AMDGPU::NoSubRegister);
1020  } else if (SrcSize == 32) {
1021  auto MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
1022  TII->get(AMDGPU::V_READFIRSTLANE_B32), DstReg);
1023  MIB.addReg(SrcReg, 0, SubReg);
1024  } else {
1025  auto Result = BuildMI(*MBB, MI, MI->getDebugLoc(),
1026  TII->get(AMDGPU::REG_SEQUENCE), DstReg);
1027  int N = TRI->getRegSizeInBits(*SrcRC) / 32;
1028  for (int i = 0; i < N; i++) {
1029  Register PartialSrc = TII->buildExtractSubReg(
1030  Result, *MRI, MI->getOperand(1), SrcRC,
1031  TRI->getSubRegFromChannel(i), &AMDGPU::VGPR_32RegClass);
1032  Register PartialDst =
1033  MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
1034  BuildMI(*MBB, *Result, Result->getDebugLoc(),
1035  TII->get(AMDGPU::V_READFIRSTLANE_B32), PartialDst)
1036  .addReg(PartialSrc);
1037  Result.addReg(PartialDst).addImm(TRI->getSubRegFromChannel(i));
1038  }
1039  }
1040  MI->eraseFromParent();
1041  }
1042 }
1043 
1044 void SIFixSGPRCopies::fixSCCCopies(MachineFunction &MF) {
1045  bool IsWave32 = MF.getSubtarget<GCNSubtarget>().isWave32();
1046  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE;
1047  ++BI) {
1048  MachineBasicBlock *MBB = &*BI;
1049  for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
1050  ++I) {
1051  MachineInstr &MI = *I;
1052  // May already have been lowered.
1053  if (!MI.isCopy())
1054  continue;
1055  Register SrcReg = MI.getOperand(1).getReg();
1056  Register DstReg = MI.getOperand(0).getReg();
1057  if (SrcReg == AMDGPU::SCC) {
1058  Register SCCCopy = MRI->createVirtualRegister(
1059  TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID));
1060  I = BuildMI(*MI.getParent(), std::next(MachineBasicBlock::iterator(MI)),
1061  MI.getDebugLoc(),
1062  TII->get(IsWave32 ? AMDGPU::S_CSELECT_B32
1063  : AMDGPU::S_CSELECT_B64),
1064  SCCCopy)
1065  .addImm(-1)
1066  .addImm(0);
1067  I = BuildMI(*MI.getParent(), std::next(I), I->getDebugLoc(),
1068  TII->get(AMDGPU::COPY), DstReg)
1069  .addReg(SCCCopy);
1070  MI.eraseFromParent();
1071  continue;
1072  }
1073  if (DstReg == AMDGPU::SCC) {
1074  unsigned Opcode = IsWave32 ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
1075  Register Exec = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1076  Register Tmp = MRI->createVirtualRegister(TRI->getBoolRC());
1077  I = BuildMI(*MI.getParent(), std::next(MachineBasicBlock::iterator(MI)),
1078  MI.getDebugLoc(), TII->get(Opcode))
1079  .addReg(Tmp, getDefRegState(true))
1080  .addReg(SrcReg)
1081  .addReg(Exec);
1082  MI.eraseFromParent();
1083  }
1084  }
1085  }
1086 }
llvm::MachineDominatorTree::findNearestCommonDominator
MachineBasicBlock * findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B)
findNearestCommonDominator - Find nearest common dominator basic block for basic block A and B.
Definition: MachineDominators.h:160
i
i
Definition: README.txt:29
llvm::TargetMachine::getOptLevel
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Definition: TargetMachine.cpp:182
llvm::HexagonInstrInfo::isSchedulingBoundary
bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const override
Test if the given instruction should be considered a scheduling boundary.
Definition: HexagonInstrInfo.cpp:1785
llvm::MachineFunctionProperties::hasProperty
bool hasProperty(Property P) const
Definition: MachineFunction.h:192
llvm::MachineBasicBlock::pred_begin
pred_iterator pred_begin()
Definition: MachineBasicBlock.h:353
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:108
llvm::MachineInstr::getOperandNo
unsigned getOperandNo(const_mop_iterator I) const
Returns the number of the operand iterator I points to.
Definition: MachineInstr.h:706
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
isReachable
static bool isReachable(const MachineInstr *From, const MachineInstr *To, const MachineBasicBlock *CutOff, MachineDominatorTree &MDT)
Definition: SIFixSGPRCopies.cpp:399
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::MachineInstr::isImplicitDef
bool isImplicitDef() const
Definition: MachineInstr.h:1305
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
UseMI
MachineInstrBuilder & UseMI
Definition: AArch64ExpandPseudoInsts.cpp:107
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:156
PHI
Rewrite undef for PHI
Definition: AMDGPURewriteUndefForPHI.cpp:101
llvm::createSIFixSGPRCopiesPass
FunctionPass * createSIFixSGPRCopiesPass()
Definition: SIFixSGPRCopies.cpp:177
llvm::MachineDominatorTree::properlyDominates
bool properlyDominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
Definition: MachineDominators.h:146
SetOperations.h
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
llvm::MachineInstrBuilder::add
const MachineInstrBuilder & add(const MachineOperand &MO) const
Definition: MachineInstrBuilder.h:224
llvm::MachineInstr::isCompare
bool isCompare(QueryType Type=IgnoreBundle) const
Return true if this instruction is a comparison.
Definition: MachineInstr.h:941
llvm::SmallVector< MachineInstr *, 4 >
llvm::MachineFunction::end
iterator end()
Definition: MachineFunction.h:856
llvm::printMBBReference
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Definition: MachineBasicBlock.cpp:117
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:237
llvm::MachineInstr::getDesc
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:513
llvm::MachineRegisterInfo::use_instructions
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
Definition: MachineRegisterInfo.h:493
llvm::MachineRegisterInfo::use_operands
iterator_range< use_iterator > use_operands(Register Reg) const
Definition: MachineRegisterInfo.h:477
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:136
llvm::MachineInstr::isCopy
bool isCopy() const
Definition: MachineInstr.h:1336
llvm::GCNSubtarget
Definition: GCNSubtarget.h:31
llvm::dump
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
Definition: SparseBitVector.h:877
llvm::MachineDominatorTree::dominates
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
Definition: MachineDominators.h:114
llvm::AMDGPU::getNamedOperandIdx
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, DenseMapInfo< ValueT >, detail::DenseSetPair< ValueT > >, DenseMapInfo< ValueT > >::insert
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:167
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::Register::isPhysical
bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:97
llvm::getDefRegState
unsigned getDefRegState(bool B)
Definition: MachineInstrBuilder.h:540
llvm::MachineBasicBlock::getFirstInstrTerminator
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
Definition: MachineBasicBlock.cpp:248
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:667
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:98
llvm::MachineFunctionProperties::Property::Selected
@ Selected
TargetMachine.h
llvm::MachineOperand::ChangeToRegister
void ChangeToRegister(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value.
Definition: MachineOperand.cpp:242
GCNSubtarget.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineOperand::getImm
int64_t getImm() const
Definition: MachineOperand.h:546
llvm::set_subtract
void set_subtract(S1Ty &S1, const S2Ty &S2)
set_subtract(A, B) - Compute A := A - B
Definition: SetOperations.h:63
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
SI
@ SI
Definition: SIInstrInfo.cpp:7966
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::MachineFunction::getProperties
const MachineFunctionProperties & getProperties() const
Get the function properties.
Definition: MachineFunction.h:748
llvm::MachineRegisterInfo::reg_nodbg_operands
iterator_range< reg_nodbg_iterator > reg_nodbg_operands(Register Reg) const
Definition: MachineRegisterInfo.h:345
false
Definition: StackSlotColoring.cpp:141
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::MachineOperand::ChangeToImmediate
void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
Definition: MachineOperand.cpp:157
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
getFirstNonPrologue
static MachineBasicBlock::iterator getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII)
Definition: SIFixSGPRCopies.cpp:418
copies
SI Fix SGPR copies
Definition: SIFixSGPRCopies.cpp:171
llvm::detail::DenseSetImpl< ValueT, DenseMap< ValueT, detail::DenseSetEmpty, DenseMapInfo< ValueT >, detail::DenseSetPair< ValueT > >, DenseMapInfo< ValueT > >::size
size_type size() const
Definition: DenseSet.h:81
llvm::M0
unsigned M0(unsigned Val)
Definition: VE.h:465
llvm::MachineFunction::begin
iterator begin()
Definition: MachineFunction.h:854
llvm::SIInstrFlags::WQM
@ WQM
Definition: SIDefines.h:77
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE, "SI Fix SGPR copies", false, false) INITIALIZE_PASS_END(SIFixSGPRCopies
llvm::SIRegisterInfo
Definition: SIRegisterInfo.h:30
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
llvm::MachineRegisterInfo::getVRegDef
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition: MachineRegisterInfo.cpp:396
llvm::SetVector::empty
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:72
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::MachineRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
Definition: MachineRegisterInfo.h:647
llvm::Register::isVirtual
bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
llvm::MachineRegisterInfo::clearKillFlags
void clearKillFlags(Register Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
Definition: MachineRegisterInfo.cpp:433
llvm::DenseSet
Implements a dense probed hash-table based set.
Definition: DenseSet.h:268
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:657
llvm::cl::opt< bool >
llvm::PPC::Predicate
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
llvm::SmallSet::count
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:165
AMDGPUMCTargetDesc.h
llvm::MachineBasicBlock::pred_end
pred_iterator pred_end()
Definition: MachineBasicBlock.h:355
llvm::TargetRegisterInfo::getRegClass
const TargetRegisterClass * getRegClass(unsigned i) const
Returns the register class associated with the enumeration value.
Definition: TargetRegisterInfo.h:771
llvm::MachineOperand::isReg
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Definition: MachineOperand.h:320
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::MachineRegisterInfo::use_instr_begin
use_instr_iterator use_instr_begin(Register RegNo) const
Definition: MachineRegisterInfo.h:485
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
DEBUG_TYPE
#define DEBUG_TYPE
Definition: SIFixSGPRCopies.cpp:77
llvm::DenseMap
Definition: DenseMap.h:714
llvm::codeview::FrameCookieKind::Copy
@ Copy
llvm::MachineRegisterInfo::def_instructions
iterator_range< def_instr_iterator > def_instructions(Register Reg) const
Definition: MachineRegisterInfo.h:413
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:447
llvm::MachineBasicBlock::getFirstNonPHI
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: MachineBasicBlock.cpp:197
llvm::SIFixSGPRCopiesID
char & SIFixSGPRCopiesID
Definition: SIFixSGPRCopies.cpp:175
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MachineBasicBlock::getParent
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Definition: MachineBasicBlock.h:261
llvm::MachineInstr::isPHI
bool isPHI() const
Definition: MachineInstr.h:1300
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:359
llvm::MachineBasicBlock::predecessors
iterator_range< pred_iterator > predecessors()
Definition: MachineBasicBlock.h:386
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::SetVector::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::CodeGenOpt::None
@ None
Definition: CodeGen.h:53
llvm::AMDGPU::CPol::SCC
@ SCC
Definition: SIDefines.h:307
llvm::SetVector::pop_back_val
T pop_back_val()
Definition: SetVector.h:232
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1741
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:265
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:1009
AMDGPU.h
llvm::MachineInstr::getOpcode
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:516
isSGPRToVGPRCopy
static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC, const TargetRegisterClass *DstRC, const SIRegisterInfo &TRI)
Definition: SIFixSGPRCopies.cpp:209
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::MCID::MoveImm
@ MoveImm
Definition: MCInstrDesc.h:161
llvm::MachineInstr::getParent
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:313
llvm::Init
Definition: Record.h:281
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::MachineOperand::getSubReg
unsigned getSubReg() const
Definition: MachineOperand.h:364
hoistAndMergeSGPRInits
static bool hoistAndMergeSGPRInits(unsigned Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo *TRI, MachineDominatorTree &MDT, const TargetInstrInfo *TII)
Definition: SIFixSGPRCopies.cpp:430
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
foldVGPRCopyIntoRegSequence
static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI, const SIRegisterInfo *TRI, const SIInstrInfo *TII, MachineRegisterInfo &MRI)
Definition: SIFixSGPRCopies.cpp:257
llvm::MachineInstr::isRegSequence
bool isRegSequence() const
Definition: MachineInstr.h:1328
tryChangeVGPRtoSGPRinCopy
static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI, const SIRegisterInfo *TRI, const SIInstrInfo *TII)
Definition: SIFixSGPRCopies.cpp:216
llvm::TargetRegisterInfo::getRegSizeInBits
unsigned getRegSizeInBits(const TargetRegisterClass &RC) const
Return the size in bits of a register from class RC.
Definition: TargetRegisterInfo.h:280
llvm::MachineFunction::getTarget
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Definition: MachineFunction.h:653
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
isSafeToFoldImmIntoCopy
static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy, const MachineInstr *MoveImm, const SIInstrInfo *TII, unsigned &SMovOp, int64_t &Imm)
Definition: SIFixSGPRCopies.cpp:336
llvm::RegState::Kill
@ Kill
The last use of a register.
Definition: MachineInstrBuilder.h:48
llvm::SmallSet::insert
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:178
llvm::MachineOperand::isImm
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
Definition: MachineOperand.h:322
llvm::AMDGPU::Hwreg::Width
Width
Definition: SIDefines.h:433
llvm::RISCVMatInt::Imm
@ Imm
Definition: RISCVMatInt.h:23
llvm::SIInstrInfo
Definition: SIInstrInfo.h:44
isVGPRToSGPRCopy
static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC, const TargetRegisterClass *DstRC, const SIRegisterInfo &TRI)
Definition: SIFixSGPRCopies.cpp:202
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:357
llvm::MachineRegisterInfo::hasOneUse
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
Definition: MachineRegisterInfo.h:518
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:305
llvm::SmallSet::size
size_type size() const
Definition: SmallSet.h:160
N
#define N
DefMI
MachineInstrBuilder MachineInstrBuilder & DefMI
Definition: AArch64ExpandPseudoInsts.cpp:108
llvm::SmallVectorImpl::pop_back_val
T pop_back_val()
Definition: SmallVector.h:677
Users
iv Induction Variable Users
Definition: IVUsers.cpp:48
Fix
Falkor HW Prefetch Fix
Definition: AArch64FalkorHWPFFix.cpp:114
getCopyRegClasses
static std::pair< const TargetRegisterClass *, const TargetRegisterClass * > getCopyRegClasses(const MachineInstr &Copy, const SIRegisterInfo &TRI, const MachineRegisterInfo &MRI)
Definition: SIFixSGPRCopies.cpp:182
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
llvm::orc::SymbolState::Resolved
@ Resolved
Queried, materialization begun.
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::orc::MemProt::Exec
@ Exec
searchPredecessors
bool searchPredecessors(const MachineBasicBlock *MBB, const MachineBasicBlock *CutOff, UnaryPredicate Predicate)
Definition: SIFixSGPRCopies.cpp:371
llvm::cl::desc
Definition: CommandLine.h:413
llvm::AMDGPU::VGPRIndexMode::Id
Id
Definition: SIDefines.h:241
EnableM0Merge
static cl::opt< bool > EnableM0Merge("amdgpu-enable-merge-m0", cl::desc("Merge and hoist M0 initializations"), cl::init(true))
llvm::MachineDominatorTree
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
Definition: MachineDominators.h:51
llvm::SetVector
A vector that has set insertion semantics.
Definition: SetVector.h:40
llvm::MachineInstr::eraseFromParent
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
Definition: MachineInstr.cpp:692
llvm::MachineInstrBundleIterator< MachineInstr >
InitializePasses.h
llvm::MCInstrDesc::getNumOperands
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:230
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:307
SubReg
unsigned SubReg
Definition: AArch64AdvSIMDScalarPass.cpp:104
llvm::MachineRegisterInfo::setRegClass
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Definition: MachineRegisterInfo.cpp:56
llvm::MachineInstr::getNumExplicitDefs
unsigned getNumExplicitDefs() const
Returns the number of non-implicit definitions.
Definition: MachineInstr.cpp:740
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
MachineDominators.h