LLVM  10.0.0svn
SIFixSGPRCopies.cpp
Go to the documentation of this file.
1 //===- SIFixSGPRCopies.cpp - Remove potential VGPR => SGPR copies ---------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Copies from VGPR to SGPR registers are illegal and the register coalescer
11 /// will sometimes generate these illegal copies in situations like this:
12 ///
13 /// Register Class <vsrc> is the union of <vgpr> and <sgpr>
14 ///
15 /// BB0:
16 /// %0 <sgpr> = SCALAR_INST
17 /// %1 <vsrc> = COPY %0 <sgpr>
18 /// ...
19 /// BRANCH %cond BB1, BB2
20 /// BB1:
21 /// %2 <vgpr> = VECTOR_INST
22 /// %3 <vsrc> = COPY %2 <vgpr>
23 /// BB2:
24 /// %4 <vsrc> = PHI %1 <vsrc>, <%bb.0>, %3 <vrsc>, <%bb.1>
25 /// %5 <vgpr> = VECTOR_INST %4 <vsrc>
26 ///
27 ///
28 /// The coalescer will begin at BB0 and eliminate its copy, then the resulting
29 /// code will look like this:
30 ///
31 /// BB0:
32 /// %0 <sgpr> = SCALAR_INST
33 /// ...
34 /// BRANCH %cond BB1, BB2
35 /// BB1:
36 /// %2 <vgpr> = VECTOR_INST
37 /// %3 <vsrc> = COPY %2 <vgpr>
38 /// BB2:
39 /// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <vsrc>, <%bb.1>
40 /// %5 <vgpr> = VECTOR_INST %4 <sgpr>
41 ///
42 /// Now that the result of the PHI instruction is an SGPR, the register
43 /// allocator is now forced to constrain the register class of %3 to
44 /// <sgpr> so we end up with final code like this:
45 ///
46 /// BB0:
47 /// %0 <sgpr> = SCALAR_INST
48 /// ...
49 /// BRANCH %cond BB1, BB2
50 /// BB1:
51 /// %2 <vgpr> = VECTOR_INST
52 /// %3 <sgpr> = COPY %2 <vgpr>
53 /// BB2:
54 /// %4 <sgpr> = PHI %0 <sgpr>, <%bb.0>, %3 <sgpr>, <%bb.1>
55 /// %5 <vgpr> = VECTOR_INST %4 <sgpr>
56 ///
57 /// Now this code contains an illegal copy from a VGPR to an SGPR.
58 ///
59 /// In order to avoid this problem, this pass searches for PHI instructions
60 /// which define a <vsrc> register and constrains its definition class to
61 /// <vgpr> if the user of the PHI's definition register is a vector instruction.
62 /// If the PHI's definition class is constrained to <vgpr> then the coalescer
63 /// will be unable to perform the COPY removal from the above example which
64 /// ultimately led to the creation of an illegal COPY.
65 //===----------------------------------------------------------------------===//
66 
67 #include "AMDGPU.h"
68 #include "AMDGPUSubtarget.h"
69 #include "SIInstrInfo.h"
70 #include "SIRegisterInfo.h"
72 #include "llvm/ADT/DenseSet.h"
73 #include "llvm/ADT/STLExtras.h"
74 #include "llvm/ADT/SmallSet.h"
75 #include "llvm/ADT/SmallVector.h"
85 #include "llvm/Pass.h"
86 #include "llvm/Support/CodeGen.h"
88 #include "llvm/Support/Debug.h"
91 #include <cassert>
92 #include <cstdint>
93 #include <iterator>
94 #include <list>
95 #include <map>
96 #include <tuple>
97 #include <utility>
98 
99 using namespace llvm;
100 
101 #define DEBUG_TYPE "si-fix-sgpr-copies"
102 
104  "amdgpu-enable-merge-m0",
105  cl::desc("Merge and hoist M0 initializations"),
106  cl::init(true));
107 
108 namespace {
109 
110 class SIFixSGPRCopies : public MachineFunctionPass {
112 
113 public:
114  static char ID;
115 
117  const SIRegisterInfo *TRI;
118  const SIInstrInfo *TII;
119 
120  SIFixSGPRCopies() : MachineFunctionPass(ID) {}
121 
122  bool runOnMachineFunction(MachineFunction &MF) override;
123 
124  void processPHINode(MachineInstr &MI);
125 
126  StringRef getPassName() const override { return "SI Fix SGPR copies"; }
127 
128  void getAnalysisUsage(AnalysisUsage &AU) const override {
131  AU.setPreservesCFG();
133  }
134 };
135 
136 } // end anonymous namespace
137 
138 INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE,
139  "SI Fix SGPR copies", false, false)
142  "SI Fix SGPR copies", false, false)
143 
144 char SIFixSGPRCopies::ID = 0;
145 
146 char &llvm::SIFixSGPRCopiesID = SIFixSGPRCopies::ID;
147 
149  return new SIFixSGPRCopies();
150 }
151 
152 static bool hasVectorOperands(const MachineInstr &MI,
153  const SIRegisterInfo *TRI) {
155  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
156  if (!MI.getOperand(i).isReg() ||
158  continue;
159 
160  if (TRI->hasVectorRegisters(MRI.getRegClass(MI.getOperand(i).getReg())))
161  return true;
162  }
163  return false;
164 }
165 
166 static std::pair<const TargetRegisterClass *, const TargetRegisterClass *>
168  const SIRegisterInfo &TRI,
169  const MachineRegisterInfo &MRI) {
170  Register DstReg = Copy.getOperand(0).getReg();
171  Register SrcReg = Copy.getOperand(1).getReg();
172 
173  const TargetRegisterClass *SrcRC = Register::isVirtualRegister(SrcReg)
174  ? MRI.getRegClass(SrcReg)
175  : TRI.getPhysRegClass(SrcReg);
176 
177  // We don't really care about the subregister here.
178  // SrcRC = TRI.getSubRegClass(SrcRC, Copy.getOperand(1).getSubReg());
179 
180  const TargetRegisterClass *DstRC = Register::isVirtualRegister(DstReg)
181  ? MRI.getRegClass(DstReg)
182  : TRI.getPhysRegClass(DstReg);
183 
184  return std::make_pair(SrcRC, DstRC);
185 }
186 
187 static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC,
188  const TargetRegisterClass *DstRC,
189  const SIRegisterInfo &TRI) {
190  return SrcRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(DstRC) &&
191  TRI.hasVectorRegisters(SrcRC);
192 }
193 
194 static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC,
195  const TargetRegisterClass *DstRC,
196  const SIRegisterInfo &TRI) {
197  return DstRC != &AMDGPU::VReg_1RegClass && TRI.isSGPRClass(SrcRC) &&
198  TRI.hasVectorRegisters(DstRC);
199 }
200 
202  const SIRegisterInfo *TRI,
203  const SIInstrInfo *TII) {
205  auto &Src = MI.getOperand(1);
206  Register DstReg = MI.getOperand(0).getReg();
207  Register SrcReg = Src.getReg();
208  if (!Register::isVirtualRegister(SrcReg) ||
210  return false;
211 
212  for (const auto &MO : MRI.reg_nodbg_operands(DstReg)) {
213  const auto *UseMI = MO.getParent();
214  if (UseMI == &MI)
215  continue;
216  if (MO.isDef() || UseMI->getParent() != MI.getParent() ||
217  UseMI->getOpcode() <= TargetOpcode::GENERIC_OP_END ||
218  !TII->isOperandLegal(*UseMI, UseMI->getOperandNo(&MO), &Src))
219  return false;
220  }
221  // Change VGPR to SGPR destination.
222  MRI.setRegClass(DstReg, TRI->getEquivalentSGPRClass(MRI.getRegClass(DstReg)));
223  return true;
224 }
225 
226 // Distribute an SGPR->VGPR copy of a REG_SEQUENCE into a VGPR REG_SEQUENCE.
227 //
228 // SGPRx = ...
229 // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
230 // VGPRz = COPY SGPRy
231 //
232 // ==>
233 //
234 // VGPRx = COPY SGPRx
235 // VGPRz = REG_SEQUENCE VGPRx, sub0
236 //
237 // This exposes immediate folding opportunities when materializing 64-bit
238 // immediates.
240  const SIRegisterInfo *TRI,
241  const SIInstrInfo *TII,
243  assert(MI.isRegSequence());
244 
245  Register DstReg = MI.getOperand(0).getReg();
246  if (!TRI->isSGPRClass(MRI.getRegClass(DstReg)))
247  return false;
248 
249  if (!MRI.hasOneUse(DstReg))
250  return false;
251 
252  MachineInstr &CopyUse = *MRI.use_instr_begin(DstReg);
253  if (!CopyUse.isCopy())
254  return false;
255 
256  // It is illegal to have vreg inputs to a physreg defining reg_sequence.
258  return false;
259 
260  const TargetRegisterClass *SrcRC, *DstRC;
261  std::tie(SrcRC, DstRC) = getCopyRegClasses(CopyUse, *TRI, MRI);
262 
263  if (!isSGPRToVGPRCopy(SrcRC, DstRC, *TRI))
264  return false;
265 
266  if (tryChangeVGPRtoSGPRinCopy(CopyUse, TRI, TII))
267  return true;
268 
269  // TODO: Could have multiple extracts?
270  unsigned SubReg = CopyUse.getOperand(1).getSubReg();
271  if (SubReg != AMDGPU::NoSubRegister)
272  return false;
273 
274  MRI.setRegClass(DstReg, DstRC);
275 
276  // SGPRx = ...
277  // SGPRy = REG_SEQUENCE SGPRx, sub0 ...
278  // VGPRz = COPY SGPRy
279 
280  // =>
281  // VGPRx = COPY SGPRx
282  // VGPRz = REG_SEQUENCE VGPRx, sub0
283 
284  MI.getOperand(0).setReg(CopyUse.getOperand(0).getReg());
285  bool IsAGPR = TRI->hasAGPRs(DstRC);
286 
287  for (unsigned I = 1, N = MI.getNumOperands(); I != N; I += 2) {
288  Register SrcReg = MI.getOperand(I).getReg();
289  unsigned SrcSubReg = MI.getOperand(I).getSubReg();
290 
291  const TargetRegisterClass *SrcRC = MRI.getRegClass(SrcReg);
292  assert(TRI->isSGPRClass(SrcRC) &&
293  "Expected SGPR REG_SEQUENCE to only have SGPR inputs");
294 
295  SrcRC = TRI->getSubRegClass(SrcRC, SrcSubReg);
296  const TargetRegisterClass *NewSrcRC = TRI->getEquivalentVGPRClass(SrcRC);
297 
298  Register TmpReg = MRI.createVirtualRegister(NewSrcRC);
299 
300  BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(AMDGPU::COPY),
301  TmpReg)
302  .add(MI.getOperand(I));
303 
304  if (IsAGPR) {
305  const TargetRegisterClass *NewSrcRC = TRI->getEquivalentAGPRClass(SrcRC);
306  Register TmpAReg = MRI.createVirtualRegister(NewSrcRC);
307  unsigned Opc = NewSrcRC == &AMDGPU::AGPR_32RegClass ?
308  AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
309  BuildMI(*MI.getParent(), &MI, MI.getDebugLoc(), TII->get(Opc),
310  TmpAReg)
311  .addReg(TmpReg, RegState::Kill);
312  TmpReg = TmpAReg;
313  }
314 
315  MI.getOperand(I).setReg(TmpReg);
316  }
317 
318  CopyUse.eraseFromParent();
319  return true;
320 }
321 
322 static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy,
323  const MachineInstr *MoveImm,
324  const SIInstrInfo *TII,
325  unsigned &SMovOp,
326  int64_t &Imm) {
327  if (Copy->getOpcode() != AMDGPU::COPY)
328  return false;
329 
330  if (!MoveImm->isMoveImmediate())
331  return false;
332 
333  const MachineOperand *ImmOp =
334  TII->getNamedOperand(*MoveImm, AMDGPU::OpName::src0);
335  if (!ImmOp->isImm())
336  return false;
337 
338  // FIXME: Handle copies with sub-regs.
339  if (Copy->getOperand(0).getSubReg())
340  return false;
341 
342  switch (MoveImm->getOpcode()) {
343  default:
344  return false;
345  case AMDGPU::V_MOV_B32_e32:
346  SMovOp = AMDGPU::S_MOV_B32;
347  break;
348  case AMDGPU::V_MOV_B64_PSEUDO:
349  SMovOp = AMDGPU::S_MOV_B64;
350  break;
351  }
352  Imm = ImmOp->getImm();
353  return true;
354 }
355 
356 template <class UnaryPredicate>
358  const MachineBasicBlock *CutOff,
359  UnaryPredicate Predicate) {
360  if (MBB == CutOff)
361  return false;
362 
365  MBB->pred_end());
366 
367  while (!Worklist.empty()) {
368  MachineBasicBlock *MBB = Worklist.pop_back_val();
369 
370  if (!Visited.insert(MBB).second)
371  continue;
372  if (MBB == CutOff)
373  continue;
374  if (Predicate(MBB))
375  return true;
376 
377  Worklist.append(MBB->pred_begin(), MBB->pred_end());
378  }
379 
380  return false;
381 }
382 
383 // Checks if there is potential path From instruction To instruction.
384 // If CutOff is specified and it sits in between of that path we ignore
385 // a higher portion of the path and report it is not reachable.
386 static bool isReachable(const MachineInstr *From,
387  const MachineInstr *To,
388  const MachineBasicBlock *CutOff,
389  MachineDominatorTree &MDT) {
390  // If either From block dominates To block or instructions are in the same
391  // block and From is higher.
392  if (MDT.dominates(From, To))
393  return true;
394 
395  const MachineBasicBlock *MBBFrom = From->getParent();
396  const MachineBasicBlock *MBBTo = To->getParent();
397  if (MBBFrom == MBBTo)
398  return false;
399 
400  // Instructions are in different blocks, do predecessor search.
401  // We should almost never get here since we do not usually produce M0 stores
402  // other than -1.
403  return searchPredecessors(MBBTo, CutOff, [MBBFrom]
404  (const MachineBasicBlock *MBB) { return MBB == MBBFrom; });
405 }
406 
407 // Return the first non-prologue instruction in the block.
411  while (I != MBB->end() && TII->isBasicBlockPrologue(*I))
412  ++I;
413 
414  return I;
415 }
416 
417 // Hoist and merge identical SGPR initializations into a common predecessor.
418 // This is intended to combine M0 initializations, but can work with any
419 // SGPR. A VGPR cannot be processed since we cannot guarantee vector
420 // executioon.
421 static bool hoistAndMergeSGPRInits(unsigned Reg,
422  const MachineRegisterInfo &MRI,
423  const TargetRegisterInfo *TRI,
425  const TargetInstrInfo *TII) {
426  // List of inits by immediate value.
427  using InitListMap = std::map<unsigned, std::list<MachineInstr *>>;
428  InitListMap Inits;
429  // List of clobbering instructions.
431  // List of instructions marked for deletion.
432  SmallSet<MachineInstr*, 8> MergedInstrs;
433 
434  bool Changed = false;
435 
436  for (auto &MI : MRI.def_instructions(Reg)) {
437  MachineOperand *Imm = nullptr;
438  for (auto &MO : MI.operands()) {
439  if ((MO.isReg() && ((MO.isDef() && MO.getReg() != Reg) || !MO.isDef())) ||
440  (!MO.isImm() && !MO.isReg()) || (MO.isImm() && Imm)) {
441  Imm = nullptr;
442  break;
443  } else if (MO.isImm())
444  Imm = &MO;
445  }
446  if (Imm)
447  Inits[Imm->getImm()].push_front(&MI);
448  else
449  Clobbers.push_back(&MI);
450  }
451 
452  for (auto &Init : Inits) {
453  auto &Defs = Init.second;
454 
455  for (auto I1 = Defs.begin(), E = Defs.end(); I1 != E; ) {
456  MachineInstr *MI1 = *I1;
457 
458  for (auto I2 = std::next(I1); I2 != E; ) {
459  MachineInstr *MI2 = *I2;
460 
461  // Check any possible interference
462  auto interferes = [&](MachineBasicBlock::iterator From,
463  MachineBasicBlock::iterator To) -> bool {
464 
465  assert(MDT.dominates(&*To, &*From));
466 
467  auto interferes = [&MDT, From, To](MachineInstr* &Clobber) -> bool {
468  const MachineBasicBlock *MBBFrom = From->getParent();
469  const MachineBasicBlock *MBBTo = To->getParent();
470  bool MayClobberFrom = isReachable(Clobber, &*From, MBBTo, MDT);
471  bool MayClobberTo = isReachable(Clobber, &*To, MBBTo, MDT);
472  if (!MayClobberFrom && !MayClobberTo)
473  return false;
474  if ((MayClobberFrom && !MayClobberTo) ||
475  (!MayClobberFrom && MayClobberTo))
476  return true;
477  // Both can clobber, this is not an interference only if both are
478  // dominated by Clobber and belong to the same block or if Clobber
479  // properly dominates To, given that To >> From, so it dominates
480  // both and located in a common dominator.
481  return !((MBBFrom == MBBTo &&
482  MDT.dominates(Clobber, &*From) &&
483  MDT.dominates(Clobber, &*To)) ||
484  MDT.properlyDominates(Clobber->getParent(), MBBTo));
485  };
486 
487  return (llvm::any_of(Clobbers, interferes)) ||
488  (llvm::any_of(Inits, [&](InitListMap::value_type &C) {
489  return C.first != Init.first &&
490  llvm::any_of(C.second, interferes);
491  }));
492  };
493 
494  if (MDT.dominates(MI1, MI2)) {
495  if (!interferes(MI2, MI1)) {
496  LLVM_DEBUG(dbgs()
497  << "Erasing from "
498  << printMBBReference(*MI2->getParent()) << " " << *MI2);
499  MergedInstrs.insert(MI2);
500  Changed = true;
501  ++I2;
502  continue;
503  }
504  } else if (MDT.dominates(MI2, MI1)) {
505  if (!interferes(MI1, MI2)) {
506  LLVM_DEBUG(dbgs()
507  << "Erasing from "
508  << printMBBReference(*MI1->getParent()) << " " << *MI1);
509  MergedInstrs.insert(MI1);
510  Changed = true;
511  ++I1;
512  break;
513  }
514  } else {
515  auto *MBB = MDT.findNearestCommonDominator(MI1->getParent(),
516  MI2->getParent());
517  if (!MBB) {
518  ++I2;
519  continue;
520  }
521 
523  if (!interferes(MI1, I) && !interferes(MI2, I)) {
524  LLVM_DEBUG(dbgs()
525  << "Erasing from "
526  << printMBBReference(*MI1->getParent()) << " " << *MI1
527  << "and moving from "
528  << printMBBReference(*MI2->getParent()) << " to "
529  << printMBBReference(*I->getParent()) << " " << *MI2);
530  I->getParent()->splice(I, MI2->getParent(), MI2);
531  MergedInstrs.insert(MI1);
532  Changed = true;
533  ++I1;
534  break;
535  }
536  }
537  ++I2;
538  }
539  ++I1;
540  }
541  }
542 
543  // Remove initializations that were merged into another.
544  for (auto &Init : Inits) {
545  auto &Defs = Init.second;
546  auto I = Defs.begin();
547  while (I != Defs.end()) {
548  if (MergedInstrs.count(*I)) {
549  (*I)->eraseFromParent();
550  I = Defs.erase(I);
551  } else
552  ++I;
553  }
554  }
555 
556  // Try to schedule SGPR initializations as early as possible in the MBB.
557  for (auto &Init : Inits) {
558  auto &Defs = Init.second;
559  for (auto MI : Defs) {
560  auto MBB = MI->getParent();
561  MachineInstr &BoundaryMI = *getFirstNonPrologue(MBB, TII);
563  // Check if B should actually be a boundary. If not set the previous
564  // instruction as the boundary instead.
565  if (!TII->isBasicBlockPrologue(*B))
566  B++;
567 
568  auto R = std::next(MI->getReverseIterator());
569  const unsigned Threshold = 50;
570  // Search until B or Threshold for a place to insert the initialization.
571  for (unsigned I = 0; R != B && I < Threshold; ++R, ++I)
572  if (R->readsRegister(Reg, TRI) || R->definesRegister(Reg, TRI) ||
573  TII->isSchedulingBoundary(*R, MBB, *MBB->getParent()))
574  break;
575 
576  // Move to directly after R.
577  if (&*--R != MI)
578  MBB->splice(*R, MBB, MI);
579  }
580  }
581 
582  if (Changed)
583  MRI.clearKillFlags(Reg);
584 
585  return Changed;
586 }
587 
588 bool SIFixSGPRCopies::runOnMachineFunction(MachineFunction &MF) {
589  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
590  MRI = &MF.getRegInfo();
591  TRI = ST.getRegisterInfo();
592  TII = ST.getInstrInfo();
593  MDT = &getAnalysis<MachineDominatorTree>();
594 
596 
597  for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
598  BI != BE; ++BI) {
599  MachineBasicBlock &MBB = *BI;
600  for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
601  I != E; ++I) {
602  MachineInstr &MI = *I;
603 
604  switch (MI.getOpcode()) {
605  default:
606  continue;
607  case AMDGPU::COPY:
608  case AMDGPU::WQM:
609  case AMDGPU::SOFT_WQM:
610  case AMDGPU::WWM: {
611  Register DstReg = MI.getOperand(0).getReg();
612 
613  const TargetRegisterClass *SrcRC, *DstRC;
614  std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, *MRI);
615 
616  if (!Register::isVirtualRegister(DstReg)) {
617  // If the destination register is a physical register there isn't
618  // really much we can do to fix this.
619  // Some special instructions use M0 as an input. Some even only use
620  // the first lane. Insert a readfirstlane and hope for the best.
621  if (DstReg == AMDGPU::M0 && TRI->hasVectorRegisters(SrcRC)) {
622  Register TmpReg
623  = MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
624 
625  BuildMI(MBB, MI, MI.getDebugLoc(),
626  TII->get(AMDGPU::V_READFIRSTLANE_B32), TmpReg)
627  .add(MI.getOperand(1));
628  MI.getOperand(1).setReg(TmpReg);
629  }
630 
631  continue;
632  }
633 
634  if (isVGPRToSGPRCopy(SrcRC, DstRC, *TRI)) {
635  Register SrcReg = MI.getOperand(1).getReg();
636  if (!Register::isVirtualRegister(SrcReg)) {
637  TII->moveToVALU(MI, MDT);
638  break;
639  }
640 
641  MachineInstr *DefMI = MRI->getVRegDef(SrcReg);
642  unsigned SMovOp;
643  int64_t Imm;
644  // If we are just copying an immediate, we can replace the copy with
645  // s_mov_b32.
646  if (isSafeToFoldImmIntoCopy(&MI, DefMI, TII, SMovOp, Imm)) {
647  MI.getOperand(1).ChangeToImmediate(Imm);
649  MI.setDesc(TII->get(SMovOp));
650  break;
651  }
652  TII->moveToVALU(MI, MDT);
653  } else if (isSGPRToVGPRCopy(SrcRC, DstRC, *TRI)) {
655  }
656 
657  break;
658  }
659  case AMDGPU::PHI: {
660  processPHINode(MI);
661  break;
662  }
663  case AMDGPU::REG_SEQUENCE:
664  if (TRI->hasVectorRegisters(TII->getOpRegClass(MI, 0)) ||
665  !hasVectorOperands(MI, TRI)) {
666  foldVGPRCopyIntoRegSequence(MI, TRI, TII, *MRI);
667  continue;
668  }
669 
670  LLVM_DEBUG(dbgs() << "Fixing REG_SEQUENCE: " << MI);
671 
672  TII->moveToVALU(MI, MDT);
673  break;
674  case AMDGPU::INSERT_SUBREG: {
675  const TargetRegisterClass *DstRC, *Src0RC, *Src1RC;
676  DstRC = MRI->getRegClass(MI.getOperand(0).getReg());
677  Src0RC = MRI->getRegClass(MI.getOperand(1).getReg());
678  Src1RC = MRI->getRegClass(MI.getOperand(2).getReg());
679  if (TRI->isSGPRClass(DstRC) &&
680  (TRI->hasVectorRegisters(Src0RC) ||
681  TRI->hasVectorRegisters(Src1RC))) {
682  LLVM_DEBUG(dbgs() << " Fixing INSERT_SUBREG: " << MI);
683  TII->moveToVALU(MI, MDT);
684  }
685  break;
686  }
687  case AMDGPU::V_WRITELANE_B32: {
688  // Some architectures allow more than one constant bus access without
689  // SGPR restriction
690  if (ST.getConstantBusLimit(MI.getOpcode()) != 1)
691  break;
692 
693  // Writelane is special in that it can use SGPR and M0 (which would
694  // normally count as using the constant bus twice - but in this case it
695  // is allowed since the lane selector doesn't count as a use of the
696  // constant bus). However, it is still required to abide by the 1 SGPR
697  // rule. Apply a fix here as we might have multiple SGPRs after
698  // legalizing VGPRs to SGPRs
699  int Src0Idx =
700  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0);
701  int Src1Idx =
702  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src1);
703  MachineOperand &Src0 = MI.getOperand(Src0Idx);
704  MachineOperand &Src1 = MI.getOperand(Src1Idx);
705 
706  // Check to see if the instruction violates the 1 SGPR rule
707  if ((Src0.isReg() && TRI->isSGPRReg(*MRI, Src0.getReg()) &&
708  Src0.getReg() != AMDGPU::M0) &&
709  (Src1.isReg() && TRI->isSGPRReg(*MRI, Src1.getReg()) &&
710  Src1.getReg() != AMDGPU::M0)) {
711 
712  // Check for trivially easy constant prop into one of the operands
713  // If this is the case then perform the operation now to resolve SGPR
714  // issue. If we don't do that here we will always insert a mov to m0
715  // that can't be resolved in later operand folding pass
716  bool Resolved = false;
717  for (MachineOperand *MO : {&Src0, &Src1}) {
718  if (Register::isVirtualRegister(MO->getReg())) {
719  MachineInstr *DefMI = MRI->getVRegDef(MO->getReg());
720  if (DefMI && TII->isFoldableCopy(*DefMI)) {
721  const MachineOperand &Def = DefMI->getOperand(0);
722  if (Def.isReg() &&
723  MO->getReg() == Def.getReg() &&
724  MO->getSubReg() == Def.getSubReg()) {
725  const MachineOperand &Copied = DefMI->getOperand(1);
726  if (Copied.isImm() &&
727  TII->isInlineConstant(APInt(64, Copied.getImm(), true))) {
728  MO->ChangeToImmediate(Copied.getImm());
729  Resolved = true;
730  break;
731  }
732  }
733  }
734  }
735  }
736 
737  if (!Resolved) {
738  // Haven't managed to resolve by replacing an SGPR with an immediate
739  // Move src1 to be in M0
740  BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
741  TII->get(AMDGPU::COPY), AMDGPU::M0)
742  .add(Src1);
743  Src1.ChangeToRegister(AMDGPU::M0, false);
744  }
745  }
746  break;
747  }
748  }
749  }
750  }
751 
753  hoistAndMergeSGPRInits(AMDGPU::M0, *MRI, TRI, *MDT, TII);
754 
755  return true;
756 }
757 
758 void SIFixSGPRCopies::processPHINode(MachineInstr &MI) {
759  unsigned numVGPRUses = 0;
760  bool AllAGPRUses = true;
763  worklist.insert(&MI);
764  Visited.insert(&MI);
765  while (!worklist.empty()) {
766  const MachineInstr *Instr = worklist.pop_back_val();
767  unsigned Reg = Instr->getOperand(0).getReg();
768  for (const auto &Use : MRI->use_operands(Reg)) {
769  const MachineInstr *UseMI = Use.getParent();
770  AllAGPRUses &= (UseMI->isCopy() &&
771  TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg())) ||
772  TRI->isAGPR(*MRI, Use.getReg());
773  if (UseMI->isCopy() || UseMI->isRegSequence()) {
774  if (UseMI->isCopy() &&
775  UseMI->getOperand(0).getReg().isPhysical() &&
776  !TRI->isSGPRReg(*MRI, UseMI->getOperand(0).getReg())) {
777  numVGPRUses++;
778  }
779  if (Visited.insert(UseMI).second)
780  worklist.insert(UseMI);
781 
782  continue;
783  }
784 
785  if (UseMI->isPHI()) {
786  const TargetRegisterClass *UseRC = MRI->getRegClass(Use.getReg());
787  if (!TRI->isSGPRReg(*MRI, Use.getReg()) &&
788  UseRC != &AMDGPU::VReg_1RegClass)
789  numVGPRUses++;
790  continue;
791  }
792 
793  const TargetRegisterClass *OpRC =
794  TII->getOpRegClass(*UseMI, UseMI->getOperandNo(&Use));
795  if (!TRI->isSGPRClass(OpRC) && OpRC != &AMDGPU::VS_32RegClass &&
796  OpRC != &AMDGPU::VS_64RegClass) {
797  numVGPRUses++;
798  }
799  }
800  }
801 
802  Register PHIRes = MI.getOperand(0).getReg();
803  const TargetRegisterClass *RC0 = MRI->getRegClass(PHIRes);
804  if (AllAGPRUses && numVGPRUses && !TRI->hasAGPRs(RC0)) {
805  LLVM_DEBUG(dbgs() << "Moving PHI to AGPR: " << MI);
806  MRI->setRegClass(PHIRes, TRI->getEquivalentAGPRClass(RC0));
807  }
808 
809  bool hasVGPRInput = false;
810  for (unsigned i = 1; i < MI.getNumOperands(); i += 2) {
811  unsigned InputReg = MI.getOperand(i).getReg();
812  MachineInstr *Def = MRI->getVRegDef(InputReg);
813  if (TRI->isVectorRegister(*MRI, InputReg)) {
814  if (Def->isCopy()) {
815  unsigned SrcReg = Def->getOperand(1).getReg();
816  const TargetRegisterClass *RC =
817  TRI->getRegClassForReg(*MRI, SrcReg);
818  if (TRI->isSGPRClass(RC))
819  continue;
820  }
821  hasVGPRInput = true;
822  break;
823  }
824  else if (Def->isCopy() &&
825  TRI->isVectorRegister(*MRI, Def->getOperand(1).getReg())) {
826  hasVGPRInput = true;
827  break;
828  }
829  }
830 
831  if ((!TRI->isVectorRegister(*MRI, PHIRes) &&
832  RC0 != &AMDGPU::VReg_1RegClass) &&
833  (hasVGPRInput || numVGPRUses > 1)) {
834  LLVM_DEBUG(dbgs() << "Fixing PHI: " << MI);
835  TII->moveToVALU(MI);
836  }
837  else {
838  LLVM_DEBUG(dbgs() << "Legalizing PHI: " << MI);
839  TII->legalizeOperands(MI, MDT);
840  }
841 
842 }
uint64_t CallInst * C
Interface definition for SIRegisterInfo.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
AMDGPU specific subclass of TargetSubtarget.
static bool isSafeToFoldImmIntoCopy(const MachineInstr *Copy, const MachineInstr *MoveImm, const SIInstrInfo *TII, unsigned &SMovOp, int64_t &Imm)
LLVM_NODISCARD T pop_back_val()
Definition: SetVector.h:227
This class represents lattice values for constants.
Definition: AllocatorList.h:23
bool isOperandLegal(const MachineInstr &MI, unsigned OpIdx, const MachineOperand *MO=nullptr) const
Check if MO is a legal operand if it was the OpIdx Operand for MI.
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
const TargetRegisterClass * getEquivalentVGPRClass(const TargetRegisterClass *SRC) const
Implements a dense probed hash-table based set.
Definition: DenseSet.h:249
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:384
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:63
unsigned getOperandNo(const_mop_iterator I) const
Returns the number of the operand iterator I points to.
Definition: MachineInstr.h:526
iterator_range< reg_nodbg_iterator > reg_nodbg_operands(unsigned Reg) const
unsigned Reg
unsigned getSubReg() const
bool isRegSequence() const
const SIInstrInfo * getInstrInfo() const override
unsigned const TargetRegisterInfo * TRI
const TargetRegisterClass * getEquivalentSGPRClass(const TargetRegisterClass *VRC) const
bool isPHI() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
bool isMoveImmediate(QueryType Type=IgnoreBundle) const
Return true if this instruction is a move immediate (including conditional moves) instruction...
Definition: MachineInstr.h:723
const TargetRegisterClass * getRegClass(Register Reg) const
Return the register class of the specified virtual register.
static bool hoistAndMergeSGPRInits(unsigned Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo *TRI, MachineDominatorTree &MDT, const TargetInstrInfo *TII)
static bool foldVGPRCopyIntoRegSequence(MachineInstr &MI, const SIRegisterInfo *TRI, const SIInstrInfo *TII, MachineRegisterInfo &MRI)
LLVM_READONLY int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIdx)
static bool tryChangeVGPRtoSGPRinCopy(MachineInstr &MI, const SIRegisterInfo *TRI, const SIInstrInfo *TII)
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
const HexagonInstrInfo * TII
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:413
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
MachineBasicBlock iterator that automatically skips over MIs that are inside bundles (i...
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
unsigned SubReg
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:410
virtual bool isBasicBlockPrologue(const MachineInstr &MI) const
True if the instruction is bound to the top of its basic block and no other instructions shall be ins...
bool isSGPRClass(const TargetRegisterClass *RC) const
SI Fix SGPR copies
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
void clearKillFlags(unsigned Reg) const
clearKillFlags - Iterate over all the uses of the given register and clear the kill flag from the Mac...
INITIALIZE_PASS_BEGIN(SIFixSGPRCopies, DEBUG_TYPE, "SI Fix SGPR copies", false, false) INITIALIZE_PASS_END(SIFixSGPRCopies
LLVM_READONLY MachineOperand * getNamedOperand(MachineInstr &MI, unsigned OperandName) const
Returns the operand named Op.
bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
const TargetRegisterClass * getEquivalentAGPRClass(const TargetRegisterClass *SRC) const
void setReg(Register Reg)
Change the register this operand corresponds to.
MachineBasicBlock * findNearestCommonDominator(MachineBasicBlock *A, MachineBasicBlock *B)
findNearestCommonDominator - Find nearest common dominator basic block for basic block A and B...
static bool isSGPRToVGPRCopy(const TargetRegisterClass *SrcRC, const TargetRegisterClass *DstRC, const SIRegisterInfo &TRI)
void ChangeToImmediate(int64_t ImmVal)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value...
TargetInstrInfo - Interface to description of machine instruction set.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
unsigned const MachineRegisterInfo * MRI
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
MachineInstrBuilder & UseMI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
Represent the analysis usage information of a pass.
use_instr_iterator use_instr_begin(unsigned RegNo) const
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1172
constexpr double e
Definition: MathExtras.h:57
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
const TargetRegisterClass * getSubRegClass(const TargetRegisterClass *RC, unsigned SubIdx) const
bool isCopy() const
Falkor HW Prefetch Fix
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
virtual bool isSchedulingBoundary(const MachineInstr &MI, const MachineBasicBlock *MBB, const MachineFunction &MF) const
Test if the given instruction should be considered a scheduling boundary.
Iterator for intrusive lists based on ilist_node.
static std::pair< const TargetRegisterClass *, const TargetRegisterClass * > getCopyRegClasses(const MachineInstr &Copy, const SIRegisterInfo &TRI, const MachineRegisterInfo &MRI)
#define DEBUG_TYPE
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
BlockVerifier::State From
bool searchPredecessors(const MachineBasicBlock *MBB, const MachineBasicBlock *CutOff, UnaryPredicate Predicate)
bool hasAGPRs(const TargetRegisterClass *RC) const
static uint64_t add(uint64_t LeftOp, uint64_t RightOp)
Definition: FileCheck.cpp:215
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
MachineInstrBuilder MachineInstrBuilder & DefMI
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
bool properlyDominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:301
static cl::opt< bool > EnableM0Merge("amdgpu-enable-merge-m0", cl::desc("Merge and hoist M0 initializations"), cl::init(true))
int64_t getImm() const
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Class for arbitrary precision integers.
Definition: APInt.h:69
static cl::opt< unsigned > Threshold("loop-unswitch-threshold", cl::desc("Max loop size to unswitch"), cl::init(100), cl::Hidden)
void addImplicitDefUseOperands(MachineFunction &MF)
Add all implicit def and use operands to this instruction.
FunctionPass * createSIFixSGPRCopiesPass()
static bool isVGPRToSGPRCopy(const TargetRegisterClass *SrcRC, const TargetRegisterClass *DstRC, const SIRegisterInfo &TRI)
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:255
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
Provides AMDGPU specific target descriptions.
Representation of each machine instruction.
Definition: MachineInstr.h:63
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
Interface definition for SIInstrInfo.
bool hasOneUse(unsigned RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
char & SIFixSGPRCopiesID
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:72
void ChangeToRegister(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isDebug=false)
ChangeToRegister - Replace this operand with a new register operand of the specified value...
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static MachineBasicBlock::iterator getFirstNonPrologue(MachineBasicBlock *MBB, const TargetInstrInfo *TII)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool hasVectorRegisters(const TargetRegisterClass *RC) const
A vector that has set insertion semantics.
Definition: SetVector.h:40
const TargetRegisterClass * getPhysRegClass(unsigned Reg) const
Return the &#39;base&#39; register class for this register.
static bool isVirtualRegister(unsigned Reg)
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:69
iterator_range< def_instr_iterator > def_instructions(unsigned Reg) const
IRTranslator LLVM IR MI
void setRegClass(unsigned Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
unsigned getConstantBusLimit(unsigned Opcode) const
Register getReg() const
getReg - Returns the register number.
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:415
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
static bool isReachable(const MachineInstr *From, const MachineInstr *To, const MachineBasicBlock *CutOff, MachineDominatorTree &MDT)
static bool hasVectorOperands(const MachineInstr &MI, const SIRegisterInfo *TRI)
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
const SIRegisterInfo * getRegisterInfo() const override