LLVM  10.0.0svn
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI Implementation of TargetInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SIInstrInfo.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "SIRegisterInfo.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/MC/MCInstrDesc.h"
52 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/Compiler.h"
59 #include <cassert>
60 #include <cstdint>
61 #include <iterator>
62 #include <utility>
63 
64 using namespace llvm;
65 
66 #define GET_INSTRINFO_CTOR_DTOR
67 #include "AMDGPUGenInstrInfo.inc"
68 
69 namespace llvm {
70 namespace AMDGPU {
71 #define GET_D16ImageDimIntrinsics_IMPL
72 #define GET_ImageDimIntrinsicTable_IMPL
73 #define GET_RsrcIntrinsics_IMPL
74 #include "AMDGPUGenSearchableTables.inc"
75 }
76 }
77 
78 
79 // Must be at least 4 to be able to branch over minimum unconditional branch
80 // code. This is only for making it possible to write reasonably small tests for
81 // long branches.
82 static cl::opt<unsigned>
83 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
84  cl::desc("Restrict range of branch instructions (DEBUG)"));
85 
87  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
88  RI(ST), ST(ST) {}
89 
90 //===----------------------------------------------------------------------===//
91 // TargetInstrInfo callbacks
92 //===----------------------------------------------------------------------===//
93 
94 static unsigned getNumOperandsNoGlue(SDNode *Node) {
95  unsigned N = Node->getNumOperands();
96  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
97  --N;
98  return N;
99 }
100 
101 /// Returns true if both nodes have the same value for the given
102 /// operand \p Op, or if both nodes do not have this operand.
103 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
104  unsigned Opc0 = N0->getMachineOpcode();
105  unsigned Opc1 = N1->getMachineOpcode();
106 
107  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
108  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
109 
110  if (Op0Idx == -1 && Op1Idx == -1)
111  return true;
112 
113 
114  if ((Op0Idx == -1 && Op1Idx != -1) ||
115  (Op1Idx == -1 && Op0Idx != -1))
116  return false;
117 
118  // getNamedOperandIdx returns the index for the MachineInstr's operands,
119  // which includes the result as the first operand. We are indexing into the
120  // MachineSDNode's operands, so we need to skip the result operand to get
121  // the real index.
122  --Op0Idx;
123  --Op1Idx;
124 
125  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
126 }
127 
129  AliasAnalysis *AA) const {
130  // TODO: The generic check fails for VALU instructions that should be
131  // rematerializable due to implicit reads of exec. We really want all of the
132  // generic logic for this except for this.
133  switch (MI.getOpcode()) {
134  case AMDGPU::V_MOV_B32_e32:
135  case AMDGPU::V_MOV_B32_e64:
136  case AMDGPU::V_MOV_B64_PSEUDO:
137  // No implicit operands.
138  return MI.getNumOperands() == MI.getDesc().getNumOperands();
139  default:
140  return false;
141  }
142 }
143 
145  int64_t &Offset0,
146  int64_t &Offset1) const {
147  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
148  return false;
149 
150  unsigned Opc0 = Load0->getMachineOpcode();
151  unsigned Opc1 = Load1->getMachineOpcode();
152 
153  // Make sure both are actually loads.
154  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
155  return false;
156 
157  if (isDS(Opc0) && isDS(Opc1)) {
158 
159  // FIXME: Handle this case:
160  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
161  return false;
162 
163  // Check base reg.
164  if (Load0->getOperand(0) != Load1->getOperand(0))
165  return false;
166 
167  // Skip read2 / write2 variants for simplicity.
168  // TODO: We should report true if the used offsets are adjacent (excluded
169  // st64 versions).
170  int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
171  int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
172  if (Offset0Idx == -1 || Offset1Idx == -1)
173  return false;
174 
175  // XXX - be careful of datalesss loads
176  // getNamedOperandIdx returns the index for MachineInstrs. Since they
177  // include the output in the operand list, but SDNodes don't, we need to
178  // subtract the index by one.
179  Offset0Idx -= get(Opc0).NumDefs;
180  Offset1Idx -= get(Opc1).NumDefs;
181  Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
182  Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
183  return true;
184  }
185 
186  if (isSMRD(Opc0) && isSMRD(Opc1)) {
187  // Skip time and cache invalidation instructions.
188  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
189  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
190  return false;
191 
193 
194  // Check base reg.
195  if (Load0->getOperand(0) != Load1->getOperand(0))
196  return false;
197 
198  const ConstantSDNode *Load0Offset =
199  dyn_cast<ConstantSDNode>(Load0->getOperand(1));
200  const ConstantSDNode *Load1Offset =
201  dyn_cast<ConstantSDNode>(Load1->getOperand(1));
202 
203  if (!Load0Offset || !Load1Offset)
204  return false;
205 
206  Offset0 = Load0Offset->getZExtValue();
207  Offset1 = Load1Offset->getZExtValue();
208  return true;
209  }
210 
211  // MUBUF and MTBUF can access the same addresses.
212  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
213 
214  // MUBUF and MTBUF have vaddr at different indices.
215  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
216  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
217  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
218  return false;
219 
220  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
221  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
222 
223  if (OffIdx0 == -1 || OffIdx1 == -1)
224  return false;
225 
226  // getNamedOperandIdx returns the index for MachineInstrs. Since they
227  // include the output in the operand list, but SDNodes don't, we need to
228  // subtract the index by one.
229  OffIdx0 -= get(Opc0).NumDefs;
230  OffIdx1 -= get(Opc1).NumDefs;
231 
232  SDValue Off0 = Load0->getOperand(OffIdx0);
233  SDValue Off1 = Load1->getOperand(OffIdx1);
234 
235  // The offset might be a FrameIndexSDNode.
236  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
237  return false;
238 
239  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
240  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
241  return true;
242  }
243 
244  return false;
245 }
246 
247 static bool isStride64(unsigned Opc) {
248  switch (Opc) {
249  case AMDGPU::DS_READ2ST64_B32:
250  case AMDGPU::DS_READ2ST64_B64:
251  case AMDGPU::DS_WRITE2ST64_B32:
252  case AMDGPU::DS_WRITE2ST64_B64:
253  return true;
254  default:
255  return false;
256  }
257 }
258 
260  const MachineOperand *&BaseOp,
261  int64_t &Offset,
262  const TargetRegisterInfo *TRI) const {
263  unsigned Opc = LdSt.getOpcode();
264 
265  if (isDS(LdSt)) {
266  const MachineOperand *OffsetImm =
267  getNamedOperand(LdSt, AMDGPU::OpName::offset);
268  if (OffsetImm) {
269  // Normal, single offset LDS instruction.
270  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
271  // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
272  // report that here?
273  if (!BaseOp)
274  return false;
275 
276  Offset = OffsetImm->getImm();
277  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
278  "operands of type register.");
279  return true;
280  }
281 
282  // The 2 offset instructions use offset0 and offset1 instead. We can treat
283  // these as a load with a single offset if the 2 offsets are consecutive. We
284  // will use this for some partially aligned loads.
285  const MachineOperand *Offset0Imm =
286  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
287  const MachineOperand *Offset1Imm =
288  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
289 
290  uint8_t Offset0 = Offset0Imm->getImm();
291  uint8_t Offset1 = Offset1Imm->getImm();
292 
293  if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
294  // Each of these offsets is in element sized units, so we need to convert
295  // to bytes of the individual reads.
296 
297  unsigned EltSize;
298  if (LdSt.mayLoad())
299  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
300  else {
301  assert(LdSt.mayStore());
302  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
303  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
304  }
305 
306  if (isStride64(Opc))
307  EltSize *= 64;
308 
309  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
310  Offset = EltSize * Offset0;
311  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
312  "operands of type register.");
313  return true;
314  }
315 
316  return false;
317  }
318 
319  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
320  const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
321  if (SOffset && SOffset->isReg()) {
322  // We can only handle this if it's a stack access, as any other resource
323  // would require reporting multiple base registers.
324  const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
325  if (AddrReg && !AddrReg->isFI())
326  return false;
327 
328  const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
329  const SIMachineFunctionInfo *MFI
331  if (RSrc->getReg() != MFI->getScratchRSrcReg())
332  return false;
333 
334  const MachineOperand *OffsetImm =
335  getNamedOperand(LdSt, AMDGPU::OpName::offset);
336  BaseOp = SOffset;
337  Offset = OffsetImm->getImm();
338  return true;
339  }
340 
341  const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
342  if (!AddrReg)
343  return false;
344 
345  const MachineOperand *OffsetImm =
346  getNamedOperand(LdSt, AMDGPU::OpName::offset);
347  BaseOp = AddrReg;
348  Offset = OffsetImm->getImm();
349 
350  if (SOffset) // soffset can be an inline immediate.
351  Offset += SOffset->getImm();
352 
353  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
354  "operands of type register.");
355  return true;
356  }
357 
358  if (isSMRD(LdSt)) {
359  const MachineOperand *OffsetImm =
360  getNamedOperand(LdSt, AMDGPU::OpName::offset);
361  if (!OffsetImm)
362  return false;
363 
364  const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
365  BaseOp = SBaseReg;
366  Offset = OffsetImm->getImm();
367  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
368  "operands of type register.");
369  return true;
370  }
371 
372  if (isFLAT(LdSt)) {
373  const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
374  if (VAddr) {
375  // Can't analyze 2 offsets.
376  if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
377  return false;
378 
379  BaseOp = VAddr;
380  } else {
381  // scratch instructions have either vaddr or saddr.
382  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
383  }
384 
385  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
386  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
387  "operands of type register.");
388  return true;
389  }
390 
391  return false;
392 }
393 
394 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
395  const MachineOperand &BaseOp1,
396  const MachineInstr &MI2,
397  const MachineOperand &BaseOp2) {
398  // Support only base operands with base registers.
399  // Note: this could be extended to support FI operands.
400  if (!BaseOp1.isReg() || !BaseOp2.isReg())
401  return false;
402 
403  if (BaseOp1.isIdenticalTo(BaseOp2))
404  return true;
405 
406  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
407  return false;
408 
409  auto MO1 = *MI1.memoperands_begin();
410  auto MO2 = *MI2.memoperands_begin();
411  if (MO1->getAddrSpace() != MO2->getAddrSpace())
412  return false;
413 
414  auto Base1 = MO1->getValue();
415  auto Base2 = MO2->getValue();
416  if (!Base1 || !Base2)
417  return false;
418  const MachineFunction &MF = *MI1.getParent()->getParent();
419  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
420  Base1 = GetUnderlyingObject(Base1, DL);
421  Base2 = GetUnderlyingObject(Base1, DL);
422 
423  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
424  return false;
425 
426  return Base1 == Base2;
427 }
428 
430  const MachineOperand &BaseOp2,
431  unsigned NumLoads) const {
432  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
433  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
434 
435  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
436  return false;
437 
438  const MachineOperand *FirstDst = nullptr;
439  const MachineOperand *SecondDst = nullptr;
440 
441  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
442  (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
443  (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
444  const unsigned MaxGlobalLoadCluster = 6;
445  if (NumLoads > MaxGlobalLoadCluster)
446  return false;
447 
448  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
449  if (!FirstDst)
450  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
451  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
452  if (!SecondDst)
453  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
454  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
455  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
456  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
457  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
458  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
459  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
460  }
461 
462  if (!FirstDst || !SecondDst)
463  return false;
464 
465  // Try to limit clustering based on the total number of bytes loaded
466  // rather than the number of instructions. This is done to help reduce
467  // register pressure. The method used is somewhat inexact, though,
468  // because it assumes that all loads in the cluster will load the
469  // same number of bytes as FirstLdSt.
470 
471  // The unit of this value is bytes.
472  // FIXME: This needs finer tuning.
473  unsigned LoadClusterThreshold = 16;
474 
475  const MachineRegisterInfo &MRI =
476  FirstLdSt.getParent()->getParent()->getRegInfo();
477 
478  const Register Reg = FirstDst->getReg();
479 
481  ? MRI.getRegClass(Reg)
482  : RI.getPhysRegClass(Reg);
483 
484  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
485 }
486 
487 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
488 // the first 16 loads will be interleaved with the stores, and the next 16 will
489 // be clustered as expected. It should really split into 2 16 store batches.
490 //
491 // Loads are clustered until this returns false, rather than trying to schedule
492 // groups of stores. This also means we have to deal with saying different
493 // address space loads should be clustered, and ones which might cause bank
494 // conflicts.
495 //
496 // This might be deprecated so it might not be worth that much effort to fix.
498  int64_t Offset0, int64_t Offset1,
499  unsigned NumLoads) const {
500  assert(Offset1 > Offset0 &&
501  "Second offset should be larger than first offset!");
502  // If we have less than 16 loads in a row, and the offsets are within 64
503  // bytes, then schedule together.
504 
505  // A cacheline is 64 bytes (for global memory).
506  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
507 }
508 
511  const DebugLoc &DL, unsigned DestReg,
512  unsigned SrcReg, bool KillSrc) {
513  MachineFunction *MF = MBB.getParent();
514  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
515  "illegal SGPR to VGPR copy",
516  DL, DS_Error);
517  LLVMContext &C = MF->getFunction().getContext();
518  C.diagnose(IllegalCopy);
519 
520  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
521  .addReg(SrcReg, getKillRegState(KillSrc));
522 }
523 
526  const DebugLoc &DL, unsigned DestReg,
527  unsigned SrcReg, bool KillSrc) const {
528  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
529 
530  if (RC == &AMDGPU::VGPR_32RegClass) {
531  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
532  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
533  AMDGPU::AGPR_32RegClass.contains(SrcReg));
534  unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
535  AMDGPU::V_ACCVGPR_READ_B32 : AMDGPU::V_MOV_B32_e32;
536  BuildMI(MBB, MI, DL, get(Opc), DestReg)
537  .addReg(SrcReg, getKillRegState(KillSrc));
538  return;
539  }
540 
541  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
542  RC == &AMDGPU::SReg_32RegClass) {
543  if (SrcReg == AMDGPU::SCC) {
544  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
545  .addImm(-1)
546  .addImm(0);
547  return;
548  }
549 
550  if (DestReg == AMDGPU::VCC_LO) {
551  if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
552  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
553  .addReg(SrcReg, getKillRegState(KillSrc));
554  } else {
555  // FIXME: Hack until VReg_1 removed.
556  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
557  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
558  .addImm(0)
559  .addReg(SrcReg, getKillRegState(KillSrc));
560  }
561 
562  return;
563  }
564 
565  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
566  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
567  return;
568  }
569 
570  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
571  .addReg(SrcReg, getKillRegState(KillSrc));
572  return;
573  }
574 
575  if (RC == &AMDGPU::SReg_64RegClass) {
576  if (DestReg == AMDGPU::VCC) {
577  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
578  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
579  .addReg(SrcReg, getKillRegState(KillSrc));
580  } else {
581  // FIXME: Hack until VReg_1 removed.
582  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
583  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
584  .addImm(0)
585  .addReg(SrcReg, getKillRegState(KillSrc));
586  }
587 
588  return;
589  }
590 
591  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
592  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
593  return;
594  }
595 
596  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
597  .addReg(SrcReg, getKillRegState(KillSrc));
598  return;
599  }
600 
601  if (DestReg == AMDGPU::SCC) {
602  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
603  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
604  .addReg(SrcReg, getKillRegState(KillSrc))
605  .addImm(0);
606  return;
607  }
608 
609  if (RC == &AMDGPU::AGPR_32RegClass) {
610  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
611  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
612  AMDGPU::AGPR_32RegClass.contains(SrcReg));
613  if (!AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
614  // First try to find defining accvgpr_write to avoid temporary registers.
615  for (auto Def = MI, E = MBB.begin(); Def != E; ) {
616  --Def;
617  if (!Def->definesRegister(SrcReg, &RI))
618  continue;
619  if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32)
620  break;
621 
622  MachineOperand &DefOp = Def->getOperand(1);
623  assert(DefOp.isReg() || DefOp.isImm());
624 
625  if (DefOp.isReg()) {
626  // Check that register source operand if not clobbered before MI.
627  // Immediate operands are always safe to propagate.
628  bool SafeToPropagate = true;
629  for (auto I = Def; I != MI && SafeToPropagate; ++I)
630  if (I->modifiesRegister(DefOp.getReg(), &RI))
631  SafeToPropagate = false;
632 
633  if (!SafeToPropagate)
634  break;
635 
636  DefOp.setIsKill(false);
637  }
638 
639  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
640  .add(DefOp);
641  return;
642  }
643 
644  RegScavenger RS;
645  RS.enterBasicBlock(MBB);
646  RS.forward(MI);
647 
648  // Ideally we want to have three registers for a long reg_sequence copy
649  // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
650  unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
651  *MBB.getParent());
652 
653  // Registers in the sequence are allocated contiguously so we can just
654  // use register number to pick one of three round-robin temps.
655  unsigned RegNo = DestReg % 3;
656  unsigned Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
657  if (!Tmp)
658  report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
659  RS.setRegUsed(Tmp);
660  // Only loop through if there are any free registers left, otherwise
661  // scavenger may report a fatal error without emergency spill slot
662  // or spill with the slot.
663  while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
664  unsigned Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
665  if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
666  break;
667  Tmp = Tmp2;
668  RS.setRegUsed(Tmp);
669  }
670  copyPhysReg(MBB, MI, DL, Tmp, SrcReg, KillSrc);
671  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
672  .addReg(Tmp, RegState::Kill);
673  return;
674  }
675 
676  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
677  .addReg(SrcReg, getKillRegState(KillSrc));
678  return;
679  }
680 
681  unsigned EltSize = 4;
682  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
683  if (RI.isSGPRClass(RC)) {
684  // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
685  if (!(RI.getRegSizeInBits(*RC) % 64)) {
686  Opcode = AMDGPU::S_MOV_B64;
687  EltSize = 8;
688  } else {
689  Opcode = AMDGPU::S_MOV_B32;
690  EltSize = 4;
691  }
692 
693  if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
694  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
695  return;
696  }
697  } else if (RI.hasAGPRs(RC)) {
698  Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ?
699  AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
700  } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) {
701  Opcode = AMDGPU::V_ACCVGPR_READ_B32;
702  }
703 
704  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
705  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
706 
707  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
708  unsigned SubIdx;
709  if (Forward)
710  SubIdx = SubIndices[Idx];
711  else
712  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
713 
714  if (Opcode == TargetOpcode::COPY) {
715  copyPhysReg(MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
716  RI.getSubReg(SrcReg, SubIdx), KillSrc);
717  continue;
718  }
719 
720  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
721  get(Opcode), RI.getSubReg(DestReg, SubIdx));
722 
723  Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
724 
725  if (Idx == 0)
726  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
727 
728  bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
729  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
730  }
731 }
732 
733 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
734  int NewOpc;
735 
736  // Try to map original to commuted opcode
737  NewOpc = AMDGPU::getCommuteRev(Opcode);
738  if (NewOpc != -1)
739  // Check if the commuted (REV) opcode exists on the target.
740  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
741 
742  // Try to map commuted to original opcode
743  NewOpc = AMDGPU::getCommuteOrig(Opcode);
744  if (NewOpc != -1)
745  // Check if the original (non-REV) opcode exists on the target.
746  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
747 
748  return Opcode;
749 }
750 
753  const DebugLoc &DL, unsigned DestReg,
754  int64_t Value) const {
756  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
757  if (RegClass == &AMDGPU::SReg_32RegClass ||
758  RegClass == &AMDGPU::SGPR_32RegClass ||
759  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
760  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
761  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
762  .addImm(Value);
763  return;
764  }
765 
766  if (RegClass == &AMDGPU::SReg_64RegClass ||
767  RegClass == &AMDGPU::SGPR_64RegClass ||
768  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
769  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
770  .addImm(Value);
771  return;
772  }
773 
774  if (RegClass == &AMDGPU::VGPR_32RegClass) {
775  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
776  .addImm(Value);
777  return;
778  }
779  if (RegClass == &AMDGPU::VReg_64RegClass) {
780  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
781  .addImm(Value);
782  return;
783  }
784 
785  unsigned EltSize = 4;
786  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
787  if (RI.isSGPRClass(RegClass)) {
788  if (RI.getRegSizeInBits(*RegClass) > 32) {
789  Opcode = AMDGPU::S_MOV_B64;
790  EltSize = 8;
791  } else {
792  Opcode = AMDGPU::S_MOV_B32;
793  EltSize = 4;
794  }
795  }
796 
797  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
798  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
799  int64_t IdxValue = Idx == 0 ? Value : 0;
800 
801  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
802  get(Opcode), RI.getSubReg(DestReg, Idx));
803  Builder.addImm(IdxValue);
804  }
805 }
806 
807 const TargetRegisterClass *
809  return &AMDGPU::VGPR_32RegClass;
810 }
811 
814  const DebugLoc &DL, unsigned DstReg,
816  unsigned TrueReg,
817  unsigned FalseReg) const {
819  MachineFunction *MF = MBB.getParent();
820  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
821  const TargetRegisterClass *BoolXExecRC =
822  RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
823  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
824  "Not a VGPR32 reg");
825 
826  if (Cond.size() == 1) {
827  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
828  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
829  .add(Cond[0]);
830  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
831  .addImm(0)
832  .addReg(FalseReg)
833  .addImm(0)
834  .addReg(TrueReg)
835  .addReg(SReg);
836  } else if (Cond.size() == 2) {
837  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
838  switch (Cond[0].getImm()) {
839  case SIInstrInfo::SCC_TRUE: {
840  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
841  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
842  : AMDGPU::S_CSELECT_B64), SReg)
843  .addImm(-1)
844  .addImm(0);
845  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
846  .addImm(0)
847  .addReg(FalseReg)
848  .addImm(0)
849  .addReg(TrueReg)
850  .addReg(SReg);
851  break;
852  }
853  case SIInstrInfo::SCC_FALSE: {
854  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
855  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
856  : AMDGPU::S_CSELECT_B64), SReg)
857  .addImm(0)
858  .addImm(-1);
859  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
860  .addImm(0)
861  .addReg(FalseReg)
862  .addImm(0)
863  .addReg(TrueReg)
864  .addReg(SReg);
865  break;
866  }
867  case SIInstrInfo::VCCNZ: {
868  MachineOperand RegOp = Cond[1];
869  RegOp.setImplicit(false);
870  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
871  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
872  .add(RegOp);
873  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
874  .addImm(0)
875  .addReg(FalseReg)
876  .addImm(0)
877  .addReg(TrueReg)
878  .addReg(SReg);
879  break;
880  }
881  case SIInstrInfo::VCCZ: {
882  MachineOperand RegOp = Cond[1];
883  RegOp.setImplicit(false);
884  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
885  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
886  .add(RegOp);
887  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
888  .addImm(0)
889  .addReg(TrueReg)
890  .addImm(0)
891  .addReg(FalseReg)
892  .addReg(SReg);
893  break;
894  }
895  case SIInstrInfo::EXECNZ: {
896  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
897  Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
898  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
899  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
900  .addImm(0);
901  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
902  : AMDGPU::S_CSELECT_B64), SReg)
903  .addImm(-1)
904  .addImm(0);
905  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
906  .addImm(0)
907  .addReg(FalseReg)
908  .addImm(0)
909  .addReg(TrueReg)
910  .addReg(SReg);
911  break;
912  }
913  case SIInstrInfo::EXECZ: {
914  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
915  Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
916  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
917  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
918  .addImm(0);
919  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
920  : AMDGPU::S_CSELECT_B64), SReg)
921  .addImm(0)
922  .addImm(-1);
923  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
924  .addImm(0)
925  .addReg(FalseReg)
926  .addImm(0)
927  .addReg(TrueReg)
928  .addReg(SReg);
929  llvm_unreachable("Unhandled branch predicate EXECZ");
930  break;
931  }
932  default:
933  llvm_unreachable("invalid branch predicate");
934  }
935  } else {
936  llvm_unreachable("Can only handle Cond size 1 or 2");
937  }
938 }
939 
942  const DebugLoc &DL,
943  unsigned SrcReg, int Value) const {
946  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
947  .addImm(Value)
948  .addReg(SrcReg);
949 
950  return Reg;
951 }
952 
955  const DebugLoc &DL,
956  unsigned SrcReg, int Value) const {
959  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
960  .addImm(Value)
961  .addReg(SrcReg);
962 
963  return Reg;
964 }
965 
966 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
967 
968  if (RI.hasAGPRs(DstRC))
969  return AMDGPU::COPY;
970  if (RI.getRegSizeInBits(*DstRC) == 32) {
971  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
972  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
973  return AMDGPU::S_MOV_B64;
974  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
975  return AMDGPU::V_MOV_B64_PSEUDO;
976  }
977  return AMDGPU::COPY;
978 }
979 
980 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
981  switch (Size) {
982  case 4:
983  return AMDGPU::SI_SPILL_S32_SAVE;
984  case 8:
985  return AMDGPU::SI_SPILL_S64_SAVE;
986  case 12:
987  return AMDGPU::SI_SPILL_S96_SAVE;
988  case 16:
989  return AMDGPU::SI_SPILL_S128_SAVE;
990  case 20:
991  return AMDGPU::SI_SPILL_S160_SAVE;
992  case 32:
993  return AMDGPU::SI_SPILL_S256_SAVE;
994  case 64:
995  return AMDGPU::SI_SPILL_S512_SAVE;
996  case 128:
997  return AMDGPU::SI_SPILL_S1024_SAVE;
998  default:
999  llvm_unreachable("unknown register size");
1000  }
1001 }
1002 
1003 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1004  switch (Size) {
1005  case 4:
1006  return AMDGPU::SI_SPILL_V32_SAVE;
1007  case 8:
1008  return AMDGPU::SI_SPILL_V64_SAVE;
1009  case 12:
1010  return AMDGPU::SI_SPILL_V96_SAVE;
1011  case 16:
1012  return AMDGPU::SI_SPILL_V128_SAVE;
1013  case 20:
1014  return AMDGPU::SI_SPILL_V160_SAVE;
1015  case 32:
1016  return AMDGPU::SI_SPILL_V256_SAVE;
1017  case 64:
1018  return AMDGPU::SI_SPILL_V512_SAVE;
1019  case 128:
1020  return AMDGPU::SI_SPILL_V1024_SAVE;
1021  default:
1022  llvm_unreachable("unknown register size");
1023  }
1024 }
1025 
1026 static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1027  switch (Size) {
1028  case 4:
1029  return AMDGPU::SI_SPILL_A32_SAVE;
1030  case 8:
1031  return AMDGPU::SI_SPILL_A64_SAVE;
1032  case 16:
1033  return AMDGPU::SI_SPILL_A128_SAVE;
1034  case 64:
1035  return AMDGPU::SI_SPILL_A512_SAVE;
1036  case 128:
1037  return AMDGPU::SI_SPILL_A1024_SAVE;
1038  default:
1039  llvm_unreachable("unknown register size");
1040  }
1041 }
1042 
1045  unsigned SrcReg, bool isKill,
1046  int FrameIndex,
1047  const TargetRegisterClass *RC,
1048  const TargetRegisterInfo *TRI) const {
1049  MachineFunction *MF = MBB.getParent();
1051  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1052  const DebugLoc &DL = MBB.findDebugLoc(MI);
1053 
1054  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1055  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1056  MachinePointerInfo PtrInfo
1057  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1058  MachineMemOperand *MMO
1060  Size, Align);
1061  unsigned SpillSize = TRI->getSpillSize(*RC);
1062 
1063  if (RI.isSGPRClass(RC)) {
1064  MFI->setHasSpilledSGPRs();
1065 
1066  // We are only allowed to create one new instruction when spilling
1067  // registers, so we need to use pseudo instruction for spilling SGPRs.
1068  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1069 
1070  // The SGPR spill/restore instructions only work on number sgprs, so we need
1071  // to make sure we are using the correct register class.
1072  if (Register::isVirtualRegister(SrcReg) && SpillSize == 4) {
1074  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
1075  }
1076 
1077  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
1078  .addReg(SrcReg, getKillRegState(isKill)) // data
1079  .addFrameIndex(FrameIndex) // addr
1080  .addMemOperand(MMO)
1082  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1083  // Add the scratch resource registers as implicit uses because we may end up
1084  // needing them, and need to ensure that the reserved registers are
1085  // correctly handled.
1086  if (RI.spillSGPRToVGPR())
1087  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1088  if (ST.hasScalarStores()) {
1089  // m0 is used for offset to scalar stores if used to spill.
1090  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1091  }
1092 
1093  return;
1094  }
1095 
1096  unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1097  : getVGPRSpillSaveOpcode(SpillSize);
1098  MFI->setHasSpilledVGPRs();
1099 
1100  auto MIB = BuildMI(MBB, MI, DL, get(Opcode));
1101  if (RI.hasAGPRs(RC)) {
1103  Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1104  MIB.addReg(Tmp, RegState::Define);
1105  }
1106  MIB.addReg(SrcReg, getKillRegState(isKill)) // data
1107  .addFrameIndex(FrameIndex) // addr
1108  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1109  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1110  .addImm(0) // offset
1111  .addMemOperand(MMO);
1112 }
1113 
1114 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1115  switch (Size) {
1116  case 4:
1117  return AMDGPU::SI_SPILL_S32_RESTORE;
1118  case 8:
1119  return AMDGPU::SI_SPILL_S64_RESTORE;
1120  case 12:
1121  return AMDGPU::SI_SPILL_S96_RESTORE;
1122  case 16:
1123  return AMDGPU::SI_SPILL_S128_RESTORE;
1124  case 20:
1125  return AMDGPU::SI_SPILL_S160_RESTORE;
1126  case 32:
1127  return AMDGPU::SI_SPILL_S256_RESTORE;
1128  case 64:
1129  return AMDGPU::SI_SPILL_S512_RESTORE;
1130  case 128:
1131  return AMDGPU::SI_SPILL_S1024_RESTORE;
1132  default:
1133  llvm_unreachable("unknown register size");
1134  }
1135 }
1136 
1137 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1138  switch (Size) {
1139  case 4:
1140  return AMDGPU::SI_SPILL_V32_RESTORE;
1141  case 8:
1142  return AMDGPU::SI_SPILL_V64_RESTORE;
1143  case 12:
1144  return AMDGPU::SI_SPILL_V96_RESTORE;
1145  case 16:
1146  return AMDGPU::SI_SPILL_V128_RESTORE;
1147  case 20:
1148  return AMDGPU::SI_SPILL_V160_RESTORE;
1149  case 32:
1150  return AMDGPU::SI_SPILL_V256_RESTORE;
1151  case 64:
1152  return AMDGPU::SI_SPILL_V512_RESTORE;
1153  case 128:
1154  return AMDGPU::SI_SPILL_V1024_RESTORE;
1155  default:
1156  llvm_unreachable("unknown register size");
1157  }
1158 }
1159 
1160 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1161  switch (Size) {
1162  case 4:
1163  return AMDGPU::SI_SPILL_A32_RESTORE;
1164  case 8:
1165  return AMDGPU::SI_SPILL_A64_RESTORE;
1166  case 16:
1167  return AMDGPU::SI_SPILL_A128_RESTORE;
1168  case 64:
1169  return AMDGPU::SI_SPILL_A512_RESTORE;
1170  case 128:
1171  return AMDGPU::SI_SPILL_A1024_RESTORE;
1172  default:
1173  llvm_unreachable("unknown register size");
1174  }
1175 }
1176 
1179  unsigned DestReg, int FrameIndex,
1180  const TargetRegisterClass *RC,
1181  const TargetRegisterInfo *TRI) const {
1182  MachineFunction *MF = MBB.getParent();
1184  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1185  const DebugLoc &DL = MBB.findDebugLoc(MI);
1186  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1187  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1188  unsigned SpillSize = TRI->getSpillSize(*RC);
1189 
1190  MachinePointerInfo PtrInfo
1191  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1192 
1194  PtrInfo, MachineMemOperand::MOLoad, Size, Align);
1195 
1196  if (RI.isSGPRClass(RC)) {
1197  MFI->setHasSpilledSGPRs();
1198 
1199  // FIXME: Maybe this should not include a memoperand because it will be
1200  // lowered to non-memory instructions.
1201  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1202  if (Register::isVirtualRegister(DestReg) && SpillSize == 4) {
1204  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1205  }
1206 
1207  if (RI.spillSGPRToVGPR())
1208  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1209  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
1210  .addFrameIndex(FrameIndex) // addr
1211  .addMemOperand(MMO)
1213  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1214 
1215  if (ST.hasScalarStores()) {
1216  // m0 is used for offset to scalar stores if used to spill.
1217  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1218  }
1219 
1220  return;
1221  }
1222 
1223  unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1224  : getVGPRSpillRestoreOpcode(SpillSize);
1225  auto MIB = BuildMI(MBB, MI, DL, get(Opcode), DestReg);
1226  if (RI.hasAGPRs(RC)) {
1228  Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1229  MIB.addReg(Tmp, RegState::Define);
1230  }
1231  MIB.addFrameIndex(FrameIndex) // vaddr
1232  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1233  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1234  .addImm(0) // offset
1235  .addMemOperand(MMO);
1236 }
1237 
1238 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1240  MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1241  unsigned FrameOffset, unsigned Size) const {
1242  MachineFunction *MF = MBB.getParent();
1244  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1245  const DebugLoc &DL = MBB.findDebugLoc(MI);
1246  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1247  unsigned WavefrontSize = ST.getWavefrontSize();
1248 
1249  unsigned TIDReg = MFI->getTIDReg();
1250  if (!MFI->hasCalculatedTID()) {
1251  MachineBasicBlock &Entry = MBB.getParent()->front();
1252  MachineBasicBlock::iterator Insert = Entry.front();
1253  const DebugLoc &DL = Insert->getDebugLoc();
1254 
1255  TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1256  *MF);
1257  if (TIDReg == AMDGPU::NoRegister)
1258  return TIDReg;
1259 
1261  WorkGroupSize > WavefrontSize) {
1262  Register TIDIGXReg =
1264  Register TIDIGYReg =
1266  Register TIDIGZReg =
1268  Register InputPtrReg =
1270  for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1271  if (!Entry.isLiveIn(Reg))
1272  Entry.addLiveIn(Reg);
1273  }
1274 
1275  RS->enterBasicBlock(Entry);
1276  // FIXME: Can we scavenge an SReg_64 and access the subregs?
1277  unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1278  unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1279  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1280  .addReg(InputPtrReg)
1282  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1283  .addReg(InputPtrReg)
1285 
1286  // NGROUPS.X * NGROUPS.Y
1287  BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1288  .addReg(STmp1)
1289  .addReg(STmp0);
1290  // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1291  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1292  .addReg(STmp1)
1293  .addReg(TIDIGXReg);
1294  // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1295  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1296  .addReg(STmp0)
1297  .addReg(TIDIGYReg)
1298  .addReg(TIDReg);
1299  // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1300  getAddNoCarry(Entry, Insert, DL, TIDReg)
1301  .addReg(TIDReg)
1302  .addReg(TIDIGZReg)
1303  .addImm(0); // clamp bit
1304  } else {
1305  // Get the wave id
1306  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1307  TIDReg)
1308  .addImm(-1)
1309  .addImm(0);
1310 
1311  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1312  TIDReg)
1313  .addImm(-1)
1314  .addReg(TIDReg);
1315  }
1316 
1317  BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1318  TIDReg)
1319  .addImm(2)
1320  .addReg(TIDReg);
1321  MFI->setTIDReg(TIDReg);
1322  }
1323 
1324  // Add FrameIndex to LDS offset
1325  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1326  getAddNoCarry(MBB, MI, DL, TmpReg)
1327  .addImm(LDSOffset)
1328  .addReg(TIDReg)
1329  .addImm(0); // clamp bit
1330 
1331  return TmpReg;
1332 }
1333 
1336  int Count) const {
1337  DebugLoc DL = MBB.findDebugLoc(MI);
1338  while (Count > 0) {
1339  int Arg;
1340  if (Count >= 8)
1341  Arg = 7;
1342  else
1343  Arg = Count - 1;
1344  Count -= 8;
1345  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1346  .addImm(Arg);
1347  }
1348 }
1349 
1352  insertWaitStates(MBB, MI, 1);
1353 }
1354 
1356  auto MF = MBB.getParent();
1358 
1359  assert(Info->isEntryFunction());
1360 
1361  if (MBB.succ_empty()) {
1362  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1363  if (HasNoTerminator) {
1364  if (Info->returnsVoid()) {
1365  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1366  } else {
1367  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1368  }
1369  }
1370  }
1371 }
1372 
1374  switch (MI.getOpcode()) {
1375  default: return 1; // FIXME: Do wait states equal cycles?
1376 
1377  case AMDGPU::S_NOP:
1378  return MI.getOperand(0).getImm() + 1;
1379  }
1380 }
1381 
1383  MachineBasicBlock &MBB = *MI.getParent();
1384  DebugLoc DL = MBB.findDebugLoc(MI);
1385  switch (MI.getOpcode()) {
1386  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1387  case AMDGPU::S_MOV_B64_term:
1388  // This is only a terminator to get the correct spill code placement during
1389  // register allocation.
1390  MI.setDesc(get(AMDGPU::S_MOV_B64));
1391  break;
1392 
1393  case AMDGPU::S_MOV_B32_term:
1394  // This is only a terminator to get the correct spill code placement during
1395  // register allocation.
1396  MI.setDesc(get(AMDGPU::S_MOV_B32));
1397  break;
1398 
1399  case AMDGPU::S_XOR_B64_term:
1400  // This is only a terminator to get the correct spill code placement during
1401  // register allocation.
1402  MI.setDesc(get(AMDGPU::S_XOR_B64));
1403  break;
1404 
1405  case AMDGPU::S_XOR_B32_term:
1406  // This is only a terminator to get the correct spill code placement during
1407  // register allocation.
1408  MI.setDesc(get(AMDGPU::S_XOR_B32));
1409  break;
1410 
1411  case AMDGPU::S_OR_B32_term:
1412  // This is only a terminator to get the correct spill code placement during
1413  // register allocation.
1414  MI.setDesc(get(AMDGPU::S_OR_B32));
1415  break;
1416 
1417  case AMDGPU::S_ANDN2_B64_term:
1418  // This is only a terminator to get the correct spill code placement during
1419  // register allocation.
1420  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1421  break;
1422 
1423  case AMDGPU::S_ANDN2_B32_term:
1424  // This is only a terminator to get the correct spill code placement during
1425  // register allocation.
1426  MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1427  break;
1428 
1429  case AMDGPU::V_MOV_B64_PSEUDO: {
1430  Register Dst = MI.getOperand(0).getReg();
1431  Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1432  Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1433 
1434  const MachineOperand &SrcOp = MI.getOperand(1);
1435  // FIXME: Will this work for 64-bit floating point immediates?
1436  assert(!SrcOp.isFPImm());
1437  if (SrcOp.isImm()) {
1438  APInt Imm(64, SrcOp.getImm());
1439  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1440  .addImm(Imm.getLoBits(32).getZExtValue())
1441  .addReg(Dst, RegState::Implicit | RegState::Define);
1442  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1443  .addImm(Imm.getHiBits(32).getZExtValue())
1444  .addReg(Dst, RegState::Implicit | RegState::Define);
1445  } else {
1446  assert(SrcOp.isReg());
1447  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1448  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1450  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1451  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1453  }
1454  MI.eraseFromParent();
1455  break;
1456  }
1457  case AMDGPU::V_SET_INACTIVE_B32: {
1458  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1459  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1460  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1461  .addReg(Exec);
1462  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1463  .add(MI.getOperand(2));
1464  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1465  .addReg(Exec);
1466  MI.eraseFromParent();
1467  break;
1468  }
1469  case AMDGPU::V_SET_INACTIVE_B64: {
1470  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1471  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1472  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1473  .addReg(Exec);
1474  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1475  MI.getOperand(0).getReg())
1476  .add(MI.getOperand(2));
1477  expandPostRAPseudo(*Copy);
1478  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1479  .addReg(Exec);
1480  MI.eraseFromParent();
1481  break;
1482  }
1483  case AMDGPU::V_MOVRELD_B32_V1:
1484  case AMDGPU::V_MOVRELD_B32_V2:
1485  case AMDGPU::V_MOVRELD_B32_V4:
1486  case AMDGPU::V_MOVRELD_B32_V8:
1487  case AMDGPU::V_MOVRELD_B32_V16: {
1488  const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1489  Register VecReg = MI.getOperand(0).getReg();
1490  bool IsUndef = MI.getOperand(1).isUndef();
1491  unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1492  assert(VecReg == MI.getOperand(1).getReg());
1493 
1494  MachineInstr *MovRel =
1495  BuildMI(MBB, MI, DL, MovRelDesc)
1496  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1497  .add(MI.getOperand(2))
1498  .addReg(VecReg, RegState::ImplicitDefine)
1499  .addReg(VecReg,
1500  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1501 
1502  const int ImpDefIdx =
1503  MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1504  const int ImpUseIdx = ImpDefIdx + 1;
1505  MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1506 
1507  MI.eraseFromParent();
1508  break;
1509  }
1510  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1511  MachineFunction &MF = *MBB.getParent();
1512  Register Reg = MI.getOperand(0).getReg();
1513  Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1514  Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1515 
1516  // Create a bundle so these instructions won't be re-ordered by the
1517  // post-RA scheduler.
1518  MIBundleBuilder Bundler(MBB, MI);
1519  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1520 
1521  // Add 32-bit offset from this instruction to the start of the
1522  // constant data.
1523  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1524  .addReg(RegLo)
1525  .add(MI.getOperand(1)));
1526 
1527  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1528  .addReg(RegHi);
1529  MIB.add(MI.getOperand(2));
1530 
1531  Bundler.append(MIB);
1532  finalizeBundle(MBB, Bundler.begin());
1533 
1534  MI.eraseFromParent();
1535  break;
1536  }
1537  case AMDGPU::ENTER_WWM: {
1538  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1539  // WWM is entered.
1540  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1541  : AMDGPU::S_OR_SAVEEXEC_B64));
1542  break;
1543  }
1544  case AMDGPU::EXIT_WWM: {
1545  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1546  // WWM is exited.
1547  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
1548  break;
1549  }
1550  case TargetOpcode::BUNDLE: {
1551  if (!MI.mayLoad() || MI.hasUnmodeledSideEffects())
1552  return false;
1553 
1554  // If it is a load it must be a memory clause
1556  I->isBundledWithSucc(); ++I) {
1557  I->unbundleFromSucc();
1558  for (MachineOperand &MO : I->operands())
1559  if (MO.isReg())
1560  MO.setIsInternalRead(false);
1561  }
1562 
1563  MI.eraseFromParent();
1564  break;
1565  }
1566  }
1567  return true;
1568 }
1569 
1571  MachineOperand &Src0,
1572  unsigned Src0OpName,
1573  MachineOperand &Src1,
1574  unsigned Src1OpName) const {
1575  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1576  if (!Src0Mods)
1577  return false;
1578 
1579  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1580  assert(Src1Mods &&
1581  "All commutable instructions have both src0 and src1 modifiers");
1582 
1583  int Src0ModsVal = Src0Mods->getImm();
1584  int Src1ModsVal = Src1Mods->getImm();
1585 
1586  Src1Mods->setImm(Src0ModsVal);
1587  Src0Mods->setImm(Src1ModsVal);
1588  return true;
1589 }
1590 
1592  MachineOperand &RegOp,
1593  MachineOperand &NonRegOp) {
1594  Register Reg = RegOp.getReg();
1595  unsigned SubReg = RegOp.getSubReg();
1596  bool IsKill = RegOp.isKill();
1597  bool IsDead = RegOp.isDead();
1598  bool IsUndef = RegOp.isUndef();
1599  bool IsDebug = RegOp.isDebug();
1600 
1601  if (NonRegOp.isImm())
1602  RegOp.ChangeToImmediate(NonRegOp.getImm());
1603  else if (NonRegOp.isFI())
1604  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1605  else
1606  return nullptr;
1607 
1608  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1609  NonRegOp.setSubReg(SubReg);
1610 
1611  return &MI;
1612 }
1613 
1615  unsigned Src0Idx,
1616  unsigned Src1Idx) const {
1617  assert(!NewMI && "this should never be used");
1618 
1619  unsigned Opc = MI.getOpcode();
1620  int CommutedOpcode = commuteOpcode(Opc);
1621  if (CommutedOpcode == -1)
1622  return nullptr;
1623 
1624  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1625  static_cast<int>(Src0Idx) &&
1626  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1627  static_cast<int>(Src1Idx) &&
1628  "inconsistency with findCommutedOpIndices");
1629 
1630  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1631  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1632 
1633  MachineInstr *CommutedMI = nullptr;
1634  if (Src0.isReg() && Src1.isReg()) {
1635  if (isOperandLegal(MI, Src1Idx, &Src0)) {
1636  // Be sure to copy the source modifiers to the right place.
1637  CommutedMI
1638  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1639  }
1640 
1641  } else if (Src0.isReg() && !Src1.isReg()) {
1642  // src0 should always be able to support any operand type, so no need to
1643  // check operand legality.
1644  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1645  } else if (!Src0.isReg() && Src1.isReg()) {
1646  if (isOperandLegal(MI, Src1Idx, &Src0))
1647  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1648  } else {
1649  // FIXME: Found two non registers to commute. This does happen.
1650  return nullptr;
1651  }
1652 
1653  if (CommutedMI) {
1654  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1655  Src1, AMDGPU::OpName::src1_modifiers);
1656 
1657  CommutedMI->setDesc(get(CommutedOpcode));
1658  }
1659 
1660  return CommutedMI;
1661 }
1662 
1663 // This needs to be implemented because the source modifiers may be inserted
1664 // between the true commutable operands, and the base
1665 // TargetInstrInfo::commuteInstruction uses it.
1667  unsigned &SrcOpIdx1) const {
1668  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1669 }
1670 
1671 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1672  unsigned &SrcOpIdx1) const {
1673  if (!Desc.isCommutable())
1674  return false;
1675 
1676  unsigned Opc = Desc.getOpcode();
1677  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1678  if (Src0Idx == -1)
1679  return false;
1680 
1681  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1682  if (Src1Idx == -1)
1683  return false;
1684 
1685  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1686 }
1687 
1688 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1689  int64_t BrOffset) const {
1690  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1691  // block is unanalyzable.
1692  assert(BranchOp != AMDGPU::S_SETPC_B64);
1693 
1694  // Convert to dwords.
1695  BrOffset /= 4;
1696 
1697  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1698  // from the next instruction.
1699  BrOffset -= 1;
1700 
1701  return isIntN(BranchOffsetBits, BrOffset);
1702 }
1703 
1705  const MachineInstr &MI) const {
1706  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1707  // This would be a difficult analysis to perform, but can always be legal so
1708  // there's no need to analyze it.
1709  return nullptr;
1710  }
1711 
1712  return MI.getOperand(0).getMBB();
1713 }
1714 
1716  MachineBasicBlock &DestBB,
1717  const DebugLoc &DL,
1718  int64_t BrOffset,
1719  RegScavenger *RS) const {
1720  assert(RS && "RegScavenger required for long branching");
1721  assert(MBB.empty() &&
1722  "new block should be inserted for expanding unconditional branch");
1723  assert(MBB.pred_size() == 1);
1724 
1725  MachineFunction *MF = MBB.getParent();
1726  MachineRegisterInfo &MRI = MF->getRegInfo();
1727 
1728  // FIXME: Virtual register workaround for RegScavenger not working with empty
1729  // blocks.
1730  Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1731 
1732  auto I = MBB.end();
1733 
1734  // We need to compute the offset relative to the instruction immediately after
1735  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1736  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1737 
1738  // TODO: Handle > 32-bit block address.
1739  if (BrOffset >= 0) {
1740  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1741  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1742  .addReg(PCReg, 0, AMDGPU::sub0)
1743  .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
1744  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1745  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1746  .addReg(PCReg, 0, AMDGPU::sub1)
1747  .addImm(0);
1748  } else {
1749  // Backwards branch.
1750  BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1751  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1752  .addReg(PCReg, 0, AMDGPU::sub0)
1753  .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
1754  BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1755  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1756  .addReg(PCReg, 0, AMDGPU::sub1)
1757  .addImm(0);
1758  }
1759 
1760  // Insert the indirect branch after the other terminator.
1761  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1762  .addReg(PCReg);
1763 
1764  // FIXME: If spilling is necessary, this will fail because this scavenger has
1765  // no emergency stack slots. It is non-trivial to spill in this situation,
1766  // because the restore code needs to be specially placed after the
1767  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1768  // block.
1769  //
1770  // If a spill is needed for the pc register pair, we need to insert a spill
1771  // restore block right before the destination block, and insert a short branch
1772  // into the old destination block's fallthrough predecessor.
1773  // e.g.:
1774  //
1775  // s_cbranch_scc0 skip_long_branch:
1776  //
1777  // long_branch_bb:
1778  // spill s[8:9]
1779  // s_getpc_b64 s[8:9]
1780  // s_add_u32 s8, s8, restore_bb
1781  // s_addc_u32 s9, s9, 0
1782  // s_setpc_b64 s[8:9]
1783  //
1784  // skip_long_branch:
1785  // foo;
1786  //
1787  // .....
1788  //
1789  // dest_bb_fallthrough_predecessor:
1790  // bar;
1791  // s_branch dest_bb
1792  //
1793  // restore_bb:
1794  // restore s[8:9]
1795  // fallthrough dest_bb
1796  ///
1797  // dest_bb:
1798  // buzz;
1799 
1800  RS->enterBasicBlockEnd(MBB);
1801  unsigned Scav = RS->scavengeRegisterBackwards(
1802  AMDGPU::SReg_64RegClass,
1803  MachineBasicBlock::iterator(GetPC), false, 0);
1804  MRI.replaceRegWith(PCReg, Scav);
1805  MRI.clearVirtRegs();
1806  RS->setRegUsed(Scav);
1807 
1808  return 4 + 8 + 4 + 4;
1809 }
1810 
1811 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1812  switch (Cond) {
1813  case SIInstrInfo::SCC_TRUE:
1814  return AMDGPU::S_CBRANCH_SCC1;
1815  case SIInstrInfo::SCC_FALSE:
1816  return AMDGPU::S_CBRANCH_SCC0;
1817  case SIInstrInfo::VCCNZ:
1818  return AMDGPU::S_CBRANCH_VCCNZ;
1819  case SIInstrInfo::VCCZ:
1820  return AMDGPU::S_CBRANCH_VCCZ;
1821  case SIInstrInfo::EXECNZ:
1822  return AMDGPU::S_CBRANCH_EXECNZ;
1823  case SIInstrInfo::EXECZ:
1824  return AMDGPU::S_CBRANCH_EXECZ;
1825  default:
1826  llvm_unreachable("invalid branch predicate");
1827  }
1828 }
1829 
1830 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1831  switch (Opcode) {
1832  case AMDGPU::S_CBRANCH_SCC0:
1833  return SCC_FALSE;
1834  case AMDGPU::S_CBRANCH_SCC1:
1835  return SCC_TRUE;
1836  case AMDGPU::S_CBRANCH_VCCNZ:
1837  return VCCNZ;
1838  case AMDGPU::S_CBRANCH_VCCZ:
1839  return VCCZ;
1840  case AMDGPU::S_CBRANCH_EXECNZ:
1841  return EXECNZ;
1842  case AMDGPU::S_CBRANCH_EXECZ:
1843  return EXECZ;
1844  default:
1845  return INVALID_BR;
1846  }
1847 }
1848 
1851  MachineBasicBlock *&TBB,
1852  MachineBasicBlock *&FBB,
1854  bool AllowModify) const {
1855  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1856  // Unconditional Branch
1857  TBB = I->getOperand(0).getMBB();
1858  return false;
1859  }
1860 
1861  MachineBasicBlock *CondBB = nullptr;
1862 
1863  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1864  CondBB = I->getOperand(1).getMBB();
1865  Cond.push_back(I->getOperand(0));
1866  } else {
1867  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1868  if (Pred == INVALID_BR)
1869  return true;
1870 
1871  CondBB = I->getOperand(0).getMBB();
1873  Cond.push_back(I->getOperand(1)); // Save the branch register.
1874  }
1875  ++I;
1876 
1877  if (I == MBB.end()) {
1878  // Conditional branch followed by fall-through.
1879  TBB = CondBB;
1880  return false;
1881  }
1882 
1883  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1884  TBB = CondBB;
1885  FBB = I->getOperand(0).getMBB();
1886  return false;
1887  }
1888 
1889  return true;
1890 }
1891 
1893  MachineBasicBlock *&FBB,
1895  bool AllowModify) const {
1897  auto E = MBB.end();
1898  if (I == E)
1899  return false;
1900 
1901  // Skip over the instructions that are artificially terminators for special
1902  // exec management.
1903  while (I != E && !I->isBranch() && !I->isReturn() &&
1904  I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1905  switch (I->getOpcode()) {
1906  case AMDGPU::SI_MASK_BRANCH:
1907  case AMDGPU::S_MOV_B64_term:
1908  case AMDGPU::S_XOR_B64_term:
1909  case AMDGPU::S_ANDN2_B64_term:
1910  case AMDGPU::S_MOV_B32_term:
1911  case AMDGPU::S_XOR_B32_term:
1912  case AMDGPU::S_OR_B32_term:
1913  case AMDGPU::S_ANDN2_B32_term:
1914  break;
1915  case AMDGPU::SI_IF:
1916  case AMDGPU::SI_ELSE:
1917  case AMDGPU::SI_KILL_I1_TERMINATOR:
1918  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1919  // FIXME: It's messy that these need to be considered here at all.
1920  return true;
1921  default:
1922  llvm_unreachable("unexpected non-branch terminator inst");
1923  }
1924 
1925  ++I;
1926  }
1927 
1928  if (I == E)
1929  return false;
1930 
1931  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1932  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1933 
1934  ++I;
1935 
1936  // TODO: Should be able to treat as fallthrough?
1937  if (I == MBB.end())
1938  return true;
1939 
1940  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1941  return true;
1942 
1943  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1944 
1945  // Specifically handle the case where the conditional branch is to the same
1946  // destination as the mask branch. e.g.
1947  //
1948  // si_mask_branch BB8
1949  // s_cbranch_execz BB8
1950  // s_cbranch BB9
1951  //
1952  // This is required to understand divergent loops which may need the branches
1953  // to be relaxed.
1954  if (TBB != MaskBrDest || Cond.empty())
1955  return true;
1956 
1957  auto Pred = Cond[0].getImm();
1958  return (Pred != EXECZ && Pred != EXECNZ);
1959 }
1960 
1962  int *BytesRemoved) const {
1964 
1965  unsigned Count = 0;
1966  unsigned RemovedSize = 0;
1967  while (I != MBB.end()) {
1968  MachineBasicBlock::iterator Next = std::next(I);
1969  if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1970  I = Next;
1971  continue;
1972  }
1973 
1974  RemovedSize += getInstSizeInBytes(*I);
1975  I->eraseFromParent();
1976  ++Count;
1977  I = Next;
1978  }
1979 
1980  if (BytesRemoved)
1981  *BytesRemoved = RemovedSize;
1982 
1983  return Count;
1984 }
1985 
1986 // Copy the flags onto the implicit condition register operand.
1988  const MachineOperand &OrigCond) {
1989  CondReg.setIsUndef(OrigCond.isUndef());
1990  CondReg.setIsKill(OrigCond.isKill());
1991 }
1992 
1994  MachineBasicBlock *TBB,
1995  MachineBasicBlock *FBB,
1997  const DebugLoc &DL,
1998  int *BytesAdded) const {
1999  if (!FBB && Cond.empty()) {
2000  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2001  .addMBB(TBB);
2002  if (BytesAdded)
2003  *BytesAdded = 4;
2004  return 1;
2005  }
2006 
2007  if(Cond.size() == 1 && Cond[0].isReg()) {
2008  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
2009  .add(Cond[0])
2010  .addMBB(TBB);
2011  return 1;
2012  }
2013 
2014  assert(TBB && Cond[0].isImm());
2015 
2016  unsigned Opcode
2017  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2018 
2019  if (!FBB) {
2020  Cond[1].isUndef();
2021  MachineInstr *CondBr =
2022  BuildMI(&MBB, DL, get(Opcode))
2023  .addMBB(TBB);
2024 
2025  // Copy the flags onto the implicit condition register operand.
2026  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2027 
2028  if (BytesAdded)
2029  *BytesAdded = 4;
2030  return 1;
2031  }
2032 
2033  assert(TBB && FBB);
2034 
2035  MachineInstr *CondBr =
2036  BuildMI(&MBB, DL, get(Opcode))
2037  .addMBB(TBB);
2038  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2039  .addMBB(FBB);
2040 
2041  MachineOperand &CondReg = CondBr->getOperand(1);
2042  CondReg.setIsUndef(Cond[1].isUndef());
2043  CondReg.setIsKill(Cond[1].isKill());
2044 
2045  if (BytesAdded)
2046  *BytesAdded = 8;
2047 
2048  return 2;
2049 }
2050 
2052  SmallVectorImpl<MachineOperand> &Cond) const {
2053  if (Cond.size() != 2) {
2054  return true;
2055  }
2056 
2057  if (Cond[0].isImm()) {
2058  Cond[0].setImm(-Cond[0].getImm());
2059  return false;
2060  }
2061 
2062  return true;
2063 }
2064 
2067  unsigned TrueReg, unsigned FalseReg,
2068  int &CondCycles,
2069  int &TrueCycles, int &FalseCycles) const {
2070  switch (Cond[0].getImm()) {
2071  case VCCNZ:
2072  case VCCZ: {
2073  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2074  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2075  assert(MRI.getRegClass(FalseReg) == RC);
2076 
2077  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2078  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2079 
2080  // Limit to equal cost for branch vs. N v_cndmask_b32s.
2081  return RI.hasVGPRs(RC) && NumInsts <= 6;
2082  }
2083  case SCC_TRUE:
2084  case SCC_FALSE: {
2085  // FIXME: We could insert for VGPRs if we could replace the original compare
2086  // with a vector one.
2087  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2088  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2089  assert(MRI.getRegClass(FalseReg) == RC);
2090 
2091  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2092 
2093  // Multiples of 8 can do s_cselect_b64
2094  if (NumInsts % 2 == 0)
2095  NumInsts /= 2;
2096 
2097  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2098  return RI.isSGPRClass(RC);
2099  }
2100  default:
2101  return false;
2102  }
2103 }
2104 
2107  unsigned DstReg, ArrayRef<MachineOperand> Cond,
2108  unsigned TrueReg, unsigned FalseReg) const {
2109  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2110  if (Pred == VCCZ || Pred == SCC_FALSE) {
2111  Pred = static_cast<BranchPredicate>(-Pred);
2112  std::swap(TrueReg, FalseReg);
2113  }
2114 
2116  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2117  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2118 
2119  if (DstSize == 32) {
2120  unsigned SelOp = Pred == SCC_TRUE ?
2121  AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
2122 
2123  // Instruction's operands are backwards from what is expected.
2124  MachineInstr *Select =
2125  BuildMI(MBB, I, DL, get(SelOp), DstReg)
2126  .addReg(FalseReg)
2127  .addReg(TrueReg);
2128 
2129  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2130  return;
2131  }
2132 
2133  if (DstSize == 64 && Pred == SCC_TRUE) {
2134  MachineInstr *Select =
2135  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2136  .addReg(FalseReg)
2137  .addReg(TrueReg);
2138 
2139  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2140  return;
2141  }
2142 
2143  static const int16_t Sub0_15[] = {
2144  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2145  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2146  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2147  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2148  };
2149 
2150  static const int16_t Sub0_15_64[] = {
2151  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2152  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2153  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2154  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2155  };
2156 
2157  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2158  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2159  const int16_t *SubIndices = Sub0_15;
2160  int NElts = DstSize / 32;
2161 
2162  // 64-bit select is only available for SALU.
2163  // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2164  if (Pred == SCC_TRUE) {
2165  if (NElts % 2) {
2166  SelOp = AMDGPU::S_CSELECT_B32;
2167  EltRC = &AMDGPU::SGPR_32RegClass;
2168  } else {
2169  SelOp = AMDGPU::S_CSELECT_B64;
2170  EltRC = &AMDGPU::SGPR_64RegClass;
2171  SubIndices = Sub0_15_64;
2172  NElts /= 2;
2173  }
2174  }
2175 
2177  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2178 
2179  I = MIB->getIterator();
2180 
2182  for (int Idx = 0; Idx != NElts; ++Idx) {
2183  Register DstElt = MRI.createVirtualRegister(EltRC);
2184  Regs.push_back(DstElt);
2185 
2186  unsigned SubIdx = SubIndices[Idx];
2187 
2188  MachineInstr *Select =
2189  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2190  .addReg(FalseReg, 0, SubIdx)
2191  .addReg(TrueReg, 0, SubIdx);
2192  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2193  fixImplicitOperands(*Select);
2194 
2195  MIB.addReg(DstElt)
2196  .addImm(SubIdx);
2197  }
2198 }
2199 
2201  switch (MI.getOpcode()) {
2202  case AMDGPU::V_MOV_B32_e32:
2203  case AMDGPU::V_MOV_B32_e64:
2204  case AMDGPU::V_MOV_B64_PSEUDO: {
2205  // If there are additional implicit register operands, this may be used for
2206  // register indexing so the source register operand isn't simply copied.
2207  unsigned NumOps = MI.getDesc().getNumOperands() +
2208  MI.getDesc().getNumImplicitUses();
2209 
2210  return MI.getNumOperands() == NumOps;
2211  }
2212  case AMDGPU::S_MOV_B32:
2213  case AMDGPU::S_MOV_B64:
2214  case AMDGPU::COPY:
2215  case AMDGPU::V_ACCVGPR_WRITE_B32:
2216  case AMDGPU::V_ACCVGPR_READ_B32:
2217  return true;
2218  default:
2219  return false;
2220  }
2221 }
2222 
2224  unsigned Kind) const {
2225  switch(Kind) {
2236  }
2237  return AMDGPUAS::FLAT_ADDRESS;
2238 }
2239 
2241  unsigned Opc = MI.getOpcode();
2242  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2243  AMDGPU::OpName::src0_modifiers);
2244  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2245  AMDGPU::OpName::src1_modifiers);
2246  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2247  AMDGPU::OpName::src2_modifiers);
2248 
2249  MI.RemoveOperand(Src2ModIdx);
2250  MI.RemoveOperand(Src1ModIdx);
2251  MI.RemoveOperand(Src0ModIdx);
2252 }
2253 
2255  unsigned Reg, MachineRegisterInfo *MRI) const {
2256  if (!MRI->hasOneNonDBGUse(Reg))
2257  return false;
2258 
2259  switch (DefMI.getOpcode()) {
2260  default:
2261  return false;
2262  case AMDGPU::S_MOV_B64:
2263  // TODO: We could fold 64-bit immediates, but this get compilicated
2264  // when there are sub-registers.
2265  return false;
2266 
2267  case AMDGPU::V_MOV_B32_e32:
2268  case AMDGPU::S_MOV_B32:
2269  case AMDGPU::V_ACCVGPR_WRITE_B32:
2270  break;
2271  }
2272 
2273  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2274  assert(ImmOp);
2275  // FIXME: We could handle FrameIndex values here.
2276  if (!ImmOp->isImm())
2277  return false;
2278 
2279  unsigned Opc = UseMI.getOpcode();
2280  if (Opc == AMDGPU::COPY) {
2281  bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2282  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2283  if (RI.isAGPR(*MRI, UseMI.getOperand(0).getReg())) {
2285  return false;
2286  NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32;
2287  }
2288  UseMI.setDesc(get(NewOpc));
2289  UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2290  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2291  return true;
2292  }
2293 
2294  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2295  Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64 ||
2296  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2297  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64) {
2298  // Don't fold if we are using source or output modifiers. The new VOP2
2299  // instructions don't have them.
2300  if (hasAnyModifiersSet(UseMI))
2301  return false;
2302 
2303  // If this is a free constant, there's no reason to do this.
2304  // TODO: We could fold this here instead of letting SIFoldOperands do it
2305  // later.
2306  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2307 
2308  // Any src operand can be used for the legality check.
2309  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2310  return false;
2311 
2312  bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2313  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64;
2314  bool IsFMA = Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2315  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64;
2316  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2317  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2318 
2319  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2320  // We should only expect these to be on src0 due to canonicalizations.
2321  if (Src0->isReg() && Src0->getReg() == Reg) {
2322  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2323  return false;
2324 
2325  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2326  return false;
2327 
2328  unsigned NewOpc =
2329  IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
2330  : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
2331  if (pseudoToMCOpcode(NewOpc) == -1)
2332  return false;
2333 
2334  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2335 
2336  const int64_t Imm = ImmOp->getImm();
2337 
2338  // FIXME: This would be a lot easier if we could return a new instruction
2339  // instead of having to modify in place.
2340 
2341  // Remove these first since they are at the end.
2342  UseMI.RemoveOperand(
2343  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2344  UseMI.RemoveOperand(
2345  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2346 
2347  Register Src1Reg = Src1->getReg();
2348  unsigned Src1SubReg = Src1->getSubReg();
2349  Src0->setReg(Src1Reg);
2350  Src0->setSubReg(Src1SubReg);
2351  Src0->setIsKill(Src1->isKill());
2352 
2353  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2354  Opc == AMDGPU::V_MAC_F16_e64 ||
2355  Opc == AMDGPU::V_FMAC_F32_e64 ||
2356  Opc == AMDGPU::V_FMAC_F16_e64)
2357  UseMI.untieRegOperand(
2358  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2359 
2360  Src1->ChangeToImmediate(Imm);
2361 
2362  removeModOperands(UseMI);
2363  UseMI.setDesc(get(NewOpc));
2364 
2365  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2366  if (DeleteDef)
2367  DefMI.eraseFromParent();
2368 
2369  return true;
2370  }
2371 
2372  // Added part is the constant: Use v_madak_{f16, f32}.
2373  if (Src2->isReg() && Src2->getReg() == Reg) {
2374  // Not allowed to use constant bus for another operand.
2375  // We can however allow an inline immediate as src0.
2376  bool Src0Inlined = false;
2377  if (Src0->isReg()) {
2378  // Try to inline constant if possible.
2379  // If the Def moves immediate and the use is single
2380  // We are saving VGPR here.
2381  MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2382  if (Def && Def->isMoveImmediate() &&
2383  isInlineConstant(Def->getOperand(1)) &&
2384  MRI->hasOneUse(Src0->getReg())) {
2385  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2386  Src0Inlined = true;
2387  } else if ((Register::isPhysicalRegister(Src0->getReg()) &&
2388  (ST.getConstantBusLimit(Opc) <= 1 &&
2389  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2390  (Register::isVirtualRegister(Src0->getReg()) &&
2391  (ST.getConstantBusLimit(Opc) <= 1 &&
2392  RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
2393  return false;
2394  // VGPR is okay as Src0 - fallthrough
2395  }
2396 
2397  if (Src1->isReg() && !Src0Inlined ) {
2398  // We have one slot for inlinable constant so far - try to fill it
2399  MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2400  if (Def && Def->isMoveImmediate() &&
2401  isInlineConstant(Def->getOperand(1)) &&
2402  MRI->hasOneUse(Src1->getReg()) &&
2403  commuteInstruction(UseMI)) {
2404  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2405  } else if ((Register::isPhysicalRegister(Src1->getReg()) &&
2406  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2407  (Register::isVirtualRegister(Src1->getReg()) &&
2408  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2409  return false;
2410  // VGPR is okay as Src1 - fallthrough
2411  }
2412 
2413  unsigned NewOpc =
2414  IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
2415  : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
2416  if (pseudoToMCOpcode(NewOpc) == -1)
2417  return false;
2418 
2419  const int64_t Imm = ImmOp->getImm();
2420 
2421  // FIXME: This would be a lot easier if we could return a new instruction
2422  // instead of having to modify in place.
2423 
2424  // Remove these first since they are at the end.
2425  UseMI.RemoveOperand(
2426  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2427  UseMI.RemoveOperand(
2428  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2429 
2430  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2431  Opc == AMDGPU::V_MAC_F16_e64 ||
2432  Opc == AMDGPU::V_FMAC_F32_e64 ||
2433  Opc == AMDGPU::V_FMAC_F16_e64)
2434  UseMI.untieRegOperand(
2435  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2436 
2437  // ChangingToImmediate adds Src2 back to the instruction.
2438  Src2->ChangeToImmediate(Imm);
2439 
2440  // These come before src2.
2441  removeModOperands(UseMI);
2442  UseMI.setDesc(get(NewOpc));
2443  // It might happen that UseMI was commuted
2444  // and we now have SGPR as SRC1. If so 2 inlined
2445  // constant and SGPR are illegal.
2446  legalizeOperands(UseMI);
2447 
2448  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2449  if (DeleteDef)
2450  DefMI.eraseFromParent();
2451 
2452  return true;
2453  }
2454  }
2455 
2456  return false;
2457 }
2458 
2459 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2460  int WidthB, int OffsetB) {
2461  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2462  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2463  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2464  return LowOffset + LowWidth <= HighOffset;
2465 }
2466 
2467 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2468  const MachineInstr &MIb) const {
2469  const MachineOperand *BaseOp0, *BaseOp1;
2470  int64_t Offset0, Offset1;
2471 
2472  if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2473  getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)) {
2474  if (!BaseOp0->isIdenticalTo(*BaseOp1))
2475  return false;
2476 
2477  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2478  // FIXME: Handle ds_read2 / ds_write2.
2479  return false;
2480  }
2481  unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2482  unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2483  if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2484  return true;
2485  }
2486  }
2487 
2488  return false;
2489 }
2490 
2492  const MachineInstr &MIb,
2493  AliasAnalysis *AA) const {
2494  assert((MIa.mayLoad() || MIa.mayStore()) &&
2495  "MIa must load from or modify a memory location");
2496  assert((MIb.mayLoad() || MIb.mayStore()) &&
2497  "MIb must load from or modify a memory location");
2498 
2500  return false;
2501 
2502  // XXX - Can we relax this between address spaces?
2503  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2504  return false;
2505 
2506  // TODO: Should we check the address space from the MachineMemOperand? That
2507  // would allow us to distinguish objects we know don't alias based on the
2508  // underlying address space, even if it was lowered to a different one,
2509  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2510  // buffer.
2511  if (isDS(MIa)) {
2512  if (isDS(MIb))
2513  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2514 
2515  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2516  }
2517 
2518  if (isMUBUF(MIa) || isMTBUF(MIa)) {
2519  if (isMUBUF(MIb) || isMTBUF(MIb))
2520  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2521 
2522  return !isFLAT(MIb) && !isSMRD(MIb);
2523  }
2524 
2525  if (isSMRD(MIa)) {
2526  if (isSMRD(MIb))
2527  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2528 
2529  return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2530  }
2531 
2532  if (isFLAT(MIa)) {
2533  if (isFLAT(MIb))
2534  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2535 
2536  return false;
2537  }
2538 
2539  return false;
2540 }
2541 
2542 static int64_t getFoldableImm(const MachineOperand* MO) {
2543  if (!MO->isReg())
2544  return false;
2545  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2546  const MachineRegisterInfo &MRI = MF->getRegInfo();
2547  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2548  if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2549  Def->getOperand(1).isImm())
2550  return Def->getOperand(1).getImm();
2551  return AMDGPU::NoRegister;
2552 }
2553 
2555  MachineInstr &MI,
2556  LiveVariables *LV) const {
2557  unsigned Opc = MI.getOpcode();
2558  bool IsF16 = false;
2559  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2560  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64;
2561 
2562  switch (Opc) {
2563  default:
2564  return nullptr;
2565  case AMDGPU::V_MAC_F16_e64:
2566  case AMDGPU::V_FMAC_F16_e64:
2567  IsF16 = true;
2569  case AMDGPU::V_MAC_F32_e64:
2570  case AMDGPU::V_FMAC_F32_e64:
2571  break;
2572  case AMDGPU::V_MAC_F16_e32:
2573  case AMDGPU::V_FMAC_F16_e32:
2574  IsF16 = true;
2576  case AMDGPU::V_MAC_F32_e32:
2577  case AMDGPU::V_FMAC_F32_e32: {
2578  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2579  AMDGPU::OpName::src0);
2580  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2581  if (!Src0->isReg() && !Src0->isImm())
2582  return nullptr;
2583 
2584  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2585  return nullptr;
2586 
2587  break;
2588  }
2589  }
2590 
2591  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2592  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2593  const MachineOperand *Src0Mods =
2594  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2595  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2596  const MachineOperand *Src1Mods =
2597  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2598  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2599  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2600  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2601 
2602  if (!Src0Mods && !Src1Mods && !Clamp && !Omod &&
2603  // If we have an SGPR input, we will violate the constant bus restriction.
2604  (ST.getConstantBusLimit(Opc) > 1 ||
2605  !Src0->isReg() ||
2606  !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2607  if (auto Imm = getFoldableImm(Src2)) {
2608  unsigned NewOpc =
2609  IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
2610  : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
2611  if (pseudoToMCOpcode(NewOpc) != -1)
2612  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2613  .add(*Dst)
2614  .add(*Src0)
2615  .add(*Src1)
2616  .addImm(Imm);
2617  }
2618  unsigned NewOpc =
2619  IsFMA ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
2620  : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
2621  if (auto Imm = getFoldableImm(Src1)) {
2622  if (pseudoToMCOpcode(NewOpc) != -1)
2623  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2624  .add(*Dst)
2625  .add(*Src0)
2626  .addImm(Imm)
2627  .add(*Src2);
2628  }
2629  if (auto Imm = getFoldableImm(Src0)) {
2630  if (pseudoToMCOpcode(NewOpc) != -1 &&
2632  AMDGPU::OpName::src0), Src1))
2633  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2634  .add(*Dst)
2635  .add(*Src1)
2636  .addImm(Imm)
2637  .add(*Src2);
2638  }
2639  }
2640 
2641  unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16 : AMDGPU::V_FMA_F32)
2642  : (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2643  if (pseudoToMCOpcode(NewOpc) == -1)
2644  return nullptr;
2645 
2646  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2647  .add(*Dst)
2648  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2649  .add(*Src0)
2650  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2651  .add(*Src1)
2652  .addImm(0) // Src mods
2653  .add(*Src2)
2654  .addImm(Clamp ? Clamp->getImm() : 0)
2655  .addImm(Omod ? Omod->getImm() : 0);
2656 }
2657 
2658 // It's not generally safe to move VALU instructions across these since it will
2659 // start using the register as a base index rather than directly.
2660 // XXX - Why isn't hasSideEffects sufficient for these?
2662  switch (MI.getOpcode()) {
2663  case AMDGPU::S_SET_GPR_IDX_ON:
2664  case AMDGPU::S_SET_GPR_IDX_MODE:
2665  case AMDGPU::S_SET_GPR_IDX_OFF:
2666  return true;
2667  default:
2668  return false;
2669  }
2670 }
2671 
2673  const MachineBasicBlock *MBB,
2674  const MachineFunction &MF) const {
2675  // XXX - Do we want the SP check in the base implementation?
2676 
2677  // Target-independent instructions do not have an implicit-use of EXEC, even
2678  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2679  // boundaries prevents incorrect movements of such instructions.
2680  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2681  MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2682  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2683  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2684  MI.getOpcode() == AMDGPU::S_DENORM_MODE ||
2686 }
2687 
2688 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
2689  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
2690  Opcode == AMDGPU::DS_GWS_INIT ||
2691  Opcode == AMDGPU::DS_GWS_SEMA_V ||
2692  Opcode == AMDGPU::DS_GWS_SEMA_BR ||
2693  Opcode == AMDGPU::DS_GWS_SEMA_P ||
2694  Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
2695  Opcode == AMDGPU::DS_GWS_BARRIER;
2696 }
2697 
2699  unsigned Opcode = MI.getOpcode();
2700 
2701  if (MI.mayStore() && isSMRD(MI))
2702  return true; // scalar store or atomic
2703 
2704  // This will terminate the function when other lanes may need to continue.
2705  if (MI.isReturn())
2706  return true;
2707 
2708  // These instructions cause shader I/O that may cause hardware lockups
2709  // when executed with an empty EXEC mask.
2710  //
2711  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2712  // EXEC = 0, but checking for that case here seems not worth it
2713  // given the typical code patterns.
2714  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2715  Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE ||
2716  Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
2717  Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
2718  return true;
2719 
2720  if (MI.isCall() || MI.isInlineAsm())
2721  return true; // conservative assumption
2722 
2723  // These are like SALU instructions in terms of effects, so it's questionable
2724  // whether we should return true for those.
2725  //
2726  // However, executing them with EXEC = 0 causes them to operate on undefined
2727  // data, which we avoid by returning true here.
2728  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2729  return true;
2730 
2731  return false;
2732 }
2733 
2735  const MachineInstr &MI) const {
2736  if (MI.isMetaInstruction())
2737  return false;
2738 
2739  // This won't read exec if this is an SGPR->SGPR copy.
2740  if (MI.isCopyLike()) {
2741  if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
2742  return true;
2743 
2744  // Make sure this isn't copying exec as a normal operand
2745  return MI.readsRegister(AMDGPU::EXEC, &RI);
2746  }
2747 
2748  // Make a conservative assumption about the callee.
2749  if (MI.isCall())
2750  return true;
2751 
2752  // Be conservative with any unhandled generic opcodes.
2753  if (!isTargetSpecificOpcode(MI.getOpcode()))
2754  return true;
2755 
2756  return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
2757 }
2758 
2759 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2760  switch (Imm.getBitWidth()) {
2761  case 1: // This likely will be a condition code mask.
2762  return true;
2763 
2764  case 32:
2766  ST.hasInv2PiInlineImm());
2767  case 64:
2769  ST.hasInv2PiInlineImm());
2770  case 16:
2771  return ST.has16BitInsts() &&
2773  ST.hasInv2PiInlineImm());
2774  default:
2775  llvm_unreachable("invalid bitwidth");
2776  }
2777 }
2778 
2780  uint8_t OperandType) const {
2781  if (!MO.isImm() ||
2782  OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2783  OperandType > AMDGPU::OPERAND_SRC_LAST)
2784  return false;
2785 
2786  // MachineOperand provides no way to tell the true operand size, since it only
2787  // records a 64-bit value. We need to know the size to determine if a 32-bit
2788  // floating point immediate bit pattern is legal for an integer immediate. It
2789  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2790 
2791  int64_t Imm = MO.getImm();
2792  switch (OperandType) {
2799  int32_t Trunc = static_cast<int32_t>(Imm);
2801  }
2807  ST.hasInv2PiInlineImm());
2814  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2815  // A few special case instructions have 16-bit operands on subtargets
2816  // where 16-bit instructions are not legal.
2817  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2818  // constants in these cases
2819  int16_t Trunc = static_cast<int16_t>(Imm);
2820  return ST.has16BitInsts() &&
2822  }
2823 
2824  return false;
2825  }
2832  uint32_t Trunc = static_cast<uint32_t>(Imm);
2834  }
2835  default:
2836  llvm_unreachable("invalid bitwidth");
2837  }
2838 }
2839 
2841  const MCOperandInfo &OpInfo) const {
2842  switch (MO.getType()) {
2844  return false;
2846  return !isInlineConstant(MO, OpInfo);
2852  return true;
2853  default:
2854  llvm_unreachable("unexpected operand type");
2855  }
2856 }
2857 
2858 static bool compareMachineOp(const MachineOperand &Op0,
2859  const MachineOperand &Op1) {
2860  if (Op0.getType() != Op1.getType())
2861  return false;
2862 
2863  switch (Op0.getType()) {
2865  return Op0.getReg() == Op1.getReg();
2867  return Op0.getImm() == Op1.getImm();
2868  default:
2869  llvm_unreachable("Didn't expect to be comparing these operand types");
2870  }
2871 }
2872 
2874  const MachineOperand &MO) const {
2875  const MCInstrDesc &InstDesc = MI.getDesc();
2876  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
2877 
2878  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
2879 
2880  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2881  return true;
2882 
2883  if (OpInfo.RegClass < 0)
2884  return false;
2885 
2886  const MachineFunction *MF = MI.getParent()->getParent();
2887  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2888 
2889  if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
2890  if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
2892  AMDGPU::OpName::src2))
2893  return false;
2894  return RI.opCanUseInlineConstant(OpInfo.OperandType);
2895  }
2896 
2897  if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
2898  return false;
2899 
2900  if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
2901  return true;
2902 
2903  return ST.hasVOP3Literal();
2904 }
2905 
2906 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2907  int Op32 = AMDGPU::getVOPe32(Opcode);
2908  if (Op32 == -1)
2909  return false;
2910 
2911  return pseudoToMCOpcode(Op32) != -1;
2912 }
2913 
2914 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2915  // The src0_modifier operand is present on all instructions
2916  // that have modifiers.
2917 
2918  return AMDGPU::getNamedOperandIdx(Opcode,
2919  AMDGPU::OpName::src0_modifiers) != -1;
2920 }
2921 
2923  unsigned OpName) const {
2924  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2925  return Mods && Mods->getImm();
2926 }
2927 
2929  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2930  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2931  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2932  hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2933  hasModifiersSet(MI, AMDGPU::OpName::omod);
2934 }
2935 
2937  const MachineRegisterInfo &MRI) const {
2938  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2939  // Can't shrink instruction with three operands.
2940  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2941  // a special case for it. It can only be shrunk if the third operand
2942  // is vcc, and src0_modifiers and src1_modifiers are not set.
2943  // We should handle this the same way we handle vopc, by addding
2944  // a register allocation hint pre-regalloc and then do the shrinking
2945  // post-regalloc.
2946  if (Src2) {
2947  switch (MI.getOpcode()) {
2948  default: return false;
2949 
2950  case AMDGPU::V_ADDC_U32_e64:
2951  case AMDGPU::V_SUBB_U32_e64:
2952  case AMDGPU::V_SUBBREV_U32_e64: {
2953  const MachineOperand *Src1
2954  = getNamedOperand(MI, AMDGPU::OpName::src1);
2955  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
2956  return false;
2957  // Additional verification is needed for sdst/src2.
2958  return true;
2959  }
2960  case AMDGPU::V_MAC_F32_e64:
2961  case AMDGPU::V_MAC_F16_e64:
2962  case AMDGPU::V_FMAC_F32_e64:
2963  case AMDGPU::V_FMAC_F16_e64:
2964  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2965  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2966  return false;
2967  break;
2968 
2969  case AMDGPU::V_CNDMASK_B32_e64:
2970  break;
2971  }
2972  }
2973 
2974  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2975  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
2976  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
2977  return false;
2978 
2979  // We don't need to check src0, all input types are legal, so just make sure
2980  // src0 isn't using any modifiers.
2981  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2982  return false;
2983 
2984  // Can it be shrunk to a valid 32 bit opcode?
2985  if (!hasVALU32BitEncoding(MI.getOpcode()))
2986  return false;
2987 
2988  // Check output modifiers
2989  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2990  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
2991 }
2992 
2993 // Set VCC operand with all flags from \p Orig, except for setting it as
2994 // implicit.
2996  const MachineOperand &Orig) {
2997 
2998  for (MachineOperand &Use : MI.implicit_operands()) {
2999  if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
3000  Use.setIsUndef(Orig.isUndef());
3001  Use.setIsKill(Orig.isKill());
3002  return;
3003  }
3004  }
3005 }
3006 
3008  unsigned Op32) const {
3009  MachineBasicBlock *MBB = MI.getParent();;
3010  MachineInstrBuilder Inst32 =
3011  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
3012 
3013  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3014  // For VOPC instructions, this is replaced by an implicit def of vcc.
3015  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
3016  if (Op32DstIdx != -1) {
3017  // dst
3018  Inst32.add(MI.getOperand(0));
3019  } else {
3020  assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
3021  (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
3022  "Unexpected case");
3023  }
3024 
3025  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3026 
3027  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3028  if (Src1)
3029  Inst32.add(*Src1);
3030 
3031  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3032 
3033  if (Src2) {
3034  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3035  if (Op32Src2Idx != -1) {
3036  Inst32.add(*Src2);
3037  } else {
3038  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3039  // replaced with an implicit read of vcc. This was already added
3040  // during the initial BuildMI, so find it to preserve the flags.
3041  copyFlagsToImplicitVCC(*Inst32, *Src2);
3042  }
3043  }
3044 
3045  return Inst32;
3046 }
3047 
3049  const MachineOperand &MO,
3050  const MCOperandInfo &OpInfo) const {
3051  // Literal constants use the constant bus.
3052  //if (isLiteralConstantLike(MO, OpInfo))
3053  // return true;
3054  if (MO.isImm())
3055  return !isInlineConstant(MO, OpInfo);
3056 
3057  if (!MO.isReg())
3058  return true; // Misc other operands like FrameIndex
3059 
3060  if (!MO.isUse())
3061  return false;
3062 
3064  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3065 
3066  // Null is free
3067  if (MO.getReg() == AMDGPU::SGPR_NULL)
3068  return false;
3069 
3070  // SGPRs use the constant bus
3071  if (MO.isImplicit()) {
3072  return MO.getReg() == AMDGPU::M0 ||
3073  MO.getReg() == AMDGPU::VCC ||
3074  MO.getReg() == AMDGPU::VCC_LO;
3075  } else {
3076  return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3077  AMDGPU::SReg_64RegClass.contains(MO.getReg());
3078  }
3079 }
3080 
3081 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
3082  for (const MachineOperand &MO : MI.implicit_operands()) {
3083  // We only care about reads.
3084  if (MO.isDef())
3085  continue;
3086 
3087  switch (MO.getReg()) {
3088  case AMDGPU::VCC:
3089  case AMDGPU::VCC_LO:
3090  case AMDGPU::VCC_HI:
3091  case AMDGPU::M0:
3092  case AMDGPU::FLAT_SCR:
3093  return MO.getReg();
3094 
3095  default:
3096  break;
3097  }
3098  }
3099 
3100  return AMDGPU::NoRegister;
3101 }
3102 
3103 static bool shouldReadExec(const MachineInstr &MI) {
3104  if (SIInstrInfo::isVALU(MI)) {
3105  switch (MI.getOpcode()) {
3106  case AMDGPU::V_READLANE_B32:
3107  case AMDGPU::V_READLANE_B32_gfx6_gfx7:
3108  case AMDGPU::V_READLANE_B32_gfx10:
3109  case AMDGPU::V_READLANE_B32_vi:
3110  case AMDGPU::V_WRITELANE_B32:
3111  case AMDGPU::V_WRITELANE_B32_gfx6_gfx7:
3112  case AMDGPU::V_WRITELANE_B32_gfx10:
3113  case AMDGPU::V_WRITELANE_B32_vi:
3114  return false;
3115  }
3116 
3117  return true;
3118  }
3119 
3120  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3121  SIInstrInfo::isSALU(MI) ||
3122  SIInstrInfo::isSMRD(MI))
3123  return false;
3124 
3125  return true;
3126 }
3127 
3128 static bool isSubRegOf(const SIRegisterInfo &TRI,
3129  const MachineOperand &SuperVec,
3130  const MachineOperand &SubReg) {
3131  if (Register::isPhysicalRegister(SubReg.getReg()))
3132  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3133 
3134  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3135  SubReg.getReg() == SuperVec.getReg();
3136 }
3137 
3139  StringRef &ErrInfo) const {
3140  uint16_t Opcode = MI.getOpcode();
3141  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3142  return true;
3143 
3144  const MachineFunction *MF = MI.getParent()->getParent();
3145  const MachineRegisterInfo &MRI = MF->getRegInfo();
3146 
3147  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3148  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3149  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3150 
3151  // Make sure the number of operands is correct.
3152  const MCInstrDesc &Desc = get(Opcode);
3153  if (!Desc.isVariadic() &&
3154  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
3155  ErrInfo = "Instruction has wrong number of operands.";
3156  return false;
3157  }
3158 
3159  if (MI.isInlineAsm()) {
3160  // Verify register classes for inlineasm constraints.
3161  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
3162  I != E; ++I) {
3163  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
3164  if (!RC)
3165  continue;
3166 
3167  const MachineOperand &Op = MI.getOperand(I);
3168  if (!Op.isReg())
3169  continue;
3170 
3171  Register Reg = Op.getReg();
3172  if (!Register::isVirtualRegister(Reg) && !RC->contains(Reg)) {
3173  ErrInfo = "inlineasm operand has incorrect register class.";
3174  return false;
3175  }
3176  }
3177 
3178  return true;
3179  }
3180 
3181  // Make sure the register classes are correct.
3182  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
3183  if (MI.getOperand(i).isFPImm()) {
3184  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3185  "all fp values to integers.";
3186  return false;
3187  }
3188 
3189  int RegClass = Desc.OpInfo[i].RegClass;
3190 
3191  switch (Desc.OpInfo[i].OperandType) {
3193  if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
3194  ErrInfo = "Illegal immediate value for operand.";
3195  return false;
3196  }
3197  break;
3200  break;
3211  const MachineOperand &MO = MI.getOperand(i);
3212  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
3213  ErrInfo = "Illegal immediate value for operand.";
3214  return false;
3215  }
3216  break;
3217  }
3220  // Check if this operand is an immediate.
3221  // FrameIndex operands will be replaced by immediates, so they are
3222  // allowed.
3223  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
3224  ErrInfo = "Expected immediate, but got non-immediate";
3225  return false;
3226  }
3228  default:
3229  continue;
3230  }
3231 
3232  if (!MI.getOperand(i).isReg())
3233  continue;
3234 
3235  if (RegClass != -1) {
3236  Register Reg = MI.getOperand(i).getReg();
3237  if (Reg == AMDGPU::NoRegister || Register::isVirtualRegister(Reg))
3238  continue;
3239 
3240  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3241  if (!RC->contains(Reg)) {
3242  ErrInfo = "Operand has incorrect register class.";
3243  return false;
3244  }
3245  }
3246  }
3247 
3248  // Verify SDWA
3249  if (isSDWA(MI)) {
3250  if (!ST.hasSDWA()) {
3251  ErrInfo = "SDWA is not supported on this target";
3252  return false;
3253  }
3254 
3255  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3256 
3257  const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3258 
3259  for (int OpIdx: OpIndicies) {
3260  if (OpIdx == -1)
3261  continue;
3262  const MachineOperand &MO = MI.getOperand(OpIdx);
3263 
3264  if (!ST.hasSDWAScalar()) {
3265  // Only VGPRS on VI
3266  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3267  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3268  return false;
3269  }
3270  } else {
3271  // No immediates on GFX9
3272  if (!MO.isReg()) {
3273  ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
3274  return false;
3275  }
3276  }
3277  }
3278 
3279  if (!ST.hasSDWAOmod()) {
3280  // No omod allowed on VI
3281  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3282  if (OMod != nullptr &&
3283  (!OMod->isImm() || OMod->getImm() != 0)) {
3284  ErrInfo = "OMod not allowed in SDWA instructions on VI";
3285  return false;
3286  }
3287  }
3288 
3289  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3290  if (isVOPC(BasicOpcode)) {
3291  if (!ST.hasSDWASdst() && DstIdx != -1) {
3292  // Only vcc allowed as dst on VI for VOPC
3293  const MachineOperand &Dst = MI.getOperand(DstIdx);
3294  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3295  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3296  return false;
3297  }
3298  } else if (!ST.hasSDWAOutModsVOPC()) {
3299  // No clamp allowed on GFX9 for VOPC
3300  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3301  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3302  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3303  return false;
3304  }
3305 
3306  // No omod allowed on GFX9 for VOPC
3307  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3308  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
3309  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3310  return false;
3311  }
3312  }
3313  }
3314 
3315  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3316  if (DstUnused && DstUnused->isImm() &&
3317  DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
3318  const MachineOperand &Dst = MI.getOperand(DstIdx);
3319  if (!Dst.isReg() || !Dst.isTied()) {
3320  ErrInfo = "Dst register should have tied register";
3321  return false;
3322  }
3323 
3324  const MachineOperand &TiedMO =
3325  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3326  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3327  ErrInfo =
3328  "Dst register should be tied to implicit use of preserved register";
3329  return false;
3330  } else if (Register::isPhysicalRegister(TiedMO.getReg()) &&
3331  Dst.getReg() != TiedMO.getReg()) {
3332  ErrInfo = "Dst register should use same physical register as preserved";
3333  return false;
3334  }
3335  }
3336  }
3337 
3338  // Verify MIMG
3339  if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
3340  // Ensure that the return type used is large enough for all the options
3341  // being used TFE/LWE require an extra result register.
3342  const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3343  if (DMask) {
3344  uint64_t DMaskImm = DMask->getImm();
3345  uint32_t RegCount =
3346  isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
3347  const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3348  const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3349  const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3350 
3351  // Adjust for packed 16 bit values
3352  if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
3353  RegCount >>= 1;
3354 
3355  // Adjust if using LWE or TFE
3356  if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
3357  RegCount += 1;
3358 
3359  const uint32_t DstIdx =
3360  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3361  const MachineOperand &Dst = MI.getOperand(DstIdx);
3362  if (Dst.isReg()) {
3363  const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3364  uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3365  if (RegCount > DstSize) {
3366  ErrInfo = "MIMG instruction returns too many registers for dst "
3367  "register class";
3368  return false;
3369  }
3370  }
3371  }
3372  }
3373 
3374  // Verify VOP*. Ignore multiple sgpr operands on writelane.
3375  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3376  && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
3377  // Only look at the true operands. Only a real operand can use the constant
3378  // bus, and we don't want to check pseudo-operands like the source modifier
3379  // flags.
3380  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3381 
3382  unsigned ConstantBusCount = 0;
3383  unsigned LiteralCount = 0;
3384 
3385  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3386  ++ConstantBusCount;
3387 
3388  SmallVector<unsigned, 2> SGPRsUsed;
3389  unsigned SGPRUsed = findImplicitSGPRRead(MI);
3390  if (SGPRUsed != AMDGPU::NoRegister) {
3391  ++ConstantBusCount;
3392  SGPRsUsed.push_back(SGPRUsed);
3393  }
3394 
3395  for (int OpIdx : OpIndices) {
3396  if (OpIdx == -1)
3397  break;
3398  const MachineOperand &MO = MI.getOperand(OpIdx);
3399  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3400  if (MO.isReg()) {
3401  SGPRUsed = MO.getReg();
3402  if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
3403  return !RI.regsOverlap(SGPRUsed, SGPR);
3404  })) {
3405  ++ConstantBusCount;
3406  SGPRsUsed.push_back(SGPRUsed);
3407  }
3408  } else {
3409  ++ConstantBusCount;
3410  ++LiteralCount;
3411  }
3412  }
3413  }
3414  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3415  // v_writelane_b32 is an exception from constant bus restriction:
3416  // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3417  if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
3418  Opcode != AMDGPU::V_WRITELANE_B32) {
3419  ErrInfo = "VOP* instruction violates constant bus restriction";
3420  return false;
3421  }
3422 
3423  if (isVOP3(MI) && LiteralCount) {
3424  if (LiteralCount && !ST.hasVOP3Literal()) {
3425  ErrInfo = "VOP3 instruction uses literal";
3426  return false;
3427  }
3428  if (LiteralCount > 1) {
3429  ErrInfo = "VOP3 instruction uses more than one literal";
3430  return false;
3431  }
3432  }
3433  }
3434 
3435  // Verify misc. restrictions on specific instructions.
3436  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3437  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3438  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3439  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3440  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3441  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3442  if (!compareMachineOp(Src0, Src1) &&
3443  !compareMachineOp(Src0, Src2)) {
3444  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3445  return false;
3446  }
3447  }
3448  }
3449 
3450  if (isSOP2(MI) || isSOPC(MI)) {
3451  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3452  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3453  unsigned Immediates = 0;
3454 
3455  if (!Src0.isReg() &&
3456  !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType))
3457  Immediates++;
3458  if (!Src1.isReg() &&
3459  !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType))
3460  Immediates++;
3461 
3462  if (Immediates > 1) {
3463  ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
3464  return false;
3465  }
3466  }
3467 
3468  if (isSOPK(MI)) {
3469  auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
3470  if (Desc.isBranch()) {
3471  if (!Op->isMBB()) {
3472  ErrInfo = "invalid branch target for SOPK instruction";
3473  return false;
3474  }
3475  } else {
3476  uint64_t Imm = Op->getImm();
3477  if (sopkIsZext(MI)) {
3478  if (!isUInt<16>(Imm)) {
3479  ErrInfo = "invalid immediate for SOPK instruction";
3480  return false;
3481  }
3482  } else {
3483  if (!isInt<16>(Imm)) {
3484  ErrInfo = "invalid immediate for SOPK instruction";
3485  return false;
3486  }
3487  }
3488  }
3489  }
3490 
3491  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3492  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3493  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3494  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3495  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3496  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3497 
3498  const unsigned StaticNumOps = Desc.getNumOperands() +
3499  Desc.getNumImplicitUses();
3500  const unsigned NumImplicitOps = IsDst ? 2 : 1;
3501 
3502  // Allow additional implicit operands. This allows a fixup done by the post
3503  // RA scheduler where the main implicit operand is killed and implicit-defs
3504  // are added for sub-registers that remain live after this instruction.
3505  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3506  ErrInfo = "missing implicit register operands";
3507  return false;
3508  }
3509 
3510  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3511  if (IsDst) {
3512  if (!Dst->isUse()) {
3513  ErrInfo = "v_movreld_b32 vdst should be a use operand";
3514  return false;
3515  }
3516 
3517  unsigned UseOpIdx;
3518  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3519  UseOpIdx != StaticNumOps + 1) {
3520  ErrInfo = "movrel implicit operands should be tied";
3521  return false;
3522  }
3523  }
3524 
3525  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3526  const MachineOperand &ImpUse
3527  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3528  if (!ImpUse.isReg() || !ImpUse.isUse() ||
3529  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3530  ErrInfo = "src0 should be subreg of implicit vector use";
3531  return false;
3532  }
3533  }
3534 
3535  // Make sure we aren't losing exec uses in the td files. This mostly requires
3536  // being careful when using let Uses to try to add other use registers.
3537  if (shouldReadExec(MI)) {
3538  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3539  ErrInfo = "VALU instruction does not implicitly read exec mask";
3540  return false;
3541  }
3542  }
3543 
3544  if (isSMRD(MI)) {
3545  if (MI.mayStore()) {
3546  // The register offset form of scalar stores may only use m0 as the
3547  // soffset register.
3548  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3549  if (Soff && Soff->getReg() != AMDGPU::M0) {
3550  ErrInfo = "scalar stores must use m0 as offset register";
3551  return false;
3552  }
3553  }
3554  }
3555 
3556  if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3557  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3558  if (Offset->getImm() != 0) {
3559  ErrInfo = "subtarget does not support offsets in flat instructions";
3560  return false;
3561  }
3562  }
3563 
3564  if (isMIMG(MI)) {
3565  const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
3566  if (DimOp) {
3567  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
3568  AMDGPU::OpName::vaddr0);
3569  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
3570  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
3571  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3573  const AMDGPU::MIMGDimInfo *Dim =
3575 
3576  if (!Dim) {
3577  ErrInfo = "dim is out of range";
3578  return false;
3579  }
3580 
3581  bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
3582  unsigned AddrWords = BaseOpcode->NumExtraArgs +
3583  (BaseOpcode->Gradients ? Dim->NumGradients : 0) +
3584  (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
3585  (BaseOpcode->LodOrClampOrMip ? 1 : 0);
3586 
3587  unsigned VAddrWords;
3588  if (IsNSA) {
3589  VAddrWords = SRsrcIdx - VAddr0Idx;
3590  } else {
3591  const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
3592  VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
3593  if (AddrWords > 8)
3594  AddrWords = 16;
3595  else if (AddrWords > 4)
3596  AddrWords = 8;
3597  else if (AddrWords == 3 && VAddrWords == 4) {
3598  // CodeGen uses the V4 variant of instructions for three addresses,
3599  // because the selection DAG does not support non-power-of-two types.
3600  AddrWords = 4;
3601  }
3602  }
3603 
3604  if (VAddrWords != AddrWords) {
3605  ErrInfo = "bad vaddr size";
3606  return false;
3607  }
3608  }
3609  }
3610 
3611  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3612  if (DppCt) {
3613  using namespace AMDGPU::DPP;
3614 
3615  unsigned DC = DppCt->getImm();
3616  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3617  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3623  ErrInfo = "Invalid dpp_ctrl value";
3624  return false;
3625  }
3626  if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
3627  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3628  ErrInfo = "Invalid dpp_ctrl value: "
3629  "wavefront shifts are not supported on GFX10+";
3630  return false;
3631  }
3632  if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
3633  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3634  ErrInfo = "Invalid dpp_ctrl value: "
3635  "broadcasts are not supported on GFX10+";
3636  return false;
3637  }
3639  ST.getGeneration() < AMDGPUSubtarget::GFX10) {
3640  ErrInfo = "Invalid dpp_ctrl value: "
3641  "row_share and row_xmask are not supported before GFX10";
3642  return false;
3643  }
3644  }
3645 
3646  return true;
3647 }
3648 
3649 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3650  switch (MI.getOpcode()) {
3651  default: return AMDGPU::INSTRUCTION_LIST_END;
3652  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3653  case AMDGPU::COPY: return AMDGPU::COPY;
3654  case AMDGPU::PHI: return AMDGPU::PHI;
3655  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3656  case AMDGPU::WQM: return AMDGPU::WQM;
3657  case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
3658  case AMDGPU::WWM: return AMDGPU::WWM;
3659  case AMDGPU::S_MOV_B32: {
3661  return MI.getOperand(1).isReg() ||
3662  RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
3663  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3664  }
3665  case AMDGPU::S_ADD_I32:
3666  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3667  case AMDGPU::S_ADDC_U32:
3668  return AMDGPU::V_ADDC_U32_e32;
3669  case AMDGPU::S_SUB_I32:
3670  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3671  // FIXME: These are not consistently handled, and selected when the carry is
3672  // used.
3673  case AMDGPU::S_ADD_U32:
3674  return AMDGPU::V_ADD_I32_e32;
3675  case AMDGPU::S_SUB_U32:
3676  return AMDGPU::V_SUB_I32_e32;
3677  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3678  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32;
3679  case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32;
3680  case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32;
3681  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3682  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3683  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3684  case AMDGPU::S_XNOR_B32:
3685  return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
3686  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3687  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3688  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3689  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3690  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3691  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3692  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3693  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3694  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3695  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3696  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3697  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3698  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3699  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3700  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3701  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3702  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3703  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3704  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3705  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3706  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3707  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3708  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3709  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3710  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3711  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3712  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3713  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3714  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3715  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3716  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3717  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3718  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3719  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3720  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3721  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3722  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3723  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3724  }
3726  "Unexpected scalar opcode without corresponding vector one!");
3727 }
3728 
3730  unsigned OpNo) const {
3732  const MCInstrDesc &Desc = get(MI.getOpcode());
3733  if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3734  Desc.OpInfo[OpNo].RegClass == -1) {
3735  Register Reg = MI.getOperand(OpNo).getReg();
3736 
3737  if (Register::isVirtualRegister(Reg))
3738  return MRI.getRegClass(Reg);
3739  return RI.getPhysRegClass(Reg);
3740  }
3741 
3742  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3743  return RI.getRegClass(RCID);
3744 }
3745 
3746 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3748  MachineBasicBlock *MBB = MI.getParent();
3749  MachineOperand &MO = MI.getOperand(OpIdx);
3751  const SIRegisterInfo *TRI =
3752  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3753  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3754  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3755  unsigned Size = TRI->getRegSizeInBits(*RC);
3756  unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
3757  if (MO.isReg())
3758  Opcode = AMDGPU::COPY;
3759  else if (RI.isSGPRClass(RC))
3760  Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
3761 
3762  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3763  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3764  VRC = &AMDGPU::VReg_64RegClass;
3765  else
3766  VRC = &AMDGPU::VGPR_32RegClass;
3767 
3768  Register Reg = MRI.createVirtualRegister(VRC);
3769  DebugLoc DL = MBB->findDebugLoc(I);
3770  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3771  MO.ChangeToRegister(Reg, false);
3772 }
3773 
3776  MachineOperand &SuperReg,
3777  const TargetRegisterClass *SuperRC,
3778  unsigned SubIdx,
3779  const TargetRegisterClass *SubRC)
3780  const {
3781  MachineBasicBlock *MBB = MI->getParent();
3782  DebugLoc DL = MI->getDebugLoc();
3783  Register SubReg = MRI.createVirtualRegister(SubRC);
3784 
3785  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3786  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3787  .addReg(SuperReg.getReg(), 0, SubIdx);
3788  return SubReg;
3789  }
3790 
3791  // Just in case the super register is itself a sub-register, copy it to a new
3792  // value so we don't need to worry about merging its subreg index with the
3793  // SubIdx passed to this function. The register coalescer should be able to
3794  // eliminate this extra copy.
3795  Register NewSuperReg = MRI.createVirtualRegister(SuperRC);
3796 
3797  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3798  .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3799 
3800  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3801  .addReg(NewSuperReg, 0, SubIdx);
3802 
3803  return SubReg;
3804 }
3805 
3809  MachineOperand &Op,
3810  const TargetRegisterClass *SuperRC,
3811  unsigned SubIdx,
3812  const TargetRegisterClass *SubRC) const {
3813  if (Op.isImm()) {
3814  if (SubIdx == AMDGPU::sub0)
3815  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3816  if (SubIdx == AMDGPU::sub1)
3817  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3818 
3819  llvm_unreachable("Unhandled register index for immediate");
3820  }
3821 
3822  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3823  SubIdx, SubRC);
3824  return MachineOperand::CreateReg(SubReg, false);
3825 }
3826 
3827 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3828 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3829  assert(Inst.getNumExplicitOperands() == 3);
3830  MachineOperand Op1 = Inst.getOperand(1);
3831  Inst.RemoveOperand(1);
3832  Inst.addOperand(Op1);
3833 }
3834 
3836  const MCOperandInfo &OpInfo,
3837  const MachineOperand &MO) const {
3838  if (!MO.isReg())
3839  return false;
3840 
3841  Register Reg = MO.getReg();
3843  ? MRI.getRegClass(Reg)
3844  : RI.getPhysRegClass(Reg);
3845 
3846  const SIRegisterInfo *TRI =
3847  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3848  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3849 
3850  // In order to be legal, the common sub-class must be equal to the
3851  // class of the current operand. For example:
3852  //
3853  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3854  // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3855  //
3856  // s_sendmsg 0, s0 ; Operand defined as m0reg
3857  // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3858 
3859  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3860 }
3861 
3863  const MCOperandInfo &OpInfo,
3864  const MachineOperand &MO) const {
3865  if (MO.isReg())
3866  return isLegalRegOperand(MRI, OpInfo, MO);
3867 
3868  // Handle non-register types that are treated like immediates.
3869  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3870  return true;
3871 }
3872 
3873 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3874  const MachineOperand *MO) const {
3875  const MachineFunction &MF = *MI.getParent()->getParent();
3876  const MachineRegisterInfo &MRI = MF.getRegInfo();
3877  const MCInstrDesc &InstDesc = MI.getDesc();
3878  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3879  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3880  const TargetRegisterClass *DefinedRC =
3881  OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3882  if (!MO)
3883  MO = &MI.getOperand(OpIdx);
3884 
3885  int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
3886  int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
3887  if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3888  if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--)
3889  return false;
3890 
3891  SmallDenseSet<RegSubRegPair> SGPRsUsed;
3892  if (MO->isReg())
3893  SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
3894 
3895  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3896  if (i == OpIdx)
3897  continue;
3898  const MachineOperand &Op = MI.getOperand(i);
3899  if (Op.isReg()) {
3900  RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
3901  if (!SGPRsUsed.count(SGPR) &&
3902  usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3903  if (--ConstantBusLimit <= 0)
3904  return false;
3905  SGPRsUsed.insert(SGPR);
3906  }
3907  } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3908  if (--ConstantBusLimit <= 0)
3909  return false;
3910  } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) &&
3911  isLiteralConstantLike(Op, InstDesc.OpInfo[i])) {
3912  if (!VOP3LiteralLimit--)
3913  return false;
3914  if (--ConstantBusLimit <= 0)
3915  return false;
3916  }
3917  }
3918  }
3919 
3920  if (MO->isReg()) {
3921  assert(DefinedRC);
3922  return isLegalRegOperand(MRI, OpInfo, *MO);
3923  }
3924 
3925  // Handle non-register types that are treated like immediates.
3926  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
3927 
3928  if (!DefinedRC) {
3929  // This operand expects an immediate.
3930  return true;
3931  }
3932 
3933  return isImmOperandLegal(MI, OpIdx, *MO);
3934 }
3935 
3937  MachineInstr &MI) const {
3938  unsigned Opc = MI.getOpcode();
3939  const MCInstrDesc &InstrDesc = get(Opc);
3940 
3941  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3942  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3943 
3944  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3945  MachineOperand &Src1 = MI.getOperand(Src1Idx);
3946 
3947  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3948  // we need to only have one constant bus use before GFX10.
3949  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3950  if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 &&
3951  Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) ||
3952  isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])))
3953  legalizeOpWithMove(MI, Src0Idx);
3954 
3955  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3956  // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3957  // src0/src1 with V_READFIRSTLANE.
3958  if (Opc == AMDGPU::V_WRITELANE_B32) {
3959  const DebugLoc &DL = MI.getDebugLoc();
3960  if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3961  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3962  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3963  .add(Src0);
3964  Src0.ChangeToRegister(Reg, false);
3965  }
3966  if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3967  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3968  const DebugLoc &DL = MI.getDebugLoc();
3969  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3970  .add(Src1);
3971  Src1.ChangeToRegister(Reg, false);
3972  }
3973  return;
3974  }
3975 
3976  // No VOP2 instructions support AGPRs.
3977  if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg()))
3978  legalizeOpWithMove(MI, Src0Idx);
3979 
3980  if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg()))
3981  legalizeOpWithMove(MI, Src1Idx);
3982 
3983  // VOP2 src0 instructions support all operand types, so we don't need to check
3984  // their legality. If src1 is already legal, we don't need to do anything.
3985  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3986  return;
3987 
3988  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3989  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3990  // select is uniform.
3991  if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3992  RI.isVGPR(MRI, Src1.getReg())) {
3993  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3994  const DebugLoc &DL = MI.getDebugLoc();
3995  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3996  .add(Src1);
3997  Src1.ChangeToRegister(Reg, false);
3998  return;
3999  }
4000 
4001  // We do not use commuteInstruction here because it is too aggressive and will
4002  // commute if it is possible. We only want to commute here if it improves
4003  // legality. This can be called a fairly large number of times so don't waste
4004  // compile time pointlessly swapping and checking legality again.
4005  if (HasImplicitSGPR || !MI.isCommutable()) {
4006  legalizeOpWithMove(MI, Src1Idx);
4007  return;
4008  }
4009 
4010  // If src0 can be used as src1, commuting will make the operands legal.
4011  // Otherwise we have to give up and insert a move.
4012  //
4013  // TODO: Other immediate-like operand kinds could be commuted if there was a
4014  // MachineOperand::ChangeTo* for them.
4015  if ((!Src1.isImm() && !Src1.isReg()) ||
4016  !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
4017  legalizeOpWithMove(MI, Src1Idx);
4018  return;
4019  }
4020 
4021  int CommutedOpc = commuteOpcode(MI);
4022  if (CommutedOpc == -1) {
4023  legalizeOpWithMove(MI, Src1Idx);
4024  return;
4025  }
4026 
4027  MI.setDesc(get(CommutedOpc));
4028 
4029  Register Src0Reg = Src0.getReg();
4030  unsigned Src0SubReg = Src0.getSubReg();
4031  bool Src0Kill = Src0.isKill();
4032 
4033  if (Src1.isImm())
4034  Src0.ChangeToImmediate(Src1.getImm());
4035  else if (Src1.isReg()) {
4036  Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
4037  Src0.setSubReg(Src1.getSubReg());
4038  } else
4039  llvm_unreachable("Should only have register or immediate operands");
4040 
4041  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
4042  Src1.setSubReg(Src0SubReg);
4043  fixImplicitOperands(MI);
4044 }
4045 
4046 // Legalize VOP3 operands. All operand types are supported for any operand
4047 // but only one literal constant and only starting from GFX10.
4049  MachineInstr &MI) const {
4050  unsigned Opc = MI.getOpcode();
4051 
4052  int VOP3Idx[3] = {
4053  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
4054  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
4055  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
4056  };
4057 
4058  if (Opc == AMDGPU::V_PERMLANE16_B32 ||
4059  Opc == AMDGPU::V_PERMLANEX16_B32) {
4060  // src1 and src2 must be scalar
4061  MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
4062  MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
4063  const DebugLoc &DL = MI.getDebugLoc();
4064  if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
4065  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4066  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4067  .add(Src1);
4068  Src1.ChangeToRegister(Reg, false);
4069  }
4070  if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
4071  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4072  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4073  .add(Src2);
4074  Src2.ChangeToRegister(Reg, false);
4075  }
4076  }
4077 
4078  // Find the one SGPR operand we are allowed to use.
4079  int ConstantBusLimit = ST.getConstantBusLimit(Opc);
4080  int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
4081  SmallDenseSet<unsigned> SGPRsUsed;
4082  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
4083  if (SGPRReg != AMDGPU::NoRegister) {
4084  SGPRsUsed.insert(SGPRReg);
4085  --ConstantBusLimit;
4086  }
4087 
4088  for (unsigned i = 0; i < 3; ++i) {
4089  int Idx = VOP3Idx[i];
4090  if (Idx == -1)
4091  break;
4092  MachineOperand &MO = MI.getOperand(Idx);
4093 
4094  if (!MO.isReg()) {
4095  if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
4096  continue;
4097 
4098  if (LiteralLimit > 0 && ConstantBusLimit > 0) {
4099  --LiteralLimit;
4100  --ConstantBusLimit;
4101  continue;
4102  }
4103 
4104  --LiteralLimit;
4105  --ConstantBusLimit;
4106  legalizeOpWithMove(MI, Idx);
4107  continue;
4108  }
4109 
4110  if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) &&
4111  !isOperandLegal(MI, Idx, &MO)) {
4112  legalizeOpWithMove(MI, Idx);
4113  continue;
4114  }
4115 
4116  if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
4117  continue; // VGPRs are legal
4118 
4119  // We can use one SGPR in each VOP3 instruction prior to GFX10
4120  // and two starting from GFX10.
4121  if (SGPRsUsed.count(MO.getReg()))
4122  continue;
4123  if (ConstantBusLimit > 0) {
4124  SGPRsUsed.insert(MO.getReg());
4125  --ConstantBusLimit;
4126  continue;
4127  }
4128 
4129  // If we make it this far, then the operand is not legal and we must
4130  // legalize it.
4131  legalizeOpWithMove(MI, Idx);
4132  }
4133 }
4134 
4136  MachineRegisterInfo &MRI) const {
4137  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
4138  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
4139  Register DstReg = MRI.createVirtualRegister(SRC);
4140  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
4141 
4142  if (RI.hasAGPRs(VRC)) {
4143  VRC = RI.getEquivalentVGPRClass(VRC);
4144  Register NewSrcReg = MRI.createVirtualRegister(VRC);
4145  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4146  get(TargetOpcode::COPY), NewSrcReg)
4147  .addReg(SrcReg);
4148  SrcReg = NewSrcReg;
4149  }
4150 
4151  if (SubRegs == 1) {
4152  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4153  get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
4154  .addReg(SrcReg);
4155  return DstReg;
4156  }
4157 
4159  for (unsigned i = 0; i < SubRegs; ++i) {
4160  Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4161  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4162  get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
4163  .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
4164  SRegs.push_back(SGPR);
4165  }
4166 
4167  MachineInstrBuilder MIB =
4168  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4169  get(AMDGPU::REG_SEQUENCE), DstReg);
4170  for (unsigned i = 0; i < SubRegs; ++i) {
4171  MIB.addReg(SRegs[i]);
4172  MIB.addImm(RI.getSubRegFromChannel(i));
4173  }
4174  return DstReg;
4175 }
4176 
4178  MachineInstr &MI) const {
4179 
4180  // If the pointer is store in VGPRs, then we need to move them to
4181  // SGPRs using v_readfirstlane. This is safe because we only select
4182  // loads with uniform pointers to SMRD instruction so we know the
4183  // pointer value is uniform.
4184  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
4185  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
4186  unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
4187  SBase->setReg(SGPR);
4188  }
4189  MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
4190  if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
4191  unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
4192  SOff->setReg(SGPR);
4193  }
4194 }
4195 
4198  const TargetRegisterClass *DstRC,
4199  MachineOperand &Op,
4201  const DebugLoc &DL) const {
4202  Register OpReg = Op.getReg();
4203  unsigned OpSubReg = Op.getSubReg();
4204 
4205  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
4206  RI.getRegClassForReg(MRI, OpReg), OpSubReg);
4207 
4208  // Check if operand is already the correct register class.
4209  if (DstRC == OpRC)
4210  return;
4211 
4212  Register DstReg = MRI.createVirtualRegister(DstRC);
4213  MachineInstr *Copy =
4214  BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
4215 
4216  Op.setReg(DstReg);
4217  Op.setSubReg(0);
4218 
4219  MachineInstr *Def = MRI.getVRegDef(OpReg);
4220  if (!Def)
4221  return;
4222 
4223  // Try to eliminate the copy if it is copying an immediate value.
4224  if (Def->isMoveImmediate())
4225  FoldImmediate(*Copy, *Def, OpReg, &MRI);
4226 
4227  bool ImpDef = Def->isImplicitDef();
4228  while (!ImpDef && Def && Def->isCopy()) {
4229  Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
4230  ImpDef = Def && Def->isImplicitDef();
4231  }
4232  if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
4233  !ImpDef)
4234  Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
4235 }
4236 
4237 // Emit the actual waterfall loop, executing the wrapped instruction for each
4238 // unique value of \p Rsrc across all lanes. In the best case we execute 1
4239 // iteration, in the worst case we execute 64 (once per lane).
4240 static void
4242  MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
4243  const DebugLoc &DL, MachineOperand &Rsrc) {
4244  MachineFunction &MF = *OrigBB.getParent();
4245  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4246  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4247  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4248  unsigned SaveExecOpc =
4249  ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
4250  unsigned XorTermOpc =
4251  ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
4252  unsigned AndOpc =
4253  ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
4254  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4255 
4256  MachineBasicBlock::iterator I = LoopBB.begin();
4257 
4258  Register VRsrc = Rsrc.getReg();
4259  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
4260 
4261  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4262  Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4263  Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4264  Register AndCond = MRI.createVirtualRegister(BoolXExecRC);
4265  Register SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4266  Register SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4267  Register SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4268  Register SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4269  Register SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4270 
4271  // Beginning of the loop, read the next Rsrc variant.
4272  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
4273  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
4274  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
4275  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
4276  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
4277  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
4278  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
4279  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
4280 
4281  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
4282  .addReg(SRsrcSub0)
4283  .addImm(AMDGPU::sub0)
4284  .addReg(SRsrcSub1)
4285  .addImm(AMDGPU::sub1)
4286  .addReg(SRsrcSub2)
4287  .addImm(AMDGPU::sub2)
4288  .addReg(SRsrcSub3)
4289  .addImm(AMDGPU::sub3);
4290 
4291  // Update Rsrc operand to use the SGPR Rsrc.
4292  Rsrc.setReg(SRsrc);
4293  Rsrc.setIsKill(true);
4294 
4295  // Identify all lanes with identical Rsrc operands in their VGPRs.
4296  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
4297  .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
4298  .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
4299  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
4300  .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
4301  .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
4302  BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndCond)
4303  .addReg(CondReg0)
4304  .addReg(CondReg1);
4305 
4306  MRI.setSimpleHint(SaveExec, AndCond);
4307 
4308  // Update EXEC to matching lanes, saving original to SaveExec.
4309  BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
4310  .addReg(AndCond, RegState::Kill);
4311 
4312  // The original instruction is here; we insert the terminators after it.
4313  I = LoopBB.end();
4314 
4315  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4316  BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
4317  .addReg(Exec)
4318  .addReg(SaveExec);
4319  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
4320 }
4321 
4322 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4323 // with SGPRs by iterating over all unique values across all lanes.
4325  MachineOperand &Rsrc, MachineDominatorTree *MDT) {
4326  MachineBasicBlock &MBB = *MI.getParent();
4327  MachineFunction &MF = *MBB.getParent();
4328  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4329  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4332  const DebugLoc &DL = MI.getDebugLoc();
4333  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4334  unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4335  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4336 
4337  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4338 
4339  // Save the EXEC mask
4340  BuildMI(MBB, I, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
4341 
4342  // Killed uses in the instruction we are waterfalling around will be
4343  // incorrect due to the added control-flow.
4344  for (auto &MO : MI.uses()) {
4345  if (MO.isReg() && MO.isUse()) {
4346  MRI.clearKillFlags(MO.getReg());
4347  }
4348  }
4349 
4350  // To insert the loop we need to split the block. Move everything after this
4351  // point to a new block, and insert a new empty block between the two.
4353  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
4354  MachineFunction::iterator MBBI(MBB);
4355  ++MBBI;
4356 
4357  MF.insert(MBBI, LoopBB);
4358  MF.insert(MBBI, RemainderBB);
4359 
4360  LoopBB->addSuccessor(LoopBB);
4361  LoopBB->addSuccessor(RemainderBB);
4362 
4363  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4365  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
4366  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
4367  LoopBB->splice(LoopBB->begin(), &MBB, J);
4368 
4369  MBB.addSuccessor(LoopBB);
4370 
4371  // Update dominators. We know that MBB immediately dominates LoopBB, that
4372  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4373  // dominates all of the successors transferred to it from MBB that MBB used
4374  // to dominate.
4375  if (MDT) {
4376  MDT->addNewBlock(LoopBB, &MBB);
4377  MDT->addNewBlock(RemainderBB, LoopBB);
4378  for (auto &Succ : RemainderBB->successors()) {
4379  if (MDT->dominates(&MBB, Succ)) {
4380  MDT->changeImmediateDominator(Succ, RemainderBB);
4381  }
4382  }
4383  }
4384 
4385  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
4386 
4387  // Restore the EXEC mask
4388  MachineBasicBlock::iterator First = RemainderBB->begin();
4389  BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
4390 }
4391 
4392 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4393 static std::tuple<unsigned, unsigned>
4395  MachineBasicBlock &MBB = *MI.getParent();
4396  MachineFunction &MF = *MBB.getParent();
4398 
4399  // Extract the ptr from the resource descriptor.
4400  unsigned RsrcPtr =
4401  TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
4402  AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
4403 
4404  // Create an empty resource descriptor
4405  Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4406  Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4407  Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4408  Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4409  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
4410 
4411  // Zero64 = 0
4412  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
4413  .addImm(0);
4414 
4415  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4416  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
4417  .addImm(RsrcDataFormat & 0xFFFFFFFF);
4418 
4419  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4420  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
4421  .addImm(RsrcDataFormat >> 32);
4422 
4423  // NewSRsrc = {Zero64, SRsrcFormat}
4424  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
4425  .addReg(Zero64)
4426  .addImm(AMDGPU::sub0_sub1)
4427  .addReg(SRsrcFormatLo)
4428  .addImm(AMDGPU::sub2)
4429  .addReg(SRsrcFormatHi)
4430  .addImm(AMDGPU::sub3);
4431 
4432  return std::make_tuple(RsrcPtr, NewSRsrc);
4433 }
4434 
4436  MachineDominatorTree *MDT) const {
4437  MachineFunction &MF = *MI.getParent()->getParent();
4439 
4440  // Legalize VOP2
4441  if (isVOP2(MI) || isVOPC(MI)) {
4442  legalizeOperandsVOP2(MRI, MI);
4443  return;
4444  }
4445 
4446  // Legalize VOP3
4447  if (isVOP3(MI)) {
4448  legalizeOperandsVOP3(MRI, MI);
4449  return;
4450  }
4451 
4452  // Legalize SMRD
4453  if (isSMRD(MI)) {
4454  legalizeOperandsSMRD(MRI, MI);
4455  return;
4456  }
4457 
4458  // Legalize REG_SEQUENCE and PHI
4459  // The register class of the operands much be the same type as the register
4460  // class of the output.
4461  if (MI.getOpcode() == AMDGPU::PHI) {
4462  const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
4463  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
4464  if (!MI.getOperand(i).isReg() ||
4466  continue;
4467  const TargetRegisterClass *OpRC =
4468  MRI.getRegClass(MI.getOperand(i).getReg());
4469  if (RI.hasVectorRegisters(OpRC)) {
4470  VRC = OpRC;
4471  } else {
4472  SRC = OpRC;
4473  }
4474  }
4475 
4476  // If any of the operands are VGPR registers, then they all most be
4477  // otherwise we will create illegal VGPR->SGPR copies when legalizing
4478  // them.
4479  if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
4480  if (!VRC) {
4481  assert(SRC);
4482  VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) ? RI.getEquivalentAGPRClass(SRC)
4483  : RI.getEquivalentVGPRClass(SRC);
4484  }
4485  RC = VRC;
4486  } else {
4487  RC = SRC;
4488  }
4489 
4490  // Update all the operands so they have the same type.
4491  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4492  MachineOperand &Op = MI.getOperand(I);
4493  if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4494  continue;
4495 
4496  // MI is a PHI instruction.
4497  MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
4498  MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
4499 
4500  // Avoid creating no-op copies with the same src and dst reg class. These
4501  // confuse some of the machine passes.
4502  legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
4503  }
4504  }
4505 
4506  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4507  // VGPR dest type and SGPR sources, insert copies so all operands are
4508  // VGPRs. This seems to help operand folding / the register coalescer.
4509  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
4510  MachineBasicBlock *MBB = MI.getParent();
4511  const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
4512  if (RI.hasVGPRs(DstRC)) {
4513  // Update all the operands so they are VGPR register classes. These may
4514  // not be the same register class because REG_SEQUENCE supports mixing
4515  // subregister index types e.g. sub0_sub1 + sub2 + sub3
4516  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4517  MachineOperand &Op = MI.getOperand(I);
4518  if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4519  continue;
4520 
4521  const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
4522  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
4523  if (VRC == OpRC)
4524  continue;
4525 
4526  legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
4527  Op.setIsKill();
4528  }
4529  }
4530 
4531  return;
4532  }
4533 
4534  // Legalize INSERT_SUBREG
4535  // src0 must have the same register class as dst
4536  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
4537  Register Dst = MI.getOperand(0).getReg();
4538  Register Src0 = MI.getOperand(1).getReg();
4539  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
4540  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
4541  if (DstRC != Src0RC) {
4542  MachineBasicBlock *MBB = MI.getParent();
4543  MachineOperand &Op = MI.getOperand(1);
4544  legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
4545  }
4546  return;
4547  }
4548 
4549  // Legalize SI_INIT_M0
4550  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
4551  MachineOperand &Src = MI.getOperand(0);
4552  if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
4553  Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
4554  return;
4555  }
4556 
4557  // Legalize MIMG and MUBUF/MTBUF for shaders.
4558  //
4559  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4560  // scratch memory access. In both cases, the legalization never involves
4561  // conversion to the addr64 form.
4562  if (isMIMG(MI) ||
4564  (isMUBUF(MI) || isMTBUF(MI)))) {
4565  MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
4566  if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
4567  unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
4568  SRsrc->setReg(SGPR);
4569  }
4570 
4571  MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
4572  if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
4573  unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
4574  SSamp->setReg(SGPR);
4575  }
4576  return;
4577  }
4578 
4579  // Legalize MUBUF* instructions.
4580  int RsrcIdx =
4581  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
4582  if (RsrcIdx != -1) {
4583  // We have an MUBUF instruction
4584  MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
4585  unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
4586  if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
4587  RI.getRegClass(RsrcRC))) {
4588  // The operands are legal.
4589  // FIXME: We may need to legalize operands besided srsrc.
4590  return;
4591  }
4592 
4593  // Legalize a VGPR Rsrc.
4594  //
4595  // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4596  // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4597  // a zero-value SRsrc.
4598  //
4599  // If the instruction is _OFFSET (both idxen and offen disabled), and we
4600  // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4601  // above.
4602  //
4603  // Otherwise we are on non-ADDR64 hardware, and/or we have
4604  // idxen/offen/bothen and we fall back to a waterfall loop.
4605 
4606  MachineBasicBlock &MBB = *MI.getParent();
4607 
4608  MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4609  if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
4610  // This is already an ADDR64 instruction so we need to add the pointer
4611  // extracted from the resource descriptor to the current value of VAddr.
4612  Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4613  Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4614  Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4615 
4616  const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4617  Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4618  Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4619 
4620  unsigned RsrcPtr, NewSRsrc;
4621  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4622 
4623  // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4624  const DebugLoc &DL = MI.getDebugLoc();
4625  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e64), NewVAddrLo)
4626  .addDef(CondReg0)
4627  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4628  .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
4629  .addImm(0);
4630 
4631  // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4632  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
4633  .addDef(CondReg1, RegState::Dead)
4634  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4635  .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
4636  .addReg(CondReg0, RegState::Kill)
4637  .addImm(0);
4638 
4639  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4640  BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4641  .addReg(NewVAddrLo)
4642  .addImm(AMDGPU::sub0)
4643  .addReg(NewVAddrHi)
4644  .addImm(AMDGPU::sub1);
4645 
4646  VAddr->setReg(NewVAddr);
4647  Rsrc->setReg(NewSRsrc);
4648  } else if (!VAddr && ST.hasAddr64()) {
4649  // This instructions is the _OFFSET variant, so we need to convert it to
4650  // ADDR64.
4651  assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4653  "FIXME: Need to emit flat atomics here");
4654 
4655  unsigned RsrcPtr, NewSRsrc;
4656  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4657 
4658  Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4659  MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4660  MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4661  MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4662  unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4663 
4664  // Atomics rith return have have an additional tied operand and are
4665  // missing some of the special bits.
4666  MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4667  MachineInstr *Addr64;
4668 
4669  if (!VDataIn) {
4670  // Regular buffer load / store.
4671  MachineInstrBuilder MIB =
4672  BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4673  .add(*VData)
4674  .addReg(NewVAddr)
4675  .addReg(NewSRsrc)
4676  .add(*SOffset)
4677  .add(*Offset);
4678 
4679  // Atomics do not have this operand.
4680  if (const MachineOperand *GLC =
4681  getNamedOperand(MI, AMDGPU::OpName::glc)) {
4682  MIB.addImm(GLC->getImm());
4683  }
4684  if (const MachineOperand *DLC =
4685  getNamedOperand(MI, AMDGPU::OpName::dlc)) {
4686  MIB.addImm(DLC->getImm());
4687  }
4688 
4689  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4690 
4691  if (const MachineOperand *TFE =
4692  getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4693  MIB.addImm(TFE->getImm());
4694  }
4695 
4696  MIB.cloneMemRefs(MI);
4697  Addr64 = MIB;
4698  } else {
4699  // Atomics with return.
4700  Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4701  .add(*VData)
4702  .add(*VDataIn)
4703  .addReg(NewVAddr)
4704  .addReg(NewSRsrc)
4705  .add(*SOffset)
4706  .add(*Offset)
4707  .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4708  .cloneMemRefs(MI);
4709  }
4710 
4711  MI.removeFromParent();
4712 
4713  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4714  BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4715  NewVAddr)
4716  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4717  .addImm(AMDGPU::sub0)
4718  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4719  .addImm(AMDGPU::sub1);
4720  } else {
4721  // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4722  // to SGPRs.
4723  loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4724  }
4725  }
4726 }
4727 
4729  MachineDominatorTree *MDT) const {
4730  SetVectorType Worklist;
4731  Worklist.insert(&TopInst);
4732 
4733  while (!Worklist.empty()) {
4734  MachineInstr &Inst = *Worklist.pop_back_val();
4735  MachineBasicBlock *MBB = Inst.getParent();
4737 
4738  unsigned Opcode = Inst.getOpcode();
4739  unsigned NewOpcode = getVALUOp(Inst);
4740 
4741  // Handle some special cases
4742  switch (Opcode) {
4743  default:
4744  break;
4745  case AMDGPU::S_ADD_U64_PSEUDO:
4746  case AMDGPU::S_SUB_U64_PSEUDO:
4747  splitScalar64BitAddSub(Worklist, Inst, MDT);
4748  Inst.eraseFromParent();
4749  continue;
4750  case AMDGPU::S_ADD_I32:
4751  case AMDGPU::S_SUB_I32:
4752  // FIXME: The u32 versions currently selected use the carry.
4753  if (moveScalarAddSub(Worklist, Inst, MDT))
4754  continue;
4755 
4756  // Default handling
4757  break;
4758  case AMDGPU::S_AND_B64:
4759  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4760  Inst.eraseFromParent();
4761  continue;
4762 
4763  case AMDGPU::S_OR_B64:
4764  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4765  Inst.eraseFromParent();
4766  continue;
4767 
4768  case AMDGPU::S_XOR_B64:
4769  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4770  Inst.eraseFromParent();
4771  continue;
4772 
4773  case AMDGPU::S_NAND_B64:
4774  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4775  Inst.eraseFromParent();
4776  continue;
4777 
4778  case AMDGPU::S_NOR_B64:
4779  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4780  Inst.eraseFromParent();
4781  continue;
4782 
4783  case AMDGPU::S_XNOR_B64:
4784  if (ST.hasDLInsts())
4785  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4786  else
4787  splitScalar64BitXnor(Worklist, Inst, MDT);
4788  Inst.eraseFromParent();
4789  continue;
4790 
4791  case AMDGPU::S_ANDN2_B64:
4792  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4793  Inst.eraseFromParent();
4794  continue;
4795 
4796  case AMDGPU::S_ORN2_B64:
4797  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4798  Inst.eraseFromParent();
4799  continue;
4800 
4801  case AMDGPU::S_NOT_B64:
4802  splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4803  Inst.eraseFromParent();
4804  continue;
4805 
4806  case AMDGPU::S_BCNT1_I32_B64:
4807  splitScalar64BitBCNT(Worklist, Inst);
4808  Inst.eraseFromParent();
4809  continue;
4810 
4811  case AMDGPU::S_BFE_I64:
4812  splitScalar64BitBFE(Worklist, Inst);
4813  Inst.eraseFromParent();
4814  continue;
4815 
4816  case AMDGPU::S_LSHL_B32:
4817  if (ST.hasOnlyRevVALUShifts()) {
4818  NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4819  swapOperands(Inst);
4820  }
4821  break;
4822  case AMDGPU::S_ASHR_I32:
4823  if (ST.hasOnlyRevVALUShifts()) {
4824  NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4825  swapOperands(Inst);
4826  }
4827  break;
4828  case AMDGPU::S_LSHR_B32:
4829  if (ST.hasOnlyRevVALUShifts()) {
4830  NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4831  swapOperands(Inst);
4832  }
4833  break;
4834  case AMDGPU::S_LSHL_B64:
4835  if (ST.hasOnlyRevVALUShifts()) {
4836  NewOpcode = AMDGPU::V_LSHLREV_B64;
4837  swapOperands(Inst);
4838  }
4839  break;
4840  case AMDGPU::S_ASHR_I64:
4841  if (ST.hasOnlyRevVALUShifts()) {
4842  NewOpcode = AMDGPU::V_ASHRREV_I64;
4843  swapOperands(Inst);
4844  }
4845  break;
4846  case AMDGPU::S_LSHR_B64:
4847  if (ST.hasOnlyRevVALUShifts()) {
4848  NewOpcode = AMDGPU::V_LSHRREV_B64;
4849  swapOperands(Inst);
4850  }
4851  break;
4852 
4853  case AMDGPU::S_ABS_I32:
4854  lowerScalarAbs(Worklist, Inst);
4855  Inst.eraseFromParent();
4856  continue;
4857 
4858  case AMDGPU::S_CBRANCH_SCC0:
4859  case AMDGPU::S_CBRANCH_SCC1:
4860  // Clear unused bits of vcc
4861  if (ST.isWave32())
4862  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
4863  AMDGPU::VCC_LO)
4864  .addReg(AMDGPU::EXEC_LO)
4865  .addReg(AMDGPU::VCC_LO);
4866  else
4867  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4868  AMDGPU::VCC)
4869  .addReg(AMDGPU::EXEC)
4870  .addReg(AMDGPU::VCC);
4871  break;
4872 
4873  case AMDGPU::S_BFE_U64:
4874  case AMDGPU::S_BFM_B64:
4875  llvm_unreachable("Moving this op to VALU not implemented");
4876 
4877  case AMDGPU::S_PACK_LL_B32_B16:
4878  case AMDGPU::S_PACK_LH_B32_B16:
4879  case AMDGPU::S_PACK_HH_B32_B16:
4880  movePackToVALU(Worklist, MRI, Inst);
4881  Inst.eraseFromParent();
4882  continue;
4883 
4884  case AMDGPU::S_XNOR_B32:
4885  lowerScalarXnor(Worklist, Inst);
4886  Inst.eraseFromParent();
4887  continue;
4888 
4889  case AMDGPU::S_NAND_B32:
4890  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4891  Inst.eraseFromParent();
4892  continue;
4893 
4894  case AMDGPU::S_NOR_B32:
4895  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4896  Inst.eraseFromParent();
4897  continue;
4898 
4899  case AMDGPU::S_ANDN2_B32:
4900  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4901  Inst.eraseFromParent();
4902  continue;
4903 
4904  case AMDGPU::S_ORN2_B32:
4905  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
4906  Inst.eraseFromParent();
4907  continue;
4908  }
4909 
4910  if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4911  // We cannot move this instruction to the VALU, so we should try to
4912  // legalize its operands instead.
4913  legalizeOperands(Inst, MDT);
4914  continue;
4915  }
4916 
4917  // Use the new VALU Opcode.
4918  const MCInstrDesc &NewDesc = get(NewOpcode);
4919  Inst.setDesc(NewDesc);
4920 
4921  // Remove any references to SCC. Vector instructions can't read from it, and
4922  // We're just about to add the implicit use / defs of VCC, and we don't want
4923  // both.
4924  for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
4925  MachineOperand &Op = Inst.getOperand(i);
4926  if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
4927  // Only propagate through live-def of SCC.
4928  if (Op.isDef() && !Op.isDead())
4929  addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
4930  Inst.RemoveOperand(i);
4931  }
4932  }
4933 
4934  if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4935  // We are converting these to a BFE, so we need to add the missing
4936  // operands for the size and offset.
4937  unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4940 
4941  } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4942  // The VALU version adds the second operand to the result, so insert an
4943  // extra 0 operand.
4945  }
4946 
4948  fixImplicitOperands(Inst);
4949 
4950  if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4951  const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4952  // If we need to move this to VGPRs, we need to unpack the second operand
4953  // back into the 2 separate ones for bit offset and width.
4954  assert(OffsetWidthOp.isImm() &&
4955  "Scalar BFE is only implemented for constant width and offset");
4956  uint32_t Imm = OffsetWidthOp.getImm();
4957 
4958  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4959  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4960  Inst.RemoveOperand(2); // Remove old immediate.
4961  Inst.addOperand(MachineOperand::CreateImm(Offset));
4962  Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4963  }
4964 
4965  bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4966  unsigned NewDstReg = AMDGPU::NoRegister;
4967  if (HasDst) {
4968  Register DstReg = Inst.getOperand(0).getReg();
4969  if (Register::isPhysicalRegister(DstReg))
4970  continue;
4971 
4972  // Update the destination register class.
4973  const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4974  if (!NewDstRC)
4975  continue;
4976 
4977  if (Inst.isCopy() &&
4979  NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4980  // Instead of creating a copy where src and dst are the same register
4981  // class, we just replace all uses of dst with src. These kinds of
4982  // copies interfere with the heuristics MachineSink uses to decide
4983  // whether or not to split a critical edge. Since the pass assumes
4984  // that copies will end up as machine instructions and not be
4985  // eliminated.
4986  addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4987  MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4988  MRI.clearKillFlags(Inst.getOperand(1).getReg());
4989  Inst.getOperand(0).setReg(DstReg);
4990 
4991  // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4992  // these are deleted later, but at -O0 it would leave a suspicious
4993  // looking illegal copy of an undef register.
4994  for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4995  Inst.RemoveOperand(I);
4996  Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4997  continue;
4998  }
4999 
5000  NewDstReg = MRI.createVirtualRegister(NewDstRC);
5001  MRI.replaceRegWith(DstReg, NewDstReg);
5002  }
5003 
5004  // Legalize the operands
5005  legalizeOperands(Inst, MDT);
5006 
5007  if (HasDst)
5008  addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
5009  }
5010 }
5011 
5012 // Add/sub require special handling to deal with carry outs.
5013 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
5014  MachineDominatorTree *MDT) const {
5015  if (ST.hasAddNoCarry()) {
5016  // Assume there is no user of scc since we don't select this in that case.
5017  // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
5018  // is used.
5019 
5020  MachineBasicBlock &MBB = *Inst.getParent();
5022 
5023  Register OldDstReg = Inst.getOperand(0).getReg();
5024  Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5025 
5026  unsigned Opc = Inst.getOpcode();
5027  assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
5028 
5029  unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
5030  AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
5031 
5032  assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
5033  Inst.RemoveOperand(3);
5034 
5035  Inst.setDesc(get(NewOpc));
5036  Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
5037  Inst.addImplicitDefUseOperands(*MBB.getParent());
5038  MRI.replaceRegWith(OldDstReg, ResultReg);
5039  legalizeOperands(Inst, MDT);
5040 
5041  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5042  return true;
5043  }
5044 
5045  return false;
5046 }
5047 
5048 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
5049  MachineInstr &Inst) const {
5050  MachineBasicBlock &MBB = *Inst.getParent();
5052  MachineBasicBlock::iterator MII = Inst;
5053  DebugLoc DL = Inst.getDebugLoc();
5054 
5055  MachineOperand &Dest = Inst.getOperand(0);
5056  MachineOperand &Src = Inst.getOperand(1);
5057  Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5058  Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5059 
5060  unsigned SubOp = ST.hasAddNoCarry() ?
5061  AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
5062 
5063  BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
5064  .addImm(0)
5065  .addReg(Src.getReg());
5066 
5067  BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
5068  .addReg(Src.getReg())
5069  .addReg(TmpReg);
5070 
5071  MRI.replaceRegWith(Dest.getReg(), ResultReg);
5072  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5073 }
5074 
5075 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
5076  MachineInstr &Inst) const {
5077  MachineBasicBlock &MBB = *Inst.getParent();
5079  MachineBasicBlock::iterator MII = Inst;
5080  const DebugLoc &DL = Inst.getDebugLoc();
5081 
5082  MachineOperand &Dest = Inst.getOperand(0);
5083  MachineOperand &Src0 = Inst.getOperand(1);
5084  MachineOperand &Src1 = Inst.getOperand(2);
5085 
5086  if (ST.hasDLInsts()) {
5087  Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5088  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
5089  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
5090 
5091  BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
5092  .add(Src0)
5093  .add(Src1);
5094 
5095  MRI.replaceRegWith(Dest.getReg(), NewDest);
5096  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5097  } else {
5098  // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
5099  // invert either source and then perform the XOR. If either source is a
5100  // scalar register, then we can leave the inversion on the scalar unit to
5101  // acheive a better distrubution of scalar and vector instructions.
5102  bool Src0IsSGPR = Src0.isReg() &&
5103  RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
5104  bool Src1IsSGPR = Src1.isReg() &&
5105  RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
5106  MachineInstr *Xor;
5107  Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5108  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5109 
5110  // Build a pair of scalar instructions and add them to the work list.
5111  // The next iteration over the work list will lower these to the vector
5112  // unit as necessary.
5113  if (Src0IsSGPR) {
5114  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
5115  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5116  .addReg(Temp)
5117  .add(Src1);
5118  } else if (Src1IsSGPR) {
5119  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
5120  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5121  .add(Src0)
5122  .addReg(Temp);
5123  } else {
5124  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
5125  .add(Src0)
5126  .add(Src1);
5127  MachineInstr *Not =
5128  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
5129  Worklist.insert(Not);
5130  }
5131 
5132  MRI.replaceRegWith(Dest.getReg(), NewDest);
5133 
5134  Worklist.insert(Xor);
5135 
5136  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5137  }
5138 }
5139 
5140 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist,
5141  MachineInstr &Inst,
5142  unsigned Opcode) const {
5143  MachineBasicBlock &MBB = *Inst.getParent();
5145  MachineBasicBlock::iterator MII = Inst;
5146  const DebugLoc &DL = Inst.getDebugLoc();
5147 
5148  MachineOperand &Dest = Inst.getOperand(0);
5149  MachineOperand &Src0 = Inst.getOperand(1);
5150  MachineOperand &Src1 = Inst.getOperand(2);
5151 
5152  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5153  Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5154 
5155  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
5156  .add(Src0)
5157  .add(Src1);
5158 
5159  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
5160  .addReg(Interm);
5161 
5162  Worklist.insert(&Op);
5163  Worklist.insert(&Not);
5164 
5165  MRI.replaceRegWith(Dest.getReg(), NewDest);
5166  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5167 }
5168 
5169 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist,
5170  MachineInstr &Inst,
5171  unsigned Opcode) const {
5172  MachineBasicBlock &MBB = *Inst.getParent();
5174  MachineBasicBlock::iterator MII = Inst;
5175  const DebugLoc &DL = Inst.getDebugLoc();
5176 
5177  MachineOperand &Dest = Inst.getOperand(0);
5178  MachineOperand &Src0 = Inst.getOperand(1);
5179  MachineOperand &Src1 = Inst.getOperand(2);
5180 
5181  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5182  Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5183 
5184  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
5185  .add(Src1);
5186 
5187  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
5188  .add(Src0)
5189  .addReg(Interm);
5190 
5191  Worklist.insert(&Not);
5192  Worklist.insert(&Op);
5193 
5194  MRI.replaceRegWith(Dest.getReg(), NewDest);
5195  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5196 }
5197 
5198 void SIInstrInfo::splitScalar64BitUnaryOp(
5199  SetVectorType &Worklist, MachineInstr &Inst,
5200  unsigned Opcode) const {
5201  MachineBasicBlock &MBB = *Inst.getParent();
5203 
5204  MachineOperand &Dest = Inst.getOperand(0);
5205  MachineOperand &Src0 = Inst.getOperand(1);
5206  DebugLoc DL = Inst.getDebugLoc();
5207 
5208  MachineBasicBlock::iterator MII = Inst;
5209 
5210  const MCInstrDesc &InstDesc = get(Opcode);
5211  const TargetRegisterClass *Src0RC = Src0.isReg() ?
5212  MRI.getRegClass(Src0.getReg()) :
5213  &AMDGPU::SGPR_32RegClass;
5214 
5215  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5216 
5217  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5218  AMDGPU::sub0, Src0SubRC);
5219 
5220  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5221  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5222  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5223 
5224  Register DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5225  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
5226 
5227  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5228  AMDGPU::sub1, Src0SubRC);
5229 
5230  Register DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5231  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
5232 
5233  Register FullDestReg = MRI.createVirtualRegister(NewDestRC);
5234  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5235  .addReg(DestSub0)
5236  .addImm(AMDGPU::sub0)
5237  .addReg(DestSub1)
5238  .addImm(AMDGPU::sub1);
5239 
5240  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5241 
5242  Worklist.insert(&LoHalf);
5243  Worklist.insert(&HiHalf);
5244 
5245  // We don't need to legalizeOperands here because for a single operand, src0
5246  // will support any kind of input.
5247 
5248  // Move all users of this moved value.
5249  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5250 }
5251 
5252 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
5253  MachineInstr &Inst,
5254  MachineDominatorTree *MDT) const {
5255  bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5256 
5257  MachineBasicBlock &MBB = *Inst.getParent();
5259  const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5260 
5261  Register FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5262  Register DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5263  Register DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5264 
5265  Register CarryReg = MRI.createVirtualRegister(CarryRC);
5266  Register DeadCarryReg = MRI.createVirtualRegister(CarryRC);
5267 
5268  MachineOperand &Dest = Inst.getOperand(0);
5269  MachineOperand &Src0 = Inst.getOperand(1);
5270  MachineOperand &Src1 = Inst.getOperand(2);
5271  const DebugLoc &DL = Inst.getDebugLoc();
5272  MachineBasicBlock::iterator MII = Inst;
5273 
5274  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
5275  const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
5276  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5277  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
5278 
5279  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
52