LLVM  10.0.0svn
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI Implementation of TargetInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SIInstrInfo.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "SIRegisterInfo.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/MC/MCInstrDesc.h"
52 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/Compiler.h"
59 #include <cassert>
60 #include <cstdint>
61 #include <iterator>
62 #include <utility>
63 
64 using namespace llvm;
65 
66 #define GET_INSTRINFO_CTOR_DTOR
67 #include "AMDGPUGenInstrInfo.inc"
68 
69 namespace llvm {
70 namespace AMDGPU {
71 #define GET_D16ImageDimIntrinsics_IMPL
72 #define GET_ImageDimIntrinsicTable_IMPL
73 #define GET_RsrcIntrinsics_IMPL
74 #include "AMDGPUGenSearchableTables.inc"
75 }
76 }
77 
78 
79 // Must be at least 4 to be able to branch over minimum unconditional branch
80 // code. This is only for making it possible to write reasonably small tests for
81 // long branches.
82 static cl::opt<unsigned>
83 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
84  cl::desc("Restrict range of branch instructions (DEBUG)"));
85 
87  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
88  RI(ST), ST(ST) {}
89 
90 //===----------------------------------------------------------------------===//
91 // TargetInstrInfo callbacks
92 //===----------------------------------------------------------------------===//
93 
94 static unsigned getNumOperandsNoGlue(SDNode *Node) {
95  unsigned N = Node->getNumOperands();
96  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
97  --N;
98  return N;
99 }
100 
101 /// Returns true if both nodes have the same value for the given
102 /// operand \p Op, or if both nodes do not have this operand.
103 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
104  unsigned Opc0 = N0->getMachineOpcode();
105  unsigned Opc1 = N1->getMachineOpcode();
106 
107  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
108  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
109 
110  if (Op0Idx == -1 && Op1Idx == -1)
111  return true;
112 
113 
114  if ((Op0Idx == -1 && Op1Idx != -1) ||
115  (Op1Idx == -1 && Op0Idx != -1))
116  return false;
117 
118  // getNamedOperandIdx returns the index for the MachineInstr's operands,
119  // which includes the result as the first operand. We are indexing into the
120  // MachineSDNode's operands, so we need to skip the result operand to get
121  // the real index.
122  --Op0Idx;
123  --Op1Idx;
124 
125  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
126 }
127 
129  AliasAnalysis *AA) const {
130  // TODO: The generic check fails for VALU instructions that should be
131  // rematerializable due to implicit reads of exec. We really want all of the
132  // generic logic for this except for this.
133  switch (MI.getOpcode()) {
134  case AMDGPU::V_MOV_B32_e32:
135  case AMDGPU::V_MOV_B32_e64:
136  case AMDGPU::V_MOV_B64_PSEUDO:
137  // No implicit operands.
138  return MI.getNumOperands() == MI.getDesc().getNumOperands();
139  default:
140  return false;
141  }
142 }
143 
145  int64_t &Offset0,
146  int64_t &Offset1) const {
147  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
148  return false;
149 
150  unsigned Opc0 = Load0->getMachineOpcode();
151  unsigned Opc1 = Load1->getMachineOpcode();
152 
153  // Make sure both are actually loads.
154  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
155  return false;
156 
157  if (isDS(Opc0) && isDS(Opc1)) {
158 
159  // FIXME: Handle this case:
160  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
161  return false;
162 
163  // Check base reg.
164  if (Load0->getOperand(0) != Load1->getOperand(0))
165  return false;
166 
167  // Skip read2 / write2 variants for simplicity.
168  // TODO: We should report true if the used offsets are adjacent (excluded
169  // st64 versions).
170  int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
171  int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
172  if (Offset0Idx == -1 || Offset1Idx == -1)
173  return false;
174 
175  // XXX - be careful of datalesss loads
176  // getNamedOperandIdx returns the index for MachineInstrs. Since they
177  // include the output in the operand list, but SDNodes don't, we need to
178  // subtract the index by one.
179  Offset0Idx -= get(Opc0).NumDefs;
180  Offset1Idx -= get(Opc1).NumDefs;
181  Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
182  Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
183  return true;
184  }
185 
186  if (isSMRD(Opc0) && isSMRD(Opc1)) {
187  // Skip time and cache invalidation instructions.
188  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
189  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
190  return false;
191 
193 
194  // Check base reg.
195  if (Load0->getOperand(0) != Load1->getOperand(0))
196  return false;
197 
198  const ConstantSDNode *Load0Offset =
199  dyn_cast<ConstantSDNode>(Load0->getOperand(1));
200  const ConstantSDNode *Load1Offset =
201  dyn_cast<ConstantSDNode>(Load1->getOperand(1));
202 
203  if (!Load0Offset || !Load1Offset)
204  return false;
205 
206  Offset0 = Load0Offset->getZExtValue();
207  Offset1 = Load1Offset->getZExtValue();
208  return true;
209  }
210 
211  // MUBUF and MTBUF can access the same addresses.
212  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
213 
214  // MUBUF and MTBUF have vaddr at different indices.
215  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
216  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
217  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
218  return false;
219 
220  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
221  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
222 
223  if (OffIdx0 == -1 || OffIdx1 == -1)
224  return false;
225 
226  // getNamedOperandIdx returns the index for MachineInstrs. Since they
227  // include the output in the operand list, but SDNodes don't, we need to
228  // subtract the index by one.
229  OffIdx0 -= get(Opc0).NumDefs;
230  OffIdx1 -= get(Opc1).NumDefs;
231 
232  SDValue Off0 = Load0->getOperand(OffIdx0);
233  SDValue Off1 = Load1->getOperand(OffIdx1);
234 
235  // The offset might be a FrameIndexSDNode.
236  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
237  return false;
238 
239  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
240  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
241  return true;
242  }
243 
244  return false;
245 }
246 
247 static bool isStride64(unsigned Opc) {
248  switch (Opc) {
249  case AMDGPU::DS_READ2ST64_B32:
250  case AMDGPU::DS_READ2ST64_B64:
251  case AMDGPU::DS_WRITE2ST64_B32:
252  case AMDGPU::DS_WRITE2ST64_B64:
253  return true;
254  default:
255  return false;
256  }
257 }
258 
260  const MachineOperand *&BaseOp,
261  int64_t &Offset,
262  const TargetRegisterInfo *TRI) const {
263  unsigned Opc = LdSt.getOpcode();
264 
265  if (isDS(LdSt)) {
266  const MachineOperand *OffsetImm =
267  getNamedOperand(LdSt, AMDGPU::OpName::offset);
268  if (OffsetImm) {
269  // Normal, single offset LDS instruction.
270  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
271  // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
272  // report that here?
273  if (!BaseOp)
274  return false;
275 
276  Offset = OffsetImm->getImm();
277  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
278  "operands of type register.");
279  return true;
280  }
281 
282  // The 2 offset instructions use offset0 and offset1 instead. We can treat
283  // these as a load with a single offset if the 2 offsets are consecutive. We
284  // will use this for some partially aligned loads.
285  const MachineOperand *Offset0Imm =
286  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
287  const MachineOperand *Offset1Imm =
288  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
289 
290  uint8_t Offset0 = Offset0Imm->getImm();
291  uint8_t Offset1 = Offset1Imm->getImm();
292 
293  if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
294  // Each of these offsets is in element sized units, so we need to convert
295  // to bytes of the individual reads.
296 
297  unsigned EltSize;
298  if (LdSt.mayLoad())
299  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
300  else {
301  assert(LdSt.mayStore());
302  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
303  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
304  }
305 
306  if (isStride64(Opc))
307  EltSize *= 64;
308 
309  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
310  Offset = EltSize * Offset0;
311  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
312  "operands of type register.");
313  return true;
314  }
315 
316  return false;
317  }
318 
319  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
320  const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
321  if (SOffset && SOffset->isReg()) {
322  // We can only handle this if it's a stack access, as any other resource
323  // would require reporting multiple base registers.
324  const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
325  if (AddrReg && !AddrReg->isFI())
326  return false;
327 
328  const MachineOperand *RSrc = getNamedOperand(LdSt, AMDGPU::OpName::srsrc);
329  const SIMachineFunctionInfo *MFI
331  if (RSrc->getReg() != MFI->getScratchRSrcReg())
332  return false;
333 
334  const MachineOperand *OffsetImm =
335  getNamedOperand(LdSt, AMDGPU::OpName::offset);
336  BaseOp = SOffset;
337  Offset = OffsetImm->getImm();
338  return true;
339  }
340 
341  const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
342  if (!AddrReg)
343  return false;
344 
345  const MachineOperand *OffsetImm =
346  getNamedOperand(LdSt, AMDGPU::OpName::offset);
347  BaseOp = AddrReg;
348  Offset = OffsetImm->getImm();
349 
350  if (SOffset) // soffset can be an inline immediate.
351  Offset += SOffset->getImm();
352 
353  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
354  "operands of type register.");
355  return true;
356  }
357 
358  if (isSMRD(LdSt)) {
359  const MachineOperand *OffsetImm =
360  getNamedOperand(LdSt, AMDGPU::OpName::offset);
361  if (!OffsetImm)
362  return false;
363 
364  const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
365  BaseOp = SBaseReg;
366  Offset = OffsetImm->getImm();
367  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
368  "operands of type register.");
369  return true;
370  }
371 
372  if (isFLAT(LdSt)) {
373  const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
374  if (VAddr) {
375  // Can't analyze 2 offsets.
376  if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
377  return false;
378 
379  BaseOp = VAddr;
380  } else {
381  // scratch instructions have either vaddr or saddr.
382  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
383  }
384 
385  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
386  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
387  "operands of type register.");
388  return true;
389  }
390 
391  return false;
392 }
393 
394 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
395  const MachineOperand &BaseOp1,
396  const MachineInstr &MI2,
397  const MachineOperand &BaseOp2) {
398  // Support only base operands with base registers.
399  // Note: this could be extended to support FI operands.
400  if (!BaseOp1.isReg() || !BaseOp2.isReg())
401  return false;
402 
403  if (BaseOp1.isIdenticalTo(BaseOp2))
404  return true;
405 
406  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
407  return false;
408 
409  auto MO1 = *MI1.memoperands_begin();
410  auto MO2 = *MI2.memoperands_begin();
411  if (MO1->getAddrSpace() != MO2->getAddrSpace())
412  return false;
413 
414  auto Base1 = MO1->getValue();
415  auto Base2 = MO2->getValue();
416  if (!Base1 || !Base2)
417  return false;
418  const MachineFunction &MF = *MI1.getParent()->getParent();
419  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
420  Base1 = GetUnderlyingObject(Base1, DL);
421  Base2 = GetUnderlyingObject(Base1, DL);
422 
423  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
424  return false;
425 
426  return Base1 == Base2;
427 }
428 
430  const MachineOperand &BaseOp2,
431  unsigned NumLoads) const {
432  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
433  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
434 
435  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
436  return false;
437 
438  const MachineOperand *FirstDst = nullptr;
439  const MachineOperand *SecondDst = nullptr;
440 
441  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
442  (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
443  (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
444  const unsigned MaxGlobalLoadCluster = 6;
445  if (NumLoads > MaxGlobalLoadCluster)
446  return false;
447 
448  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
449  if (!FirstDst)
450  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
451  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
452  if (!SecondDst)
453  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
454  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
455  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
456  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
457  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
458  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
459  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
460  }
461 
462  if (!FirstDst || !SecondDst)
463  return false;
464 
465  // Try to limit clustering based on the total number of bytes loaded
466  // rather than the number of instructions. This is done to help reduce
467  // register pressure. The method used is somewhat inexact, though,
468  // because it assumes that all loads in the cluster will load the
469  // same number of bytes as FirstLdSt.
470 
471  // The unit of this value is bytes.
472  // FIXME: This needs finer tuning.
473  unsigned LoadClusterThreshold = 16;
474 
475  const MachineRegisterInfo &MRI =
476  FirstLdSt.getParent()->getParent()->getRegInfo();
477 
478  const Register Reg = FirstDst->getReg();
479 
481  ? MRI.getRegClass(Reg)
482  : RI.getPhysRegClass(Reg);
483 
484  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
485 }
486 
487 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
488 // the first 16 loads will be interleaved with the stores, and the next 16 will
489 // be clustered as expected. It should really split into 2 16 store batches.
490 //
491 // Loads are clustered until this returns false, rather than trying to schedule
492 // groups of stores. This also means we have to deal with saying different
493 // address space loads should be clustered, and ones which might cause bank
494 // conflicts.
495 //
496 // This might be deprecated so it might not be worth that much effort to fix.
498  int64_t Offset0, int64_t Offset1,
499  unsigned NumLoads) const {
500  assert(Offset1 > Offset0 &&
501  "Second offset should be larger than first offset!");
502  // If we have less than 16 loads in a row, and the offsets are within 64
503  // bytes, then schedule together.
504 
505  // A cacheline is 64 bytes (for global memory).
506  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
507 }
508 
511  const DebugLoc &DL, unsigned DestReg,
512  unsigned SrcReg, bool KillSrc) {
513  MachineFunction *MF = MBB.getParent();
514  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
515  "illegal SGPR to VGPR copy",
516  DL, DS_Error);
517  LLVMContext &C = MF->getFunction().getContext();
518  C.diagnose(IllegalCopy);
519 
520  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
521  .addReg(SrcReg, getKillRegState(KillSrc));
522 }
523 
526  const DebugLoc &DL, unsigned DestReg,
527  unsigned SrcReg, bool KillSrc) const {
528  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
529 
530  if (RC == &AMDGPU::VGPR_32RegClass) {
531  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
532  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
533  AMDGPU::AGPR_32RegClass.contains(SrcReg));
534  unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
535  AMDGPU::V_ACCVGPR_READ_B32 : AMDGPU::V_MOV_B32_e32;
536  BuildMI(MBB, MI, DL, get(Opc), DestReg)
537  .addReg(SrcReg, getKillRegState(KillSrc));
538  return;
539  }
540 
541  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
542  RC == &AMDGPU::SReg_32RegClass) {
543  if (SrcReg == AMDGPU::SCC) {
544  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
545  .addImm(-1)
546  .addImm(0);
547  return;
548  }
549 
550  if (DestReg == AMDGPU::VCC_LO) {
551  if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
552  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
553  .addReg(SrcReg, getKillRegState(KillSrc));
554  } else {
555  // FIXME: Hack until VReg_1 removed.
556  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
557  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
558  .addImm(0)
559  .addReg(SrcReg, getKillRegState(KillSrc));
560  }
561 
562  return;
563  }
564 
565  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
566  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
567  return;
568  }
569 
570  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
571  .addReg(SrcReg, getKillRegState(KillSrc));
572  return;
573  }
574 
575  if (RC == &AMDGPU::SReg_64RegClass) {
576  if (DestReg == AMDGPU::VCC) {
577  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
578  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
579  .addReg(SrcReg, getKillRegState(KillSrc));
580  } else {
581  // FIXME: Hack until VReg_1 removed.
582  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
583  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
584  .addImm(0)
585  .addReg(SrcReg, getKillRegState(KillSrc));
586  }
587 
588  return;
589  }
590 
591  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
592  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
593  return;
594  }
595 
596  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
597  .addReg(SrcReg, getKillRegState(KillSrc));
598  return;
599  }
600 
601  if (DestReg == AMDGPU::SCC) {
602  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
603  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
604  .addReg(SrcReg, getKillRegState(KillSrc))
605  .addImm(0);
606  return;
607  }
608 
609  if (RC == &AMDGPU::AGPR_32RegClass) {
610  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
611  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
612  AMDGPU::AGPR_32RegClass.contains(SrcReg));
613  if (!AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
614  // First try to find defining accvgpr_write to avoid temporary registers.
615  for (auto Def = MI, E = MBB.begin(); Def != E; ) {
616  --Def;
617  if (!Def->definesRegister(SrcReg, &RI))
618  continue;
619  if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32)
620  break;
621 
622  MachineOperand &DefOp = Def->getOperand(1);
623  assert(DefOp.isReg() || DefOp.isImm());
624 
625  if (DefOp.isReg()) {
626  // Check that register source operand if not clobbered before MI.
627  // Immediate operands are always safe to propagate.
628  bool SafeToPropagate = true;
629  for (auto I = Def; I != MI && SafeToPropagate; ++I)
630  if (I->modifiesRegister(DefOp.getReg(), &RI))
631  SafeToPropagate = false;
632 
633  if (!SafeToPropagate)
634  break;
635 
636  DefOp.setIsKill(false);
637  }
638 
639  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
640  .add(DefOp);
641  return;
642  }
643 
644  RegScavenger RS;
645  RS.enterBasicBlock(MBB);
646  RS.forward(MI);
647 
648  // Ideally we want to have three registers for a long reg_sequence copy
649  // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
650  unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
651  *MBB.getParent());
652 
653  // Registers in the sequence are allocated contiguously so we can just
654  // use register number to pick one of three round-robin temps.
655  unsigned RegNo = DestReg % 3;
656  unsigned Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
657  if (!Tmp)
658  report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
659  RS.setRegUsed(Tmp);
660  // Only loop through if there are any free registers left, otherwise
661  // scavenger may report a fatal error without emergency spill slot
662  // or spill with the slot.
663  while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
664  unsigned Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
665  if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
666  break;
667  Tmp = Tmp2;
668  RS.setRegUsed(Tmp);
669  }
670  copyPhysReg(MBB, MI, DL, Tmp, SrcReg, KillSrc);
671  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
672  .addReg(Tmp, RegState::Kill);
673  return;
674  }
675 
676  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
677  .addReg(SrcReg, getKillRegState(KillSrc));
678  return;
679  }
680 
681  unsigned EltSize = 4;
682  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
683  if (RI.isSGPRClass(RC)) {
684  // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
685  if (!(RI.getRegSizeInBits(*RC) % 64)) {
686  Opcode = AMDGPU::S_MOV_B64;
687  EltSize = 8;
688  } else {
689  Opcode = AMDGPU::S_MOV_B32;
690  EltSize = 4;
691  }
692 
693  if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
694  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
695  return;
696  }
697  } else if (RI.hasAGPRs(RC)) {
698  Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ?
699  AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
700  } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) {
701  Opcode = AMDGPU::V_ACCVGPR_READ_B32;
702  }
703 
704  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
705  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
706 
707  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
708  unsigned SubIdx;
709  if (Forward)
710  SubIdx = SubIndices[Idx];
711  else
712  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
713 
714  if (Opcode == TargetOpcode::COPY) {
715  copyPhysReg(MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
716  RI.getSubReg(SrcReg, SubIdx), KillSrc);
717  continue;
718  }
719 
720  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
721  get(Opcode), RI.getSubReg(DestReg, SubIdx));
722 
723  Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
724 
725  if (Idx == 0)
726  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
727 
728  bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
729  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
730  }
731 }
732 
733 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
734  int NewOpc;
735 
736  // Try to map original to commuted opcode
737  NewOpc = AMDGPU::getCommuteRev(Opcode);
738  if (NewOpc != -1)
739  // Check if the commuted (REV) opcode exists on the target.
740  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
741 
742  // Try to map commuted to original opcode
743  NewOpc = AMDGPU::getCommuteOrig(Opcode);
744  if (NewOpc != -1)
745  // Check if the original (non-REV) opcode exists on the target.
746  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
747 
748  return Opcode;
749 }
750 
753  const DebugLoc &DL, unsigned DestReg,
754  int64_t Value) const {
756  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
757  if (RegClass == &AMDGPU::SReg_32RegClass ||
758  RegClass == &AMDGPU::SGPR_32RegClass ||
759  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
760  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
761  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
762  .addImm(Value);
763  return;
764  }
765 
766  if (RegClass == &AMDGPU::SReg_64RegClass ||
767  RegClass == &AMDGPU::SGPR_64RegClass ||
768  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
769  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
770  .addImm(Value);
771  return;
772  }
773 
774  if (RegClass == &AMDGPU::VGPR_32RegClass) {
775  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
776  .addImm(Value);
777  return;
778  }
779  if (RegClass == &AMDGPU::VReg_64RegClass) {
780  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
781  .addImm(Value);
782  return;
783  }
784 
785  unsigned EltSize = 4;
786  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
787  if (RI.isSGPRClass(RegClass)) {
788  if (RI.getRegSizeInBits(*RegClass) > 32) {
789  Opcode = AMDGPU::S_MOV_B64;
790  EltSize = 8;
791  } else {
792  Opcode = AMDGPU::S_MOV_B32;
793  EltSize = 4;
794  }
795  }
796 
797  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
798  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
799  int64_t IdxValue = Idx == 0 ? Value : 0;
800 
801  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
802  get(Opcode), RI.getSubReg(DestReg, Idx));
803  Builder.addImm(IdxValue);
804  }
805 }
806 
807 const TargetRegisterClass *
809  return &AMDGPU::VGPR_32RegClass;
810 }
811 
814  const DebugLoc &DL, unsigned DstReg,
816  unsigned TrueReg,
817  unsigned FalseReg) const {
819  MachineFunction *MF = MBB.getParent();
820  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
821  const TargetRegisterClass *BoolXExecRC =
822  RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
823  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
824  "Not a VGPR32 reg");
825 
826  if (Cond.size() == 1) {
827  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
828  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
829  .add(Cond[0]);
830  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
831  .addImm(0)
832  .addReg(FalseReg)
833  .addImm(0)
834  .addReg(TrueReg)
835  .addReg(SReg);
836  } else if (Cond.size() == 2) {
837  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
838  switch (Cond[0].getImm()) {
839  case SIInstrInfo::SCC_TRUE: {
840  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
841  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
842  : AMDGPU::S_CSELECT_B64), SReg)
843  .addImm(-1)
844  .addImm(0);
845  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
846  .addImm(0)
847  .addReg(FalseReg)
848  .addImm(0)
849  .addReg(TrueReg)
850  .addReg(SReg);
851  break;
852  }
853  case SIInstrInfo::SCC_FALSE: {
854  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
855  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
856  : AMDGPU::S_CSELECT_B64), SReg)
857  .addImm(0)
858  .addImm(-1);
859  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
860  .addImm(0)
861  .addReg(FalseReg)
862  .addImm(0)
863  .addReg(TrueReg)
864  .addReg(SReg);
865  break;
866  }
867  case SIInstrInfo::VCCNZ: {
868  MachineOperand RegOp = Cond[1];
869  RegOp.setImplicit(false);
870  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
871  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
872  .add(RegOp);
873  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
874  .addImm(0)
875  .addReg(FalseReg)
876  .addImm(0)
877  .addReg(TrueReg)
878  .addReg(SReg);
879  break;
880  }
881  case SIInstrInfo::VCCZ: {
882  MachineOperand RegOp = Cond[1];
883  RegOp.setImplicit(false);
884  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
885  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
886  .add(RegOp);
887  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
888  .addImm(0)
889  .addReg(TrueReg)
890  .addImm(0)
891  .addReg(FalseReg)
892  .addReg(SReg);
893  break;
894  }
895  case SIInstrInfo::EXECNZ: {
896  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
897  Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
898  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
899  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
900  .addImm(0);
901  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
902  : AMDGPU::S_CSELECT_B64), SReg)
903  .addImm(-1)
904  .addImm(0);
905  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
906  .addImm(0)
907  .addReg(FalseReg)
908  .addImm(0)
909  .addReg(TrueReg)
910  .addReg(SReg);
911  break;
912  }
913  case SIInstrInfo::EXECZ: {
914  Register SReg = MRI.createVirtualRegister(BoolXExecRC);
915  Register SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
916  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
917  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
918  .addImm(0);
919  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
920  : AMDGPU::S_CSELECT_B64), SReg)
921  .addImm(0)
922  .addImm(-1);
923  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
924  .addImm(0)
925  .addReg(FalseReg)
926  .addImm(0)
927  .addReg(TrueReg)
928  .addReg(SReg);
929  llvm_unreachable("Unhandled branch predicate EXECZ");
930  break;
931  }
932  default:
933  llvm_unreachable("invalid branch predicate");
934  }
935  } else {
936  llvm_unreachable("Can only handle Cond size 1 or 2");
937  }
938 }
939 
942  const DebugLoc &DL,
943  unsigned SrcReg, int Value) const {
946  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
947  .addImm(Value)
948  .addReg(SrcReg);
949 
950  return Reg;
951 }
952 
955  const DebugLoc &DL,
956  unsigned SrcReg, int Value) const {
959  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
960  .addImm(Value)
961  .addReg(SrcReg);
962 
963  return Reg;
964 }
965 
966 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
967 
968  if (RI.hasAGPRs(DstRC))
969  return AMDGPU::COPY;
970  if (RI.getRegSizeInBits(*DstRC) == 32) {
971  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
972  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
973  return AMDGPU::S_MOV_B64;
974  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
975  return AMDGPU::V_MOV_B64_PSEUDO;
976  }
977  return AMDGPU::COPY;
978 }
979 
980 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
981  switch (Size) {
982  case 4:
983  return AMDGPU::SI_SPILL_S32_SAVE;
984  case 8:
985  return AMDGPU::SI_SPILL_S64_SAVE;
986  case 12:
987  return AMDGPU::SI_SPILL_S96_SAVE;
988  case 16:
989  return AMDGPU::SI_SPILL_S128_SAVE;
990  case 20:
991  return AMDGPU::SI_SPILL_S160_SAVE;
992  case 32:
993  return AMDGPU::SI_SPILL_S256_SAVE;
994  case 64:
995  return AMDGPU::SI_SPILL_S512_SAVE;
996  case 128:
997  return AMDGPU::SI_SPILL_S1024_SAVE;
998  default:
999  llvm_unreachable("unknown register size");
1000  }
1001 }
1002 
1003 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
1004  switch (Size) {
1005  case 4:
1006  return AMDGPU::SI_SPILL_V32_SAVE;
1007  case 8:
1008  return AMDGPU::SI_SPILL_V64_SAVE;
1009  case 12:
1010  return AMDGPU::SI_SPILL_V96_SAVE;
1011  case 16:
1012  return AMDGPU::SI_SPILL_V128_SAVE;
1013  case 20:
1014  return AMDGPU::SI_SPILL_V160_SAVE;
1015  case 32:
1016  return AMDGPU::SI_SPILL_V256_SAVE;
1017  case 64:
1018  return AMDGPU::SI_SPILL_V512_SAVE;
1019  case 128:
1020  return AMDGPU::SI_SPILL_V1024_SAVE;
1021  default:
1022  llvm_unreachable("unknown register size");
1023  }
1024 }
1025 
1026 static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1027  switch (Size) {
1028  case 4:
1029  return AMDGPU::SI_SPILL_A32_SAVE;
1030  case 8:
1031  return AMDGPU::SI_SPILL_A64_SAVE;
1032  case 16:
1033  return AMDGPU::SI_SPILL_A128_SAVE;
1034  case 64:
1035  return AMDGPU::SI_SPILL_A512_SAVE;
1036  case 128:
1037  return AMDGPU::SI_SPILL_A1024_SAVE;
1038  default:
1039  llvm_unreachable("unknown register size");
1040  }
1041 }
1042 
1045  unsigned SrcReg, bool isKill,
1046  int FrameIndex,
1047  const TargetRegisterClass *RC,
1048  const TargetRegisterInfo *TRI) const {
1049  MachineFunction *MF = MBB.getParent();
1051  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1052  const DebugLoc &DL = MBB.findDebugLoc(MI);
1053 
1054  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1055  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1056  MachinePointerInfo PtrInfo
1057  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1058  MachineMemOperand *MMO
1060  Size, Align);
1061  unsigned SpillSize = TRI->getSpillSize(*RC);
1062 
1063  if (RI.isSGPRClass(RC)) {
1064  MFI->setHasSpilledSGPRs();
1065 
1066  // We are only allowed to create one new instruction when spilling
1067  // registers, so we need to use pseudo instruction for spilling SGPRs.
1068  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1069 
1070  // The SGPR spill/restore instructions only work on number sgprs, so we need
1071  // to make sure we are using the correct register class.
1072  if (Register::isVirtualRegister(SrcReg) && SpillSize == 4) {
1074  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
1075  }
1076 
1077  BuildMI(MBB, MI, DL, OpDesc)
1078  .addReg(SrcReg, getKillRegState(isKill)) // data
1079  .addFrameIndex(FrameIndex) // addr
1080  .addMemOperand(MMO)
1082  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1083  // Add the scratch resource registers as implicit uses because we may end up
1084  // needing them, and need to ensure that the reserved registers are
1085  // correctly handled.
1086  if (RI.spillSGPRToVGPR())
1087  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1088  return;
1089  }
1090 
1091  unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1092  : getVGPRSpillSaveOpcode(SpillSize);
1093  MFI->setHasSpilledVGPRs();
1094 
1095  auto MIB = BuildMI(MBB, MI, DL, get(Opcode));
1096  if (RI.hasAGPRs(RC)) {
1098  Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1099  MIB.addReg(Tmp, RegState::Define);
1100  }
1101  MIB.addReg(SrcReg, getKillRegState(isKill)) // data
1102  .addFrameIndex(FrameIndex) // addr
1103  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1104  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1105  .addImm(0) // offset
1106  .addMemOperand(MMO);
1107 }
1108 
1109 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1110  switch (Size) {
1111  case 4:
1112  return AMDGPU::SI_SPILL_S32_RESTORE;
1113  case 8:
1114  return AMDGPU::SI_SPILL_S64_RESTORE;
1115  case 12:
1116  return AMDGPU::SI_SPILL_S96_RESTORE;
1117  case 16:
1118  return AMDGPU::SI_SPILL_S128_RESTORE;
1119  case 20:
1120  return AMDGPU::SI_SPILL_S160_RESTORE;
1121  case 32:
1122  return AMDGPU::SI_SPILL_S256_RESTORE;
1123  case 64:
1124  return AMDGPU::SI_SPILL_S512_RESTORE;
1125  case 128:
1126  return AMDGPU::SI_SPILL_S1024_RESTORE;
1127  default:
1128  llvm_unreachable("unknown register size");
1129  }
1130 }
1131 
1132 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1133  switch (Size) {
1134  case 4:
1135  return AMDGPU::SI_SPILL_V32_RESTORE;
1136  case 8:
1137  return AMDGPU::SI_SPILL_V64_RESTORE;
1138  case 12:
1139  return AMDGPU::SI_SPILL_V96_RESTORE;
1140  case 16:
1141  return AMDGPU::SI_SPILL_V128_RESTORE;
1142  case 20:
1143  return AMDGPU::SI_SPILL_V160_RESTORE;
1144  case 32:
1145  return AMDGPU::SI_SPILL_V256_RESTORE;
1146  case 64:
1147  return AMDGPU::SI_SPILL_V512_RESTORE;
1148  case 128:
1149  return AMDGPU::SI_SPILL_V1024_RESTORE;
1150  default:
1151  llvm_unreachable("unknown register size");
1152  }
1153 }
1154 
1155 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1156  switch (Size) {
1157  case 4:
1158  return AMDGPU::SI_SPILL_A32_RESTORE;
1159  case 8:
1160  return AMDGPU::SI_SPILL_A64_RESTORE;
1161  case 16:
1162  return AMDGPU::SI_SPILL_A128_RESTORE;
1163  case 64:
1164  return AMDGPU::SI_SPILL_A512_RESTORE;
1165  case 128:
1166  return AMDGPU::SI_SPILL_A1024_RESTORE;
1167  default:
1168  llvm_unreachable("unknown register size");
1169  }
1170 }
1171 
1174  unsigned DestReg, int FrameIndex,
1175  const TargetRegisterClass *RC,
1176  const TargetRegisterInfo *TRI) const {
1177  MachineFunction *MF = MBB.getParent();
1179  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1180  const DebugLoc &DL = MBB.findDebugLoc(MI);
1181  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1182  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1183  unsigned SpillSize = TRI->getSpillSize(*RC);
1184 
1185  MachinePointerInfo PtrInfo
1186  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1187 
1189  PtrInfo, MachineMemOperand::MOLoad, Size, Align);
1190 
1191  if (RI.isSGPRClass(RC)) {
1192  MFI->setHasSpilledSGPRs();
1193 
1194  // FIXME: Maybe this should not include a memoperand because it will be
1195  // lowered to non-memory instructions.
1196  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1197  if (Register::isVirtualRegister(DestReg) && SpillSize == 4) {
1199  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1200  }
1201 
1202  if (RI.spillSGPRToVGPR())
1203  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1204  BuildMI(MBB, MI, DL, OpDesc, DestReg)
1205  .addFrameIndex(FrameIndex) // addr
1206  .addMemOperand(MMO)
1208  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1209  return;
1210  }
1211 
1212  unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1213  : getVGPRSpillRestoreOpcode(SpillSize);
1214  auto MIB = BuildMI(MBB, MI, DL, get(Opcode), DestReg);
1215  if (RI.hasAGPRs(RC)) {
1217  Register Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1218  MIB.addReg(Tmp, RegState::Define);
1219  }
1220  MIB.addFrameIndex(FrameIndex) // vaddr
1221  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1222  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1223  .addImm(0) // offset
1224  .addMemOperand(MMO);
1225 }
1226 
1227 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1229  MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1230  unsigned FrameOffset, unsigned Size) const {
1231  MachineFunction *MF = MBB.getParent();
1233  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1234  const DebugLoc &DL = MBB.findDebugLoc(MI);
1235  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1236  unsigned WavefrontSize = ST.getWavefrontSize();
1237 
1238  unsigned TIDReg = MFI->getTIDReg();
1239  if (!MFI->hasCalculatedTID()) {
1240  MachineBasicBlock &Entry = MBB.getParent()->front();
1241  MachineBasicBlock::iterator Insert = Entry.front();
1242  const DebugLoc &DL = Insert->getDebugLoc();
1243 
1244  TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1245  *MF);
1246  if (TIDReg == AMDGPU::NoRegister)
1247  return TIDReg;
1248 
1250  WorkGroupSize > WavefrontSize) {
1251  Register TIDIGXReg =
1253  Register TIDIGYReg =
1255  Register TIDIGZReg =
1257  Register InputPtrReg =
1259  for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1260  if (!Entry.isLiveIn(Reg))
1261  Entry.addLiveIn(Reg);
1262  }
1263 
1264  RS->enterBasicBlock(Entry);
1265  // FIXME: Can we scavenge an SReg_64 and access the subregs?
1266  unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1267  unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1268  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1269  .addReg(InputPtrReg)
1271  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1272  .addReg(InputPtrReg)
1274 
1275  // NGROUPS.X * NGROUPS.Y
1276  BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1277  .addReg(STmp1)
1278  .addReg(STmp0);
1279  // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1280  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1281  .addReg(STmp1)
1282  .addReg(TIDIGXReg);
1283  // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1284  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1285  .addReg(STmp0)
1286  .addReg(TIDIGYReg)
1287  .addReg(TIDReg);
1288  // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1289  getAddNoCarry(Entry, Insert, DL, TIDReg)
1290  .addReg(TIDReg)
1291  .addReg(TIDIGZReg)
1292  .addImm(0); // clamp bit
1293  } else {
1294  // Get the wave id
1295  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1296  TIDReg)
1297  .addImm(-1)
1298  .addImm(0);
1299 
1300  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1301  TIDReg)
1302  .addImm(-1)
1303  .addReg(TIDReg);
1304  }
1305 
1306  BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1307  TIDReg)
1308  .addImm(2)
1309  .addReg(TIDReg);
1310  MFI->setTIDReg(TIDReg);
1311  }
1312 
1313  // Add FrameIndex to LDS offset
1314  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1315  getAddNoCarry(MBB, MI, DL, TmpReg)
1316  .addImm(LDSOffset)
1317  .addReg(TIDReg)
1318  .addImm(0); // clamp bit
1319 
1320  return TmpReg;
1321 }
1322 
1325  int Count) const {
1326  DebugLoc DL = MBB.findDebugLoc(MI);
1327  while (Count > 0) {
1328  int Arg;
1329  if (Count >= 8)
1330  Arg = 7;
1331  else
1332  Arg = Count - 1;
1333  Count -= 8;
1334  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1335  .addImm(Arg);
1336  }
1337 }
1338 
1341  insertWaitStates(MBB, MI, 1);
1342 }
1343 
1345  auto MF = MBB.getParent();
1347 
1348  assert(Info->isEntryFunction());
1349 
1350  if (MBB.succ_empty()) {
1351  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1352  if (HasNoTerminator) {
1353  if (Info->returnsVoid()) {
1354  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1355  } else {
1356  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1357  }
1358  }
1359  }
1360 }
1361 
1363  switch (MI.getOpcode()) {
1364  default: return 1; // FIXME: Do wait states equal cycles?
1365 
1366  case AMDGPU::S_NOP:
1367  return MI.getOperand(0).getImm() + 1;
1368  }
1369 }
1370 
1372  MachineBasicBlock &MBB = *MI.getParent();
1373  DebugLoc DL = MBB.findDebugLoc(MI);
1374  switch (MI.getOpcode()) {
1375  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1376  case AMDGPU::S_MOV_B64_term:
1377  // This is only a terminator to get the correct spill code placement during
1378  // register allocation.
1379  MI.setDesc(get(AMDGPU::S_MOV_B64));
1380  break;
1381 
1382  case AMDGPU::S_MOV_B32_term:
1383  // This is only a terminator to get the correct spill code placement during
1384  // register allocation.
1385  MI.setDesc(get(AMDGPU::S_MOV_B32));
1386  break;
1387 
1388  case AMDGPU::S_XOR_B64_term:
1389  // This is only a terminator to get the correct spill code placement during
1390  // register allocation.
1391  MI.setDesc(get(AMDGPU::S_XOR_B64));
1392  break;
1393 
1394  case AMDGPU::S_XOR_B32_term:
1395  // This is only a terminator to get the correct spill code placement during
1396  // register allocation.
1397  MI.setDesc(get(AMDGPU::S_XOR_B32));
1398  break;
1399 
1400  case AMDGPU::S_OR_B32_term:
1401  // This is only a terminator to get the correct spill code placement during
1402  // register allocation.
1403  MI.setDesc(get(AMDGPU::S_OR_B32));
1404  break;
1405 
1406  case AMDGPU::S_ANDN2_B64_term:
1407  // This is only a terminator to get the correct spill code placement during
1408  // register allocation.
1409  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1410  break;
1411 
1412  case AMDGPU::S_ANDN2_B32_term:
1413  // This is only a terminator to get the correct spill code placement during
1414  // register allocation.
1415  MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1416  break;
1417 
1418  case AMDGPU::V_MOV_B64_PSEUDO: {
1419  Register Dst = MI.getOperand(0).getReg();
1420  Register DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1421  Register DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1422 
1423  const MachineOperand &SrcOp = MI.getOperand(1);
1424  // FIXME: Will this work for 64-bit floating point immediates?
1425  assert(!SrcOp.isFPImm());
1426  if (SrcOp.isImm()) {
1427  APInt Imm(64, SrcOp.getImm());
1428  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1429  .addImm(Imm.getLoBits(32).getZExtValue())
1430  .addReg(Dst, RegState::Implicit | RegState::Define);
1431  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1432  .addImm(Imm.getHiBits(32).getZExtValue())
1433  .addReg(Dst, RegState::Implicit | RegState::Define);
1434  } else {
1435  assert(SrcOp.isReg());
1436  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1437  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1439  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1440  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1442  }
1443  MI.eraseFromParent();
1444  break;
1445  }
1446  case AMDGPU::V_MOV_B64_DPP_PSEUDO: {
1447  expandMovDPP64(MI);
1448  break;
1449  }
1450  case AMDGPU::V_SET_INACTIVE_B32: {
1451  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1452  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1453  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1454  .addReg(Exec);
1455  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1456  .add(MI.getOperand(2));
1457  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1458  .addReg(Exec);
1459  MI.eraseFromParent();
1460  break;
1461  }
1462  case AMDGPU::V_SET_INACTIVE_B64: {
1463  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1464  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1465  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1466  .addReg(Exec);
1467  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1468  MI.getOperand(0).getReg())
1469  .add(MI.getOperand(2));
1470  expandPostRAPseudo(*Copy);
1471  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1472  .addReg(Exec);
1473  MI.eraseFromParent();
1474  break;
1475  }
1476  case AMDGPU::V_MOVRELD_B32_V1:
1477  case AMDGPU::V_MOVRELD_B32_V2:
1478  case AMDGPU::V_MOVRELD_B32_V4:
1479  case AMDGPU::V_MOVRELD_B32_V8:
1480  case AMDGPU::V_MOVRELD_B32_V16: {
1481  const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1482  Register VecReg = MI.getOperand(0).getReg();
1483  bool IsUndef = MI.getOperand(1).isUndef();
1484  unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1485  assert(VecReg == MI.getOperand(1).getReg());
1486 
1487  MachineInstr *MovRel =
1488  BuildMI(MBB, MI, DL, MovRelDesc)
1489  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1490  .add(MI.getOperand(2))
1491  .addReg(VecReg, RegState::ImplicitDefine)
1492  .addReg(VecReg,
1493  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1494 
1495  const int ImpDefIdx =
1496  MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1497  const int ImpUseIdx = ImpDefIdx + 1;
1498  MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1499 
1500  MI.eraseFromParent();
1501  break;
1502  }
1503  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1504  MachineFunction &MF = *MBB.getParent();
1505  Register Reg = MI.getOperand(0).getReg();
1506  Register RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1507  Register RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1508 
1509  // Create a bundle so these instructions won't be re-ordered by the
1510  // post-RA scheduler.
1511  MIBundleBuilder Bundler(MBB, MI);
1512  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1513 
1514  // Add 32-bit offset from this instruction to the start of the
1515  // constant data.
1516  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1517  .addReg(RegLo)
1518  .add(MI.getOperand(1)));
1519 
1520  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1521  .addReg(RegHi);
1522  MIB.add(MI.getOperand(2));
1523 
1524  Bundler.append(MIB);
1525  finalizeBundle(MBB, Bundler.begin());
1526 
1527  MI.eraseFromParent();
1528  break;
1529  }
1530  case AMDGPU::ENTER_WWM: {
1531  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1532  // WWM is entered.
1533  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1534  : AMDGPU::S_OR_SAVEEXEC_B64));
1535  break;
1536  }
1537  case AMDGPU::EXIT_WWM: {
1538  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1539  // WWM is exited.
1540  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
1541  break;
1542  }
1543  case TargetOpcode::BUNDLE: {
1544  if (!MI.mayLoad() || MI.hasUnmodeledSideEffects())
1545  return false;
1546 
1547  // If it is a load it must be a memory clause
1549  I->isBundledWithSucc(); ++I) {
1550  I->unbundleFromSucc();
1551  for (MachineOperand &MO : I->operands())
1552  if (MO.isReg())
1553  MO.setIsInternalRead(false);
1554  }
1555 
1556  MI.eraseFromParent();
1557  break;
1558  }
1559  }
1560  return true;
1561 }
1562 
1563 std::pair<MachineInstr*, MachineInstr*>
1565  assert (MI.getOpcode() == AMDGPU::V_MOV_B64_DPP_PSEUDO);
1566 
1567  MachineBasicBlock &MBB = *MI.getParent();
1568  DebugLoc DL = MBB.findDebugLoc(MI);
1569  MachineFunction *MF = MBB.getParent();
1571  Register Dst = MI.getOperand(0).getReg();
1572  unsigned Part = 0;
1573  MachineInstr *Split[2];
1574 
1575 
1576  for (auto Sub : { AMDGPU::sub0, AMDGPU::sub1 }) {
1577  auto MovDPP = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_dpp));
1578  if (Dst.isPhysical()) {
1579  MovDPP.addDef(RI.getSubReg(Dst, Sub));
1580  } else {
1581  assert(MRI.isSSA());
1582  auto Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1583  MovDPP.addDef(Tmp);
1584  }
1585 
1586  for (unsigned I = 1; I <= 2; ++I) { // old and src operands.
1587  const MachineOperand &SrcOp = MI.getOperand(I);
1588  assert(!SrcOp.isFPImm());
1589  if (SrcOp.isImm()) {
1590  APInt Imm(64, SrcOp.getImm());
1591  Imm.ashrInPlace(Part * 32);
1592  MovDPP.addImm(Imm.getLoBits(32).getZExtValue());
1593  } else {
1594  assert(SrcOp.isReg());
1595  Register Src = SrcOp.getReg();
1596  if (Src.isPhysical())
1597  MovDPP.addReg(RI.getSubReg(Src, Sub));
1598  else
1599  MovDPP.addReg(Src, SrcOp.isUndef() ? RegState::Undef : 0, Sub);
1600  }
1601  }
1602 
1603  for (unsigned I = 3; I < MI.getNumExplicitOperands(); ++I)
1604  MovDPP.addImm(MI.getOperand(I).getImm());
1605 
1606  Split[Part] = MovDPP;
1607  ++Part;
1608  }
1609 
1610  if (Dst.isVirtual())
1611  BuildMI(MBB, MI, DL, get(AMDGPU::REG_SEQUENCE), Dst)
1612  .addReg(Split[0]->getOperand(0).getReg())
1613  .addImm(AMDGPU::sub0)
1614  .addReg(Split[1]->getOperand(0).getReg())
1615  .addImm(AMDGPU::sub1);
1616 
1617  MI.eraseFromParent();
1618  return std::make_pair(Split[0], Split[1]);
1619 }
1620 
1622  MachineOperand &Src0,
1623  unsigned Src0OpName,
1624  MachineOperand &Src1,
1625  unsigned Src1OpName) const {
1626  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1627  if (!Src0Mods)
1628  return false;
1629 
1630  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1631  assert(Src1Mods &&
1632  "All commutable instructions have both src0 and src1 modifiers");
1633 
1634  int Src0ModsVal = Src0Mods->getImm();
1635  int Src1ModsVal = Src1Mods->getImm();
1636 
1637  Src1Mods->setImm(Src0ModsVal);
1638  Src0Mods->setImm(Src1ModsVal);
1639  return true;
1640 }
1641 
1643  MachineOperand &RegOp,
1644  MachineOperand &NonRegOp) {
1645  Register Reg = RegOp.getReg();
1646  unsigned SubReg = RegOp.getSubReg();
1647  bool IsKill = RegOp.isKill();
1648  bool IsDead = RegOp.isDead();
1649  bool IsUndef = RegOp.isUndef();
1650  bool IsDebug = RegOp.isDebug();
1651 
1652  if (NonRegOp.isImm())
1653  RegOp.ChangeToImmediate(NonRegOp.getImm());
1654  else if (NonRegOp.isFI())
1655  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1656  else
1657  return nullptr;
1658 
1659  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1660  NonRegOp.setSubReg(SubReg);
1661 
1662  return &MI;
1663 }
1664 
1666  unsigned Src0Idx,
1667  unsigned Src1Idx) const {
1668  assert(!NewMI && "this should never be used");
1669 
1670  unsigned Opc = MI.getOpcode();
1671  int CommutedOpcode = commuteOpcode(Opc);
1672  if (CommutedOpcode == -1)
1673  return nullptr;
1674 
1675  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1676  static_cast<int>(Src0Idx) &&
1677  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1678  static_cast<int>(Src1Idx) &&
1679  "inconsistency with findCommutedOpIndices");
1680 
1681  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1682  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1683 
1684  MachineInstr *CommutedMI = nullptr;
1685  if (Src0.isReg() && Src1.isReg()) {
1686  if (isOperandLegal(MI, Src1Idx, &Src0)) {
1687  // Be sure to copy the source modifiers to the right place.
1688  CommutedMI
1689  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1690  }
1691 
1692  } else if (Src0.isReg() && !Src1.isReg()) {
1693  // src0 should always be able to support any operand type, so no need to
1694  // check operand legality.
1695  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1696  } else if (!Src0.isReg() && Src1.isReg()) {
1697  if (isOperandLegal(MI, Src1Idx, &Src0))
1698  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1699  } else {
1700  // FIXME: Found two non registers to commute. This does happen.
1701  return nullptr;
1702  }
1703 
1704  if (CommutedMI) {
1705  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1706  Src1, AMDGPU::OpName::src1_modifiers);
1707 
1708  CommutedMI->setDesc(get(CommutedOpcode));
1709  }
1710 
1711  return CommutedMI;
1712 }
1713 
1714 // This needs to be implemented because the source modifiers may be inserted
1715 // between the true commutable operands, and the base
1716 // TargetInstrInfo::commuteInstruction uses it.
1718  unsigned &SrcOpIdx0,
1719  unsigned &SrcOpIdx1) const {
1720  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1721 }
1722 
1723 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1724  unsigned &SrcOpIdx1) const {
1725  if (!Desc.isCommutable())
1726  return false;
1727 
1728  unsigned Opc = Desc.getOpcode();
1729  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1730  if (Src0Idx == -1)
1731  return false;
1732 
1733  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1734  if (Src1Idx == -1)
1735  return false;
1736 
1737  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1738 }
1739 
1740 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1741  int64_t BrOffset) const {
1742  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1743  // block is unanalyzable.
1744  assert(BranchOp != AMDGPU::S_SETPC_B64);
1745 
1746  // Convert to dwords.
1747  BrOffset /= 4;
1748 
1749  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1750  // from the next instruction.
1751  BrOffset -= 1;
1752 
1753  return isIntN(BranchOffsetBits, BrOffset);
1754 }
1755 
1757  const MachineInstr &MI) const {
1758  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1759  // This would be a difficult analysis to perform, but can always be legal so
1760  // there's no need to analyze it.
1761  return nullptr;
1762  }
1763 
1764  return MI.getOperand(0).getMBB();
1765 }
1766 
1768  MachineBasicBlock &DestBB,
1769  const DebugLoc &DL,
1770  int64_t BrOffset,
1771  RegScavenger *RS) const {
1772  assert(RS && "RegScavenger required for long branching");
1773  assert(MBB.empty() &&
1774  "new block should be inserted for expanding unconditional branch");
1775  assert(MBB.pred_size() == 1);
1776 
1777  MachineFunction *MF = MBB.getParent();
1778  MachineRegisterInfo &MRI = MF->getRegInfo();
1779 
1780  // FIXME: Virtual register workaround for RegScavenger not working with empty
1781  // blocks.
1782  Register PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1783 
1784  auto I = MBB.end();
1785 
1786  // We need to compute the offset relative to the instruction immediately after
1787  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1788  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1789 
1790  // TODO: Handle > 32-bit block address.
1791  if (BrOffset >= 0) {
1792  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1793  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1794  .addReg(PCReg, 0, AMDGPU::sub0)
1795  .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
1796  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1797  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1798  .addReg(PCReg, 0, AMDGPU::sub1)
1799  .addImm(0);
1800  } else {
1801  // Backwards branch.
1802  BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1803  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1804  .addReg(PCReg, 0, AMDGPU::sub0)
1805  .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
1806  BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1807  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1808  .addReg(PCReg, 0, AMDGPU::sub1)
1809  .addImm(0);
1810  }
1811 
1812  // Insert the indirect branch after the other terminator.
1813  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1814  .addReg(PCReg);
1815 
1816  // FIXME: If spilling is necessary, this will fail because this scavenger has
1817  // no emergency stack slots. It is non-trivial to spill in this situation,
1818  // because the restore code needs to be specially placed after the
1819  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1820  // block.
1821  //
1822  // If a spill is needed for the pc register pair, we need to insert a spill
1823  // restore block right before the destination block, and insert a short branch
1824  // into the old destination block's fallthrough predecessor.
1825  // e.g.:
1826  //
1827  // s_cbranch_scc0 skip_long_branch:
1828  //
1829  // long_branch_bb:
1830  // spill s[8:9]
1831  // s_getpc_b64 s[8:9]
1832  // s_add_u32 s8, s8, restore_bb
1833  // s_addc_u32 s9, s9, 0
1834  // s_setpc_b64 s[8:9]
1835  //
1836  // skip_long_branch:
1837  // foo;
1838  //
1839  // .....
1840  //
1841  // dest_bb_fallthrough_predecessor:
1842  // bar;
1843  // s_branch dest_bb
1844  //
1845  // restore_bb:
1846  // restore s[8:9]
1847  // fallthrough dest_bb
1848  ///
1849  // dest_bb:
1850  // buzz;
1851 
1852  RS->enterBasicBlockEnd(MBB);
1853  unsigned Scav = RS->scavengeRegisterBackwards(
1854  AMDGPU::SReg_64RegClass,
1855  MachineBasicBlock::iterator(GetPC), false, 0);
1856  MRI.replaceRegWith(PCReg, Scav);
1857  MRI.clearVirtRegs();
1858  RS->setRegUsed(Scav);
1859 
1860  return 4 + 8 + 4 + 4;
1861 }
1862 
1863 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1864  switch (Cond) {
1865  case SIInstrInfo::SCC_TRUE:
1866  return AMDGPU::S_CBRANCH_SCC1;
1867  case SIInstrInfo::SCC_FALSE:
1868  return AMDGPU::S_CBRANCH_SCC0;
1869  case SIInstrInfo::VCCNZ:
1870  return AMDGPU::S_CBRANCH_VCCNZ;
1871  case SIInstrInfo::VCCZ:
1872  return AMDGPU::S_CBRANCH_VCCZ;
1873  case SIInstrInfo::EXECNZ:
1874  return AMDGPU::S_CBRANCH_EXECNZ;
1875  case SIInstrInfo::EXECZ:
1876  return AMDGPU::S_CBRANCH_EXECZ;
1877  default:
1878  llvm_unreachable("invalid branch predicate");
1879  }
1880 }
1881 
1882 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1883  switch (Opcode) {
1884  case AMDGPU::S_CBRANCH_SCC0:
1885  return SCC_FALSE;
1886  case AMDGPU::S_CBRANCH_SCC1:
1887  return SCC_TRUE;
1888  case AMDGPU::S_CBRANCH_VCCNZ:
1889  return VCCNZ;
1890  case AMDGPU::S_CBRANCH_VCCZ:
1891  return VCCZ;
1892  case AMDGPU::S_CBRANCH_EXECNZ:
1893  return EXECNZ;
1894  case AMDGPU::S_CBRANCH_EXECZ:
1895  return EXECZ;
1896  default:
1897  return INVALID_BR;
1898  }
1899 }
1900 
1903  MachineBasicBlock *&TBB,
1904  MachineBasicBlock *&FBB,
1906  bool AllowModify) const {
1907  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1908  // Unconditional Branch
1909  TBB = I->getOperand(0).getMBB();
1910  return false;
1911  }
1912 
1913  MachineBasicBlock *CondBB = nullptr;
1914 
1915  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1916  CondBB = I->getOperand(1).getMBB();
1917  Cond.push_back(I->getOperand(0));
1918  } else {
1919  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1920  if (Pred == INVALID_BR)
1921  return true;
1922 
1923  CondBB = I->getOperand(0).getMBB();
1925  Cond.push_back(I->getOperand(1)); // Save the branch register.
1926  }
1927  ++I;
1928 
1929  if (I == MBB.end()) {
1930  // Conditional branch followed by fall-through.
1931  TBB = CondBB;
1932  return false;
1933  }
1934 
1935  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1936  TBB = CondBB;
1937  FBB = I->getOperand(0).getMBB();
1938  return false;
1939  }
1940 
1941  return true;
1942 }
1943 
1945  MachineBasicBlock *&FBB,
1947  bool AllowModify) const {
1949  auto E = MBB.end();
1950  if (I == E)
1951  return false;
1952 
1953  // Skip over the instructions that are artificially terminators for special
1954  // exec management.
1955  while (I != E && !I->isBranch() && !I->isReturn() &&
1956  I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1957  switch (I->getOpcode()) {
1958  case AMDGPU::SI_MASK_BRANCH:
1959  case AMDGPU::S_MOV_B64_term:
1960  case AMDGPU::S_XOR_B64_term:
1961  case AMDGPU::S_ANDN2_B64_term:
1962  case AMDGPU::S_MOV_B32_term:
1963  case AMDGPU::S_XOR_B32_term:
1964  case AMDGPU::S_OR_B32_term:
1965  case AMDGPU::S_ANDN2_B32_term:
1966  break;
1967  case AMDGPU::SI_IF:
1968  case AMDGPU::SI_ELSE:
1969  case AMDGPU::SI_KILL_I1_TERMINATOR:
1970  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1971  // FIXME: It's messy that these need to be considered here at all.
1972  return true;
1973  default:
1974  llvm_unreachable("unexpected non-branch terminator inst");
1975  }
1976 
1977  ++I;
1978  }
1979 
1980  if (I == E)
1981  return false;
1982 
1983  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1984  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1985 
1986  ++I;
1987 
1988  // TODO: Should be able to treat as fallthrough?
1989  if (I == MBB.end())
1990  return true;
1991 
1992  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1993  return true;
1994 
1995  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1996 
1997  // Specifically handle the case where the conditional branch is to the same
1998  // destination as the mask branch. e.g.
1999  //
2000  // si_mask_branch BB8
2001  // s_cbranch_execz BB8
2002  // s_cbranch BB9
2003  //
2004  // This is required to understand divergent loops which may need the branches
2005  // to be relaxed.
2006  if (TBB != MaskBrDest || Cond.empty())
2007  return true;
2008 
2009  auto Pred = Cond[0].getImm();
2010  return (Pred != EXECZ && Pred != EXECNZ);
2011 }
2012 
2014  int *BytesRemoved) const {
2016 
2017  unsigned Count = 0;
2018  unsigned RemovedSize = 0;
2019  while (I != MBB.end()) {
2020  MachineBasicBlock::iterator Next = std::next(I);
2021  if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
2022  I = Next;
2023  continue;
2024  }
2025 
2026  RemovedSize += getInstSizeInBytes(*I);
2027  I->eraseFromParent();
2028  ++Count;
2029  I = Next;
2030  }
2031 
2032  if (BytesRemoved)
2033  *BytesRemoved = RemovedSize;
2034 
2035  return Count;
2036 }
2037 
2038 // Copy the flags onto the implicit condition register operand.
2040  const MachineOperand &OrigCond) {
2041  CondReg.setIsUndef(OrigCond.isUndef());
2042  CondReg.setIsKill(OrigCond.isKill());
2043 }
2044 
2046  MachineBasicBlock *TBB,
2047  MachineBasicBlock *FBB,
2049  const DebugLoc &DL,
2050  int *BytesAdded) const {
2051  if (!FBB && Cond.empty()) {
2052  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2053  .addMBB(TBB);
2054  if (BytesAdded)
2055  *BytesAdded = 4;
2056  return 1;
2057  }
2058 
2059  if(Cond.size() == 1 && Cond[0].isReg()) {
2060  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
2061  .add(Cond[0])
2062  .addMBB(TBB);
2063  return 1;
2064  }
2065 
2066  assert(TBB && Cond[0].isImm());
2067 
2068  unsigned Opcode
2069  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2070 
2071  if (!FBB) {
2072  Cond[1].isUndef();
2073  MachineInstr *CondBr =
2074  BuildMI(&MBB, DL, get(Opcode))
2075  .addMBB(TBB);
2076 
2077  // Copy the flags onto the implicit condition register operand.
2078  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2079 
2080  if (BytesAdded)
2081  *BytesAdded = 4;
2082  return 1;
2083  }
2084 
2085  assert(TBB && FBB);
2086 
2087  MachineInstr *CondBr =
2088  BuildMI(&MBB, DL, get(Opcode))
2089  .addMBB(TBB);
2090  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2091  .addMBB(FBB);
2092 
2093  MachineOperand &CondReg = CondBr->getOperand(1);
2094  CondReg.setIsUndef(Cond[1].isUndef());
2095  CondReg.setIsKill(Cond[1].isKill());
2096 
2097  if (BytesAdded)
2098  *BytesAdded = 8;
2099 
2100  return 2;
2101 }
2102 
2104  SmallVectorImpl<MachineOperand> &Cond) const {
2105  if (Cond.size() != 2) {
2106  return true;
2107  }
2108 
2109  if (Cond[0].isImm()) {
2110  Cond[0].setImm(-Cond[0].getImm());
2111  return false;
2112  }
2113 
2114  return true;
2115 }
2116 
2119  unsigned TrueReg, unsigned FalseReg,
2120  int &CondCycles,
2121  int &TrueCycles, int &FalseCycles) const {
2122  switch (Cond[0].getImm()) {
2123  case VCCNZ:
2124  case VCCZ: {
2125  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2126  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2127  assert(MRI.getRegClass(FalseReg) == RC);
2128 
2129  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2130  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2131 
2132  // Limit to equal cost for branch vs. N v_cndmask_b32s.
2133  return RI.hasVGPRs(RC) && NumInsts <= 6;
2134  }
2135  case SCC_TRUE:
2136  case SCC_FALSE: {
2137  // FIXME: We could insert for VGPRs if we could replace the original compare
2138  // with a vector one.
2139  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2140  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2141  assert(MRI.getRegClass(FalseReg) == RC);
2142 
2143  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2144 
2145  // Multiples of 8 can do s_cselect_b64
2146  if (NumInsts % 2 == 0)
2147  NumInsts /= 2;
2148 
2149  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2150  return RI.isSGPRClass(RC);
2151  }
2152  default:
2153  return false;
2154  }
2155 }
2156 
2159  unsigned DstReg, ArrayRef<MachineOperand> Cond,
2160  unsigned TrueReg, unsigned FalseReg) const {
2161  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2162  if (Pred == VCCZ || Pred == SCC_FALSE) {
2163  Pred = static_cast<BranchPredicate>(-Pred);
2164  std::swap(TrueReg, FalseReg);
2165  }
2166 
2168  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2169  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2170 
2171  if (DstSize == 32) {
2172  unsigned SelOp = Pred == SCC_TRUE ?
2173  AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
2174 
2175  // Instruction's operands are backwards from what is expected.
2176  MachineInstr *Select =
2177  BuildMI(MBB, I, DL, get(SelOp), DstReg)
2178  .addReg(FalseReg)
2179  .addReg(TrueReg);
2180 
2181  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2182  return;
2183  }
2184 
2185  if (DstSize == 64 && Pred == SCC_TRUE) {
2186  MachineInstr *Select =
2187  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2188  .addReg(FalseReg)
2189  .addReg(TrueReg);
2190 
2191  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2192  return;
2193  }
2194 
2195  static const int16_t Sub0_15[] = {
2196  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2197  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2198  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2199  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2200  };
2201 
2202  static const int16_t Sub0_15_64[] = {
2203  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2204  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2205  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2206  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2207  };
2208 
2209  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2210  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2211  const int16_t *SubIndices = Sub0_15;
2212  int NElts = DstSize / 32;
2213 
2214  // 64-bit select is only available for SALU.
2215  // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2216  if (Pred == SCC_TRUE) {
2217  if (NElts % 2) {
2218  SelOp = AMDGPU::S_CSELECT_B32;
2219  EltRC = &AMDGPU::SGPR_32RegClass;
2220  } else {
2221  SelOp = AMDGPU::S_CSELECT_B64;
2222  EltRC = &AMDGPU::SGPR_64RegClass;
2223  SubIndices = Sub0_15_64;
2224  NElts /= 2;
2225  }
2226  }
2227 
2229  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2230 
2231  I = MIB->getIterator();
2232 
2234  for (int Idx = 0; Idx != NElts; ++Idx) {
2235  Register DstElt = MRI.createVirtualRegister(EltRC);
2236  Regs.push_back(DstElt);
2237 
2238  unsigned SubIdx = SubIndices[Idx];
2239 
2240  MachineInstr *Select =
2241  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2242  .addReg(FalseReg, 0, SubIdx)
2243  .addReg(TrueReg, 0, SubIdx);
2244  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2245  fixImplicitOperands(*Select);
2246 
2247  MIB.addReg(DstElt)
2248  .addImm(SubIdx);
2249  }
2250 }
2251 
2253  switch (MI.getOpcode()) {
2254  case AMDGPU::V_MOV_B32_e32:
2255  case AMDGPU::V_MOV_B32_e64:
2256  case AMDGPU::V_MOV_B64_PSEUDO: {
2257  // If there are additional implicit register operands, this may be used for
2258  // register indexing so the source register operand isn't simply copied.
2259  unsigned NumOps = MI.getDesc().getNumOperands() +
2260  MI.getDesc().getNumImplicitUses();
2261 
2262  return MI.getNumOperands() == NumOps;
2263  }
2264  case AMDGPU::S_MOV_B32:
2265  case AMDGPU::S_MOV_B64:
2266  case AMDGPU::COPY:
2267  case AMDGPU::V_ACCVGPR_WRITE_B32:
2268  case AMDGPU::V_ACCVGPR_READ_B32:
2269  return true;
2270  default:
2271  return false;
2272  }
2273 }
2274 
2276  unsigned Kind) const {
2277  switch(Kind) {
2288  }
2289  return AMDGPUAS::FLAT_ADDRESS;
2290 }
2291 
2293  unsigned Opc = MI.getOpcode();
2294  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2295  AMDGPU::OpName::src0_modifiers);
2296  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2297  AMDGPU::OpName::src1_modifiers);
2298  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2299  AMDGPU::OpName::src2_modifiers);
2300 
2301  MI.RemoveOperand(Src2ModIdx);
2302  MI.RemoveOperand(Src1ModIdx);
2303  MI.RemoveOperand(Src0ModIdx);
2304 }
2305 
2307  unsigned Reg, MachineRegisterInfo *MRI) const {
2308  if (!MRI->hasOneNonDBGUse(Reg))
2309  return false;
2310 
2311  switch (DefMI.getOpcode()) {
2312  default:
2313  return false;
2314  case AMDGPU::S_MOV_B64:
2315  // TODO: We could fold 64-bit immediates, but this get compilicated
2316  // when there are sub-registers.
2317  return false;
2318 
2319  case AMDGPU::V_MOV_B32_e32:
2320  case AMDGPU::S_MOV_B32:
2321  case AMDGPU::V_ACCVGPR_WRITE_B32:
2322  break;
2323  }
2324 
2325  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2326  assert(ImmOp);
2327  // FIXME: We could handle FrameIndex values here.
2328  if (!ImmOp->isImm())
2329  return false;
2330 
2331  unsigned Opc = UseMI.getOpcode();
2332  if (Opc == AMDGPU::COPY) {
2333  bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2334  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2335  if (RI.isAGPR(*MRI, UseMI.getOperand(0).getReg())) {
2337  return false;
2338  NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32;
2339  }
2340  UseMI.setDesc(get(NewOpc));
2341  UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2342  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2343  return true;
2344  }
2345 
2346  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2347  Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64 ||
2348  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2349  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64) {
2350  // Don't fold if we are using source or output modifiers. The new VOP2
2351  // instructions don't have them.
2352  if (hasAnyModifiersSet(UseMI))
2353  return false;
2354 
2355  // If this is a free constant, there's no reason to do this.
2356  // TODO: We could fold this here instead of letting SIFoldOperands do it
2357  // later.
2358  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2359 
2360  // Any src operand can be used for the legality check.
2361  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2362  return false;
2363 
2364  bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2365  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64;
2366  bool IsFMA = Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2367  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64;
2368  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2369  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2370 
2371  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2372  // We should only expect these to be on src0 due to canonicalizations.
2373  if (Src0->isReg() && Src0->getReg() == Reg) {
2374  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2375  return false;
2376 
2377  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2378  return false;
2379 
2380  unsigned NewOpc =
2381  IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
2382  : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
2383  if (pseudoToMCOpcode(NewOpc) == -1)
2384  return false;
2385 
2386  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2387 
2388  const int64_t Imm = ImmOp->getImm();
2389 
2390  // FIXME: This would be a lot easier if we could return a new instruction
2391  // instead of having to modify in place.
2392 
2393  // Remove these first since they are at the end.
2394  UseMI.RemoveOperand(
2395  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2396  UseMI.RemoveOperand(
2397  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2398 
2399  Register Src1Reg = Src1->getReg();
2400  unsigned Src1SubReg = Src1->getSubReg();
2401  Src0->setReg(Src1Reg);
2402  Src0->setSubReg(Src1SubReg);
2403  Src0->setIsKill(Src1->isKill());
2404 
2405  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2406  Opc == AMDGPU::V_MAC_F16_e64 ||
2407  Opc == AMDGPU::V_FMAC_F32_e64 ||
2408  Opc == AMDGPU::V_FMAC_F16_e64)
2409  UseMI.untieRegOperand(
2410  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2411 
2412  Src1->ChangeToImmediate(Imm);
2413 
2414  removeModOperands(UseMI);
2415  UseMI.setDesc(get(NewOpc));
2416 
2417  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2418  if (DeleteDef)
2419  DefMI.eraseFromParent();
2420 
2421  return true;
2422  }
2423 
2424  // Added part is the constant: Use v_madak_{f16, f32}.
2425  if (Src2->isReg() && Src2->getReg() == Reg) {
2426  // Not allowed to use constant bus for another operand.
2427  // We can however allow an inline immediate as src0.
2428  bool Src0Inlined = false;
2429  if (Src0->isReg()) {
2430  // Try to inline constant if possible.
2431  // If the Def moves immediate and the use is single
2432  // We are saving VGPR here.
2433  MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2434  if (Def && Def->isMoveImmediate() &&
2435  isInlineConstant(Def->getOperand(1)) &&
2436  MRI->hasOneUse(Src0->getReg())) {
2437  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2438  Src0Inlined = true;
2439  } else if ((Register::isPhysicalRegister(Src0->getReg()) &&
2440  (ST.getConstantBusLimit(Opc) <= 1 &&
2441  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2442  (Register::isVirtualRegister(Src0->getReg()) &&
2443  (ST.getConstantBusLimit(Opc) <= 1 &&
2444  RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
2445  return false;
2446  // VGPR is okay as Src0 - fallthrough
2447  }
2448 
2449  if (Src1->isReg() && !Src0Inlined ) {
2450  // We have one slot for inlinable constant so far - try to fill it
2451  MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2452  if (Def && Def->isMoveImmediate() &&
2453  isInlineConstant(Def->getOperand(1)) &&
2454  MRI->hasOneUse(Src1->getReg()) &&
2455  commuteInstruction(UseMI)) {
2456  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2457  } else if ((Register::isPhysicalRegister(Src1->getReg()) &&
2458  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2459  (Register::isVirtualRegister(Src1->getReg()) &&
2460  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2461  return false;
2462  // VGPR is okay as Src1 - fallthrough
2463  }
2464 
2465  unsigned NewOpc =
2466  IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
2467  : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
2468  if (pseudoToMCOpcode(NewOpc) == -1)
2469  return false;
2470 
2471  const int64_t Imm = ImmOp->getImm();
2472 
2473  // FIXME: This would be a lot easier if we could return a new instruction
2474  // instead of having to modify in place.
2475 
2476  // Remove these first since they are at the end.
2477  UseMI.RemoveOperand(
2478  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2479  UseMI.RemoveOperand(
2480  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2481 
2482  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2483  Opc == AMDGPU::V_MAC_F16_e64 ||
2484  Opc == AMDGPU::V_FMAC_F32_e64 ||
2485  Opc == AMDGPU::V_FMAC_F16_e64)
2486  UseMI.untieRegOperand(
2487  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2488 
2489  // ChangingToImmediate adds Src2 back to the instruction.
2490  Src2->ChangeToImmediate(Imm);
2491 
2492  // These come before src2.
2493  removeModOperands(UseMI);
2494  UseMI.setDesc(get(NewOpc));
2495  // It might happen that UseMI was commuted
2496  // and we now have SGPR as SRC1. If so 2 inlined
2497  // constant and SGPR are illegal.
2498  legalizeOperands(UseMI);
2499 
2500  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2501  if (DeleteDef)
2502  DefMI.eraseFromParent();
2503 
2504  return true;
2505  }
2506  }
2507 
2508  return false;
2509 }
2510 
2511 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2512  int WidthB, int OffsetB) {
2513  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2514  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2515  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2516  return LowOffset + LowWidth <= HighOffset;
2517 }
2518 
2519 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2520  const MachineInstr &MIb) const {
2521  const MachineOperand *BaseOp0, *BaseOp1;
2522  int64_t Offset0, Offset1;
2523 
2524  if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2525  getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)) {
2526  if (!BaseOp0->isIdenticalTo(*BaseOp1))
2527  return false;
2528 
2529  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2530  // FIXME: Handle ds_read2 / ds_write2.
2531  return false;
2532  }
2533  unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2534  unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2535  if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2536  return true;
2537  }
2538  }
2539 
2540  return false;
2541 }
2542 
2544  const MachineInstr &MIb) const {
2545  assert((MIa.mayLoad() || MIa.mayStore()) &&
2546  "MIa must load from or modify a memory location");
2547  assert((MIb.mayLoad() || MIb.mayStore()) &&
2548  "MIb must load from or modify a memory location");
2549 
2551  return false;
2552 
2553  // XXX - Can we relax this between address spaces?
2554  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2555  return false;
2556 
2557  // TODO: Should we check the address space from the MachineMemOperand? That
2558  // would allow us to distinguish objects we know don't alias based on the
2559  // underlying address space, even if it was lowered to a different one,
2560  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2561  // buffer.
2562  if (isDS(MIa)) {
2563  if (isDS(MIb))
2564  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2565 
2566  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2567  }
2568 
2569  if (isMUBUF(MIa) || isMTBUF(MIa)) {
2570  if (isMUBUF(MIb) || isMTBUF(MIb))
2571  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2572 
2573  return !isFLAT(MIb) && !isSMRD(MIb);
2574  }
2575 
2576  if (isSMRD(MIa)) {
2577  if (isSMRD(MIb))
2578  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2579 
2580  return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2581  }
2582 
2583  if (isFLAT(MIa)) {
2584  if (isFLAT(MIb))
2585  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2586 
2587  return false;
2588  }
2589 
2590  return false;
2591 }
2592 
2593 static int64_t getFoldableImm(const MachineOperand* MO) {
2594  if (!MO->isReg())
2595  return false;
2596  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2597  const MachineRegisterInfo &MRI = MF->getRegInfo();
2598  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2599  if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2600  Def->getOperand(1).isImm())
2601  return Def->getOperand(1).getImm();
2602  return AMDGPU::NoRegister;
2603 }
2604 
2606  MachineInstr &MI,
2607  LiveVariables *LV) const {
2608  unsigned Opc = MI.getOpcode();
2609  bool IsF16 = false;
2610  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2611  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64;
2612 
2613  switch (Opc) {
2614  default:
2615  return nullptr;
2616  case AMDGPU::V_MAC_F16_e64:
2617  case AMDGPU::V_FMAC_F16_e64:
2618  IsF16 = true;
2620  case AMDGPU::V_MAC_F32_e64:
2621  case AMDGPU::V_FMAC_F32_e64:
2622  break;
2623  case AMDGPU::V_MAC_F16_e32:
2624  case AMDGPU::V_FMAC_F16_e32:
2625  IsF16 = true;
2627  case AMDGPU::V_MAC_F32_e32:
2628  case AMDGPU::V_FMAC_F32_e32: {
2629  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2630  AMDGPU::OpName::src0);
2631  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2632  if (!Src0->isReg() && !Src0->isImm())
2633  return nullptr;
2634 
2635  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2636  return nullptr;
2637 
2638  break;
2639  }
2640  }
2641 
2642  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2643  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2644  const MachineOperand *Src0Mods =
2645  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2646  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2647  const MachineOperand *Src1Mods =
2648  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2649  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2650  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2651  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2652 
2653  if (!Src0Mods && !Src1Mods && !Clamp && !Omod &&
2654  // If we have an SGPR input, we will violate the constant bus restriction.
2655  (ST.getConstantBusLimit(Opc) > 1 ||
2656  !Src0->isReg() ||
2657  !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2658  if (auto Imm = getFoldableImm(Src2)) {
2659  unsigned NewOpc =
2660  IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
2661  : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
2662  if (pseudoToMCOpcode(NewOpc) != -1)
2663  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2664  .add(*Dst)
2665  .add(*Src0)
2666  .add(*Src1)
2667  .addImm(Imm);
2668  }
2669  unsigned NewOpc =
2670  IsFMA ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
2671  : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
2672  if (auto Imm = getFoldableImm(Src1)) {
2673  if (pseudoToMCOpcode(NewOpc) != -1)
2674  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2675  .add(*Dst)
2676  .add(*Src0)
2677  .addImm(Imm)
2678  .add(*Src2);
2679  }
2680  if (auto Imm = getFoldableImm(Src0)) {
2681  if (pseudoToMCOpcode(NewOpc) != -1 &&
2683  AMDGPU::OpName::src0), Src1))
2684  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2685  .add(*Dst)
2686  .add(*Src1)
2687  .addImm(Imm)
2688  .add(*Src2);
2689  }
2690  }
2691 
2692  unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16 : AMDGPU::V_FMA_F32)
2693  : (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2694  if (pseudoToMCOpcode(NewOpc) == -1)
2695  return nullptr;
2696 
2697  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2698  .add(*Dst)
2699  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2700  .add(*Src0)
2701  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2702  .add(*Src1)
2703  .addImm(0) // Src mods
2704  .add(*Src2)
2705  .addImm(Clamp ? Clamp->getImm() : 0)
2706  .addImm(Omod ? Omod->getImm() : 0);
2707 }
2708 
2709 // It's not generally safe to move VALU instructions across these since it will
2710 // start using the register as a base index rather than directly.
2711 // XXX - Why isn't hasSideEffects sufficient for these?
2713  switch (MI.getOpcode()) {
2714  case AMDGPU::S_SET_GPR_IDX_ON:
2715  case AMDGPU::S_SET_GPR_IDX_MODE:
2716  case AMDGPU::S_SET_GPR_IDX_OFF:
2717  return true;
2718  default:
2719  return false;
2720  }
2721 }
2722 
2724  const MachineBasicBlock *MBB,
2725  const MachineFunction &MF) const {
2726  // XXX - Do we want the SP check in the base implementation?
2727 
2728  // Target-independent instructions do not have an implicit-use of EXEC, even
2729  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2730  // boundaries prevents incorrect movements of such instructions.
2731  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2732  MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2733  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2734  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2735  MI.getOpcode() == AMDGPU::S_DENORM_MODE ||
2737 }
2738 
2739 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
2740  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
2741  Opcode == AMDGPU::DS_GWS_INIT ||
2742  Opcode == AMDGPU::DS_GWS_SEMA_V ||
2743  Opcode == AMDGPU::DS_GWS_SEMA_BR ||
2744  Opcode == AMDGPU::DS_GWS_SEMA_P ||
2745  Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
2746  Opcode == AMDGPU::DS_GWS_BARRIER;
2747 }
2748 
2750  unsigned Opcode = MI.getOpcode();
2751 
2752  if (MI.mayStore() && isSMRD(MI))
2753  return true; // scalar store or atomic
2754 
2755  // This will terminate the function when other lanes may need to continue.
2756  if (MI.isReturn())
2757  return true;
2758 
2759  // These instructions cause shader I/O that may cause hardware lockups
2760  // when executed with an empty EXEC mask.
2761  //
2762  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2763  // EXEC = 0, but checking for that case here seems not worth it
2764  // given the typical code patterns.
2765  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2766  Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE ||
2767  Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
2768  Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
2769  return true;
2770 
2771  if (MI.isCall() || MI.isInlineAsm())
2772  return true; // conservative assumption
2773 
2774  // These are like SALU instructions in terms of effects, so it's questionable
2775  // whether we should return true for those.
2776  //
2777  // However, executing them with EXEC = 0 causes them to operate on undefined
2778  // data, which we avoid by returning true here.
2779  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2780  return true;
2781 
2782  return false;
2783 }
2784 
2786  const MachineInstr &MI) const {
2787  if (MI.isMetaInstruction())
2788  return false;
2789 
2790  // This won't read exec if this is an SGPR->SGPR copy.
2791  if (MI.isCopyLike()) {
2792  if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
2793  return true;
2794 
2795  // Make sure this isn't copying exec as a normal operand
2796  return MI.readsRegister(AMDGPU::EXEC, &RI);
2797  }
2798 
2799  // Make a conservative assumption about the callee.
2800  if (MI.isCall())
2801  return true;
2802 
2803  // Be conservative with any unhandled generic opcodes.
2804  if (!isTargetSpecificOpcode(MI.getOpcode()))
2805  return true;
2806 
2807  return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
2808 }
2809 
2810 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2811  switch (Imm.getBitWidth()) {
2812  case 1: // This likely will be a condition code mask.
2813  return true;
2814 
2815  case 32:
2817  ST.hasInv2PiInlineImm());
2818  case 64:
2820  ST.hasInv2PiInlineImm());
2821  case 16:
2822  return ST.has16BitInsts() &&
2824  ST.hasInv2PiInlineImm());
2825  default:
2826  llvm_unreachable("invalid bitwidth");
2827  }
2828 }
2829 
2831  uint8_t OperandType) const {
2832  if (!MO.isImm() ||
2833  OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2834  OperandType > AMDGPU::OPERAND_SRC_LAST)
2835  return false;
2836 
2837  // MachineOperand provides no way to tell the true operand size, since it only
2838  // records a 64-bit value. We need to know the size to determine if a 32-bit
2839  // floating point immediate bit pattern is legal for an integer immediate. It
2840  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2841 
2842  int64_t Imm = MO.getImm();
2843  switch (OperandType) {
2850  int32_t Trunc = static_cast<int32_t>(Imm);
2852  }
2858  ST.hasInv2PiInlineImm());
2865  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2866  // A few special case instructions have 16-bit operands on subtargets
2867  // where 16-bit instructions are not legal.
2868  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2869  // constants in these cases
2870  int16_t Trunc = static_cast<int16_t>(Imm);
2871  return ST.has16BitInsts() &&
2873  }
2874 
2875  return false;
2876  }
2883  uint32_t Trunc = static_cast<uint32_t>(Imm);
2885  }
2886  default:
2887  llvm_unreachable("invalid bitwidth");
2888  }
2889 }
2890 
2892  const MCOperandInfo &OpInfo) const {
2893  switch (MO.getType()) {
2895  return false;
2897  return !isInlineConstant(MO, OpInfo);
2903  return true;
2904  default:
2905  llvm_unreachable("unexpected operand type");
2906  }
2907 }
2908 
2909 static bool compareMachineOp(const MachineOperand &Op0,
2910  const MachineOperand &Op1) {
2911  if (Op0.getType() != Op1.getType())
2912  return false;
2913 
2914  switch (Op0.getType()) {
2916  return Op0.getReg() == Op1.getReg();
2918  return Op0.getImm() == Op1.getImm();
2919  default:
2920  llvm_unreachable("Didn't expect to be comparing these operand types");
2921  }
2922 }
2923 
2925  const MachineOperand &MO) const {
2926  const MCInstrDesc &InstDesc = MI.getDesc();
2927  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
2928 
2929  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
2930 
2931  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2932  return true;
2933 
2934  if (OpInfo.RegClass < 0)
2935  return false;
2936 
2937  const MachineFunction *MF = MI.getParent()->getParent();
2938  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2939 
2940  if (MO.isImm() && isInlineConstant(MO, OpInfo)) {
2941  if (isMAI(MI) && ST.hasMFMAInlineLiteralBug() &&
2943  AMDGPU::OpName::src2))
2944  return false;
2945  return RI.opCanUseInlineConstant(OpInfo.OperandType);
2946  }
2947 
2948  if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
2949  return false;
2950 
2951  if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
2952  return true;
2953 
2954  return ST.hasVOP3Literal();
2955 }
2956 
2957 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2958  int Op32 = AMDGPU::getVOPe32(Opcode);
2959  if (Op32 == -1)
2960  return false;
2961 
2962  return pseudoToMCOpcode(Op32) != -1;
2963 }
2964 
2965 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2966  // The src0_modifier operand is present on all instructions
2967  // that have modifiers.
2968 
2969  return AMDGPU::getNamedOperandIdx(Opcode,
2970  AMDGPU::OpName::src0_modifiers) != -1;
2971 }
2972 
2974  unsigned OpName) const {
2975  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2976  return Mods && Mods->getImm();
2977 }
2978 
2980  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2981  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2982  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2983  hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2984  hasModifiersSet(MI, AMDGPU::OpName::omod);
2985 }
2986 
2988  const MachineRegisterInfo &MRI) const {
2989  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2990  // Can't shrink instruction with three operands.
2991  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2992  // a special case for it. It can only be shrunk if the third operand
2993  // is vcc, and src0_modifiers and src1_modifiers are not set.
2994  // We should handle this the same way we handle vopc, by addding
2995  // a register allocation hint pre-regalloc and then do the shrinking
2996  // post-regalloc.
2997  if (Src2) {
2998  switch (MI.getOpcode()) {
2999  default: return false;
3000 
3001  case AMDGPU::V_ADDC_U32_e64:
3002  case AMDGPU::V_SUBB_U32_e64:
3003  case AMDGPU::V_SUBBREV_U32_e64: {
3004  const MachineOperand *Src1
3005  = getNamedOperand(MI, AMDGPU::OpName::src1);
3006  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
3007  return false;
3008  // Additional verification is needed for sdst/src2.
3009  return true;
3010  }
3011  case AMDGPU::V_MAC_F32_e64:
3012  case AMDGPU::V_MAC_F16_e64:
3013  case AMDGPU::V_FMAC_F32_e64:
3014  case AMDGPU::V_FMAC_F16_e64:
3015  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
3016  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
3017  return false;
3018  break;
3019 
3020  case AMDGPU::V_CNDMASK_B32_e64:
3021  break;
3022  }
3023  }
3024 
3025  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3026  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
3027  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
3028  return false;
3029 
3030  // We don't need to check src0, all input types are legal, so just make sure
3031  // src0 isn't using any modifiers.
3032  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
3033  return false;
3034 
3035  // Can it be shrunk to a valid 32 bit opcode?
3036  if (!hasVALU32BitEncoding(MI.getOpcode()))
3037  return false;
3038 
3039  // Check output modifiers
3040  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
3041  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
3042 }
3043 
3044 // Set VCC operand with all flags from \p Orig, except for setting it as
3045 // implicit.
3047  const MachineOperand &Orig) {
3048 
3049  for (MachineOperand &Use : MI.implicit_operands()) {
3050  if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
3051  Use.setIsUndef(Orig.isUndef());
3052  Use.setIsKill(Orig.isKill());
3053  return;
3054  }
3055  }
3056 }
3057 
3059  unsigned Op32) const {
3060  MachineBasicBlock *MBB = MI.getParent();;
3061  MachineInstrBuilder Inst32 =
3062  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
3063 
3064  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
3065  // For VOPC instructions, this is replaced by an implicit def of vcc.
3066  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
3067  if (Op32DstIdx != -1) {
3068  // dst
3069  Inst32.add(MI.getOperand(0));
3070  } else {
3071  assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
3072  (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
3073  "Unexpected case");
3074  }
3075 
3076  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3077 
3078  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3079  if (Src1)
3080  Inst32.add(*Src1);
3081 
3082  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3083 
3084  if (Src2) {
3085  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3086  if (Op32Src2Idx != -1) {
3087  Inst32.add(*Src2);
3088  } else {
3089  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3090  // replaced with an implicit read of vcc. This was already added
3091  // during the initial BuildMI, so find it to preserve the flags.
3092  copyFlagsToImplicitVCC(*Inst32, *Src2);
3093  }
3094  }
3095 
3096  return Inst32;
3097 }
3098 
3100  const MachineOperand &MO,
3101  const MCOperandInfo &OpInfo) const {
3102  // Literal constants use the constant bus.
3103  //if (isLiteralConstantLike(MO, OpInfo))
3104  // return true;
3105  if (MO.isImm())
3106  return !isInlineConstant(MO, OpInfo);
3107 
3108  if (!MO.isReg())
3109  return true; // Misc other operands like FrameIndex
3110 
3111  if (!MO.isUse())
3112  return false;
3113 
3115  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3116 
3117  // Null is free
3118  if (MO.getReg() == AMDGPU::SGPR_NULL)
3119  return false;
3120 
3121  // SGPRs use the constant bus
3122  if (MO.isImplicit()) {
3123  return MO.getReg() == AMDGPU::M0 ||
3124  MO.getReg() == AMDGPU::VCC ||
3125  MO.getReg() == AMDGPU::VCC_LO;
3126  } else {
3127  return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3128  AMDGPU::SReg_64RegClass.contains(MO.getReg());
3129  }
3130 }
3131 
3132 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
3133  for (const MachineOperand &MO : MI.implicit_operands()) {
3134  // We only care about reads.
3135  if (MO.isDef())
3136  continue;
3137 
3138  switch (MO.getReg()) {
3139  case AMDGPU::VCC:
3140  case AMDGPU::VCC_LO:
3141  case AMDGPU::VCC_HI:
3142  case AMDGPU::M0:
3143  case AMDGPU::FLAT_SCR:
3144  return MO.getReg();
3145 
3146  default:
3147  break;
3148  }
3149  }
3150 
3151  return AMDGPU::NoRegister;
3152 }
3153 
3154 static bool shouldReadExec(const MachineInstr &MI) {
3155  if (SIInstrInfo::isVALU(MI)) {
3156  switch (MI.getOpcode()) {
3157  case AMDGPU::V_READLANE_B32:
3158  case AMDGPU::V_READLANE_B32_gfx6_gfx7:
3159  case AMDGPU::V_READLANE_B32_gfx10:
3160  case AMDGPU::V_READLANE_B32_vi:
3161  case AMDGPU::V_WRITELANE_B32:
3162  case AMDGPU::V_WRITELANE_B32_gfx6_gfx7:
3163  case AMDGPU::V_WRITELANE_B32_gfx10:
3164  case AMDGPU::V_WRITELANE_B32_vi:
3165  return false;
3166  }
3167 
3168  return true;
3169  }
3170 
3171  if (MI.isPreISelOpcode() ||
3172  SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3173  SIInstrInfo::isSALU(MI) ||
3174  SIInstrInfo::isSMRD(MI))
3175  return false;
3176 
3177  return true;
3178 }
3179 
3180 static bool isSubRegOf(const SIRegisterInfo &TRI,
3181  const MachineOperand &SuperVec,
3182  const MachineOperand &SubReg) {
3183  if (Register::isPhysicalRegister(SubReg.getReg()))
3184  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3185 
3186  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3187  SubReg.getReg() == SuperVec.getReg();
3188 }
3189 
3191  StringRef &ErrInfo) const {
3192  uint16_t Opcode = MI.getOpcode();
3193  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3194  return true;
3195 
3196  const MachineFunction *MF = MI.getParent()->getParent();
3197  const MachineRegisterInfo &MRI = MF->getRegInfo();
3198 
3199  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3200  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3201  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3202 
3203  // Make sure the number of operands is correct.
3204  const MCInstrDesc &Desc = get(Opcode);
3205  if (!Desc.isVariadic() &&
3206  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
3207  ErrInfo = "Instruction has wrong number of operands.";
3208  return false;
3209  }
3210 
3211  if (MI.isInlineAsm()) {
3212  // Verify register classes for inlineasm constraints.
3213  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
3214  I != E; ++I) {
3215  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
3216  if (!RC)
3217  continue;
3218 
3219  const MachineOperand &Op = MI.getOperand(I);
3220  if (!Op.isReg())
3221  continue;
3222 
3223  Register Reg = Op.getReg();
3224  if (!Register::isVirtualRegister(Reg) && !RC->contains(Reg)) {
3225  ErrInfo = "inlineasm operand has incorrect register class.";
3226  return false;
3227  }
3228  }
3229 
3230  return true;
3231  }
3232 
3233  // Make sure the register classes are correct.
3234  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
3235  if (MI.getOperand(i).isFPImm()) {
3236  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3237  "all fp values to integers.";
3238  return false;
3239  }
3240 
3241  int RegClass = Desc.OpInfo[i].RegClass;
3242 
3243  switch (Desc.OpInfo[i].OperandType) {
3245  if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
3246  ErrInfo = "Illegal immediate value for operand.";
3247  return false;
3248  }
3249  break;
3252  break;
3263  const MachineOperand &MO = MI.getOperand(i);
3264  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
3265  ErrInfo = "Illegal immediate value for operand.";
3266  return false;
3267  }
3268  break;
3269  }
3272  // Check if this operand is an immediate.
3273  // FrameIndex operands will be replaced by immediates, so they are
3274  // allowed.
3275  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
3276  ErrInfo = "Expected immediate, but got non-immediate";
3277  return false;
3278  }
3280  default:
3281  continue;
3282  }
3283 
3284  if (!MI.getOperand(i).isReg())
3285  continue;
3286 
3287  if (RegClass != -1) {
3288  Register Reg = MI.getOperand(i).getReg();
3289  if (Reg == AMDGPU::NoRegister || Register::isVirtualRegister(Reg))
3290  continue;
3291 
3292  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3293  if (!RC->contains(Reg)) {
3294  ErrInfo = "Operand has incorrect register class.";
3295  return false;
3296  }
3297  }
3298  }
3299 
3300  // Verify SDWA
3301  if (isSDWA(MI)) {
3302  if (!ST.hasSDWA()) {
3303  ErrInfo = "SDWA is not supported on this target";
3304  return false;
3305  }
3306 
3307  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3308 
3309  const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3310 
3311  for (int OpIdx: OpIndicies) {
3312  if (OpIdx == -1)
3313  continue;
3314  const MachineOperand &MO = MI.getOperand(OpIdx);
3315 
3316  if (!ST.hasSDWAScalar()) {
3317  // Only VGPRS on VI
3318  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3319  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3320  return false;
3321  }
3322  } else {
3323  // No immediates on GFX9
3324  if (!MO.isReg()) {
3325  ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
3326  return false;
3327  }
3328  }
3329  }
3330 
3331  if (!ST.hasSDWAOmod()) {
3332  // No omod allowed on VI
3333  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3334  if (OMod != nullptr &&
3335  (!OMod->isImm() || OMod->getImm() != 0)) {
3336  ErrInfo = "OMod not allowed in SDWA instructions on VI";
3337  return false;
3338  }
3339  }
3340 
3341  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3342  if (isVOPC(BasicOpcode)) {
3343  if (!ST.hasSDWASdst() && DstIdx != -1) {
3344  // Only vcc allowed as dst on VI for VOPC
3345  const MachineOperand &Dst = MI.getOperand(DstIdx);
3346  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3347  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3348  return false;
3349  }
3350  } else if (!ST.hasSDWAOutModsVOPC()) {
3351  // No clamp allowed on GFX9 for VOPC
3352  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3353  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3354  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3355  return false;
3356  }
3357 
3358  // No omod allowed on GFX9 for VOPC
3359  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3360  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
3361  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3362  return false;
3363  }
3364  }
3365  }
3366 
3367  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3368  if (DstUnused && DstUnused->isImm() &&
3369  DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
3370  const MachineOperand &Dst = MI.getOperand(DstIdx);
3371  if (!Dst.isReg() || !Dst.isTied()) {
3372  ErrInfo = "Dst register should have tied register";
3373  return false;
3374  }
3375 
3376  const MachineOperand &TiedMO =
3377  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3378  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3379  ErrInfo =
3380  "Dst register should be tied to implicit use of preserved register";
3381  return false;
3382  } else if (Register::isPhysicalRegister(TiedMO.getReg()) &&
3383  Dst.getReg() != TiedMO.getReg()) {
3384  ErrInfo = "Dst register should use same physical register as preserved";
3385  return false;
3386  }
3387  }
3388  }
3389 
3390  // Verify MIMG
3391  if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
3392  // Ensure that the return type used is large enough for all the options
3393  // being used TFE/LWE require an extra result register.
3394  const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3395  if (DMask) {
3396  uint64_t DMaskImm = DMask->getImm();
3397  uint32_t RegCount =
3398  isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
3399  const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3400  const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3401  const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3402 
3403  // Adjust for packed 16 bit values
3404  if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
3405  RegCount >>= 1;
3406 
3407  // Adjust if using LWE or TFE
3408  if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
3409  RegCount += 1;
3410 
3411  const uint32_t DstIdx =
3412  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3413  const MachineOperand &Dst = MI.getOperand(DstIdx);
3414  if (Dst.isReg()) {
3415  const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3416  uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3417  if (RegCount > DstSize) {
3418  ErrInfo = "MIMG instruction returns too many registers for dst "
3419  "register class";
3420  return false;
3421  }
3422  }
3423  }
3424  }
3425 
3426  // Verify VOP*. Ignore multiple sgpr operands on writelane.
3427  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3428  && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
3429  // Only look at the true operands. Only a real operand can use the constant
3430  // bus, and we don't want to check pseudo-operands like the source modifier
3431  // flags.
3432  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3433 
3434  unsigned ConstantBusCount = 0;
3435  unsigned LiteralCount = 0;
3436 
3437  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3438  ++ConstantBusCount;
3439 
3440  SmallVector<unsigned, 2> SGPRsUsed;
3441  unsigned SGPRUsed = findImplicitSGPRRead(MI);
3442  if (SGPRUsed != AMDGPU::NoRegister) {
3443  ++ConstantBusCount;
3444  SGPRsUsed.push_back(SGPRUsed);
3445  }
3446 
3447  for (int OpIdx : OpIndices) {
3448  if (OpIdx == -1)
3449  break;
3450  const MachineOperand &MO = MI.getOperand(OpIdx);
3451  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3452  if (MO.isReg()) {
3453  SGPRUsed = MO.getReg();
3454  if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
3455  return !RI.regsOverlap(SGPRUsed, SGPR);
3456  })) {
3457  ++ConstantBusCount;
3458  SGPRsUsed.push_back(SGPRUsed);
3459  }
3460  } else {
3461  ++ConstantBusCount;
3462  ++LiteralCount;
3463  }
3464  }
3465  }
3466  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3467  // v_writelane_b32 is an exception from constant bus restriction:
3468  // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3469  if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
3470  Opcode != AMDGPU::V_WRITELANE_B32) {
3471  ErrInfo = "VOP* instruction violates constant bus restriction";
3472  return false;
3473  }
3474 
3475  if (isVOP3(MI) && LiteralCount) {
3476  if (LiteralCount && !ST.hasVOP3Literal()) {
3477  ErrInfo = "VOP3 instruction uses literal";
3478  return false;
3479  }
3480  if (LiteralCount > 1) {
3481  ErrInfo = "VOP3 instruction uses more than one literal";
3482  return false;
3483  }
3484  }
3485  }
3486 
3487  // Special case for writelane - this can break the multiple constant bus rule,
3488  // but still can't use more than one SGPR register
3489  if (Desc.getOpcode() == AMDGPU::V_WRITELANE_B32) {
3490  unsigned SGPRCount = 0;
3491  Register SGPRUsed = AMDGPU::NoRegister;
3492 
3493  for (int OpIdx : {Src0Idx, Src1Idx, Src2Idx}) {
3494  if (OpIdx == -1)
3495  break;
3496 
3497  const MachineOperand &MO = MI.getOperand(OpIdx);
3498 
3499  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3500  if (MO.isReg() && MO.getReg() != AMDGPU::M0) {
3501  if (MO.getReg() != SGPRUsed)
3502  ++SGPRCount;
3503  SGPRUsed = MO.getReg();
3504  }
3505  }
3506  if (SGPRCount > ST.getConstantBusLimit(Opcode)) {
3507  ErrInfo = "WRITELANE instruction violates constant bus restriction";
3508  return false;
3509  }
3510  }
3511  }
3512 
3513  // Verify misc. restrictions on specific instructions.
3514  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3515  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3516  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3517  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3518  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3519  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3520  if (!compareMachineOp(Src0, Src1) &&
3521  !compareMachineOp(Src0, Src2)) {
3522  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3523  return false;
3524  }
3525  }
3526  }
3527 
3528  if (isSOP2(MI) || isSOPC(MI)) {
3529  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3530  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3531  unsigned Immediates = 0;
3532 
3533  if (!Src0.isReg() &&
3534  !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType))
3535  Immediates++;
3536  if (!Src1.isReg() &&
3537  !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType))
3538  Immediates++;
3539 
3540  if (Immediates > 1) {
3541  ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
3542  return false;
3543  }
3544  }
3545 
3546  if (isSOPK(MI)) {
3547  auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
3548  if (Desc.isBranch()) {
3549  if (!Op->isMBB()) {
3550  ErrInfo = "invalid branch target for SOPK instruction";
3551  return false;
3552  }
3553  } else {
3554  uint64_t Imm = Op->getImm();
3555  if (sopkIsZext(MI)) {
3556  if (!isUInt<16>(Imm)) {
3557  ErrInfo = "invalid immediate for SOPK instruction";
3558  return false;
3559  }
3560  } else {
3561  if (!isInt<16>(Imm)) {
3562  ErrInfo = "invalid immediate for SOPK instruction";
3563  return false;
3564  }
3565  }
3566  }
3567  }
3568 
3569  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3570  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3571  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3572  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3573  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3574  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3575 
3576  const unsigned StaticNumOps = Desc.getNumOperands() +
3577  Desc.getNumImplicitUses();
3578  const unsigned NumImplicitOps = IsDst ? 2 : 1;
3579 
3580  // Allow additional implicit operands. This allows a fixup done by the post
3581  // RA scheduler where the main implicit operand is killed and implicit-defs
3582  // are added for sub-registers that remain live after this instruction.
3583  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3584  ErrInfo = "missing implicit register operands";
3585  return false;
3586  }
3587 
3588  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3589  if (IsDst) {
3590  if (!Dst->isUse()) {
3591  ErrInfo = "v_movreld_b32 vdst should be a use operand";
3592  return false;
3593  }
3594 
3595  unsigned UseOpIdx;
3596  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3597  UseOpIdx != StaticNumOps + 1) {
3598  ErrInfo = "movrel implicit operands should be tied";
3599  return false;
3600  }
3601  }
3602 
3603  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3604  const MachineOperand &ImpUse
3605  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3606  if (!ImpUse.isReg() || !ImpUse.isUse() ||
3607  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3608  ErrInfo = "src0 should be subreg of implicit vector use";
3609  return false;
3610  }
3611  }
3612 
3613  // Make sure we aren't losing exec uses in the td files. This mostly requires
3614  // being careful when using let Uses to try to add other use registers.
3615  if (shouldReadExec(MI)) {
3616  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3617  ErrInfo = "VALU instruction does not implicitly read exec mask";
3618  return false;
3619  }
3620  }
3621 
3622  if (isSMRD(MI)) {
3623  if (MI.mayStore()) {
3624  // The register offset form of scalar stores may only use m0 as the
3625  // soffset register.
3626  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3627  if (Soff && Soff->getReg() != AMDGPU::M0) {
3628  ErrInfo = "scalar stores must use m0 as offset register";
3629  return false;
3630  }
3631  }
3632  }
3633 
3634  if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3635  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3636  if (Offset->getImm() != 0) {
3637  ErrInfo = "subtarget does not support offsets in flat instructions";
3638  return false;
3639  }
3640  }
3641 
3642  if (isMIMG(MI)) {
3643  const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
3644  if (DimOp) {
3645  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
3646  AMDGPU::OpName::vaddr0);
3647  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
3648  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
3649  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3651  const AMDGPU::MIMGDimInfo *Dim =
3653 
3654  if (!Dim) {
3655  ErrInfo = "dim is out of range";
3656  return false;
3657  }
3658 
3659  bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
3660  unsigned AddrWords = BaseOpcode->NumExtraArgs +
3661  (BaseOpcode->Gradients ? Dim->NumGradients : 0) +
3662  (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
3663  (BaseOpcode->LodOrClampOrMip ? 1 : 0);
3664 
3665  unsigned VAddrWords;
3666  if (IsNSA) {
3667  VAddrWords = SRsrcIdx - VAddr0Idx;
3668  } else {
3669  const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
3670  VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
3671  if (AddrWords > 8)
3672  AddrWords = 16;
3673  else if (AddrWords > 4)
3674  AddrWords = 8;
3675  else if (AddrWords == 3 && VAddrWords == 4) {
3676  // CodeGen uses the V4 variant of instructions for three addresses,
3677  // because the selection DAG does not support non-power-of-two types.
3678  AddrWords = 4;
3679  }
3680  }
3681 
3682  if (VAddrWords != AddrWords) {
3683  ErrInfo = "bad vaddr size";
3684  return false;
3685  }
3686  }
3687  }
3688 
3689  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3690  if (DppCt) {
3691  using namespace AMDGPU::DPP;
3692 
3693  unsigned DC = DppCt->getImm();
3694  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3695  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3701  ErrInfo = "Invalid dpp_ctrl value";
3702  return false;
3703  }
3704  if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
3705  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3706  ErrInfo = "Invalid dpp_ctrl value: "
3707  "wavefront shifts are not supported on GFX10+";
3708  return false;
3709  }
3710  if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
3711  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3712  ErrInfo = "Invalid dpp_ctrl value: "
3713  "broadcasts are not supported on GFX10+";
3714  return false;
3715  }
3717  ST.getGeneration() < AMDGPUSubtarget::GFX10) {
3718  ErrInfo = "Invalid dpp_ctrl value: "
3719  "row_share and row_xmask are not supported before GFX10";
3720  return false;
3721  }
3722  }
3723 
3724  return true;
3725 }
3726 
3727 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3728  switch (MI.getOpcode()) {
3729  default: return AMDGPU::INSTRUCTION_LIST_END;
3730  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3731  case AMDGPU::COPY: return AMDGPU::COPY;
3732  case AMDGPU::PHI: return AMDGPU::PHI;
3733  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3734  case AMDGPU::WQM: return AMDGPU::WQM;
3735  case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
3736  case AMDGPU::WWM: return AMDGPU::WWM;
3737  case AMDGPU::S_MOV_B32: {
3739  return MI.getOperand(1).isReg() ||
3740  RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
3741  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3742  }
3743  case AMDGPU::S_ADD_I32:
3744  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3745  case AMDGPU::S_ADDC_U32:
3746  return AMDGPU::V_ADDC_U32_e32;
3747  case AMDGPU::S_SUB_I32:
3748  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3749  // FIXME: These are not consistently handled, and selected when the carry is
3750  // used.
3751  case AMDGPU::S_ADD_U32:
3752  return AMDGPU::V_ADD_I32_e32;
3753  case AMDGPU::S_SUB_U32:
3754  return AMDGPU::V_SUB_I32_e32;
3755  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3756  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32;
3757  case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32;
3758  case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32;
3759  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3760  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3761  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3762  case AMDGPU::S_XNOR_B32:
3763  return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
3764  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3765  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3766  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3767  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3768  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3769  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3770  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3771  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3772  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3773  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3774  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3775  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3776  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3777  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3778  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3779  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3780  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3781  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3782  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3783  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3784  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3785  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3786  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3787  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3788  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3789  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3790  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3791  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3792  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3793  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3794  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3795  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3796  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3797  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3798  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3799  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3800  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3801  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3802  }
3804  "Unexpected scalar opcode without corresponding vector one!");
3805 }
3806 
3808  unsigned OpNo) const {
3810  const MCInstrDesc &Desc = get(MI.getOpcode());
3811  if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3812  Desc.OpInfo[OpNo].RegClass == -1) {
3813  Register Reg = MI.getOperand(OpNo).getReg();
3814 
3815  if (Register::isVirtualRegister(Reg))
3816  return MRI.getRegClass(Reg);
3817  return RI.getPhysRegClass(Reg);
3818  }
3819 
3820  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3821  return RI.getRegClass(RCID);
3822 }
3823 
3824 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3826  MachineBasicBlock *MBB = MI.getParent();
3827  MachineOperand &MO = MI.getOperand(OpIdx);
3829  const SIRegisterInfo *TRI =
3830  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3831  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3832  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3833  unsigned Size = TRI->getRegSizeInBits(*RC);
3834  unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
3835  if (MO.isReg())
3836  Opcode = AMDGPU::COPY;
3837  else if (RI.isSGPRClass(RC))
3838  Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
3839 
3840  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3841  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3842  VRC = &AMDGPU::VReg_64RegClass;
3843  else
3844  VRC = &AMDGPU::VGPR_32RegClass;
3845 
3846  Register Reg = MRI.createVirtualRegister(VRC);
3847  DebugLoc DL = MBB->findDebugLoc(I);
3848  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3849  MO.ChangeToRegister(Reg, false);
3850 }
3851 
3854  MachineOperand &SuperReg,
3855  const TargetRegisterClass *SuperRC,
3856  unsigned SubIdx,
3857  const TargetRegisterClass *SubRC)
3858  const {
3859  MachineBasicBlock *MBB = MI->getParent();
3860  DebugLoc DL = MI->getDebugLoc();
3861  Register SubReg = MRI.createVirtualRegister(SubRC);
3862 
3863  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3864  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3865  .addReg(SuperReg.getReg(), 0, SubIdx);
3866  return SubReg;
3867  }
3868 
3869  // Just in case the super register is itself a sub-register, copy it to a new
3870  // value so we don't need to worry about merging its subreg index with the
3871  // SubIdx passed to this function. The register coalescer should be able to
3872  // eliminate this extra copy.
3873  Register NewSuperReg = MRI.createVirtualRegister(SuperRC);
3874 
3875  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3876  .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3877 
3878  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3879  .addReg(NewSuperReg, 0, SubIdx);
3880 
3881  return SubReg;
3882 }
3883 
3887  MachineOperand &Op,
3888  const TargetRegisterClass *SuperRC,
3889  unsigned SubIdx,
3890  const TargetRegisterClass *SubRC) const {
3891  if (Op.isImm()) {
3892  if (SubIdx == AMDGPU::sub0)
3893  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3894  if (SubIdx == AMDGPU::sub1)
3895  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3896 
3897  llvm_unreachable("Unhandled register index for immediate");
3898  }
3899 
3900  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3901  SubIdx, SubRC);
3902  return MachineOperand::CreateReg(SubReg, false);
3903 }
3904 
3905 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3906 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3907  assert(Inst.getNumExplicitOperands() == 3);
3908  MachineOperand Op1 = Inst.getOperand(1);
3909  Inst.RemoveOperand(1);
3910  Inst.addOperand(Op1);
3911 }
3912 
3914  const MCOperandInfo &OpInfo,
3915  const MachineOperand &MO) const {
3916  if (!MO.isReg())
3917  return false;
3918 
3919  Register Reg = MO.getReg();
3921  ? MRI.getRegClass(Reg)
3922  : RI.getPhysRegClass(Reg);
3923 
3924  const SIRegisterInfo *TRI =
3925  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3926  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3927 
3928  // In order to be legal, the common sub-class must be equal to the
3929  // class of the current operand. For example:
3930  //
3931  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3932  // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3933  //
3934  // s_sendmsg 0, s0 ; Operand defined as m0reg
3935  // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3936 
3937  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3938 }
3939 
3941  const MCOperandInfo &OpInfo,
3942  const MachineOperand &MO) const {
3943  if (MO.isReg())
3944  return isLegalRegOperand(MRI, OpInfo, MO);
3945 
3946  // Handle non-register types that are treated like immediates.
3947  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3948  return true;
3949 }
3950 
3951 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3952  const MachineOperand *MO) const {
3953  const MachineFunction &MF = *MI.getParent()->getParent();
3954  const MachineRegisterInfo &MRI = MF.getRegInfo();
3955  const MCInstrDesc &InstDesc = MI.getDesc();
3956  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3957  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3958  const TargetRegisterClass *DefinedRC =
3959  OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3960  if (!MO)
3961  MO = &MI.getOperand(OpIdx);
3962 
3963  int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
3964  int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
3965  if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3966  if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--)
3967  return false;
3968 
3969  SmallDenseSet<RegSubRegPair> SGPRsUsed;
3970  if (MO->isReg())
3971  SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
3972 
3973  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3974  if (i == OpIdx)
3975  continue;
3976  const MachineOperand &Op = MI.getOperand(i);
3977  if (Op.isReg()) {
3978  RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
3979  if (!SGPRsUsed.count(SGPR) &&
3980  usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3981  if (--ConstantBusLimit <= 0)
3982  return false;
3983  SGPRsUsed.insert(SGPR);
3984  }
3985  } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3986  if (--ConstantBusLimit <= 0)
3987  return false;
3988  } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) &&
3989  isLiteralConstantLike(Op, InstDesc.OpInfo[i])) {
3990  if (!VOP3LiteralLimit--)
3991  return false;
3992  if (--ConstantBusLimit <= 0)
3993  return false;
3994  }
3995  }
3996  }
3997 
3998  if (MO->isReg()) {
3999  assert(DefinedRC);
4000  return isLegalRegOperand(MRI, OpInfo, *MO);
4001  }
4002 
4003  // Handle non-register types that are treated like immediates.
4004  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
4005 
4006  if (!DefinedRC) {
4007  // This operand expects an immediate.
4008  return true;
4009  }
4010 
4011  return isImmOperandLegal(MI, OpIdx, *MO);
4012 }
4013 
4015  MachineInstr &MI) const {
4016  unsigned Opc = MI.getOpcode();
4017  const MCInstrDesc &InstrDesc = get(Opc);
4018 
4019  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
4020  MachineOperand &Src0 = MI.getOperand(Src0Idx);
4021 
4022  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
4023  MachineOperand &Src1 = MI.getOperand(Src1Idx);
4024 
4025  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
4026  // we need to only have one constant bus use before GFX10.
4027  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
4028  if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 &&
4029  Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) ||
4030  isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])))
4031  legalizeOpWithMove(MI, Src0Idx);
4032 
4033  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
4034  // both the value to write (src0) and lane select (src1). Fix up non-SGPR
4035  // src0/src1 with V_READFIRSTLANE.
4036  if (Opc == AMDGPU::V_WRITELANE_B32) {
4037  const DebugLoc &DL = MI.getDebugLoc();
4038  if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
4039  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4040  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4041  .add(Src0);
4042  Src0.ChangeToRegister(Reg, false);
4043  }
4044  if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
4045  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4046  const DebugLoc &DL = MI.getDebugLoc();
4047  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4048  .add(Src1);
4049  Src1.ChangeToRegister(Reg, false);
4050  }
4051  return;
4052  }
4053 
4054  // No VOP2 instructions support AGPRs.
4055  if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg()))
4056  legalizeOpWithMove(MI, Src0Idx);
4057 
4058  if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg()))
4059  legalizeOpWithMove(MI, Src1Idx);
4060 
4061  // VOP2 src0 instructions support all operand types, so we don't need to check
4062  // their legality. If src1 is already legal, we don't need to do anything.
4063  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
4064  return;
4065 
4066  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
4067  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
4068  // select is uniform.
4069  if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
4070  RI.isVGPR(MRI, Src1.getReg())) {
4071  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4072  const DebugLoc &DL = MI.getDebugLoc();
4073  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4074  .add(Src1);
4075  Src1.ChangeToRegister(Reg, false);
4076  return;
4077  }
4078 
4079  // We do not use commuteInstruction here because it is too aggressive and will
4080  // commute if it is possible. We only want to commute here if it improves
4081  // legality. This can be called a fairly large number of times so don't waste
4082  // compile time pointlessly swapping and checking legality again.
4083  if (HasImplicitSGPR || !MI.isCommutable()) {
4084  legalizeOpWithMove(MI, Src1Idx);
4085  return;
4086  }
4087 
4088  // If src0 can be used as src1, commuting will make the operands legal.
4089  // Otherwise we have to give up and insert a move.
4090  //
4091  // TODO: Other immediate-like operand kinds could be commuted if there was a
4092  // MachineOperand::ChangeTo* for them.
4093  if ((!Src1.isImm() && !Src1.isReg()) ||
4094  !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
4095  legalizeOpWithMove(MI, Src1Idx);
4096  return;
4097  }
4098 
4099  int CommutedOpc = commuteOpcode(MI);
4100  if (CommutedOpc == -1) {
4101  legalizeOpWithMove(MI, Src1Idx);
4102  return;
4103  }
4104 
4105  MI.setDesc(get(CommutedOpc));
4106 
4107  Register Src0Reg = Src0.getReg();
4108  unsigned Src0SubReg = Src0.getSubReg();
4109  bool Src0Kill = Src0.isKill();
4110 
4111  if (Src1.isImm())
4112  Src0.ChangeToImmediate(Src1.getImm());
4113  else if (Src1.isReg()) {
4114  Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
4115  Src0.setSubReg(Src1.getSubReg());
4116  } else
4117  llvm_unreachable("Should only have register or immediate operands");
4118 
4119  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
4120  Src1.setSubReg(Src0SubReg);
4121  fixImplicitOperands(MI);
4122 }
4123 
4124 // Legalize VOP3 operands. All operand types are supported for any operand
4125 // but only one literal constant and only starting from GFX10.
4127  MachineInstr &MI) const {
4128  unsigned Opc = MI.getOpcode();
4129 
4130  int VOP3Idx[3] = {
4131  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
4132  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
4133  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
4134  };
4135 
4136  if (Opc == AMDGPU::V_PERMLANE16_B32 ||
4137  Opc == AMDGPU::V_PERMLANEX16_B32) {
4138  // src1 and src2 must be scalar
4139  MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
4140  MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
4141  const DebugLoc &DL = MI.getDebugLoc();
4142  if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
4143  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4144  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4145  .add(Src1);
4146  Src1.ChangeToRegister(Reg, false);
4147  }
4148  if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
4149  Register Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4150  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4151  .add(Src2);
4152  Src2.ChangeToRegister(Reg, false);
4153  }
4154  }
4155 
4156  // Find the one SGPR operand we are allowed to use.
4157  int ConstantBusLimit = ST.getConstantBusLimit(Opc);
4158  int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
4159  SmallDenseSet<unsigned> SGPRsUsed;
4160  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
4161  if (SGPRReg != AMDGPU::NoRegister) {
4162  SGPRsUsed.insert(SGPRReg);
4163  --ConstantBusLimit;
4164  }
4165 
4166  for (unsigned i = 0; i < 3; ++i) {
4167  int Idx = VOP3Idx[i];
4168  if (Idx == -1)
4169  break;
4170  MachineOperand &MO = MI.getOperand(Idx);
4171 
4172  if (!MO.isReg()) {
4173  if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
4174  continue;
4175 
4176  if (LiteralLimit > 0 && ConstantBusLimit > 0) {
4177  --LiteralLimit;
4178  --ConstantBusLimit;
4179  continue;
4180  }
4181 
4182  --LiteralLimit;
4183  --ConstantBusLimit;
4184  legalizeOpWithMove(MI, Idx);
4185  continue;
4186  }
4187 
4188  if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) &&
4189  !isOperandLegal(MI, Idx, &MO)) {
4190  legalizeOpWithMove(MI, Idx);
4191  continue;
4192  }
4193 
4194  if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
4195  continue; // VGPRs are legal
4196 
4197  // We can use one SGPR in each VOP3 instruction prior to GFX10
4198  // and two starting from GFX10.
4199  if (SGPRsUsed.count(MO.getReg()))
4200  continue;
4201  if (ConstantBusLimit > 0) {
4202  SGPRsUsed.insert(MO.getReg());
4203  --ConstantBusLimit;
4204  continue;
4205  }
4206 
4207  // If we make it this far, then the operand is not legal and we must
4208  // legalize it.
4209  legalizeOpWithMove(MI, Idx);
4210  }
4211 }
4212 
4214  MachineRegisterInfo &MRI) const {
4215  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
4216  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
4217  Register DstReg = MRI.createVirtualRegister(SRC);
4218  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
4219 
4220  if (RI.hasAGPRs(VRC)) {
4221  VRC = RI.getEquivalentVGPRClass(VRC);
4222  Register NewSrcReg = MRI.createVirtualRegister(VRC);
4223  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4224  get(TargetOpcode::COPY), NewSrcReg)
4225  .addReg(SrcReg);
4226  SrcReg = NewSrcReg;
4227  }
4228 
4229  if (SubRegs == 1) {
4230  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4231  get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
4232  .addReg(SrcReg);
4233  return DstReg;
4234  }
4235 
4237  for (unsigned i = 0; i < SubRegs; ++i) {
4238  Register SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4239  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4240  get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
4241  .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
4242  SRegs.push_back(SGPR);
4243  }
4244 
4245  MachineInstrBuilder MIB =
4246  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4247  get(AMDGPU::REG_SEQUENCE), DstReg);
4248  for (unsigned i = 0; i < SubRegs; ++i) {
4249  MIB.addReg(SRegs[i]);
4250  MIB.addImm(RI.getSubRegFromChannel(i));
4251  }
4252  return DstReg;
4253 }
4254 
4256  MachineInstr &MI) const {
4257 
4258  // If the pointer is store in VGPRs, then we need to move them to
4259  // SGPRs using v_readfirstlane. This is safe because we only select
4260  // loads with uniform pointers to SMRD instruction so we know the
4261  // pointer value is uniform.
4262  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
4263  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
4264  unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
4265  SBase->setReg(SGPR);
4266  }
4267  MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
4268  if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
4269  unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
4270  SOff->setReg(SGPR);
4271  }
4272 }
4273 
4276  const TargetRegisterClass *DstRC,
4277  MachineOperand &Op,
4279  const DebugLoc &DL) const {
4280  Register OpReg = Op.getReg();
4281  unsigned OpSubReg = Op.getSubReg();
4282 
4283  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
4284  RI.getRegClassForReg(MRI, OpReg), OpSubReg);
4285 
4286  // Check if operand is already the correct register class.
4287  if (DstRC == OpRC)
4288  return;
4289 
4290  Register DstReg = MRI.createVirtualRegister(DstRC);
4291  MachineInstr *Copy =
4292  BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
4293 
4294  Op.setReg(DstReg);
4295  Op.setSubReg(0);
4296 
4297  MachineInstr *Def = MRI.getVRegDef(OpReg);
4298  if (!Def)
4299  return;
4300 
4301  // Try to eliminate the copy if it is copying an immediate value.
4302  if (Def->isMoveImmediate() && DstRC != &AMDGPU::VReg_1RegClass)
4303  FoldImmediate(*Copy, *Def, OpReg, &MRI);
4304 
4305  bool ImpDef = Def->isImplicitDef();
4306  while (!ImpDef && Def && Def->isCopy()) {
4307  if (Def->getOperand(1).getReg().isPhysical())
4308  break;
4309  Def = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
4310  ImpDef = Def && Def->isImplicitDef();
4311  }
4312  if (!RI.isSGPRClass(DstRC) && !Copy->readsRegister(AMDGPU::EXEC, &RI) &&
4313  !ImpDef)
4314  Copy->addOperand(MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
4315 }
4316 
4317 // Emit the actual waterfall loop, executing the wrapped instruction for each
4318 // unique value of \p Rsrc across all lanes. In the best case we execute 1
4319 // iteration, in the worst case we execute 64 (once per lane).
4320 static void
4322  MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
4323  const DebugLoc &DL, MachineOperand &Rsrc) {
4324  MachineFunction &MF = *OrigBB.getParent();
4325  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4326  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4327  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4328  unsigned SaveExecOpc =
4329  ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
4330  unsigned XorTermOpc =
4331  ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
4332  unsigned AndOpc =
4333  ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
4334  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4335 
4336  MachineBasicBlock::iterator I = LoopBB.begin();
4337 
4338  Register VRsrc = Rsrc.getReg();
4339  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
4340 
4341  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4342  Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4343  Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4344  Register AndCond = MRI.createVirtualRegister(BoolXExecRC);
4345  Register SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4346  Register SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4347  Register SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4348  Register SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4349  Register SRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
4350 
4351  // Beginning of the loop, read the next Rsrc variant.
4352  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
4353  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
4354  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
4355  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
4356  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
4357  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
4358  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
4359  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
4360 
4361  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
4362  .addReg(SRsrcSub0)
4363  .addImm(AMDGPU::sub0)
4364  .addReg(SRsrcSub1)
4365  .addImm(AMDGPU::sub1)
4366  .addReg(SRsrcSub2)
4367  .addImm(AMDGPU::sub2)
4368  .addReg(SRsrcSub3)
4369  .addImm(AMDGPU::sub3);
4370 
4371  // Update Rsrc operand to use the SGPR Rsrc.
4372  Rsrc.setReg(SRsrc);
4373  Rsrc.setIsKill(true);
4374 
4375  // Identify all lanes with identical Rsrc operands in their VGPRs.
4376  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
4377  .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
4378  .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
4379  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
4380  .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
4381  .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
4382  BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndCond)
4383  .addReg(CondReg0)
4384  .addReg(CondReg1);
4385 
4386  MRI.setSimpleHint(SaveExec, AndCond);
4387 
4388  // Update EXEC to matching lanes, saving original to SaveExec.
4389  BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
4390  .addReg(AndCond, RegState::Kill);
4391 
4392  // The original instruction is here; we insert the terminators after it.
4393  I = LoopBB.end();
4394 
4395  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4396  BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
4397  .addReg(Exec)
4398  .addReg(SaveExec);
4399  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
4400 }
4401 
4402 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4403 // with SGPRs by iterating over all unique values across all lanes.
4405  MachineOperand &Rsrc, MachineDominatorTree *MDT) {
4406  MachineBasicBlock &MBB = *MI.getParent();
4407  MachineFunction &MF = *MBB.getParent();
4408  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4409  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4412  const DebugLoc &DL = MI.getDebugLoc();
4413  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4414  unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4415  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4416 
4417  Register SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4418 
4419  // Save the EXEC mask
4420  BuildMI(MBB, I, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
4421 
4422  // Killed uses in the instruction we are waterfalling around will be
4423  // incorrect due to the added control-flow.
4424  for (auto &MO : MI.uses()) {
4425  if (MO.isReg() && MO.isUse()) {
4426  MRI.clearKillFlags(MO.getReg());
4427  }
4428  }
4429 
4430  // To insert the loop we need to split the block. Move everything after this
4431  // point to a new block, and insert a new empty block between the two.
4433  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
4434  MachineFunction::iterator MBBI(MBB);
4435  ++MBBI;
4436 
4437  MF.insert(MBBI, LoopBB);
4438  MF.insert(MBBI, RemainderBB);
4439 
4440  LoopBB->addSuccessor(LoopBB);
4441  LoopBB->addSuccessor(RemainderBB);
4442 
4443  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4445  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
4446  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
4447  LoopBB->splice(LoopBB->begin(), &MBB, J);
4448 
4449  MBB.addSuccessor(LoopBB);
4450 
4451  // Update dominators. We know that MBB immediately dominates LoopBB, that
4452  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4453  // dominates all of the successors transferred to it from MBB that MBB used
4454  // to dominate.
4455  if (MDT) {
4456  MDT->addNewBlock(LoopBB, &MBB);
4457  MDT->addNewBlock(RemainderBB, LoopBB);
4458  for (auto &Succ : RemainderBB->successors()) {
4459  if (MDT->dominates(&MBB, Succ)) {
4460  MDT->changeImmediateDominator(Succ, RemainderBB);
4461  }
4462  }
4463  }
4464 
4465  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
4466 
4467  // Restore the EXEC mask
4468  MachineBasicBlock::iterator First = RemainderBB->begin();
4469  BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
4470 }
4471 
4472 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4473 static std::tuple<unsigned, unsigned>
4475  MachineBasicBlock &MBB = *MI.getParent();
4476  MachineFunction &MF = *MBB.getParent();
4478 
4479  // Extract the ptr from the resource descriptor.
4480  unsigned RsrcPtr =
4481  TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
4482  AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
4483 
4484  // Create an empty resource descriptor
4485  Register Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4486  Register SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4487  Register SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4488  Register NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SGPR_128RegClass);
4489  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
4490 
4491  // Zero64 = 0
4492  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
4493  .addImm(0);
4494 
4495  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4496  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
4497  .addImm(RsrcDataFormat & 0xFFFFFFFF);
4498 
4499  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4500  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
4501  .addImm(RsrcDataFormat >> 32);
4502 
4503  // NewSRsrc = {Zero64, SRsrcFormat}
4504  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
4505  .addReg(Zero64)
4506  .addImm(AMDGPU::sub0_sub1)
4507  .addReg(SRsrcFormatLo)
4508  .addImm(AMDGPU::sub2)
4509  .addReg(SRsrcFormatHi)
4510  .addImm(AMDGPU::sub3);
4511 
4512  return std::make_tuple(RsrcPtr, NewSRsrc);
4513 }
4514 
4516  MachineDominatorTree *MDT) const {
4517  MachineFunction &MF = *MI.getParent()->getParent();
4519 
4520  // Legalize VOP2
4521  if (isVOP2(MI) || isVOPC(MI)) {
4522  legalizeOperandsVOP2(MRI, MI);
4523  return;
4524  }
4525 
4526  // Legalize VOP3
4527  if (isVOP3(MI)) {
4528  legalizeOperandsVOP3(MRI, MI);
4529  return;
4530  }
4531 
4532  // Legalize SMRD
4533  if (isSMRD(MI)) {
4534  legalizeOperandsSMRD(MRI, MI);
4535  return;
4536  }
4537 
4538  // Legalize REG_SEQUENCE and PHI
4539  // The register class of the operands much be the same type as the register
4540  // class of the output.
4541  if (MI.getOpcode() == AMDGPU::PHI) {
4542  const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
4543  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
4544  if (!MI.getOperand(i).isReg() ||
4546  continue;
4547  const TargetRegisterClass *OpRC =
4548  MRI.getRegClass(MI.getOperand(i).getReg());
4549  if (RI.hasVectorRegisters(OpRC)) {
4550  VRC = OpRC;
4551  } else {
4552  SRC = OpRC;
4553  }
4554  }
4555 
4556  // If any of the operands are VGPR registers, then they all most be
4557  // otherwise we will create illegal VGPR->SGPR copies when legalizing
4558  // them.
4559  if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
4560  if (!VRC) {
4561  assert(SRC);
4562  if (getOpRegClass(MI, 0) == &AMDGPU::VReg_1RegClass) {
4563  VRC = &AMDGPU::VReg_1RegClass;
4564  } else
4565  VRC = RI.hasAGPRs(getOpRegClass(MI, 0))
4566  ? RI.getEquivalentAGPRClass(SRC)
4567  : RI.getEquivalentVGPRClass(SRC);
4568  } else {
4569  VRC = RI.hasAGPRs(getOpRegClass(MI, 0))
4570  ? RI.getEquivalentAGPRClass(VRC)
4571  : RI.getEquivalentVGPRClass(VRC);
4572  }
4573  RC = VRC;
4574  } else {
4575  RC = SRC;
4576  }
4577 
4578  // Update all the operands so they have the same type.
4579  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4580  MachineOperand &Op = MI.getOperand(I);
4581  if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4582  continue;
4583 
4584  // MI is a PHI instruction.
4585  MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
4586  MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
4587 
4588  // Avoid creating no-op copies with the same src and dst reg class. These
4589  // confuse some of the machine passes.
4590  legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
4591  }
4592  }
4593 
4594  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4595  // VGPR dest type and SGPR sources, insert copies so all operands are
4596  // VGPRs. This seems to help operand folding / the register coalescer.
4597  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
4598  MachineBasicBlock *MBB = MI.getParent();
4599  const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
4600  if (RI.hasVGPRs(DstRC)) {
4601  // Update all the operands so they are VGPR register classes. These may
4602  // not be the same register class because REG_SEQUENCE supports mixing
4603  // subregister index types e.g. sub0_sub1 + sub2 + sub3
4604  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4605  MachineOperand &Op = MI.getOperand(I);
4606  if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg()))
4607  continue;
4608 
4609  const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
4610  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
4611  if (VRC == OpRC)
4612  continue;
4613 
4614  legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
4615  Op.setIsKill();
4616  }
4617  }
4618 
4619  return;
4620  }
4621 
4622  // Legalize INSERT_SUBREG
4623  // src0 must have the same register class as dst
4624  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
4625  Register Dst = MI.getOperand(0).getReg();
4626  Register Src0 = MI.getOperand(1).getReg();
4627  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
4628  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
4629  if (DstRC != Src0RC) {
4630  MachineBasicBlock *MBB = MI.getParent();
4631  MachineOperand &Op = MI.getOperand(1);
4632  legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
4633  }
4634  return;
4635  }
4636 
4637  // Legalize SI_INIT_M0
4638  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
4639  MachineOperand &Src = MI.getOperand(0);
4640  if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
4641  Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
4642  return;
4643  }
4644 
4645  // Legalize MIMG and MUBUF/MTBUF for shaders.
4646  //
4647  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4648  // scratch memory access. In both cases, the legalization never involves
4649  // conversion to the addr64 form.
4650  if (isMIMG(MI) ||
4652  (isMUBUF(MI) || isMTBUF(MI)))) {
4653  MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
4654  if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
4655  unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
4656  SRsrc->setReg(SGPR);
4657  }
4658 
4659  MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
4660  if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
4661  unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
4662  SSamp->setReg(SGPR);
4663  }
4664  return;
4665  }
4666 
4667  // Legalize MUBUF* instructions.
4668  int RsrcIdx =
4669  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
4670  if (RsrcIdx != -1) {
4671  // We have an MUBUF instruction
4672  MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
4673  unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
4674  if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
4675  RI.getRegClass(RsrcRC))) {
4676  // The operands are legal.
4677  // FIXME: We may need to legalize operands besided srsrc.
4678  return;
4679  }
4680 
4681  // Legalize a VGPR Rsrc.
4682  //
4683  // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4684  // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4685  // a zero-value SRsrc.
4686  //
4687  // If the instruction is _OFFSET (both idxen and offen disabled), and we
4688  // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4689  // above.
4690  //
4691  // Otherwise we are on non-ADDR64 hardware, and/or we have
4692  // idxen/offen/bothen and we fall back to a waterfall loop.
4693 
4694  MachineBasicBlock &MBB = *MI.getParent();
4695 
4696  MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4697  if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
4698  // This is already an ADDR64 instruction so we need to add the pointer
4699  // extracted from the resource descriptor to the current value of VAddr.
4700  Register NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4701  Register NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4702  Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4703 
4704  const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4705  Register CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4706  Register CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4707 
4708  unsigned RsrcPtr, NewSRsrc;
4709  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4710 
4711  // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4712  const DebugLoc &DL = MI.getDebugLoc();
4713  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e64), NewVAddrLo)
4714  .addDef(CondReg0)
4715  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4716  .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
4717  .addImm(0);
4718 
4719  // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4720  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
4721  .addDef(CondReg1, RegState::Dead)
4722  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4723  .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
4724  .addReg(CondReg0, RegState::Kill)
4725  .addImm(0);
4726 
4727  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4728  BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4729  .addReg(NewVAddrLo)
4730  .addImm(AMDGPU::sub0)
4731  .addReg(NewVAddrHi)
4732  .addImm(AMDGPU::sub1);
4733 
4734  VAddr->setReg(NewVAddr);
4735  Rsrc->setReg(NewSRsrc);
4736  } else if (!VAddr && ST.hasAddr64()) {
4737  // This instructions is the _OFFSET variant, so we need to convert it to
4738  // ADDR64.
4739  assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4741  "FIXME: Need to emit flat atomics here");
4742 
4743  unsigned RsrcPtr, NewSRsrc;
4744  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4745 
4746  Register NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4747  MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4748  MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4749  MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4750  unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4751 
4752  // Atomics rith return have have an additional tied operand and are
4753  // missing some of the special bits.
4754  MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4755  MachineInstr *Addr64;
4756 
4757  if (!VDataIn) {
4758  // Regular buffer load / store.
4759  MachineInstrBuilder MIB =
4760  BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4761  .add(*VData)
4762  .addReg(NewVAddr)
4763  .addReg(NewSRsrc)
4764  .add(*SOffset)
4765  .add(*Offset);
4766 
4767  // Atomics do not have this operand.
4768  if (const MachineOperand *GLC =
4769  getNamedOperand(MI, AMDGPU::OpName::glc)) {
4770  MIB.addImm(GLC->getImm());
4771  }
4772  if (const MachineOperand *DLC =
4773  getNamedOperand(MI, AMDGPU::OpName::dlc)) {
4774  MIB.addImm(DLC->getImm());
4775  }
4776 
4777  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4778 
4779  if (const MachineOperand *TFE =
4780  getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4781  MIB.addImm(TFE->getImm());
4782  }
4783 
4784  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));
4785 
4786  MIB.cloneMemRefs(MI);
4787  Addr64 = MIB;
4788  } else {
4789  // Atomics with return.
4790  Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4791  .add(*VData)
4792  .add(*VDataIn)
4793  .addReg(NewVAddr)
4794  .addReg(NewSRsrc)
4795  .add(*SOffset)
4796  .add(*Offset)
4797  .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4798  .cloneMemRefs(MI);
4799  }
4800 
4801  MI.removeFromParent();
4802 
4803  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4804  BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4805  NewVAddr)
4806  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4807  .addImm(AMDGPU::sub0)
4808  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4809  .addImm(AMDGPU::sub1);
4810  } else {
4811  // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4812  // to SGPRs.
4813  loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4814  }
4815  }
4816 }
4817 
4819  MachineDominatorTree *MDT) const {
4820  SetVectorType Worklist;
4821  Worklist.insert(&TopInst);
4822 
4823  while (!Worklist.empty()) {
4824  MachineInstr &Inst = *Worklist.pop_back_val();
4825  MachineBasicBlock *MBB = Inst.getParent();
4827 
4828  unsigned Opcode = Inst.getOpcode();
4829  unsigned NewOpcode = getVALUOp(Inst);
4830 
4831  // Handle some special cases
4832  switch (Opcode) {
4833  default:
4834  break;
4835  case AMDGPU::S_ADD_U64_PSEUDO:
4836  case AMDGPU::S_SUB_U64_PSEUDO:
4837  splitScalar64BitAddSub(Worklist, Inst, MDT);
4838  Inst.eraseFromParent();
4839  continue;
4840  case AMDGPU::S_ADD_I32:
4841  case AMDGPU::S_SUB_I32:
4842  // FIXME: The u32 versions currently selected use the carry.
4843  if (moveScalarAddSub(Worklist, Inst, MDT))
4844  continue;
4845 
4846  // Default handling
4847  break;
4848  case AMDGPU::S_AND_B64:
4849  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4850  Inst.eraseFromParent();
4851  continue;
4852 
4853  case AMDGPU::S_OR_B64:
4854  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4855  Inst.eraseFromParent();
4856  continue;
4857 
4858  case AMDGPU::S_XOR_B64:
4859  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4860  Inst.eraseFromParent();
4861  continue;
4862 
4863  case AMDGPU::S_NAND_B64:
4864  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4865  Inst.eraseFromParent();
4866  continue;
4867 
4868  case AMDGPU::S_NOR_B64:
4869  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4870  Inst.eraseFromParent();
4871  continue;
4872 
4873  case AMDGPU::S_XNOR_B64:
4874  if (ST.hasDLInsts())
4875  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4876  else
4877  splitScalar64BitXnor(Worklist, Inst, MDT);
4878  Inst.eraseFromParent();
4879  continue;
4880 
4881  case AMDGPU::S_ANDN2_B64:
4882  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4883  Inst.eraseFromParent();
4884  continue;
4885 
4886  case AMDGPU::S_ORN2_B64:
4887  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4888  Inst.eraseFromParent();
4889  continue;
4890 
4891  case AMDGPU::S_NOT_B64:
4892  splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4893  Inst.eraseFromParent();
4894  continue;
4895 
4896  case AMDGPU::S_BCNT1_I32_B64:
4897  splitScalar64BitBCNT(Worklist, Inst);
4898  Inst.eraseFromParent();
4899  continue;
4900 
4901  case AMDGPU::S_BFE_I64:
4902  splitScalar64BitBFE(Worklist, Inst);
4903  Inst.eraseFromParent();
4904  continue;
4905 
4906  case AMDGPU::S_LSHL_B32:
4907  if (ST.hasOnlyRevVALUShifts()) {
4908  NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4909  swapOperands(Inst);
4910  }
4911  break;
4912  case AMDGPU::S_ASHR_I32:
4913  if (ST.hasOnlyRevVALUShifts()) {
4914  NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4915  swapOperands(Inst);
4916  }
4917  break;
4918  case AMDGPU::S_LSHR_B32:
4919  if (ST.hasOnlyRevVALUShifts()) {
4920  NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4921  swapOperands(Inst);
4922  }
4923  break;
4924  case AMDGPU::S_LSHL_B64:
4925  if (ST.hasOnlyRevVALUShifts()) {
4926  NewOpcode = AMDGPU::V_LSHLREV_B64;
4927  swapOperands(Inst);
4928  }
4929  break;
4930  case AMDGPU::S_ASHR_I64:
4931  if (ST.hasOnlyRevVALUShifts()) {
4932  NewOpcode = AMDGPU::V_ASHRREV_I64;
4933  swapOperands(Inst);
4934  }
4935  break;
4936  case AMDGPU::S_LSHR_B64:
4937  if (ST.hasOnlyRevVALUShifts()) {
4938  NewOpcode = AMDGPU::V_LSHRREV_B64;
4939  swapOperands(Inst);
4940  }
4941  break;
4942 
4943  case AMDGPU::S_ABS_I32:
4944  lowerScalarAbs(Worklist, Inst);
4945  Inst.eraseFromParent();
4946  continue;
4947 
4948  case AMDGPU::S_CBRANCH_SCC0:
4949  case AMDGPU::S_CBRANCH_SCC1:
4950  // Clear unused bits of vcc
4951  if (ST.isWave32())
4952  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
4953  AMDGPU::VCC_LO)
4954  .addReg(AMDGPU::EXEC_LO)
4955  .addReg(AMDGPU::VCC_LO);
4956  else
4957  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4958  AMDGPU::VCC)
4959  .addReg(AMDGPU::EXEC)
4960  .addReg(AMDGPU::VCC);
4961  break;
4962 
4963  case AMDGPU::S_BFE_U64:
4964  case AMDGPU::S_BFM_B64:
4965  llvm_unreachable("Moving this op to VALU not implemented");
4966 
4967  case AMDGPU::S_PACK_LL_B32_B16:
4968  case AMDGPU::S_PACK_LH_B32_B16:
4969  case AMDGPU::S_PACK_HH_B32_B16:
4970  movePackToVALU(Worklist, MRI, Inst);
4971  Inst.eraseFromParent();
4972  continue;
4973 
4974  case AMDGPU::S_XNOR_B32:
4975  lowerScalarXnor(Worklist, Inst);
4976  Inst.eraseFromParent();
4977  continue;
4978 
4979  case AMDGPU::S_NAND_B32:
4980  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4981  Inst.eraseFromParent();
4982  continue;
4983 
4984  case AMDGPU::S_NOR_B32:
4985  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4986  Inst.eraseFromParent();
4987  continue;
4988 
4989  case AMDGPU::S_ANDN2_B32:
4990  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4991  Inst.eraseFromParent();
4992  continue;
4993 
4994  case AMDGPU::S_ORN2_B32:
4995  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
4996  Inst.eraseFromParent();
4997  continue;
4998  }
4999 
5000  if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
5001  // We cannot move this instruction to the VALU, so we should try to
5002  // legalize its operands instead.
5003  legalizeOperands(Inst, MDT);
5004  continue;
5005  }
5006 
5007  // Use the new VALU Opcode.
5008  const MCInstrDesc &NewDesc = get(NewOpcode);
5009  Inst.setDesc(NewDesc);
5010 
5011  // Remove any references to SCC. Vector instructions can't read from it, and
5012  // We're just about to add the implicit use / defs of VCC, and we don't want
5013  // both.
5014  for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
5015  MachineOperand &Op = Inst.getOperand(i);
5016  if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
5017  // Only propagate through live-def of SCC.
5018  if (Op.isDef() && !Op.isDead())
5019  addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
5020  Inst.RemoveOperand(i);
5021  }
5022  }
5023 
5024  if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
5025  // We are converting these to a BFE, so we need to add the missing
5026  // operands for the size and offset.
5027  unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
5030 
5031  } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
5032  // The VALU version adds the second operand to the result, so insert an
5033  // extra 0 operand.
5035  }
5036 
5038  fixImplicitOperands(Inst);
5039 
5040  if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
5041  const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
5042  // If we need to move this to VGPRs, we need to unpack the second operand
5043  // back into the 2 separate ones for bit offset and width.
5044  assert(OffsetWidthOp.isImm() &&
5045  "Scalar BFE is only implemented for constant width and offset");
5046  uint32_t Imm = OffsetWidthOp.getImm();
5047 
5048  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
5049  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
5050  Inst.RemoveOperand(2); // Remove old immediate.
5051  Inst.addOperand(MachineOperand::CreateImm(Offset));
5052  Inst.addOperand(MachineOperand::CreateImm(BitWidth));
5053  }
5054 
5055  bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
5056  unsigned NewDstReg = AMDGPU::NoRegister;
5057  if (HasDst) {
5058  Register DstReg = Inst.getOperand(0).getReg();
5059  if (Register::isPhysicalRegister(DstReg))
5060  continue;
5061 
5062  // Update the destination register class.
5063  const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
5064  if (!NewDstRC)
5065  continue;
5066 
5067  if (Inst.isCopy() &&
5069  NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
5070  // Instead of creating a copy where src and dst are the same register
5071  // class, we just replace all uses of dst with src. These kinds of
5072  // copies interfere with the heuristics MachineSink uses to decide
5073  // whether or not to split a critical edge. Since the pass assumes
5074  // that copies will end up as machine instructions and not be
5075  // eliminated.
5076  addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
5077  MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
5078  MRI.clearKillFlags(Inst.getOperand(1).getReg());
5079  Inst.getOperand(0).setReg(DstReg);
5080 
5081  // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
5082  // these are deleted later, but at -O0 it would leave a suspicious
5083  // looking illegal copy of an undef register.
5084  for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
5085  Inst.RemoveOperand(I);
5086  Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
5087  continue;
5088  }
5089 
5090  NewDstReg = MRI.createVirtualRegister(NewDstRC);
5091  MRI.replaceRegWith(DstReg, NewDstReg);
5092  }
5093 
5094  // Legalize the operands
5095  legalizeOperands(Inst, MDT);
5096 
5097  if (HasDst)
5098  addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
5099  }
5100 }
5101 
5102 // Add/sub require special handling to deal with carry outs.
5103 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
5104  MachineDominatorTree *MDT) const {
5105  if (ST.hasAddNoCarry()) {
5106  // Assume there is no user of scc since we don't select this in that case.
5107  // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
5108  // is used.
5109 
5110  MachineBasicBlock &MBB = *Inst.getParent();
5112 
5113  Register OldDstReg = Inst.getOperand(0).getReg();
5114  Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5115 
5116  unsigned Opc = Inst.getOpcode();
5117  assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
5118 
5119  unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
5120  AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
5121 
5122  assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
5123  Inst.RemoveOperand(3);
5124 
5125  Inst.setDesc(get(NewOpc));
5126  Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
5127  Inst.addImplicitDefUseOperands(*MBB.getParent());
5128  MRI.replaceRegWith(OldDstReg, ResultReg);
5129  legalizeOperands(Inst, MDT);
5130 
5131  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5132  return true;
5133  }
5134 
5135  return false;
5136 }
5137 
5138 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
5139  MachineInstr &Inst) const {
5140  MachineBasicBlock &MBB = *Inst.getParent();
5142  MachineBasicBlock::iterator MII = Inst;
5143  DebugLoc DL = Inst.getDebugLoc();
5144 
5145  MachineOperand &Dest = Inst.getOperand(0);
5146  MachineOperand &Src = Inst.getOperand(1);
5147  Register TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5148  Register ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5149 
5150  unsigned SubOp = ST.hasAddNoCarry() ?
5151  AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
5152 
5153  BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
5154  .addImm(0)
5155  .addReg(Src.getReg());
5156 
5157  BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
5158  .addReg(Src.getReg())
5159  .addReg(TmpReg);
5160 
5161  MRI.replaceRegWith(Dest.getReg(), ResultReg);
5162  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5163 }
5164 
5165 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
5166  MachineInstr &Inst) const {
5167  MachineBasicBlock &MBB = *Inst.getParent();
5169  MachineBasicBlock::iterator MII = Inst;
5170  const DebugLoc &DL = Inst.getDebugLoc();
5171 
5172  MachineOperand &Dest = Inst.getOperand(0);
5173  MachineOperand &Src0 = Inst.getOperand(1);
5174  MachineOperand &Src1 = Inst.getOperand(2);
5175 
5176  if (ST.hasDLInsts()) {
5177  Register NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5178  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
5179  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
5180 
5181  BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
5182  .add(Src0)
5183  .add(Src1);
5184 
5185  MRI.replaceRegWith(Dest.getReg(), NewDest);
5186  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5187  } else {
5188  // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
5189  // invert either source and then perform the XOR. If either source is a
5190  // scalar register, then we can leave the inversion on the scalar unit to
5191  // acheive a better distrubution of scalar and vector instructions.
5192  bool Src0IsSGPR = Src0.isReg() &&
5193  RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
5194  bool Src1IsSGPR = Src1.isReg() &&
5195  RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
5196  MachineInstr *Xor;
5197  Register Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5198  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5199 
5200  // Build a pair of scalar instructions and add them to the work list.
5201  // The next iteration over the work list will lower these to the vector
5202  // unit as necessary.
5203  if (Src0IsSGPR) {
5204  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
5205  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5206  .addReg(Temp)
5207  .add(Src1);
5208  } else if (Src1IsSGPR) {
5209  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
5210  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5211  .add(Src0)
5212  .addReg(Temp);
5213  } else {
5214  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
5215  .add(Src0)
5216  .add(Src1);
5217  MachineInstr *Not =
5218  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
5219  Worklist.insert(Not);
5220  }
5221 
5222  MRI.replaceRegWith(Dest.getReg(), NewDest);
5223 
5224  Worklist.insert(Xor);
5225 
5226  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5227  }
5228 }
5229 
5230 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist,
5231  MachineInstr &Inst,
5232  unsigned Opcode) const {
5233  MachineBasicBlock &MBB = *Inst.getParent();
5235  MachineBasicBlock::iterator MII = Inst;
5236  const DebugLoc &DL = Inst.getDebugLoc();
5237 
5238  MachineOperand &Dest = Inst.getOperand(0);
5239  MachineOperand &Src0 = Inst.getOperand(1);
5240  MachineOperand &Src1 = Inst.getOperand(2);
5241 
5242  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5243  Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
5244 
5245  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
5246  .add(Src0)
5247  .add(Src1);
5248 
5249  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
5250  .addReg(Interm);
5251 
5252  Worklist.insert(&Op);
5253  Worklist.insert(&Not);
5254 
5255  MRI.replaceRegWith(Dest.getReg(), NewDest);
5256  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5257 }
5258 
5259 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist,
5260  MachineInstr &Inst,
5261  unsigned Opcode) const {
5262  MachineBasicBlock &MBB = *Inst.getParent();
5264  MachineBasicBlock::iterator MII = Inst;
5265  const DebugLoc &DL = Inst.getDebugLoc();
5266 
5267  MachineOperand &Dest = Inst.getOperand(0);
5268  MachineOperand &Src0 = Inst.getOperand(1);
5269  MachineOperand &Src1 = Inst.getOperand(2);
5270 
5271  Register NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5272  Register Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5273 
5274  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
5275  .add(Src1);
5276 
5277  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
5278  .add(Src0)
5279  .addReg(Interm);
5280 
5281  Worklist.insert(&Not);
5282  Worklist.insert(&Op);
5283 
5284  MRI.replaceRegWith(Dest.getReg(), NewDest);
5285  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5286 }
5287 
5288 void SIInstrInfo::splitScalar64BitUnaryOp(
5289  SetVectorType &Worklist, MachineInstr &Inst,
5290  unsigned Opcode) const {
5291  MachineBasicBlock &MBB = *Inst.getParent();