LLVM  10.0.0svn
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// SI Implementation of TargetInstrInfo.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "SIInstrInfo.h"
15 #include "AMDGPU.h"
16 #include "AMDGPUSubtarget.h"
17 #include "GCNHazardRecognizer.h"
18 #include "SIDefines.h"
19 #include "SIMachineFunctionInfo.h"
20 #include "SIRegisterInfo.h"
22 #include "Utils/AMDGPUBaseInfo.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/SmallVector.h"
26 #include "llvm/ADT/StringRef.h"
46 #include "llvm/IR/DebugLoc.h"
47 #include "llvm/IR/DiagnosticInfo.h"
48 #include "llvm/IR/Function.h"
49 #include "llvm/IR/InlineAsm.h"
50 #include "llvm/IR/LLVMContext.h"
51 #include "llvm/MC/MCInstrDesc.h"
52 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/Compiler.h"
59 #include <cassert>
60 #include <cstdint>
61 #include <iterator>
62 #include <utility>
63 
64 using namespace llvm;
65 
66 #define GET_INSTRINFO_CTOR_DTOR
67 #include "AMDGPUGenInstrInfo.inc"
68 
69 namespace llvm {
70 namespace AMDGPU {
71 #define GET_D16ImageDimIntrinsics_IMPL
72 #define GET_ImageDimIntrinsicTable_IMPL
73 #define GET_RsrcIntrinsics_IMPL
74 #include "AMDGPUGenSearchableTables.inc"
75 }
76 }
77 
78 
79 // Must be at least 4 to be able to branch over minimum unconditional branch
80 // code. This is only for making it possible to write reasonably small tests for
81 // long branches.
82 static cl::opt<unsigned>
83 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
84  cl::desc("Restrict range of branch instructions (DEBUG)"));
85 
87  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
88  RI(ST), ST(ST) {}
89 
90 //===----------------------------------------------------------------------===//
91 // TargetInstrInfo callbacks
92 //===----------------------------------------------------------------------===//
93 
94 static unsigned getNumOperandsNoGlue(SDNode *Node) {
95  unsigned N = Node->getNumOperands();
96  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
97  --N;
98  return N;
99 }
100 
101 /// Returns true if both nodes have the same value for the given
102 /// operand \p Op, or if both nodes do not have this operand.
103 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
104  unsigned Opc0 = N0->getMachineOpcode();
105  unsigned Opc1 = N1->getMachineOpcode();
106 
107  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
108  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
109 
110  if (Op0Idx == -1 && Op1Idx == -1)
111  return true;
112 
113 
114  if ((Op0Idx == -1 && Op1Idx != -1) ||
115  (Op1Idx == -1 && Op0Idx != -1))
116  return false;
117 
118  // getNamedOperandIdx returns the index for the MachineInstr's operands,
119  // which includes the result as the first operand. We are indexing into the
120  // MachineSDNode's operands, so we need to skip the result operand to get
121  // the real index.
122  --Op0Idx;
123  --Op1Idx;
124 
125  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
126 }
127 
129  AliasAnalysis *AA) const {
130  // TODO: The generic check fails for VALU instructions that should be
131  // rematerializable due to implicit reads of exec. We really want all of the
132  // generic logic for this except for this.
133  switch (MI.getOpcode()) {
134  case AMDGPU::V_MOV_B32_e32:
135  case AMDGPU::V_MOV_B32_e64:
136  case AMDGPU::V_MOV_B64_PSEUDO:
137  // No implicit operands.
138  return MI.getNumOperands() == MI.getDesc().getNumOperands();
139  default:
140  return false;
141  }
142 }
143 
145  int64_t &Offset0,
146  int64_t &Offset1) const {
147  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
148  return false;
149 
150  unsigned Opc0 = Load0->getMachineOpcode();
151  unsigned Opc1 = Load1->getMachineOpcode();
152 
153  // Make sure both are actually loads.
154  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
155  return false;
156 
157  if (isDS(Opc0) && isDS(Opc1)) {
158 
159  // FIXME: Handle this case:
160  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
161  return false;
162 
163  // Check base reg.
164  if (Load0->getOperand(0) != Load1->getOperand(0))
165  return false;
166 
167  // Skip read2 / write2 variants for simplicity.
168  // TODO: We should report true if the used offsets are adjacent (excluded
169  // st64 versions).
170  int Offset0Idx = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
171  int Offset1Idx = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
172  if (Offset0Idx == -1 || Offset1Idx == -1)
173  return false;
174 
175  // XXX - be careful of datalesss loads
176  // getNamedOperandIdx returns the index for MachineInstrs. Since they
177  // include the output in the operand list, but SDNodes don't, we need to
178  // subtract the index by one.
179  Offset0Idx -= get(Opc0).NumDefs;
180  Offset1Idx -= get(Opc1).NumDefs;
181  Offset0 = cast<ConstantSDNode>(Load0->getOperand(Offset0Idx))->getZExtValue();
182  Offset1 = cast<ConstantSDNode>(Load1->getOperand(Offset1Idx))->getZExtValue();
183  return true;
184  }
185 
186  if (isSMRD(Opc0) && isSMRD(Opc1)) {
187  // Skip time and cache invalidation instructions.
188  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
189  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
190  return false;
191 
193 
194  // Check base reg.
195  if (Load0->getOperand(0) != Load1->getOperand(0))
196  return false;
197 
198  const ConstantSDNode *Load0Offset =
199  dyn_cast<ConstantSDNode>(Load0->getOperand(1));
200  const ConstantSDNode *Load1Offset =
201  dyn_cast<ConstantSDNode>(Load1->getOperand(1));
202 
203  if (!Load0Offset || !Load1Offset)
204  return false;
205 
206  Offset0 = Load0Offset->getZExtValue();
207  Offset1 = Load1Offset->getZExtValue();
208  return true;
209  }
210 
211  // MUBUF and MTBUF can access the same addresses.
212  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
213 
214  // MUBUF and MTBUF have vaddr at different indices.
215  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
216  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
217  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
218  return false;
219 
220  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
221  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
222 
223  if (OffIdx0 == -1 || OffIdx1 == -1)
224  return false;
225 
226  // getNamedOperandIdx returns the index for MachineInstrs. Since they
227  // include the output in the operand list, but SDNodes don't, we need to
228  // subtract the index by one.
229  OffIdx0 -= get(Opc0).NumDefs;
230  OffIdx1 -= get(Opc1).NumDefs;
231 
232  SDValue Off0 = Load0->getOperand(OffIdx0);
233  SDValue Off1 = Load1->getOperand(OffIdx1);
234 
235  // The offset might be a FrameIndexSDNode.
236  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
237  return false;
238 
239  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
240  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
241  return true;
242  }
243 
244  return false;
245 }
246 
247 static bool isStride64(unsigned Opc) {
248  switch (Opc) {
249  case AMDGPU::DS_READ2ST64_B32:
250  case AMDGPU::DS_READ2ST64_B64:
251  case AMDGPU::DS_WRITE2ST64_B32:
252  case AMDGPU::DS_WRITE2ST64_B64:
253  return true;
254  default:
255  return false;
256  }
257 }
258 
260  const MachineOperand *&BaseOp,
261  int64_t &Offset,
262  const TargetRegisterInfo *TRI) const {
263  unsigned Opc = LdSt.getOpcode();
264 
265  if (isDS(LdSt)) {
266  const MachineOperand *OffsetImm =
267  getNamedOperand(LdSt, AMDGPU::OpName::offset);
268  if (OffsetImm) {
269  // Normal, single offset LDS instruction.
270  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
271  // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to
272  // report that here?
273  if (!BaseOp)
274  return false;
275 
276  Offset = OffsetImm->getImm();
277  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
278  "operands of type register.");
279  return true;
280  }
281 
282  // The 2 offset instructions use offset0 and offset1 instead. We can treat
283  // these as a load with a single offset if the 2 offsets are consecutive. We
284  // will use this for some partially aligned loads.
285  const MachineOperand *Offset0Imm =
286  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
287  const MachineOperand *Offset1Imm =
288  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
289 
290  uint8_t Offset0 = Offset0Imm->getImm();
291  uint8_t Offset1 = Offset1Imm->getImm();
292 
293  if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
294  // Each of these offsets is in element sized units, so we need to convert
295  // to bytes of the individual reads.
296 
297  unsigned EltSize;
298  if (LdSt.mayLoad())
299  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
300  else {
301  assert(LdSt.mayStore());
302  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
303  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
304  }
305 
306  if (isStride64(Opc))
307  EltSize *= 64;
308 
309  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr);
310  Offset = EltSize * Offset0;
311  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
312  "operands of type register.");
313  return true;
314  }
315 
316  return false;
317  }
318 
319  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
320  const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
321  if (SOffset && SOffset->isReg())
322  return false;
323 
324  const MachineOperand *AddrReg = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
325  if (!AddrReg)
326  return false;
327 
328  const MachineOperand *OffsetImm =
329  getNamedOperand(LdSt, AMDGPU::OpName::offset);
330  BaseOp = AddrReg;
331  Offset = OffsetImm->getImm();
332 
333  if (SOffset) // soffset can be an inline immediate.
334  Offset += SOffset->getImm();
335 
336  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
337  "operands of type register.");
338  return true;
339  }
340 
341  if (isSMRD(LdSt)) {
342  const MachineOperand *OffsetImm =
343  getNamedOperand(LdSt, AMDGPU::OpName::offset);
344  if (!OffsetImm)
345  return false;
346 
347  const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase);
348  BaseOp = SBaseReg;
349  Offset = OffsetImm->getImm();
350  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
351  "operands of type register.");
352  return true;
353  }
354 
355  if (isFLAT(LdSt)) {
356  const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
357  if (VAddr) {
358  // Can't analyze 2 offsets.
359  if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
360  return false;
361 
362  BaseOp = VAddr;
363  } else {
364  // scratch instructions have either vaddr or saddr.
365  BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::saddr);
366  }
367 
368  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
369  assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base "
370  "operands of type register.");
371  return true;
372  }
373 
374  return false;
375 }
376 
377 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
378  const MachineOperand &BaseOp1,
379  const MachineInstr &MI2,
380  const MachineOperand &BaseOp2) {
381  // Support only base operands with base registers.
382  // Note: this could be extended to support FI operands.
383  if (!BaseOp1.isReg() || !BaseOp2.isReg())
384  return false;
385 
386  if (BaseOp1.isIdenticalTo(BaseOp2))
387  return true;
388 
389  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
390  return false;
391 
392  auto MO1 = *MI1.memoperands_begin();
393  auto MO2 = *MI2.memoperands_begin();
394  if (MO1->getAddrSpace() != MO2->getAddrSpace())
395  return false;
396 
397  auto Base1 = MO1->getValue();
398  auto Base2 = MO2->getValue();
399  if (!Base1 || !Base2)
400  return false;
401  const MachineFunction &MF = *MI1.getParent()->getParent();
402  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
403  Base1 = GetUnderlyingObject(Base1, DL);
404  Base2 = GetUnderlyingObject(Base1, DL);
405 
406  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
407  return false;
408 
409  return Base1 == Base2;
410 }
411 
413  const MachineOperand &BaseOp2,
414  unsigned NumLoads) const {
415  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
416  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
417 
418  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOp1, SecondLdSt, BaseOp2))
419  return false;
420 
421  const MachineOperand *FirstDst = nullptr;
422  const MachineOperand *SecondDst = nullptr;
423 
424  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
425  (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
426  (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
427  const unsigned MaxGlobalLoadCluster = 6;
428  if (NumLoads > MaxGlobalLoadCluster)
429  return false;
430 
431  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
432  if (!FirstDst)
433  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
434  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
435  if (!SecondDst)
436  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
437  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
438  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
439  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
440  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
441  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
442  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
443  }
444 
445  if (!FirstDst || !SecondDst)
446  return false;
447 
448  // Try to limit clustering based on the total number of bytes loaded
449  // rather than the number of instructions. This is done to help reduce
450  // register pressure. The method used is somewhat inexact, though,
451  // because it assumes that all loads in the cluster will load the
452  // same number of bytes as FirstLdSt.
453 
454  // The unit of this value is bytes.
455  // FIXME: This needs finer tuning.
456  unsigned LoadClusterThreshold = 16;
457 
458  const MachineRegisterInfo &MRI =
459  FirstLdSt.getParent()->getParent()->getRegInfo();
460 
461  const unsigned Reg = FirstDst->getReg();
462 
464  ? MRI.getRegClass(Reg)
465  : RI.getPhysRegClass(Reg);
466 
467  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
468 }
469 
470 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
471 // the first 16 loads will be interleaved with the stores, and the next 16 will
472 // be clustered as expected. It should really split into 2 16 store batches.
473 //
474 // Loads are clustered until this returns false, rather than trying to schedule
475 // groups of stores. This also means we have to deal with saying different
476 // address space loads should be clustered, and ones which might cause bank
477 // conflicts.
478 //
479 // This might be deprecated so it might not be worth that much effort to fix.
481  int64_t Offset0, int64_t Offset1,
482  unsigned NumLoads) const {
483  assert(Offset1 > Offset0 &&
484  "Second offset should be larger than first offset!");
485  // If we have less than 16 loads in a row, and the offsets are within 64
486  // bytes, then schedule together.
487 
488  // A cacheline is 64 bytes (for global memory).
489  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
490 }
491 
494  const DebugLoc &DL, unsigned DestReg,
495  unsigned SrcReg, bool KillSrc) {
496  MachineFunction *MF = MBB.getParent();
497  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
498  "illegal SGPR to VGPR copy",
499  DL, DS_Error);
500  LLVMContext &C = MF->getFunction().getContext();
501  C.diagnose(IllegalCopy);
502 
503  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
504  .addReg(SrcReg, getKillRegState(KillSrc));
505 }
506 
509  const DebugLoc &DL, unsigned DestReg,
510  unsigned SrcReg, bool KillSrc) const {
511  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
512 
513  if (RC == &AMDGPU::VGPR_32RegClass) {
514  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
515  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
516  AMDGPU::AGPR_32RegClass.contains(SrcReg));
517  unsigned Opc = AMDGPU::AGPR_32RegClass.contains(SrcReg) ?
518  AMDGPU::V_ACCVGPR_READ_B32 : AMDGPU::V_MOV_B32_e32;
519  BuildMI(MBB, MI, DL, get(Opc), DestReg)
520  .addReg(SrcReg, getKillRegState(KillSrc));
521  return;
522  }
523 
524  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
525  RC == &AMDGPU::SReg_32RegClass) {
526  if (SrcReg == AMDGPU::SCC) {
527  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
528  .addImm(-1)
529  .addImm(0);
530  return;
531  }
532 
533  if (DestReg == AMDGPU::VCC_LO) {
534  if (AMDGPU::SReg_32RegClass.contains(SrcReg)) {
535  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), AMDGPU::VCC_LO)
536  .addReg(SrcReg, getKillRegState(KillSrc));
537  } else {
538  // FIXME: Hack until VReg_1 removed.
539  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
540  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
541  .addImm(0)
542  .addReg(SrcReg, getKillRegState(KillSrc));
543  }
544 
545  return;
546  }
547 
548  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
549  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
550  return;
551  }
552 
553  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
554  .addReg(SrcReg, getKillRegState(KillSrc));
555  return;
556  }
557 
558  if (RC == &AMDGPU::SReg_64RegClass) {
559  if (DestReg == AMDGPU::VCC) {
560  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
561  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
562  .addReg(SrcReg, getKillRegState(KillSrc));
563  } else {
564  // FIXME: Hack until VReg_1 removed.
565  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
566  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
567  .addImm(0)
568  .addReg(SrcReg, getKillRegState(KillSrc));
569  }
570 
571  return;
572  }
573 
574  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
575  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
576  return;
577  }
578 
579  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
580  .addReg(SrcReg, getKillRegState(KillSrc));
581  return;
582  }
583 
584  if (DestReg == AMDGPU::SCC) {
585  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
586  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
587  .addReg(SrcReg, getKillRegState(KillSrc))
588  .addImm(0);
589  return;
590  }
591 
592  if (RC == &AMDGPU::AGPR_32RegClass) {
593  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
594  AMDGPU::SReg_32RegClass.contains(SrcReg) ||
595  AMDGPU::AGPR_32RegClass.contains(SrcReg));
596  if (!AMDGPU::VGPR_32RegClass.contains(SrcReg)) {
597  // First try to find defining accvgpr_write to avoid temporary registers.
598  for (auto Def = MI, E = MBB.begin(); Def != E; ) {
599  --Def;
600  if (!Def->definesRegister(SrcReg, &RI))
601  continue;
602  if (Def->getOpcode() != AMDGPU::V_ACCVGPR_WRITE_B32)
603  break;
604 
605  MachineOperand &DefOp = Def->getOperand(1);
606  assert(DefOp.isReg() || DefOp.isImm());
607 
608  if (DefOp.isReg()) {
609  // Check that register source operand if not clobbered before MI.
610  // Immediate operands are always safe to propagate.
611  bool SafeToPropagate = true;
612  for (auto I = Def; I != MI && SafeToPropagate; ++I)
613  if (I->modifiesRegister(DefOp.getReg(), &RI))
614  SafeToPropagate = false;
615 
616  if (!SafeToPropagate)
617  break;
618 
619  DefOp.setIsKill(false);
620  }
621 
622  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
623  .add(DefOp);
624  return;
625  }
626 
627  RegScavenger RS;
628  RS.enterBasicBlock(MBB);
629  RS.forward(MI);
630 
631  // Ideally we want to have three registers for a long reg_sequence copy
632  // to hide 2 waitstates between v_mov_b32 and accvgpr_write.
633  unsigned MaxVGPRs = RI.getRegPressureLimit(&AMDGPU::VGPR_32RegClass,
634  *MBB.getParent());
635 
636  // Registers in the sequence are allocated contiguously so we can just
637  // use register number to pick one of three round-robin temps.
638  unsigned RegNo = DestReg % 3;
639  unsigned Tmp = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
640  if (!Tmp)
641  report_fatal_error("Cannot scavenge VGPR to copy to AGPR");
642  RS.setRegUsed(Tmp);
643  // Only loop through if there are any free registers left, otherwise
644  // scavenger may report a fatal error without emergency spill slot
645  // or spill with the slot.
646  while (RegNo-- && RS.FindUnusedReg(&AMDGPU::VGPR_32RegClass)) {
647  unsigned Tmp2 = RS.scavengeRegister(&AMDGPU::VGPR_32RegClass, 0);
648  if (!Tmp2 || RI.getHWRegIndex(Tmp2) >= MaxVGPRs)
649  break;
650  Tmp = Tmp2;
651  RS.setRegUsed(Tmp);
652  }
653  copyPhysReg(MBB, MI, DL, Tmp, SrcReg, KillSrc);
654  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
655  .addReg(Tmp, RegState::Kill);
656  return;
657  }
658 
659  BuildMI(MBB, MI, DL, get(AMDGPU::V_ACCVGPR_WRITE_B32), DestReg)
660  .addReg(SrcReg, getKillRegState(KillSrc));
661  return;
662  }
663 
664  unsigned EltSize = 4;
665  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
666  if (RI.isSGPRClass(RC)) {
667  // TODO: Copy vec3/vec5 with s_mov_b64s then final s_mov_b32.
668  if (!(RI.getRegSizeInBits(*RC) % 64)) {
669  Opcode = AMDGPU::S_MOV_B64;
670  EltSize = 8;
671  } else {
672  Opcode = AMDGPU::S_MOV_B32;
673  EltSize = 4;
674  }
675 
676  if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
677  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
678  return;
679  }
680  } else if (RI.hasAGPRs(RC)) {
681  Opcode = RI.hasVGPRs(RI.getPhysRegClass(SrcReg)) ?
682  AMDGPU::V_ACCVGPR_WRITE_B32 : AMDGPU::COPY;
683  } else if (RI.hasVGPRs(RC) && RI.hasAGPRs(RI.getPhysRegClass(SrcReg))) {
684  Opcode = AMDGPU::V_ACCVGPR_READ_B32;
685  }
686 
687  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
688  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
689 
690  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
691  unsigned SubIdx;
692  if (Forward)
693  SubIdx = SubIndices[Idx];
694  else
695  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
696 
697  if (Opcode == TargetOpcode::COPY) {
698  copyPhysReg(MBB, MI, DL, RI.getSubReg(DestReg, SubIdx),
699  RI.getSubReg(SrcReg, SubIdx), KillSrc);
700  continue;
701  }
702 
703  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
704  get(Opcode), RI.getSubReg(DestReg, SubIdx));
705 
706  Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
707 
708  if (Idx == 0)
709  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
710 
711  bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
712  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
713  }
714 }
715 
716 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
717  int NewOpc;
718 
719  // Try to map original to commuted opcode
720  NewOpc = AMDGPU::getCommuteRev(Opcode);
721  if (NewOpc != -1)
722  // Check if the commuted (REV) opcode exists on the target.
723  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
724 
725  // Try to map commuted to original opcode
726  NewOpc = AMDGPU::getCommuteOrig(Opcode);
727  if (NewOpc != -1)
728  // Check if the original (non-REV) opcode exists on the target.
729  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
730 
731  return Opcode;
732 }
733 
736  const DebugLoc &DL, unsigned DestReg,
737  int64_t Value) const {
739  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
740  if (RegClass == &AMDGPU::SReg_32RegClass ||
741  RegClass == &AMDGPU::SGPR_32RegClass ||
742  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
743  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
744  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
745  .addImm(Value);
746  return;
747  }
748 
749  if (RegClass == &AMDGPU::SReg_64RegClass ||
750  RegClass == &AMDGPU::SGPR_64RegClass ||
751  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
752  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
753  .addImm(Value);
754  return;
755  }
756 
757  if (RegClass == &AMDGPU::VGPR_32RegClass) {
758  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
759  .addImm(Value);
760  return;
761  }
762  if (RegClass == &AMDGPU::VReg_64RegClass) {
763  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
764  .addImm(Value);
765  return;
766  }
767 
768  unsigned EltSize = 4;
769  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
770  if (RI.isSGPRClass(RegClass)) {
771  if (RI.getRegSizeInBits(*RegClass) > 32) {
772  Opcode = AMDGPU::S_MOV_B64;
773  EltSize = 8;
774  } else {
775  Opcode = AMDGPU::S_MOV_B32;
776  EltSize = 4;
777  }
778  }
779 
780  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
781  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
782  int64_t IdxValue = Idx == 0 ? Value : 0;
783 
784  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
785  get(Opcode), RI.getSubReg(DestReg, Idx));
786  Builder.addImm(IdxValue);
787  }
788 }
789 
790 const TargetRegisterClass *
792  return &AMDGPU::VGPR_32RegClass;
793 }
794 
797  const DebugLoc &DL, unsigned DstReg,
799  unsigned TrueReg,
800  unsigned FalseReg) const {
802  MachineFunction *MF = MBB.getParent();
803  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
804  const TargetRegisterClass *BoolXExecRC =
805  RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
806  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
807  "Not a VGPR32 reg");
808 
809  if (Cond.size() == 1) {
810  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
811  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
812  .add(Cond[0]);
813  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
814  .addImm(0)
815  .addReg(FalseReg)
816  .addImm(0)
817  .addReg(TrueReg)
818  .addReg(SReg);
819  } else if (Cond.size() == 2) {
820  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
821  switch (Cond[0].getImm()) {
822  case SIInstrInfo::SCC_TRUE: {
823  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
824  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
825  : AMDGPU::S_CSELECT_B64), SReg)
826  .addImm(-1)
827  .addImm(0);
828  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
829  .addImm(0)
830  .addReg(FalseReg)
831  .addImm(0)
832  .addReg(TrueReg)
833  .addReg(SReg);
834  break;
835  }
836  case SIInstrInfo::SCC_FALSE: {
837  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
838  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
839  : AMDGPU::S_CSELECT_B64), SReg)
840  .addImm(0)
841  .addImm(-1);
842  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
843  .addImm(0)
844  .addReg(FalseReg)
845  .addImm(0)
846  .addReg(TrueReg)
847  .addReg(SReg);
848  break;
849  }
850  case SIInstrInfo::VCCNZ: {
851  MachineOperand RegOp = Cond[1];
852  RegOp.setImplicit(false);
853  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
854  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
855  .add(RegOp);
856  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
857  .addImm(0)
858  .addReg(FalseReg)
859  .addImm(0)
860  .addReg(TrueReg)
861  .addReg(SReg);
862  break;
863  }
864  case SIInstrInfo::VCCZ: {
865  MachineOperand RegOp = Cond[1];
866  RegOp.setImplicit(false);
867  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
868  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
869  .add(RegOp);
870  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
871  .addImm(0)
872  .addReg(TrueReg)
873  .addImm(0)
874  .addReg(FalseReg)
875  .addReg(SReg);
876  break;
877  }
878  case SIInstrInfo::EXECNZ: {
879  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
880  unsigned SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
881  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
882  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
883  .addImm(0);
884  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
885  : AMDGPU::S_CSELECT_B64), SReg)
886  .addImm(-1)
887  .addImm(0);
888  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
889  .addImm(0)
890  .addReg(FalseReg)
891  .addImm(0)
892  .addReg(TrueReg)
893  .addReg(SReg);
894  break;
895  }
896  case SIInstrInfo::EXECZ: {
897  unsigned SReg = MRI.createVirtualRegister(BoolXExecRC);
898  unsigned SReg2 = MRI.createVirtualRegister(RI.getBoolRC());
899  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
900  : AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
901  .addImm(0);
902  BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32
903  : AMDGPU::S_CSELECT_B64), SReg)
904  .addImm(0)
905  .addImm(-1);
906  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
907  .addImm(0)
908  .addReg(FalseReg)
909  .addImm(0)
910  .addReg(TrueReg)
911  .addReg(SReg);
912  llvm_unreachable("Unhandled branch predicate EXECZ");
913  break;
914  }
915  default:
916  llvm_unreachable("invalid branch predicate");
917  }
918  } else {
919  llvm_unreachable("Can only handle Cond size 1 or 2");
920  }
921 }
922 
925  const DebugLoc &DL,
926  unsigned SrcReg, int Value) const {
928  unsigned Reg = MRI.createVirtualRegister(RI.getBoolRC());
929  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
930  .addImm(Value)
931  .addReg(SrcReg);
932 
933  return Reg;
934 }
935 
938  const DebugLoc &DL,
939  unsigned SrcReg, int Value) const {
941  unsigned Reg = MRI.createVirtualRegister(RI.getBoolRC());
942  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
943  .addImm(Value)
944  .addReg(SrcReg);
945 
946  return Reg;
947 }
948 
949 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
950 
951  if (RI.hasAGPRs(DstRC))
952  return AMDGPU::COPY;
953  if (RI.getRegSizeInBits(*DstRC) == 32) {
954  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
955  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
956  return AMDGPU::S_MOV_B64;
957  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
958  return AMDGPU::V_MOV_B64_PSEUDO;
959  }
960  return AMDGPU::COPY;
961 }
962 
963 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
964  switch (Size) {
965  case 4:
966  return AMDGPU::SI_SPILL_S32_SAVE;
967  case 8:
968  return AMDGPU::SI_SPILL_S64_SAVE;
969  case 12:
970  return AMDGPU::SI_SPILL_S96_SAVE;
971  case 16:
972  return AMDGPU::SI_SPILL_S128_SAVE;
973  case 20:
974  return AMDGPU::SI_SPILL_S160_SAVE;
975  case 32:
976  return AMDGPU::SI_SPILL_S256_SAVE;
977  case 64:
978  return AMDGPU::SI_SPILL_S512_SAVE;
979  case 128:
980  return AMDGPU::SI_SPILL_S1024_SAVE;
981  default:
982  llvm_unreachable("unknown register size");
983  }
984 }
985 
986 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
987  switch (Size) {
988  case 4:
989  return AMDGPU::SI_SPILL_V32_SAVE;
990  case 8:
991  return AMDGPU::SI_SPILL_V64_SAVE;
992  case 12:
993  return AMDGPU::SI_SPILL_V96_SAVE;
994  case 16:
995  return AMDGPU::SI_SPILL_V128_SAVE;
996  case 20:
997  return AMDGPU::SI_SPILL_V160_SAVE;
998  case 32:
999  return AMDGPU::SI_SPILL_V256_SAVE;
1000  case 64:
1001  return AMDGPU::SI_SPILL_V512_SAVE;
1002  case 128:
1003  return AMDGPU::SI_SPILL_V1024_SAVE;
1004  default:
1005  llvm_unreachable("unknown register size");
1006  }
1007 }
1008 
1009 static unsigned getAGPRSpillSaveOpcode(unsigned Size) {
1010  switch (Size) {
1011  case 4:
1012  return AMDGPU::SI_SPILL_A32_SAVE;
1013  case 8:
1014  return AMDGPU::SI_SPILL_A64_SAVE;
1015  case 16:
1016  return AMDGPU::SI_SPILL_A128_SAVE;
1017  case 64:
1018  return AMDGPU::SI_SPILL_A512_SAVE;
1019  case 128:
1020  return AMDGPU::SI_SPILL_A1024_SAVE;
1021  default:
1022  llvm_unreachable("unknown register size");
1023  }
1024 }
1025 
1028  unsigned SrcReg, bool isKill,
1029  int FrameIndex,
1030  const TargetRegisterClass *RC,
1031  const TargetRegisterInfo *TRI) const {
1032  MachineFunction *MF = MBB.getParent();
1034  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1035  const DebugLoc &DL = MBB.findDebugLoc(MI);
1036 
1037  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1038  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1039  MachinePointerInfo PtrInfo
1040  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1041  MachineMemOperand *MMO
1043  Size, Align);
1044  unsigned SpillSize = TRI->getSpillSize(*RC);
1045 
1046  if (RI.isSGPRClass(RC)) {
1047  MFI->setHasSpilledSGPRs();
1048 
1049  // We are only allowed to create one new instruction when spilling
1050  // registers, so we need to use pseudo instruction for spilling SGPRs.
1051  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
1052 
1053  // The SGPR spill/restore instructions only work on number sgprs, so we need
1054  // to make sure we are using the correct register class.
1055  if (TargetRegisterInfo::isVirtualRegister(SrcReg) && SpillSize == 4) {
1057  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
1058  }
1059 
1060  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
1061  .addReg(SrcReg, getKillRegState(isKill)) // data
1062  .addFrameIndex(FrameIndex) // addr
1063  .addMemOperand(MMO)
1065  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1066  // Add the scratch resource registers as implicit uses because we may end up
1067  // needing them, and need to ensure that the reserved registers are
1068  // correctly handled.
1069  if (RI.spillSGPRToVGPR())
1070  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1071  if (ST.hasScalarStores()) {
1072  // m0 is used for offset to scalar stores if used to spill.
1073  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1074  }
1075 
1076  return;
1077  }
1078 
1079  unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillSaveOpcode(SpillSize)
1080  : getVGPRSpillSaveOpcode(SpillSize);
1081  MFI->setHasSpilledVGPRs();
1082 
1083  auto MIB = BuildMI(MBB, MI, DL, get(Opcode));
1084  if (RI.hasAGPRs(RC)) {
1086  unsigned Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1087  MIB.addReg(Tmp, RegState::Define);
1088  }
1089  MIB.addReg(SrcReg, getKillRegState(isKill)) // data
1090  .addFrameIndex(FrameIndex) // addr
1091  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1092  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1093  .addImm(0) // offset
1094  .addMemOperand(MMO);
1095 }
1096 
1097 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
1098  switch (Size) {
1099  case 4:
1100  return AMDGPU::SI_SPILL_S32_RESTORE;
1101  case 8:
1102  return AMDGPU::SI_SPILL_S64_RESTORE;
1103  case 12:
1104  return AMDGPU::SI_SPILL_S96_RESTORE;
1105  case 16:
1106  return AMDGPU::SI_SPILL_S128_RESTORE;
1107  case 20:
1108  return AMDGPU::SI_SPILL_S160_RESTORE;
1109  case 32:
1110  return AMDGPU::SI_SPILL_S256_RESTORE;
1111  case 64:
1112  return AMDGPU::SI_SPILL_S512_RESTORE;
1113  case 128:
1114  return AMDGPU::SI_SPILL_S1024_RESTORE;
1115  default:
1116  llvm_unreachable("unknown register size");
1117  }
1118 }
1119 
1120 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
1121  switch (Size) {
1122  case 4:
1123  return AMDGPU::SI_SPILL_V32_RESTORE;
1124  case 8:
1125  return AMDGPU::SI_SPILL_V64_RESTORE;
1126  case 12:
1127  return AMDGPU::SI_SPILL_V96_RESTORE;
1128  case 16:
1129  return AMDGPU::SI_SPILL_V128_RESTORE;
1130  case 20:
1131  return AMDGPU::SI_SPILL_V160_RESTORE;
1132  case 32:
1133  return AMDGPU::SI_SPILL_V256_RESTORE;
1134  case 64:
1135  return AMDGPU::SI_SPILL_V512_RESTORE;
1136  case 128:
1137  return AMDGPU::SI_SPILL_V1024_RESTORE;
1138  default:
1139  llvm_unreachable("unknown register size");
1140  }
1141 }
1142 
1143 static unsigned getAGPRSpillRestoreOpcode(unsigned Size) {
1144  switch (Size) {
1145  case 4:
1146  return AMDGPU::SI_SPILL_A32_RESTORE;
1147  case 8:
1148  return AMDGPU::SI_SPILL_A64_RESTORE;
1149  case 16:
1150  return AMDGPU::SI_SPILL_A128_RESTORE;
1151  case 64:
1152  return AMDGPU::SI_SPILL_A512_RESTORE;
1153  case 128:
1154  return AMDGPU::SI_SPILL_A1024_RESTORE;
1155  default:
1156  llvm_unreachable("unknown register size");
1157  }
1158 }
1159 
1162  unsigned DestReg, int FrameIndex,
1163  const TargetRegisterClass *RC,
1164  const TargetRegisterInfo *TRI) const {
1165  MachineFunction *MF = MBB.getParent();
1167  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
1168  const DebugLoc &DL = MBB.findDebugLoc(MI);
1169  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
1170  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
1171  unsigned SpillSize = TRI->getSpillSize(*RC);
1172 
1173  MachinePointerInfo PtrInfo
1174  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
1175 
1177  PtrInfo, MachineMemOperand::MOLoad, Size, Align);
1178 
1179  if (RI.isSGPRClass(RC)) {
1180  MFI->setHasSpilledSGPRs();
1181 
1182  // FIXME: Maybe this should not include a memoperand because it will be
1183  // lowered to non-memory instructions.
1184  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
1185  if (TargetRegisterInfo::isVirtualRegister(DestReg) && SpillSize == 4) {
1187  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
1188  }
1189 
1190  if (RI.spillSGPRToVGPR())
1191  FrameInfo.setStackID(FrameIndex, TargetStackID::SGPRSpill);
1192  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
1193  .addFrameIndex(FrameIndex) // addr
1194  .addMemOperand(MMO)
1196  .addReg(MFI->getStackPtrOffsetReg(), RegState::Implicit);
1197 
1198  if (ST.hasScalarStores()) {
1199  // m0 is used for offset to scalar stores if used to spill.
1200  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
1201  }
1202 
1203  return;
1204  }
1205 
1206  unsigned Opcode = RI.hasAGPRs(RC) ? getAGPRSpillRestoreOpcode(SpillSize)
1207  : getVGPRSpillRestoreOpcode(SpillSize);
1208  auto MIB = BuildMI(MBB, MI, DL, get(Opcode), DestReg);
1209  if (RI.hasAGPRs(RC)) {
1211  unsigned Tmp = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
1212  MIB.addReg(Tmp, RegState::Define);
1213  }
1214  MIB.addFrameIndex(FrameIndex) // vaddr
1215  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1216  .addReg(MFI->getStackPtrOffsetReg()) // scratch_offset
1217  .addImm(0) // offset
1218  .addMemOperand(MMO);
1219 }
1220 
1221 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1223  MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1224  unsigned FrameOffset, unsigned Size) const {
1225  MachineFunction *MF = MBB.getParent();
1227  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1228  const DebugLoc &DL = MBB.findDebugLoc(MI);
1229  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1230  unsigned WavefrontSize = ST.getWavefrontSize();
1231 
1232  unsigned TIDReg = MFI->getTIDReg();
1233  if (!MFI->hasCalculatedTID()) {
1234  MachineBasicBlock &Entry = MBB.getParent()->front();
1235  MachineBasicBlock::iterator Insert = Entry.front();
1236  const DebugLoc &DL = Insert->getDebugLoc();
1237 
1238  TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1239  *MF);
1240  if (TIDReg == AMDGPU::NoRegister)
1241  return TIDReg;
1242 
1244  WorkGroupSize > WavefrontSize) {
1245  unsigned TIDIGXReg
1247  unsigned TIDIGYReg
1249  unsigned TIDIGZReg
1251  unsigned InputPtrReg =
1253  for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1254  if (!Entry.isLiveIn(Reg))
1255  Entry.addLiveIn(Reg);
1256  }
1257 
1258  RS->enterBasicBlock(Entry);
1259  // FIXME: Can we scavenge an SReg_64 and access the subregs?
1260  unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1261  unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1262  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1263  .addReg(InputPtrReg)
1265  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1266  .addReg(InputPtrReg)
1268 
1269  // NGROUPS.X * NGROUPS.Y
1270  BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1271  .addReg(STmp1)
1272  .addReg(STmp0);
1273  // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1274  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1275  .addReg(STmp1)
1276  .addReg(TIDIGXReg);
1277  // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1278  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1279  .addReg(STmp0)
1280  .addReg(TIDIGYReg)
1281  .addReg(TIDReg);
1282  // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1283  getAddNoCarry(Entry, Insert, DL, TIDReg)
1284  .addReg(TIDReg)
1285  .addReg(TIDIGZReg)
1286  .addImm(0); // clamp bit
1287  } else {
1288  // Get the wave id
1289  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1290  TIDReg)
1291  .addImm(-1)
1292  .addImm(0);
1293 
1294  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1295  TIDReg)
1296  .addImm(-1)
1297  .addReg(TIDReg);
1298  }
1299 
1300  BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1301  TIDReg)
1302  .addImm(2)
1303  .addReg(TIDReg);
1304  MFI->setTIDReg(TIDReg);
1305  }
1306 
1307  // Add FrameIndex to LDS offset
1308  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1309  getAddNoCarry(MBB, MI, DL, TmpReg)
1310  .addImm(LDSOffset)
1311  .addReg(TIDReg)
1312  .addImm(0); // clamp bit
1313 
1314  return TmpReg;
1315 }
1316 
1319  int Count) const {
1320  DebugLoc DL = MBB.findDebugLoc(MI);
1321  while (Count > 0) {
1322  int Arg;
1323  if (Count >= 8)
1324  Arg = 7;
1325  else
1326  Arg = Count - 1;
1327  Count -= 8;
1328  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1329  .addImm(Arg);
1330  }
1331 }
1332 
1335  insertWaitStates(MBB, MI, 1);
1336 }
1337 
1339  auto MF = MBB.getParent();
1341 
1342  assert(Info->isEntryFunction());
1343 
1344  if (MBB.succ_empty()) {
1345  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1346  if (HasNoTerminator) {
1347  if (Info->returnsVoid()) {
1348  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::S_ENDPGM)).addImm(0);
1349  } else {
1350  BuildMI(MBB, MBB.end(), DebugLoc(), get(AMDGPU::SI_RETURN_TO_EPILOG));
1351  }
1352  }
1353  }
1354 }
1355 
1357  switch (MI.getOpcode()) {
1358  default: return 1; // FIXME: Do wait states equal cycles?
1359 
1360  case AMDGPU::S_NOP:
1361  return MI.getOperand(0).getImm() + 1;
1362  }
1363 }
1364 
1366  MachineBasicBlock &MBB = *MI.getParent();
1367  DebugLoc DL = MBB.findDebugLoc(MI);
1368  switch (MI.getOpcode()) {
1369  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1370  case AMDGPU::S_MOV_B64_term:
1371  // This is only a terminator to get the correct spill code placement during
1372  // register allocation.
1373  MI.setDesc(get(AMDGPU::S_MOV_B64));
1374  break;
1375 
1376  case AMDGPU::S_MOV_B32_term:
1377  // This is only a terminator to get the correct spill code placement during
1378  // register allocation.
1379  MI.setDesc(get(AMDGPU::S_MOV_B32));
1380  break;
1381 
1382  case AMDGPU::S_XOR_B64_term:
1383  // This is only a terminator to get the correct spill code placement during
1384  // register allocation.
1385  MI.setDesc(get(AMDGPU::S_XOR_B64));
1386  break;
1387 
1388  case AMDGPU::S_XOR_B32_term:
1389  // This is only a terminator to get the correct spill code placement during
1390  // register allocation.
1391  MI.setDesc(get(AMDGPU::S_XOR_B32));
1392  break;
1393 
1394  case AMDGPU::S_OR_B32_term:
1395  // This is only a terminator to get the correct spill code placement during
1396  // register allocation.
1397  MI.setDesc(get(AMDGPU::S_OR_B32));
1398  break;
1399 
1400  case AMDGPU::S_ANDN2_B64_term:
1401  // This is only a terminator to get the correct spill code placement during
1402  // register allocation.
1403  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1404  break;
1405 
1406  case AMDGPU::S_ANDN2_B32_term:
1407  // This is only a terminator to get the correct spill code placement during
1408  // register allocation.
1409  MI.setDesc(get(AMDGPU::S_ANDN2_B32));
1410  break;
1411 
1412  case AMDGPU::V_MOV_B64_PSEUDO: {
1413  unsigned Dst = MI.getOperand(0).getReg();
1414  unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1415  unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1416 
1417  const MachineOperand &SrcOp = MI.getOperand(1);
1418  // FIXME: Will this work for 64-bit floating point immediates?
1419  assert(!SrcOp.isFPImm());
1420  if (SrcOp.isImm()) {
1421  APInt Imm(64, SrcOp.getImm());
1422  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1423  .addImm(Imm.getLoBits(32).getZExtValue())
1424  .addReg(Dst, RegState::Implicit | RegState::Define);
1425  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1426  .addImm(Imm.getHiBits(32).getZExtValue())
1427  .addReg(Dst, RegState::Implicit | RegState::Define);
1428  } else {
1429  assert(SrcOp.isReg());
1430  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1431  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1433  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1434  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1436  }
1437  MI.eraseFromParent();
1438  break;
1439  }
1440  case AMDGPU::V_SET_INACTIVE_B32: {
1441  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1442  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1443  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1444  .addReg(Exec);
1445  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1446  .add(MI.getOperand(2));
1447  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1448  .addReg(Exec);
1449  MI.eraseFromParent();
1450  break;
1451  }
1452  case AMDGPU::V_SET_INACTIVE_B64: {
1453  unsigned NotOpc = ST.isWave32() ? AMDGPU::S_NOT_B32 : AMDGPU::S_NOT_B64;
1454  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
1455  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1456  .addReg(Exec);
1457  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1458  MI.getOperand(0).getReg())
1459  .add(MI.getOperand(2));
1460  expandPostRAPseudo(*Copy);
1461  BuildMI(MBB, MI, DL, get(NotOpc), Exec)
1462  .addReg(Exec);
1463  MI.eraseFromParent();
1464  break;
1465  }
1466  case AMDGPU::V_MOVRELD_B32_V1:
1467  case AMDGPU::V_MOVRELD_B32_V2:
1468  case AMDGPU::V_MOVRELD_B32_V4:
1469  case AMDGPU::V_MOVRELD_B32_V8:
1470  case AMDGPU::V_MOVRELD_B32_V16: {
1471  const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1472  unsigned VecReg = MI.getOperand(0).getReg();
1473  bool IsUndef = MI.getOperand(1).isUndef();
1474  unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1475  assert(VecReg == MI.getOperand(1).getReg());
1476 
1477  MachineInstr *MovRel =
1478  BuildMI(MBB, MI, DL, MovRelDesc)
1479  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1480  .add(MI.getOperand(2))
1481  .addReg(VecReg, RegState::ImplicitDefine)
1482  .addReg(VecReg,
1483  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1484 
1485  const int ImpDefIdx =
1486  MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1487  const int ImpUseIdx = ImpDefIdx + 1;
1488  MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1489 
1490  MI.eraseFromParent();
1491  break;
1492  }
1493  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1494  MachineFunction &MF = *MBB.getParent();
1495  unsigned Reg = MI.getOperand(0).getReg();
1496  unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1497  unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1498 
1499  // Create a bundle so these instructions won't be re-ordered by the
1500  // post-RA scheduler.
1501  MIBundleBuilder Bundler(MBB, MI);
1502  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1503 
1504  // Add 32-bit offset from this instruction to the start of the
1505  // constant data.
1506  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1507  .addReg(RegLo)
1508  .add(MI.getOperand(1)));
1509 
1510  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1511  .addReg(RegHi);
1512  MIB.add(MI.getOperand(2));
1513 
1514  Bundler.append(MIB);
1515  finalizeBundle(MBB, Bundler.begin());
1516 
1517  MI.eraseFromParent();
1518  break;
1519  }
1520  case AMDGPU::ENTER_WWM: {
1521  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1522  // WWM is entered.
1523  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
1524  : AMDGPU::S_OR_SAVEEXEC_B64));
1525  break;
1526  }
1527  case AMDGPU::EXIT_WWM: {
1528  // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
1529  // WWM is exited.
1530  MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
1531  break;
1532  }
1533  case TargetOpcode::BUNDLE: {
1534  if (!MI.mayLoad() || MI.hasUnmodeledSideEffects())
1535  return false;
1536 
1537  // If it is a load it must be a memory clause
1539  I->isBundledWithSucc(); ++I) {
1540  I->unbundleFromSucc();
1541  for (MachineOperand &MO : I->operands())
1542  if (MO.isReg())
1543  MO.setIsInternalRead(false);
1544  }
1545 
1546  MI.eraseFromParent();
1547  break;
1548  }
1549  }
1550  return true;
1551 }
1552 
1554  MachineOperand &Src0,
1555  unsigned Src0OpName,
1556  MachineOperand &Src1,
1557  unsigned Src1OpName) const {
1558  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1559  if (!Src0Mods)
1560  return false;
1561 
1562  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1563  assert(Src1Mods &&
1564  "All commutable instructions have both src0 and src1 modifiers");
1565 
1566  int Src0ModsVal = Src0Mods->getImm();
1567  int Src1ModsVal = Src1Mods->getImm();
1568 
1569  Src1Mods->setImm(Src0ModsVal);
1570  Src0Mods->setImm(Src1ModsVal);
1571  return true;
1572 }
1573 
1575  MachineOperand &RegOp,
1576  MachineOperand &NonRegOp) {
1577  unsigned Reg = RegOp.getReg();
1578  unsigned SubReg = RegOp.getSubReg();
1579  bool IsKill = RegOp.isKill();
1580  bool IsDead = RegOp.isDead();
1581  bool IsUndef = RegOp.isUndef();
1582  bool IsDebug = RegOp.isDebug();
1583 
1584  if (NonRegOp.isImm())
1585  RegOp.ChangeToImmediate(NonRegOp.getImm());
1586  else if (NonRegOp.isFI())
1587  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1588  else
1589  return nullptr;
1590 
1591  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1592  NonRegOp.setSubReg(SubReg);
1593 
1594  return &MI;
1595 }
1596 
1598  unsigned Src0Idx,
1599  unsigned Src1Idx) const {
1600  assert(!NewMI && "this should never be used");
1601 
1602  unsigned Opc = MI.getOpcode();
1603  int CommutedOpcode = commuteOpcode(Opc);
1604  if (CommutedOpcode == -1)
1605  return nullptr;
1606 
1607  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1608  static_cast<int>(Src0Idx) &&
1609  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1610  static_cast<int>(Src1Idx) &&
1611  "inconsistency with findCommutedOpIndices");
1612 
1613  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1614  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1615 
1616  MachineInstr *CommutedMI = nullptr;
1617  if (Src0.isReg() && Src1.isReg()) {
1618  if (isOperandLegal(MI, Src1Idx, &Src0)) {
1619  // Be sure to copy the source modifiers to the right place.
1620  CommutedMI
1621  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1622  }
1623 
1624  } else if (Src0.isReg() && !Src1.isReg()) {
1625  // src0 should always be able to support any operand type, so no need to
1626  // check operand legality.
1627  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1628  } else if (!Src0.isReg() && Src1.isReg()) {
1629  if (isOperandLegal(MI, Src1Idx, &Src0))
1630  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1631  } else {
1632  // FIXME: Found two non registers to commute. This does happen.
1633  return nullptr;
1634  }
1635 
1636  if (CommutedMI) {
1637  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1638  Src1, AMDGPU::OpName::src1_modifiers);
1639 
1640  CommutedMI->setDesc(get(CommutedOpcode));
1641  }
1642 
1643  return CommutedMI;
1644 }
1645 
1646 // This needs to be implemented because the source modifiers may be inserted
1647 // between the true commutable operands, and the base
1648 // TargetInstrInfo::commuteInstruction uses it.
1650  unsigned &SrcOpIdx1) const {
1651  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1652 }
1653 
1654 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1655  unsigned &SrcOpIdx1) const {
1656  if (!Desc.isCommutable())
1657  return false;
1658 
1659  unsigned Opc = Desc.getOpcode();
1660  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1661  if (Src0Idx == -1)
1662  return false;
1663 
1664  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1665  if (Src1Idx == -1)
1666  return false;
1667 
1668  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1669 }
1670 
1671 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1672  int64_t BrOffset) const {
1673  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1674  // block is unanalyzable.
1675  assert(BranchOp != AMDGPU::S_SETPC_B64);
1676 
1677  // Convert to dwords.
1678  BrOffset /= 4;
1679 
1680  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1681  // from the next instruction.
1682  BrOffset -= 1;
1683 
1684  return isIntN(BranchOffsetBits, BrOffset);
1685 }
1686 
1688  const MachineInstr &MI) const {
1689  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1690  // This would be a difficult analysis to perform, but can always be legal so
1691  // there's no need to analyze it.
1692  return nullptr;
1693  }
1694 
1695  return MI.getOperand(0).getMBB();
1696 }
1697 
1699  MachineBasicBlock &DestBB,
1700  const DebugLoc &DL,
1701  int64_t BrOffset,
1702  RegScavenger *RS) const {
1703  assert(RS && "RegScavenger required for long branching");
1704  assert(MBB.empty() &&
1705  "new block should be inserted for expanding unconditional branch");
1706  assert(MBB.pred_size() == 1);
1707 
1708  MachineFunction *MF = MBB.getParent();
1709  MachineRegisterInfo &MRI = MF->getRegInfo();
1710 
1711  // FIXME: Virtual register workaround for RegScavenger not working with empty
1712  // blocks.
1713  unsigned PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1714 
1715  auto I = MBB.end();
1716 
1717  // We need to compute the offset relative to the instruction immediately after
1718  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1719  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1720 
1721  // TODO: Handle > 32-bit block address.
1722  if (BrOffset >= 0) {
1723  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1724  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1725  .addReg(PCReg, 0, AMDGPU::sub0)
1726  .addMBB(&DestBB, MO_LONG_BRANCH_FORWARD);
1727  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1728  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1729  .addReg(PCReg, 0, AMDGPU::sub1)
1730  .addImm(0);
1731  } else {
1732  // Backwards branch.
1733  BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1734  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1735  .addReg(PCReg, 0, AMDGPU::sub0)
1736  .addMBB(&DestBB, MO_LONG_BRANCH_BACKWARD);
1737  BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1738  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1739  .addReg(PCReg, 0, AMDGPU::sub1)
1740  .addImm(0);
1741  }
1742 
1743  // Insert the indirect branch after the other terminator.
1744  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1745  .addReg(PCReg);
1746 
1747  // FIXME: If spilling is necessary, this will fail because this scavenger has
1748  // no emergency stack slots. It is non-trivial to spill in this situation,
1749  // because the restore code needs to be specially placed after the
1750  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1751  // block.
1752  //
1753  // If a spill is needed for the pc register pair, we need to insert a spill
1754  // restore block right before the destination block, and insert a short branch
1755  // into the old destination block's fallthrough predecessor.
1756  // e.g.:
1757  //
1758  // s_cbranch_scc0 skip_long_branch:
1759  //
1760  // long_branch_bb:
1761  // spill s[8:9]
1762  // s_getpc_b64 s[8:9]
1763  // s_add_u32 s8, s8, restore_bb
1764  // s_addc_u32 s9, s9, 0
1765  // s_setpc_b64 s[8:9]
1766  //
1767  // skip_long_branch:
1768  // foo;
1769  //
1770  // .....
1771  //
1772  // dest_bb_fallthrough_predecessor:
1773  // bar;
1774  // s_branch dest_bb
1775  //
1776  // restore_bb:
1777  // restore s[8:9]
1778  // fallthrough dest_bb
1779  ///
1780  // dest_bb:
1781  // buzz;
1782 
1783  RS->enterBasicBlockEnd(MBB);
1784  unsigned Scav = RS->scavengeRegisterBackwards(
1785  AMDGPU::SReg_64RegClass,
1786  MachineBasicBlock::iterator(GetPC), false, 0);
1787  MRI.replaceRegWith(PCReg, Scav);
1788  MRI.clearVirtRegs();
1789  RS->setRegUsed(Scav);
1790 
1791  return 4 + 8 + 4 + 4;
1792 }
1793 
1794 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1795  switch (Cond) {
1796  case SIInstrInfo::SCC_TRUE:
1797  return AMDGPU::S_CBRANCH_SCC1;
1798  case SIInstrInfo::SCC_FALSE:
1799  return AMDGPU::S_CBRANCH_SCC0;
1800  case SIInstrInfo::VCCNZ:
1801  return AMDGPU::S_CBRANCH_VCCNZ;
1802  case SIInstrInfo::VCCZ:
1803  return AMDGPU::S_CBRANCH_VCCZ;
1804  case SIInstrInfo::EXECNZ:
1805  return AMDGPU::S_CBRANCH_EXECNZ;
1806  case SIInstrInfo::EXECZ:
1807  return AMDGPU::S_CBRANCH_EXECZ;
1808  default:
1809  llvm_unreachable("invalid branch predicate");
1810  }
1811 }
1812 
1813 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1814  switch (Opcode) {
1815  case AMDGPU::S_CBRANCH_SCC0:
1816  return SCC_FALSE;
1817  case AMDGPU::S_CBRANCH_SCC1:
1818  return SCC_TRUE;
1819  case AMDGPU::S_CBRANCH_VCCNZ:
1820  return VCCNZ;
1821  case AMDGPU::S_CBRANCH_VCCZ:
1822  return VCCZ;
1823  case AMDGPU::S_CBRANCH_EXECNZ:
1824  return EXECNZ;
1825  case AMDGPU::S_CBRANCH_EXECZ:
1826  return EXECZ;
1827  default:
1828  return INVALID_BR;
1829  }
1830 }
1831 
1834  MachineBasicBlock *&TBB,
1835  MachineBasicBlock *&FBB,
1837  bool AllowModify) const {
1838  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1839  // Unconditional Branch
1840  TBB = I->getOperand(0).getMBB();
1841  return false;
1842  }
1843 
1844  MachineBasicBlock *CondBB = nullptr;
1845 
1846  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1847  CondBB = I->getOperand(1).getMBB();
1848  Cond.push_back(I->getOperand(0));
1849  } else {
1850  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1851  if (Pred == INVALID_BR)
1852  return true;
1853 
1854  CondBB = I->getOperand(0).getMBB();
1856  Cond.push_back(I->getOperand(1)); // Save the branch register.
1857  }
1858  ++I;
1859 
1860  if (I == MBB.end()) {
1861  // Conditional branch followed by fall-through.
1862  TBB = CondBB;
1863  return false;
1864  }
1865 
1866  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1867  TBB = CondBB;
1868  FBB = I->getOperand(0).getMBB();
1869  return false;
1870  }
1871 
1872  return true;
1873 }
1874 
1876  MachineBasicBlock *&FBB,
1878  bool AllowModify) const {
1880  auto E = MBB.end();
1881  if (I == E)
1882  return false;
1883 
1884  // Skip over the instructions that are artificially terminators for special
1885  // exec management.
1886  while (I != E && !I->isBranch() && !I->isReturn() &&
1887  I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1888  switch (I->getOpcode()) {
1889  case AMDGPU::SI_MASK_BRANCH:
1890  case AMDGPU::S_MOV_B64_term:
1891  case AMDGPU::S_XOR_B64_term:
1892  case AMDGPU::S_ANDN2_B64_term:
1893  case AMDGPU::S_MOV_B32_term:
1894  case AMDGPU::S_XOR_B32_term:
1895  case AMDGPU::S_OR_B32_term:
1896  case AMDGPU::S_ANDN2_B32_term:
1897  break;
1898  case AMDGPU::SI_IF:
1899  case AMDGPU::SI_ELSE:
1900  case AMDGPU::SI_KILL_I1_TERMINATOR:
1901  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1902  // FIXME: It's messy that these need to be considered here at all.
1903  return true;
1904  default:
1905  llvm_unreachable("unexpected non-branch terminator inst");
1906  }
1907 
1908  ++I;
1909  }
1910 
1911  if (I == E)
1912  return false;
1913 
1914  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1915  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1916 
1917  ++I;
1918 
1919  // TODO: Should be able to treat as fallthrough?
1920  if (I == MBB.end())
1921  return true;
1922 
1923  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1924  return true;
1925 
1926  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1927 
1928  // Specifically handle the case where the conditional branch is to the same
1929  // destination as the mask branch. e.g.
1930  //
1931  // si_mask_branch BB8
1932  // s_cbranch_execz BB8
1933  // s_cbranch BB9
1934  //
1935  // This is required to understand divergent loops which may need the branches
1936  // to be relaxed.
1937  if (TBB != MaskBrDest || Cond.empty())
1938  return true;
1939 
1940  auto Pred = Cond[0].getImm();
1941  return (Pred != EXECZ && Pred != EXECNZ);
1942 }
1943 
1945  int *BytesRemoved) const {
1947 
1948  unsigned Count = 0;
1949  unsigned RemovedSize = 0;
1950  while (I != MBB.end()) {
1951  MachineBasicBlock::iterator Next = std::next(I);
1952  if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1953  I = Next;
1954  continue;
1955  }
1956 
1957  RemovedSize += getInstSizeInBytes(*I);
1958  I->eraseFromParent();
1959  ++Count;
1960  I = Next;
1961  }
1962 
1963  if (BytesRemoved)
1964  *BytesRemoved = RemovedSize;
1965 
1966  return Count;
1967 }
1968 
1969 // Copy the flags onto the implicit condition register operand.
1971  const MachineOperand &OrigCond) {
1972  CondReg.setIsUndef(OrigCond.isUndef());
1973  CondReg.setIsKill(OrigCond.isKill());
1974 }
1975 
1977  MachineBasicBlock *TBB,
1978  MachineBasicBlock *FBB,
1980  const DebugLoc &DL,
1981  int *BytesAdded) const {
1982  if (!FBB && Cond.empty()) {
1983  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1984  .addMBB(TBB);
1985  if (BytesAdded)
1986  *BytesAdded = 4;
1987  return 1;
1988  }
1989 
1990  if(Cond.size() == 1 && Cond[0].isReg()) {
1991  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
1992  .add(Cond[0])
1993  .addMBB(TBB);
1994  return 1;
1995  }
1996 
1997  assert(TBB && Cond[0].isImm());
1998 
1999  unsigned Opcode
2000  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
2001 
2002  if (!FBB) {
2003  Cond[1].isUndef();
2004  MachineInstr *CondBr =
2005  BuildMI(&MBB, DL, get(Opcode))
2006  .addMBB(TBB);
2007 
2008  // Copy the flags onto the implicit condition register operand.
2009  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
2010 
2011  if (BytesAdded)
2012  *BytesAdded = 4;
2013  return 1;
2014  }
2015 
2016  assert(TBB && FBB);
2017 
2018  MachineInstr *CondBr =
2019  BuildMI(&MBB, DL, get(Opcode))
2020  .addMBB(TBB);
2021  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
2022  .addMBB(FBB);
2023 
2024  MachineOperand &CondReg = CondBr->getOperand(1);
2025  CondReg.setIsUndef(Cond[1].isUndef());
2026  CondReg.setIsKill(Cond[1].isKill());
2027 
2028  if (BytesAdded)
2029  *BytesAdded = 8;
2030 
2031  return 2;
2032 }
2033 
2035  SmallVectorImpl<MachineOperand> &Cond) const {
2036  if (Cond.size() != 2) {
2037  return true;
2038  }
2039 
2040  if (Cond[0].isImm()) {
2041  Cond[0].setImm(-Cond[0].getImm());
2042  return false;
2043  }
2044 
2045  return true;
2046 }
2047 
2050  unsigned TrueReg, unsigned FalseReg,
2051  int &CondCycles,
2052  int &TrueCycles, int &FalseCycles) const {
2053  switch (Cond[0].getImm()) {
2054  case VCCNZ:
2055  case VCCZ: {
2056  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2057  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2058  assert(MRI.getRegClass(FalseReg) == RC);
2059 
2060  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2061  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2062 
2063  // Limit to equal cost for branch vs. N v_cndmask_b32s.
2064  return RI.hasVGPRs(RC) && NumInsts <= 6;
2065  }
2066  case SCC_TRUE:
2067  case SCC_FALSE: {
2068  // FIXME: We could insert for VGPRs if we could replace the original compare
2069  // with a vector one.
2070  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2071  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
2072  assert(MRI.getRegClass(FalseReg) == RC);
2073 
2074  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
2075 
2076  // Multiples of 8 can do s_cselect_b64
2077  if (NumInsts % 2 == 0)
2078  NumInsts /= 2;
2079 
2080  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
2081  return RI.isSGPRClass(RC);
2082  }
2083  default:
2084  return false;
2085  }
2086 }
2087 
2090  unsigned DstReg, ArrayRef<MachineOperand> Cond,
2091  unsigned TrueReg, unsigned FalseReg) const {
2092  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
2093  if (Pred == VCCZ || Pred == SCC_FALSE) {
2094  Pred = static_cast<BranchPredicate>(-Pred);
2095  std::swap(TrueReg, FalseReg);
2096  }
2097 
2099  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
2100  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
2101 
2102  if (DstSize == 32) {
2103  unsigned SelOp = Pred == SCC_TRUE ?
2104  AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
2105 
2106  // Instruction's operands are backwards from what is expected.
2107  MachineInstr *Select =
2108  BuildMI(MBB, I, DL, get(SelOp), DstReg)
2109  .addReg(FalseReg)
2110  .addReg(TrueReg);
2111 
2112  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2113  return;
2114  }
2115 
2116  if (DstSize == 64 && Pred == SCC_TRUE) {
2117  MachineInstr *Select =
2118  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
2119  .addReg(FalseReg)
2120  .addReg(TrueReg);
2121 
2122  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2123  return;
2124  }
2125 
2126  static const int16_t Sub0_15[] = {
2127  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
2128  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
2129  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
2130  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
2131  };
2132 
2133  static const int16_t Sub0_15_64[] = {
2134  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
2135  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
2136  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
2137  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
2138  };
2139 
2140  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
2141  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
2142  const int16_t *SubIndices = Sub0_15;
2143  int NElts = DstSize / 32;
2144 
2145  // 64-bit select is only available for SALU.
2146  // TODO: Split 96-bit into 64-bit and 32-bit, not 3x 32-bit.
2147  if (Pred == SCC_TRUE) {
2148  if (NElts % 2) {
2149  SelOp = AMDGPU::S_CSELECT_B32;
2150  EltRC = &AMDGPU::SGPR_32RegClass;
2151  } else {
2152  SelOp = AMDGPU::S_CSELECT_B64;
2153  EltRC = &AMDGPU::SGPR_64RegClass;
2154  SubIndices = Sub0_15_64;
2155  NElts /= 2;
2156  }
2157  }
2158 
2160  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
2161 
2162  I = MIB->getIterator();
2163 
2165  for (int Idx = 0; Idx != NElts; ++Idx) {
2166  unsigned DstElt = MRI.createVirtualRegister(EltRC);
2167  Regs.push_back(DstElt);
2168 
2169  unsigned SubIdx = SubIndices[Idx];
2170 
2171  MachineInstr *Select =
2172  BuildMI(MBB, I, DL, get(SelOp), DstElt)
2173  .addReg(FalseReg, 0, SubIdx)
2174  .addReg(TrueReg, 0, SubIdx);
2175  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
2176  fixImplicitOperands(*Select);
2177 
2178  MIB.addReg(DstElt)
2179  .addImm(SubIdx);
2180  }
2181 }
2182 
2184  switch (MI.getOpcode()) {
2185  case AMDGPU::V_MOV_B32_e32:
2186  case AMDGPU::V_MOV_B32_e64:
2187  case AMDGPU::V_MOV_B64_PSEUDO: {
2188  // If there are additional implicit register operands, this may be used for
2189  // register indexing so the source register operand isn't simply copied.
2190  unsigned NumOps = MI.getDesc().getNumOperands() +
2191  MI.getDesc().getNumImplicitUses();
2192 
2193  return MI.getNumOperands() == NumOps;
2194  }
2195  case AMDGPU::S_MOV_B32:
2196  case AMDGPU::S_MOV_B64:
2197  case AMDGPU::COPY:
2198  case AMDGPU::V_ACCVGPR_WRITE_B32:
2199  case AMDGPU::V_ACCVGPR_READ_B32:
2200  return true;
2201  default:
2202  return false;
2203  }
2204 }
2205 
2207  unsigned Kind) const {
2208  switch(Kind) {
2219  }
2220  return AMDGPUAS::FLAT_ADDRESS;
2221 }
2222 
2224  unsigned Opc = MI.getOpcode();
2225  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2226  AMDGPU::OpName::src0_modifiers);
2227  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2228  AMDGPU::OpName::src1_modifiers);
2229  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
2230  AMDGPU::OpName::src2_modifiers);
2231 
2232  MI.RemoveOperand(Src2ModIdx);
2233  MI.RemoveOperand(Src1ModIdx);
2234  MI.RemoveOperand(Src0ModIdx);
2235 }
2236 
2238  unsigned Reg, MachineRegisterInfo *MRI) const {
2239  if (!MRI->hasOneNonDBGUse(Reg))
2240  return false;
2241 
2242  switch (DefMI.getOpcode()) {
2243  default:
2244  return false;
2245  case AMDGPU::S_MOV_B64:
2246  // TODO: We could fold 64-bit immediates, but this get compilicated
2247  // when there are sub-registers.
2248  return false;
2249 
2250  case AMDGPU::V_MOV_B32_e32:
2251  case AMDGPU::S_MOV_B32:
2252  case AMDGPU::V_ACCVGPR_WRITE_B32:
2253  break;
2254  }
2255 
2256  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2257  assert(ImmOp);
2258  // FIXME: We could handle FrameIndex values here.
2259  if (!ImmOp->isImm())
2260  return false;
2261 
2262  unsigned Opc = UseMI.getOpcode();
2263  if (Opc == AMDGPU::COPY) {
2264  bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2265  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2266  if (RI.isAGPR(*MRI, UseMI.getOperand(0).getReg())) {
2268  return false;
2269  NewOpc = AMDGPU::V_ACCVGPR_WRITE_B32;
2270  }
2271  UseMI.setDesc(get(NewOpc));
2272  UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2273  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2274  return true;
2275  }
2276 
2277  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2278  Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64 ||
2279  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2280  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64) {
2281  // Don't fold if we are using source or output modifiers. The new VOP2
2282  // instructions don't have them.
2283  if (hasAnyModifiersSet(UseMI))
2284  return false;
2285 
2286  // If this is a free constant, there's no reason to do this.
2287  // TODO: We could fold this here instead of letting SIFoldOperands do it
2288  // later.
2289  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2290 
2291  // Any src operand can be used for the legality check.
2292  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2293  return false;
2294 
2295  bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2296  Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64;
2297  bool IsFMA = Opc == AMDGPU::V_FMA_F32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2298  Opc == AMDGPU::V_FMA_F16 || Opc == AMDGPU::V_FMAC_F16_e64;
2299  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2300  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2301 
2302  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2303  // We should only expect these to be on src0 due to canonicalizations.
2304  if (Src0->isReg() && Src0->getReg() == Reg) {
2305  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2306  return false;
2307 
2308  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2309  return false;
2310 
2311  unsigned NewOpc =
2312  IsFMA ? (IsF32 ? AMDGPU::V_FMAMK_F32 : AMDGPU::V_FMAMK_F16)
2313  : (IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16);
2314  if (pseudoToMCOpcode(NewOpc) == -1)
2315  return false;
2316 
2317  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2318 
2319  const int64_t Imm = ImmOp->getImm();
2320 
2321  // FIXME: This would be a lot easier if we could return a new instruction
2322  // instead of having to modify in place.
2323 
2324  // Remove these first since they are at the end.
2325  UseMI.RemoveOperand(
2326  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2327  UseMI.RemoveOperand(
2328  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2329 
2330  unsigned Src1Reg = Src1->getReg();
2331  unsigned Src1SubReg = Src1->getSubReg();
2332  Src0->setReg(Src1Reg);
2333  Src0->setSubReg(Src1SubReg);
2334  Src0->setIsKill(Src1->isKill());
2335 
2336  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2337  Opc == AMDGPU::V_MAC_F16_e64 ||
2338  Opc == AMDGPU::V_FMAC_F32_e64 ||
2339  Opc == AMDGPU::V_FMAC_F16_e64)
2340  UseMI.untieRegOperand(
2341  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2342 
2343  Src1->ChangeToImmediate(Imm);
2344 
2345  removeModOperands(UseMI);
2346  UseMI.setDesc(get(NewOpc));
2347 
2348  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2349  if (DeleteDef)
2350  DefMI.eraseFromParent();
2351 
2352  return true;
2353  }
2354 
2355  // Added part is the constant: Use v_madak_{f16, f32}.
2356  if (Src2->isReg() && Src2->getReg() == Reg) {
2357  // Not allowed to use constant bus for another operand.
2358  // We can however allow an inline immediate as src0.
2359  bool Src0Inlined = false;
2360  if (Src0->isReg()) {
2361  // Try to inline constant if possible.
2362  // If the Def moves immediate and the use is single
2363  // We are saving VGPR here.
2364  MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2365  if (Def && Def->isMoveImmediate() &&
2366  isInlineConstant(Def->getOperand(1)) &&
2367  MRI->hasOneUse(Src0->getReg())) {
2368  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2369  Src0Inlined = true;
2370  } else if ((RI.isPhysicalRegister(Src0->getReg()) &&
2371  (ST.getConstantBusLimit(Opc) <= 1 &&
2372  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg())))) ||
2373  (RI.isVirtualRegister(Src0->getReg()) &&
2374  (ST.getConstantBusLimit(Opc) <= 1 &&
2375  RI.isSGPRClass(MRI->getRegClass(Src0->getReg())))))
2376  return false;
2377  // VGPR is okay as Src0 - fallthrough
2378  }
2379 
2380  if (Src1->isReg() && !Src0Inlined ) {
2381  // We have one slot for inlinable constant so far - try to fill it
2382  MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2383  if (Def && Def->isMoveImmediate() &&
2384  isInlineConstant(Def->getOperand(1)) &&
2385  MRI->hasOneUse(Src1->getReg()) &&
2386  commuteInstruction(UseMI)) {
2387  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2388  } else if ((RI.isPhysicalRegister(Src1->getReg()) &&
2389  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2390  (RI.isVirtualRegister(Src1->getReg()) &&
2391  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2392  return false;
2393  // VGPR is okay as Src1 - fallthrough
2394  }
2395 
2396  unsigned NewOpc =
2397  IsFMA ? (IsF32 ? AMDGPU::V_FMAAK_F32 : AMDGPU::V_FMAAK_F16)
2398  : (IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16);
2399  if (pseudoToMCOpcode(NewOpc) == -1)
2400  return false;
2401 
2402  const int64_t Imm = ImmOp->getImm();
2403 
2404  // FIXME: This would be a lot easier if we could return a new instruction
2405  // instead of having to modify in place.
2406 
2407  // Remove these first since they are at the end.
2408  UseMI.RemoveOperand(
2409  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2410  UseMI.RemoveOperand(
2411  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2412 
2413  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2414  Opc == AMDGPU::V_MAC_F16_e64 ||
2415  Opc == AMDGPU::V_FMAC_F32_e64 ||
2416  Opc == AMDGPU::V_FMAC_F16_e64)
2417  UseMI.untieRegOperand(
2418  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2419 
2420  // ChangingToImmediate adds Src2 back to the instruction.
2421  Src2->ChangeToImmediate(Imm);
2422 
2423  // These come before src2.
2424  removeModOperands(UseMI);
2425  UseMI.setDesc(get(NewOpc));
2426  // It might happen that UseMI was commuted
2427  // and we now have SGPR as SRC1. If so 2 inlined
2428  // constant and SGPR are illegal.
2429  legalizeOperands(UseMI);
2430 
2431  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2432  if (DeleteDef)
2433  DefMI.eraseFromParent();
2434 
2435  return true;
2436  }
2437  }
2438 
2439  return false;
2440 }
2441 
2442 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2443  int WidthB, int OffsetB) {
2444  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2445  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2446  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2447  return LowOffset + LowWidth <= HighOffset;
2448 }
2449 
2450 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
2451  const MachineInstr &MIb) const {
2452  const MachineOperand *BaseOp0, *BaseOp1;
2453  int64_t Offset0, Offset1;
2454 
2455  if (getMemOperandWithOffset(MIa, BaseOp0, Offset0, &RI) &&
2456  getMemOperandWithOffset(MIb, BaseOp1, Offset1, &RI)) {
2457  if (!BaseOp0->isIdenticalTo(*BaseOp1))
2458  return false;
2459 
2460  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2461  // FIXME: Handle ds_read2 / ds_write2.
2462  return false;
2463  }
2464  unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2465  unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2466  if (offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2467  return true;
2468  }
2469  }
2470 
2471  return false;
2472 }
2473 
2475  const MachineInstr &MIb,
2476  AliasAnalysis *AA) const {
2477  assert((MIa.mayLoad() || MIa.mayStore()) &&
2478  "MIa must load from or modify a memory location");
2479  assert((MIb.mayLoad() || MIb.mayStore()) &&
2480  "MIb must load from or modify a memory location");
2481 
2483  return false;
2484 
2485  // XXX - Can we relax this between address spaces?
2486  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2487  return false;
2488 
2489  // TODO: Should we check the address space from the MachineMemOperand? That
2490  // would allow us to distinguish objects we know don't alias based on the
2491  // underlying address space, even if it was lowered to a different one,
2492  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2493  // buffer.
2494  if (isDS(MIa)) {
2495  if (isDS(MIb))
2496  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2497 
2498  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2499  }
2500 
2501  if (isMUBUF(MIa) || isMTBUF(MIa)) {
2502  if (isMUBUF(MIb) || isMTBUF(MIb))
2503  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2504 
2505  return !isFLAT(MIb) && !isSMRD(MIb);
2506  }
2507 
2508  if (isSMRD(MIa)) {
2509  if (isSMRD(MIb))
2510  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2511 
2512  return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2513  }
2514 
2515  if (isFLAT(MIa)) {
2516  if (isFLAT(MIb))
2517  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2518 
2519  return false;
2520  }
2521 
2522  return false;
2523 }
2524 
2525 static int64_t getFoldableImm(const MachineOperand* MO) {
2526  if (!MO->isReg())
2527  return false;
2528  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2529  const MachineRegisterInfo &MRI = MF->getRegInfo();
2530  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2531  if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2532  Def->getOperand(1).isImm())
2533  return Def->getOperand(1).getImm();
2534  return AMDGPU::NoRegister;
2535 }
2536 
2538  MachineInstr &MI,
2539  LiveVariables *LV) const {
2540  unsigned Opc = MI.getOpcode();
2541  bool IsF16 = false;
2542  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64 ||
2543  Opc == AMDGPU::V_FMAC_F16_e32 || Opc == AMDGPU::V_FMAC_F16_e64;
2544 
2545  switch (Opc) {
2546  default:
2547  return nullptr;
2548  case AMDGPU::V_MAC_F16_e64:
2549  case AMDGPU::V_FMAC_F16_e64:
2550  IsF16 = true;
2552  case AMDGPU::V_MAC_F32_e64:
2553  case AMDGPU::V_FMAC_F32_e64:
2554  break;
2555  case AMDGPU::V_MAC_F16_e32:
2556  case AMDGPU::V_FMAC_F16_e32:
2557  IsF16 = true;
2559  case AMDGPU::V_MAC_F32_e32:
2560  case AMDGPU::V_FMAC_F32_e32: {
2561  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2562  AMDGPU::OpName::src0);
2563  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2564  if (!Src0->isReg() && !Src0->isImm())
2565  return nullptr;
2566 
2567  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2568  return nullptr;
2569 
2570  break;
2571  }
2572  }
2573 
2574  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2575  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2576  const MachineOperand *Src0Mods =
2577  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2578  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2579  const MachineOperand *Src1Mods =
2580  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2581  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2582  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2583  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2584 
2585  if (!Src0Mods && !Src1Mods && !Clamp && !Omod &&
2586  // If we have an SGPR input, we will violate the constant bus restriction.
2587  (ST.getConstantBusLimit(Opc) > 1 ||
2588  !Src0->isReg() ||
2589  !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2590  if (auto Imm = getFoldableImm(Src2)) {
2591  unsigned NewOpc =
2592  IsFMA ? (IsF16 ? AMDGPU::V_FMAAK_F16 : AMDGPU::V_FMAAK_F32)
2593  : (IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32);
2594  if (pseudoToMCOpcode(NewOpc) != -1)
2595  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2596  .add(*Dst)
2597  .add(*Src0)
2598  .add(*Src1)
2599  .addImm(Imm);
2600  }
2601  unsigned NewOpc =
2602  IsFMA ? (IsF16 ? AMDGPU::V_FMAMK_F16 : AMDGPU::V_FMAMK_F32)
2603  : (IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32);
2604  if (auto Imm = getFoldableImm(Src1)) {
2605  if (pseudoToMCOpcode(NewOpc) != -1)
2606  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2607  .add(*Dst)
2608  .add(*Src0)
2609  .addImm(Imm)
2610  .add(*Src2);
2611  }
2612  if (auto Imm = getFoldableImm(Src0)) {
2613  if (pseudoToMCOpcode(NewOpc) != -1 &&
2615  AMDGPU::OpName::src0), Src1))
2616  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2617  .add(*Dst)
2618  .add(*Src1)
2619  .addImm(Imm)
2620  .add(*Src2);
2621  }
2622  }
2623 
2624  unsigned NewOpc = IsFMA ? (IsF16 ? AMDGPU::V_FMA_F16 : AMDGPU::V_FMA_F32)
2625  : (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2626  if (pseudoToMCOpcode(NewOpc) == -1)
2627  return nullptr;
2628 
2629  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2630  .add(*Dst)
2631  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2632  .add(*Src0)
2633  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2634  .add(*Src1)
2635  .addImm(0) // Src mods
2636  .add(*Src2)
2637  .addImm(Clamp ? Clamp->getImm() : 0)
2638  .addImm(Omod ? Omod->getImm() : 0);
2639 }
2640 
2641 // It's not generally safe to move VALU instructions across these since it will
2642 // start using the register as a base index rather than directly.
2643 // XXX - Why isn't hasSideEffects sufficient for these?
2645  switch (MI.getOpcode()) {
2646  case AMDGPU::S_SET_GPR_IDX_ON:
2647  case AMDGPU::S_SET_GPR_IDX_MODE:
2648  case AMDGPU::S_SET_GPR_IDX_OFF:
2649  return true;
2650  default:
2651  return false;
2652  }
2653 }
2654 
2656  const MachineBasicBlock *MBB,
2657  const MachineFunction &MF) const {
2658  // XXX - Do we want the SP check in the base implementation?
2659 
2660  // Target-independent instructions do not have an implicit-use of EXEC, even
2661  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2662  // boundaries prevents incorrect movements of such instructions.
2663  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2664  MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2665  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2666  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2668 }
2669 
2670 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
2671  return Opcode == AMDGPU::DS_ORDERED_COUNT ||
2672  Opcode == AMDGPU::DS_GWS_INIT ||
2673  Opcode == AMDGPU::DS_GWS_SEMA_V ||
2674  Opcode == AMDGPU::DS_GWS_SEMA_BR ||
2675  Opcode == AMDGPU::DS_GWS_SEMA_P ||
2676  Opcode == AMDGPU::DS_GWS_SEMA_RELEASE_ALL ||
2677  Opcode == AMDGPU::DS_GWS_BARRIER;
2678 }
2679 
2681  unsigned Opcode = MI.getOpcode();
2682 
2683  if (MI.mayStore() && isSMRD(MI))
2684  return true; // scalar store or atomic
2685 
2686  // This will terminate the function when other lanes may need to continue.
2687  if (MI.isReturn())
2688  return true;
2689 
2690  // These instructions cause shader I/O that may cause hardware lockups
2691  // when executed with an empty EXEC mask.
2692  //
2693  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2694  // EXEC = 0, but checking for that case here seems not worth it
2695  // given the typical code patterns.
2696  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2697  Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE ||
2698  Opcode == AMDGPU::DS_ORDERED_COUNT || Opcode == AMDGPU::S_TRAP ||
2699  Opcode == AMDGPU::DS_GWS_INIT || Opcode == AMDGPU::DS_GWS_BARRIER)
2700  return true;
2701 
2702  if (MI.isCall() || MI.isInlineAsm())
2703  return true; // conservative assumption
2704 
2705  // These are like SALU instructions in terms of effects, so it's questionable
2706  // whether we should return true for those.
2707  //
2708  // However, executing them with EXEC = 0 causes them to operate on undefined
2709  // data, which we avoid by returning true here.
2710  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2711  return true;
2712 
2713  return false;
2714 }
2715 
2717  const MachineInstr &MI) const {
2718  if (MI.isMetaInstruction())
2719  return false;
2720 
2721  // This won't read exec if this is an SGPR->SGPR copy.
2722  if (MI.isCopyLike()) {
2723  if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg()))
2724  return true;
2725 
2726  // Make sure this isn't copying exec as a normal operand
2727  return MI.readsRegister(AMDGPU::EXEC, &RI);
2728  }
2729 
2730  // Make a conservative assumption about the callee.
2731  if (MI.isCall())
2732  return true;
2733 
2734  // Be conservative with any unhandled generic opcodes.
2735  if (!isTargetSpecificOpcode(MI.getOpcode()))
2736  return true;
2737 
2738  return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI);
2739 }
2740 
2741 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2742  switch (Imm.getBitWidth()) {
2743  case 1: // This likely will be a condition code mask.
2744  return true;
2745 
2746  case 32:
2748  ST.hasInv2PiInlineImm());
2749  case 64:
2751  ST.hasInv2PiInlineImm());
2752  case 16:
2753  return ST.has16BitInsts() &&
2755  ST.hasInv2PiInlineImm());
2756  default:
2757  llvm_unreachable("invalid bitwidth");
2758  }
2759 }
2760 
2762  uint8_t OperandType) const {
2763  if (!MO.isImm() ||
2764  OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2765  OperandType > AMDGPU::OPERAND_SRC_LAST)
2766  return false;
2767 
2768  // MachineOperand provides no way to tell the true operand size, since it only
2769  // records a 64-bit value. We need to know the size to determine if a 32-bit
2770  // floating point immediate bit pattern is legal for an integer immediate. It
2771  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2772 
2773  int64_t Imm = MO.getImm();
2774  switch (OperandType) {
2781  int32_t Trunc = static_cast<int32_t>(Imm);
2783  }
2789  ST.hasInv2PiInlineImm());
2796  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2797  // A few special case instructions have 16-bit operands on subtargets
2798  // where 16-bit instructions are not legal.
2799  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2800  // constants in these cases
2801  int16_t Trunc = static_cast<int16_t>(Imm);
2802  return ST.has16BitInsts() &&
2804  }
2805 
2806  return false;
2807  }
2814  uint32_t Trunc = static_cast<uint32_t>(Imm);
2816  }
2817  default:
2818  llvm_unreachable("invalid bitwidth");
2819  }
2820 }
2821 
2823  const MCOperandInfo &OpInfo) const {
2824  switch (MO.getType()) {
2826  return false;
2828  return !isInlineConstant(MO, OpInfo);
2834  return true;
2835  default:
2836  llvm_unreachable("unexpected operand type");
2837  }
2838 }
2839 
2840 static bool compareMachineOp(const MachineOperand &Op0,
2841  const MachineOperand &Op1) {
2842  if (Op0.getType() != Op1.getType())
2843  return false;
2844 
2845  switch (Op0.getType()) {
2847  return Op0.getReg() == Op1.getReg();
2849  return Op0.getImm() == Op1.getImm();
2850  default:
2851  llvm_unreachable("Didn't expect to be comparing these operand types");
2852  }
2853 }
2854 
2856  const MachineOperand &MO) const {
2857  const MCInstrDesc &InstDesc = MI.getDesc();
2858  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo];
2859 
2860  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
2861 
2862  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2863  return true;
2864 
2865  if (OpInfo.RegClass < 0)
2866  return false;
2867 
2868  if (MO.isImm() && isInlineConstant(MO, OpInfo))
2869  return RI.opCanUseInlineConstant(OpInfo.OperandType);
2870 
2871  if (!RI.opCanUseLiteralConstant(OpInfo.OperandType))
2872  return false;
2873 
2874  if (!isVOP3(MI) || !AMDGPU::isSISrcOperand(InstDesc, OpNo))
2875  return true;
2876 
2877  const MachineFunction *MF = MI.getParent()->getParent();
2878  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2879  return ST.hasVOP3Literal();
2880 }
2881 
2882 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2883  int Op32 = AMDGPU::getVOPe32(Opcode);
2884  if (Op32 == -1)
2885  return false;
2886 
2887  return pseudoToMCOpcode(Op32) != -1;
2888 }
2889 
2890 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2891  // The src0_modifier operand is present on all instructions
2892  // that have modifiers.
2893 
2894  return AMDGPU::getNamedOperandIdx(Opcode,
2895  AMDGPU::OpName::src0_modifiers) != -1;
2896 }
2897 
2899  unsigned OpName) const {
2900  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2901  return Mods && Mods->getImm();
2902 }
2903 
2905  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2906  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2907  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2908  hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2909  hasModifiersSet(MI, AMDGPU::OpName::omod);
2910 }
2911 
2913  const MachineRegisterInfo &MRI) const {
2914  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2915  // Can't shrink instruction with three operands.
2916  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2917  // a special case for it. It can only be shrunk if the third operand
2918  // is vcc, and src0_modifiers and src1_modifiers are not set.
2919  // We should handle this the same way we handle vopc, by addding
2920  // a register allocation hint pre-regalloc and then do the shrinking
2921  // post-regalloc.
2922  if (Src2) {
2923  switch (MI.getOpcode()) {
2924  default: return false;
2925 
2926  case AMDGPU::V_ADDC_U32_e64:
2927  case AMDGPU::V_SUBB_U32_e64:
2928  case AMDGPU::V_SUBBREV_U32_e64: {
2929  const MachineOperand *Src1
2930  = getNamedOperand(MI, AMDGPU::OpName::src1);
2931  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
2932  return false;
2933  // Additional verification is needed for sdst/src2.
2934  return true;
2935  }
2936  case AMDGPU::V_MAC_F32_e64:
2937  case AMDGPU::V_MAC_F16_e64:
2938  case AMDGPU::V_FMAC_F32_e64:
2939  case AMDGPU::V_FMAC_F16_e64:
2940  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2941  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2942  return false;
2943  break;
2944 
2945  case AMDGPU::V_CNDMASK_B32_e64:
2946  break;
2947  }
2948  }
2949 
2950  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2951  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
2952  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
2953  return false;
2954 
2955  // We don't need to check src0, all input types are legal, so just make sure
2956  // src0 isn't using any modifiers.
2957  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2958  return false;
2959 
2960  // Can it be shrunk to a valid 32 bit opcode?
2961  if (!hasVALU32BitEncoding(MI.getOpcode()))
2962  return false;
2963 
2964  // Check output modifiers
2965  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2966  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
2967 }
2968 
2969 // Set VCC operand with all flags from \p Orig, except for setting it as
2970 // implicit.
2972  const MachineOperand &Orig) {
2973 
2974  for (MachineOperand &Use : MI.implicit_operands()) {
2975  if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
2976  Use.setIsUndef(Orig.isUndef());
2977  Use.setIsKill(Orig.isKill());
2978  return;
2979  }
2980  }
2981 }
2982 
2984  unsigned Op32) const {
2985  MachineBasicBlock *MBB = MI.getParent();;
2986  MachineInstrBuilder Inst32 =
2987  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
2988 
2989  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
2990  // For VOPC instructions, this is replaced by an implicit def of vcc.
2991  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
2992  if (Op32DstIdx != -1) {
2993  // dst
2994  Inst32.add(MI.getOperand(0));
2995  } else {
2996  assert(((MI.getOperand(0).getReg() == AMDGPU::VCC) ||
2997  (MI.getOperand(0).getReg() == AMDGPU::VCC_LO)) &&
2998  "Unexpected case");
2999  }
3000 
3001  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
3002 
3003  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
3004  if (Src1)
3005  Inst32.add(*Src1);
3006 
3007  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
3008 
3009  if (Src2) {
3010  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
3011  if (Op32Src2Idx != -1) {
3012  Inst32.add(*Src2);
3013  } else {
3014  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
3015  // replaced with an implicit read of vcc. This was already added
3016  // during the initial BuildMI, so find it to preserve the flags.
3017  copyFlagsToImplicitVCC(*Inst32, *Src2);
3018  }
3019  }
3020 
3021  return Inst32;
3022 }
3023 
3025  const MachineOperand &MO,
3026  const MCOperandInfo &OpInfo) const {
3027  // Literal constants use the constant bus.
3028  //if (isLiteralConstantLike(MO, OpInfo))
3029  // return true;
3030  if (MO.isImm())
3031  return !isInlineConstant(MO, OpInfo);
3032 
3033  if (!MO.isReg())
3034  return true; // Misc other operands like FrameIndex
3035 
3036  if (!MO.isUse())
3037  return false;
3038 
3040  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
3041 
3042  // Null is free
3043  if (MO.getReg() == AMDGPU::SGPR_NULL)
3044  return false;
3045 
3046  // SGPRs use the constant bus
3047  if (MO.isImplicit()) {
3048  return MO.getReg() == AMDGPU::M0 ||
3049  MO.getReg() == AMDGPU::VCC ||
3050  MO.getReg() == AMDGPU::VCC_LO;
3051  } else {
3052  return AMDGPU::SReg_32RegClass.contains(MO.getReg()) ||
3053  AMDGPU::SReg_64RegClass.contains(MO.getReg());
3054  }
3055 }
3056 
3057 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
3058  for (const MachineOperand &MO : MI.implicit_operands()) {
3059  // We only care about reads.
3060  if (MO.isDef())
3061  continue;
3062 
3063  switch (MO.getReg()) {
3064  case AMDGPU::VCC:
3065  case AMDGPU::VCC_LO:
3066  case AMDGPU::VCC_HI:
3067  case AMDGPU::M0:
3068  case AMDGPU::FLAT_SCR:
3069  return MO.getReg();
3070 
3071  default:
3072  break;
3073  }
3074  }
3075 
3076  return AMDGPU::NoRegister;
3077 }
3078 
3079 static bool shouldReadExec(const MachineInstr &MI) {
3080  if (SIInstrInfo::isVALU(MI)) {
3081  switch (MI.getOpcode()) {
3082  case AMDGPU::V_READLANE_B32:
3083  case AMDGPU::V_READLANE_B32_gfx6_gfx7:
3084  case AMDGPU::V_READLANE_B32_gfx10:
3085  case AMDGPU::V_READLANE_B32_vi:
3086  case AMDGPU::V_WRITELANE_B32:
3087  case AMDGPU::V_WRITELANE_B32_gfx6_gfx7:
3088  case AMDGPU::V_WRITELANE_B32_gfx10:
3089  case AMDGPU::V_WRITELANE_B32_vi:
3090  return false;
3091  }
3092 
3093  return true;
3094  }
3095 
3096  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
3097  SIInstrInfo::isSALU(MI) ||
3098  SIInstrInfo::isSMRD(MI))
3099  return false;
3100 
3101  return true;
3102 }
3103 
3104 static bool isSubRegOf(const SIRegisterInfo &TRI,
3105  const MachineOperand &SuperVec,
3106  const MachineOperand &SubReg) {
3108  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
3109 
3110  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
3111  SubReg.getReg() == SuperVec.getReg();
3112 }
3113 
3115  StringRef &ErrInfo) const {
3116  uint16_t Opcode = MI.getOpcode();
3117  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
3118  return true;
3119 
3120  const MachineFunction *MF = MI.getParent()->getParent();
3121  const MachineRegisterInfo &MRI = MF->getRegInfo();
3122 
3123  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3124  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3125  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3126 
3127  // Make sure the number of operands is correct.
3128  const MCInstrDesc &Desc = get(Opcode);
3129  if (!Desc.isVariadic() &&
3130  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
3131  ErrInfo = "Instruction has wrong number of operands.";
3132  return false;
3133  }
3134 
3135  if (MI.isInlineAsm()) {
3136  // Verify register classes for inlineasm constraints.
3137  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
3138  I != E; ++I) {
3139  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
3140  if (!RC)
3141  continue;
3142 
3143  const MachineOperand &Op = MI.getOperand(I);
3144  if (!Op.isReg())
3145  continue;
3146 
3147  unsigned Reg = Op.getReg();
3148  if (!TargetRegisterInfo::isVirtualRegister(Reg) && !RC->contains(Reg)) {
3149  ErrInfo = "inlineasm operand has incorrect register class.";
3150  return false;
3151  }
3152  }
3153 
3154  return true;
3155  }
3156 
3157  // Make sure the register classes are correct.
3158  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
3159  if (MI.getOperand(i).isFPImm()) {
3160  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
3161  "all fp values to integers.";
3162  return false;
3163  }
3164 
3165  int RegClass = Desc.OpInfo[i].RegClass;
3166 
3167  switch (Desc.OpInfo[i].OperandType) {
3169  if (MI.getOperand(i).isImm() || MI.getOperand(i).isGlobal()) {
3170  ErrInfo = "Illegal immediate value for operand.";
3171  return false;
3172  }
3173  break;
3176  break;
3187  const MachineOperand &MO = MI.getOperand(i);
3188  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
3189  ErrInfo = "Illegal immediate value for operand.";
3190  return false;
3191  }
3192  break;
3193  }
3196  // Check if this operand is an immediate.
3197  // FrameIndex operands will be replaced by immediates, so they are
3198  // allowed.
3199  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
3200  ErrInfo = "Expected immediate, but got non-immediate";
3201  return false;
3202  }
3204  default:
3205  continue;
3206  }
3207 
3208  if (!MI.getOperand(i).isReg())
3209  continue;
3210 
3211  if (RegClass != -1) {
3212  unsigned Reg = MI.getOperand(i).getReg();
3213  if (Reg == AMDGPU::NoRegister ||
3215  continue;
3216 
3217  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
3218  if (!RC->contains(Reg)) {
3219  ErrInfo = "Operand has incorrect register class.";
3220  return false;
3221  }
3222  }
3223  }
3224 
3225  // Verify SDWA
3226  if (isSDWA(MI)) {
3227  if (!ST.hasSDWA()) {
3228  ErrInfo = "SDWA is not supported on this target";
3229  return false;
3230  }
3231 
3232  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
3233 
3234  const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
3235 
3236  for (int OpIdx: OpIndicies) {
3237  if (OpIdx == -1)
3238  continue;
3239  const MachineOperand &MO = MI.getOperand(OpIdx);
3240 
3241  if (!ST.hasSDWAScalar()) {
3242  // Only VGPRS on VI
3243  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
3244  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
3245  return false;
3246  }
3247  } else {
3248  // No immediates on GFX9
3249  if (!MO.isReg()) {
3250  ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
3251  return false;
3252  }
3253  }
3254  }
3255 
3256  if (!ST.hasSDWAOmod()) {
3257  // No omod allowed on VI
3258  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3259  if (OMod != nullptr &&
3260  (!OMod->isImm() || OMod->getImm() != 0)) {
3261  ErrInfo = "OMod not allowed in SDWA instructions on VI";
3262  return false;
3263  }
3264  }
3265 
3266  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
3267  if (isVOPC(BasicOpcode)) {
3268  if (!ST.hasSDWASdst() && DstIdx != -1) {
3269  // Only vcc allowed as dst on VI for VOPC
3270  const MachineOperand &Dst = MI.getOperand(DstIdx);
3271  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
3272  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
3273  return false;
3274  }
3275  } else if (!ST.hasSDWAOutModsVOPC()) {
3276  // No clamp allowed on GFX9 for VOPC
3277  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
3278  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
3279  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
3280  return false;
3281  }
3282 
3283  // No omod allowed on GFX9 for VOPC
3284  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
3285  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
3286  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
3287  return false;
3288  }
3289  }
3290  }
3291 
3292  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
3293  if (DstUnused && DstUnused->isImm() &&
3294  DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
3295  const MachineOperand &Dst = MI.getOperand(DstIdx);
3296  if (!Dst.isReg() || !Dst.isTied()) {
3297  ErrInfo = "Dst register should have tied register";
3298  return false;
3299  }
3300 
3301  const MachineOperand &TiedMO =
3302  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
3303  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
3304  ErrInfo =
3305  "Dst register should be tied to implicit use of preserved register";
3306  return false;
3307  } else if (TargetRegisterInfo::isPhysicalRegister(TiedMO.getReg()) &&
3308  Dst.getReg() != TiedMO.getReg()) {
3309  ErrInfo = "Dst register should use same physical register as preserved";
3310  return false;
3311  }
3312  }
3313  }
3314 
3315  // Verify MIMG
3316  if (isMIMG(MI.getOpcode()) && !MI.mayStore()) {
3317  // Ensure that the return type used is large enough for all the options
3318  // being used TFE/LWE require an extra result register.
3319  const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask);
3320  if (DMask) {
3321  uint64_t DMaskImm = DMask->getImm();
3322  uint32_t RegCount =
3323  isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm);
3324  const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe);
3325  const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe);
3326  const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16);
3327 
3328  // Adjust for packed 16 bit values
3329  if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem())
3330  RegCount >>= 1;
3331 
3332  // Adjust if using LWE or TFE
3333  if ((LWE && LWE->getImm()) || (TFE && TFE->getImm()))
3334  RegCount += 1;
3335 
3336  const uint32_t DstIdx =
3337  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata);
3338  const MachineOperand &Dst = MI.getOperand(DstIdx);
3339  if (Dst.isReg()) {
3340  const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx);
3341  uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32;
3342  if (RegCount > DstSize) {
3343  ErrInfo = "MIMG instruction returns too many registers for dst "
3344  "register class";
3345  return false;
3346  }
3347  }
3348  }
3349  }
3350 
3351  // Verify VOP*. Ignore multiple sgpr operands on writelane.
3352  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
3353  && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
3354  // Only look at the true operands. Only a real operand can use the constant
3355  // bus, and we don't want to check pseudo-operands like the source modifier
3356  // flags.
3357  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3358 
3359  unsigned ConstantBusCount = 0;
3360  unsigned LiteralCount = 0;
3361 
3362  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
3363  ++ConstantBusCount;
3364 
3365  SmallVector<unsigned, 2> SGPRsUsed;
3366  unsigned SGPRUsed = findImplicitSGPRRead(MI);
3367  if (SGPRUsed != AMDGPU::NoRegister) {
3368  ++ConstantBusCount;
3369  SGPRsUsed.push_back(SGPRUsed);
3370  }
3371 
3372  for (int OpIdx : OpIndices) {
3373  if (OpIdx == -1)
3374  break;
3375  const MachineOperand &MO = MI.getOperand(OpIdx);
3376  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
3377  if (MO.isReg()) {
3378  SGPRUsed = MO.getReg();
3379  if (llvm::all_of(SGPRsUsed, [this, SGPRUsed](unsigned SGPR) {
3380  return !RI.regsOverlap(SGPRUsed, SGPR);
3381  })) {
3382  ++ConstantBusCount;
3383  SGPRsUsed.push_back(SGPRUsed);
3384  }
3385  } else {
3386  ++ConstantBusCount;
3387  ++LiteralCount;
3388  }
3389  }
3390  }
3391  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
3392  // v_writelane_b32 is an exception from constant bus restriction:
3393  // vsrc0 can be sgpr, const or m0 and lane select sgpr, m0 or inline-const
3394  if (ConstantBusCount > ST.getConstantBusLimit(Opcode) &&
3395  Opcode != AMDGPU::V_WRITELANE_B32) {
3396  ErrInfo = "VOP* instruction violates constant bus restriction";
3397  return false;
3398  }
3399 
3400  if (isVOP3(MI) && LiteralCount) {
3401  if (LiteralCount && !ST.hasVOP3Literal()) {
3402  ErrInfo = "VOP3 instruction uses literal";
3403  return false;
3404  }
3405  if (LiteralCount > 1) {
3406  ErrInfo = "VOP3 instruction uses more than one literal";
3407  return false;
3408  }
3409  }
3410  }
3411 
3412  // Verify misc. restrictions on specific instructions.
3413  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3414  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3415  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3416  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3417  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3418  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3419  if (!compareMachineOp(Src0, Src1) &&
3420  !compareMachineOp(Src0, Src2)) {
3421  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3422  return false;
3423  }
3424  }
3425  }
3426 
3427  if (isSOP2(MI) || isSOPC(MI)) {
3428  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3429  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3430  unsigned Immediates = 0;
3431 
3432  if (!Src0.isReg() &&
3433  !isInlineConstant(Src0, Desc.OpInfo[Src0Idx].OperandType))
3434  Immediates++;
3435  if (!Src1.isReg() &&
3436  !isInlineConstant(Src1, Desc.OpInfo[Src1Idx].OperandType))
3437  Immediates++;
3438 
3439  if (Immediates > 1) {
3440  ErrInfo = "SOP2/SOPC instruction requires too many immediate constants";
3441  return false;
3442  }
3443  }
3444 
3445  if (isSOPK(MI)) {
3446  auto Op = getNamedOperand(MI, AMDGPU::OpName::simm16);
3447  if (Desc.isBranch()) {
3448  if (!Op->isMBB()) {
3449  ErrInfo = "invalid branch target for SOPK instruction";
3450  return false;
3451  }
3452  } else {
3453  uint64_t Imm = Op->getImm();
3454  if (sopkIsZext(MI)) {
3455  if (!isUInt<16>(Imm)) {
3456  ErrInfo = "invalid immediate for SOPK instruction";
3457  return false;
3458  }
3459  } else {
3460  if (!isInt<16>(Imm)) {
3461  ErrInfo = "invalid immediate for SOPK instruction";
3462  return false;
3463  }
3464  }
3465  }
3466  }
3467 
3468  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3469  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3470  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3471  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3472  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3473  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3474 
3475  const unsigned StaticNumOps = Desc.getNumOperands() +
3476  Desc.getNumImplicitUses();
3477  const unsigned NumImplicitOps = IsDst ? 2 : 1;
3478 
3479  // Allow additional implicit operands. This allows a fixup done by the post
3480  // RA scheduler where the main implicit operand is killed and implicit-defs
3481  // are added for sub-registers that remain live after this instruction.
3482  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3483  ErrInfo = "missing implicit register operands";
3484  return false;
3485  }
3486 
3487  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3488  if (IsDst) {
3489  if (!Dst->isUse()) {
3490  ErrInfo = "v_movreld_b32 vdst should be a use operand";
3491  return false;
3492  }
3493 
3494  unsigned UseOpIdx;
3495  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3496  UseOpIdx != StaticNumOps + 1) {
3497  ErrInfo = "movrel implicit operands should be tied";
3498  return false;
3499  }
3500  }
3501 
3502  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3503  const MachineOperand &ImpUse
3504  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3505  if (!ImpUse.isReg() || !ImpUse.isUse() ||
3506  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3507  ErrInfo = "src0 should be subreg of implicit vector use";
3508  return false;
3509  }
3510  }
3511 
3512  // Make sure we aren't losing exec uses in the td files. This mostly requires
3513  // being careful when using let Uses to try to add other use registers.
3514  if (shouldReadExec(MI)) {
3515  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3516  ErrInfo = "VALU instruction does not implicitly read exec mask";
3517  return false;
3518  }
3519  }
3520 
3521  if (isSMRD(MI)) {
3522  if (MI.mayStore()) {
3523  // The register offset form of scalar stores may only use m0 as the
3524  // soffset register.
3525  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3526  if (Soff && Soff->getReg() != AMDGPU::M0) {
3527  ErrInfo = "scalar stores must use m0 as offset register";
3528  return false;
3529  }
3530  }
3531  }
3532 
3533  if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3534  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3535  if (Offset->getImm() != 0) {
3536  ErrInfo = "subtarget does not support offsets in flat instructions";
3537  return false;
3538  }
3539  }
3540 
3541  if (isMIMG(MI)) {
3542  const MachineOperand *DimOp = getNamedOperand(MI, AMDGPU::OpName::dim);
3543  if (DimOp) {
3544  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opcode,
3545  AMDGPU::OpName::vaddr0);
3546  int SRsrcIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::srsrc);
3547  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opcode);
3548  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3550  const AMDGPU::MIMGDimInfo *Dim =
3552 
3553  if (!Dim) {
3554  ErrInfo = "dim is out of range";
3555  return false;
3556  }
3557 
3558  bool IsNSA = SRsrcIdx - VAddr0Idx > 1;
3559  unsigned AddrWords = BaseOpcode->NumExtraArgs +
3560  (BaseOpcode->Gradients ? Dim->NumGradients : 0) +
3561  (BaseOpcode->Coordinates ? Dim->NumCoords : 0) +
3562  (BaseOpcode->LodOrClampOrMip ? 1 : 0);
3563 
3564  unsigned VAddrWords;
3565  if (IsNSA) {
3566  VAddrWords = SRsrcIdx - VAddr0Idx;
3567  } else {
3568  const TargetRegisterClass *RC = getOpRegClass(MI, VAddr0Idx);
3569  VAddrWords = MRI.getTargetRegisterInfo()->getRegSizeInBits(*RC) / 32;
3570  if (AddrWords > 8)
3571  AddrWords = 16;
3572  else if (AddrWords > 4)
3573  AddrWords = 8;
3574  else if (AddrWords == 3 && VAddrWords == 4) {
3575  // CodeGen uses the V4 variant of instructions for three addresses,
3576  // because the selection DAG does not support non-power-of-two types.
3577  AddrWords = 4;
3578  }
3579  }
3580 
3581  if (VAddrWords != AddrWords) {
3582  ErrInfo = "bad vaddr size";
3583  return false;
3584  }
3585  }
3586  }
3587 
3588  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3589  if (DppCt) {
3590  using namespace AMDGPU::DPP;
3591 
3592  unsigned DC = DppCt->getImm();
3593  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3594  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3600  ErrInfo = "Invalid dpp_ctrl value";
3601  return false;
3602  }
3603  if (DC >= DppCtrl::WAVE_SHL1 && DC <= DppCtrl::WAVE_ROR1 &&
3604  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3605  ErrInfo = "Invalid dpp_ctrl value: "
3606  "wavefront shifts are not supported on GFX10+";
3607  return false;
3608  }
3609  if (DC >= DppCtrl::BCAST15 && DC <= DppCtrl::BCAST31 &&
3610  ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
3611  ErrInfo = "Invalid dpp_ctrl value: "
3612  "broadcats are not supported on GFX10+";
3613  return false;
3614  }
3616  ST.getGeneration() < AMDGPUSubtarget::GFX10) {
3617  ErrInfo = "Invalid dpp_ctrl value: "
3618  "row_share and row_xmask are not supported before GFX10";
3619  return false;
3620  }
3621  }
3622 
3623  return true;
3624 }
3625 
3626 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3627  switch (MI.getOpcode()) {
3628  default: return AMDGPU::INSTRUCTION_LIST_END;
3629  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3630  case AMDGPU::COPY: return AMDGPU::COPY;
3631  case AMDGPU::PHI: return AMDGPU::PHI;
3632  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3633  case AMDGPU::WQM: return AMDGPU::WQM;
3634  case AMDGPU::WWM: return AMDGPU::WWM;
3635  case AMDGPU::S_MOV_B32: {
3637  return MI.getOperand(1).isReg() ||
3638  RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
3639  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3640  }
3641  case AMDGPU::S_ADD_I32:
3642  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3643  case AMDGPU::S_ADDC_U32:
3644  return AMDGPU::V_ADDC_U32_e32;
3645  case AMDGPU::S_SUB_I32:
3646  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3647  // FIXME: These are not consistently handled, and selected when the carry is
3648  // used.
3649  case AMDGPU::S_ADD_U32:
3650  return AMDGPU::V_ADD_I32_e32;
3651  case AMDGPU::S_SUB_U32:
3652  return AMDGPU::V_SUB_I32_e32;
3653  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3654  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_U32;
3655  case AMDGPU::S_MUL_HI_U32: return AMDGPU::V_MUL_HI_U32;
3656  case AMDGPU::S_MUL_HI_I32: return AMDGPU::V_MUL_HI_I32;
3657  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3658  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3659  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3660  case AMDGPU::S_XNOR_B32:
3661  return ST.hasDLInsts() ? AMDGPU::V_XNOR_B32_e64 : AMDGPU::INSTRUCTION_LIST_END;
3662  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3663  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3664  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3665  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3666  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3667  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3668  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3669  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3670  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3671  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3672  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3673  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3674  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3675  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3676  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3677  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3678  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3679  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3680  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3681  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3682  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3683  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3684  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3685  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3686  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3687  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3688  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3689  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3690  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3691  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3692  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3693  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3694  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3695  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3696  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3697  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3698  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3699  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3700  }
3702  "Unexpected scalar opcode without corresponding vector one!");
3703 }
3704 
3706  unsigned OpNo) const {
3708  const MCInstrDesc &Desc = get(MI.getOpcode());
3709  if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3710  Desc.OpInfo[OpNo].RegClass == -1) {
3711  unsigned Reg = MI.getOperand(OpNo).getReg();
3712 
3714  return MRI.getRegClass(Reg);
3715  return RI.getPhysRegClass(Reg);
3716  }
3717 
3718  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3719  return RI.getRegClass(RCID);
3720 }
3721 
3722 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3724  MachineBasicBlock *MBB = MI.getParent();
3725  MachineOperand &MO = MI.getOperand(OpIdx);
3727  const SIRegisterInfo *TRI =
3728  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3729  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3730  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3731  unsigned Size = TRI->getRegSizeInBits(*RC);
3732  unsigned Opcode = (Size == 64) ? AMDGPU::V_MOV_B64_PSEUDO : AMDGPU::V_MOV_B32_e32;
3733  if (MO.isReg())
3734  Opcode = AMDGPU::COPY;
3735  else if (RI.isSGPRClass(RC))
3736  Opcode = (Size == 64) ? AMDGPU::S_MOV_B64 : AMDGPU::S_MOV_B32;
3737 
3738  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3739  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3740  VRC = &AMDGPU::VReg_64RegClass;
3741  else
3742  VRC = &AMDGPU::VGPR_32RegClass;
3743 
3744  unsigned Reg = MRI.createVirtualRegister(VRC);
3745  DebugLoc DL = MBB->findDebugLoc(I);
3746  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3747  MO.ChangeToRegister(Reg, false);
3748 }
3749 
3752  MachineOperand &SuperReg,
3753  const TargetRegisterClass *SuperRC,
3754  unsigned SubIdx,
3755  const TargetRegisterClass *SubRC)
3756  const {
3757  MachineBasicBlock *MBB = MI->getParent();
3758  DebugLoc DL = MI->getDebugLoc();
3759  unsigned SubReg = MRI.createVirtualRegister(SubRC);
3760 
3761  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3762  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3763  .addReg(SuperReg.getReg(), 0, SubIdx);
3764  return SubReg;
3765  }
3766 
3767  // Just in case the super register is itself a sub-register, copy it to a new
3768  // value so we don't need to worry about merging its subreg index with the
3769  // SubIdx passed to this function. The register coalescer should be able to
3770  // eliminate this extra copy.
3771  unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC);
3772 
3773  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3774  .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3775 
3776  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3777  .addReg(NewSuperReg, 0, SubIdx);
3778 
3779  return SubReg;
3780 }
3781 
3785  MachineOperand &Op,
3786  const TargetRegisterClass *SuperRC,
3787  unsigned SubIdx,
3788  const TargetRegisterClass *SubRC) const {
3789  if (Op.isImm()) {
3790  if (SubIdx == AMDGPU::sub0)
3791  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3792  if (SubIdx == AMDGPU::sub1)
3793  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3794 
3795  llvm_unreachable("Unhandled register index for immediate");
3796  }
3797 
3798  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3799  SubIdx, SubRC);
3800  return MachineOperand::CreateReg(SubReg, false);
3801 }
3802 
3803 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3804 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3805  assert(Inst.getNumExplicitOperands() == 3);
3806  MachineOperand Op1 = Inst.getOperand(1);
3807  Inst.RemoveOperand(1);
3808  Inst.addOperand(Op1);
3809 }
3810 
3812  const MCOperandInfo &OpInfo,
3813  const MachineOperand &MO) const {
3814  if (!MO.isReg())
3815  return false;
3816 
3817  unsigned Reg = MO.getReg();
3818  const TargetRegisterClass *RC =
3820  MRI.getRegClass(Reg) :
3821  RI.getPhysRegClass(Reg);
3822 
3823  const SIRegisterInfo *TRI =
3824  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3825  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3826 
3827  // In order to be legal, the common sub-class must be equal to the
3828  // class of the current operand. For example:
3829  //
3830  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3831  // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3832  //
3833  // s_sendmsg 0, s0 ; Operand defined as m0reg
3834  // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3835 
3836  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3837 }
3838 
3840  const MCOperandInfo &OpInfo,
3841  const MachineOperand &MO) const {
3842  if (MO.isReg())
3843  return isLegalRegOperand(MRI, OpInfo, MO);
3844 
3845  // Handle non-register types that are treated like immediates.
3846  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI() || MO.isGlobal());
3847  return true;
3848 }
3849 
3850 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3851  const MachineOperand *MO) const {
3852  const MachineFunction &MF = *MI.getParent()->getParent();
3853  const MachineRegisterInfo &MRI = MF.getRegInfo();
3854  const MCInstrDesc &InstDesc = MI.getDesc();
3855  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3856  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
3857  const TargetRegisterClass *DefinedRC =
3858  OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3859  if (!MO)
3860  MO = &MI.getOperand(OpIdx);
3861 
3862  int ConstantBusLimit = ST.getConstantBusLimit(MI.getOpcode());
3863  int VOP3LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
3864  if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3865  if (isVOP3(MI) && isLiteralConstantLike(*MO, OpInfo) && !VOP3LiteralLimit--)
3866  return false;
3867 
3868  SmallDenseSet<RegSubRegPair> SGPRsUsed;
3869  if (MO->isReg())
3870  SGPRsUsed.insert(RegSubRegPair(MO->getReg(), MO->getSubReg()));
3871 
3872  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3873  if (i == OpIdx)
3874  continue;
3875  const MachineOperand &Op = MI.getOperand(i);
3876  if (Op.isReg()) {
3877  RegSubRegPair SGPR(Op.getReg(), Op.getSubReg());
3878  if (!SGPRsUsed.count(SGPR) &&
3879  usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3880  if (--ConstantBusLimit <= 0)
3881  return false;
3882  SGPRsUsed.insert(SGPR);
3883  }
3884  } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3885  if (--ConstantBusLimit <= 0)
3886  return false;
3887  } else if (isVOP3(MI) && AMDGPU::isSISrcOperand(InstDesc, i) &&
3888  isLiteralConstantLike(Op, InstDesc.OpInfo[i])) {
3889  if (!VOP3LiteralLimit--)
3890  return false;
3891  if (--ConstantBusLimit <= 0)
3892  return false;
3893  }
3894  }
3895  }
3896 
3897  if (MO->isReg()) {
3898  assert(DefinedRC);
3899  return isLegalRegOperand(MRI, OpInfo, *MO);
3900  }
3901 
3902  // Handle non-register types that are treated like immediates.
3903  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI() || MO->isGlobal());
3904 
3905  if (!DefinedRC) {
3906  // This operand expects an immediate.
3907  return true;
3908  }
3909 
3910  return isImmOperandLegal(MI, OpIdx, *MO);
3911 }
3912 
3914  MachineInstr &MI) const {
3915  unsigned Opc = MI.getOpcode();
3916  const MCInstrDesc &InstrDesc = get(Opc);
3917 
3918  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3919  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3920 
3921  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3922  MachineOperand &Src1 = MI.getOperand(Src1Idx);
3923 
3924  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3925  // we need to only have one constant bus use before GFX10.
3926  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3927  if (HasImplicitSGPR && ST.getConstantBusLimit(Opc) <= 1 &&
3928  Src0.isReg() && (RI.isSGPRReg(MRI, Src0.getReg()) ||
3929  isLiteralConstantLike(Src0, InstrDesc.OpInfo[Src0Idx])))
3930  legalizeOpWithMove(MI, Src0Idx);
3931 
3932  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3933  // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3934  // src0/src1 with V_READFIRSTLANE.
3935  if (Opc == AMDGPU::V_WRITELANE_B32) {
3936  const DebugLoc &DL = MI.getDebugLoc();
3937  if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3938  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3939  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3940  .add(Src0);
3941  Src0.ChangeToRegister(Reg, false);
3942  }
3943  if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3944  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3945  const DebugLoc &DL = MI.getDebugLoc();
3946  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3947  .add(Src1);
3948  Src1.ChangeToRegister(Reg, false);
3949  }
3950  return;
3951  }
3952 
3953  // No VOP2 instructions support AGPRs.
3954  if (Src0.isReg() && RI.isAGPR(MRI, Src0.getReg()))
3955  legalizeOpWithMove(MI, Src0Idx);
3956 
3957  if (Src1.isReg() && RI.isAGPR(MRI, Src1.getReg()))
3958  legalizeOpWithMove(MI, Src1Idx);
3959 
3960  // VOP2 src0 instructions support all operand types, so we don't need to check
3961  // their legality. If src1 is already legal, we don't need to do anything.
3962  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3963  return;
3964 
3965  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3966  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3967  // select is uniform.
3968  if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3969  RI.isVGPR(MRI, Src1.getReg())) {
3970  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3971  const DebugLoc &DL = MI.getDebugLoc();
3972  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3973  .add(Src1);
3974  Src1.ChangeToRegister(Reg, false);
3975  return;
3976  }
3977 
3978  // We do not use commuteInstruction here because it is too aggressive and will
3979  // commute if it is possible. We only want to commute here if it improves
3980  // legality. This can be called a fairly large number of times so don't waste
3981  // compile time pointlessly swapping and checking legality again.
3982  if (HasImplicitSGPR || !MI.isCommutable()) {
3983  legalizeOpWithMove(MI, Src1Idx);
3984  return;
3985  }
3986 
3987  // If src0 can be used as src1, commuting will make the operands legal.
3988  // Otherwise we have to give up and insert a move.
3989  //
3990  // TODO: Other immediate-like operand kinds could be commuted if there was a
3991  // MachineOperand::ChangeTo* for them.
3992  if ((!Src1.isImm() && !Src1.isReg()) ||
3993  !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
3994  legalizeOpWithMove(MI, Src1Idx);
3995  return;
3996  }
3997 
3998  int CommutedOpc = commuteOpcode(MI);
3999  if (CommutedOpc == -1) {
4000  legalizeOpWithMove(MI, Src1Idx);
4001  return;
4002  }
4003 
4004  MI.setDesc(get(CommutedOpc));
4005 
4006  unsigned Src0Reg = Src0.getReg();
4007  unsigned Src0SubReg = Src0.getSubReg();
4008  bool Src0Kill = Src0.isKill();
4009 
4010  if (Src1.isImm())
4011  Src0.ChangeToImmediate(Src1.getImm());
4012  else if (Src1.isReg()) {
4013  Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
4014  Src0.setSubReg(Src1.getSubReg());
4015  } else
4016  llvm_unreachable("Should only have register or immediate operands");
4017 
4018  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
4019  Src1.setSubReg(Src0SubReg);
4020  fixImplicitOperands(MI);
4021 }
4022 
4023 // Legalize VOP3 operands. All operand types are supported for any operand
4024 // but only one literal constant and only starting from GFX10.
4026  MachineInstr &MI) const {
4027  unsigned Opc = MI.getOpcode();
4028 
4029  int VOP3Idx[3] = {
4030  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
4031  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
4032  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
4033  };
4034 
4035  if (Opc == AMDGPU::V_PERMLANE16_B32 ||
4036  Opc == AMDGPU::V_PERMLANEX16_B32) {
4037  // src1 and src2 must be scalar
4038  MachineOperand &Src1 = MI.getOperand(VOP3Idx[1]);
4039  MachineOperand &Src2 = MI.getOperand(VOP3Idx[2]);
4040  const DebugLoc &DL = MI.getDebugLoc();
4041  if (Src1.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src1.getReg()))) {
4042  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4043  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4044  .add(Src1);
4045  Src1.ChangeToRegister(Reg, false);
4046  }
4047  if (Src2.isReg() && !RI.isSGPRClass(MRI.getRegClass(Src2.getReg()))) {
4048  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
4049  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
4050  .add(Src2);
4051  Src2.ChangeToRegister(Reg, false);
4052  }
4053  }
4054 
4055  // Find the one SGPR operand we are allowed to use.
4056  int ConstantBusLimit = ST.getConstantBusLimit(Opc);
4057  int LiteralLimit = ST.hasVOP3Literal() ? 1 : 0;
4058  SmallDenseSet<unsigned> SGPRsUsed;
4059  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
4060  if (SGPRReg != AMDGPU::NoRegister) {
4061  SGPRsUsed.insert(SGPRReg);
4062  --ConstantBusLimit;
4063  }
4064 
4065  for (unsigned i = 0; i < 3; ++i) {
4066  int Idx = VOP3Idx[i];
4067  if (Idx == -1)
4068  break;
4069  MachineOperand &MO = MI.getOperand(Idx);
4070 
4071  if (!MO.isReg()) {
4072  if (!isLiteralConstantLike(MO, get(Opc).OpInfo[Idx]))
4073  continue;
4074 
4075  if (LiteralLimit > 0 && ConstantBusLimit > 0) {
4076  --LiteralLimit;
4077  --ConstantBusLimit;
4078  continue;
4079  }
4080 
4081  --LiteralLimit;
4082  --ConstantBusLimit;
4083  legalizeOpWithMove(MI, Idx);
4084  continue;
4085  }
4086 
4087  if (RI.hasAGPRs(MRI.getRegClass(MO.getReg())) &&
4088  !isOperandLegal(MI, Idx, &MO)) {
4089  legalizeOpWithMove(MI, Idx);
4090  continue;
4091  }
4092 
4093  if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
4094  continue; // VGPRs are legal
4095 
4096  // We can use one SGPR in each VOP3 instruction prior to GFX10
4097  // and two starting from GFX10.
4098  if (SGPRsUsed.count(MO.getReg()))
4099  continue;
4100  if (ConstantBusLimit > 0) {
4101  SGPRsUsed.insert(MO.getReg());
4102  --ConstantBusLimit;
4103  continue;
4104  }
4105 
4106  // If we make it this far, then the operand is not legal and we must
4107  // legalize it.
4108  legalizeOpWithMove(MI, Idx);
4109  }
4110 }
4111 
4113  MachineRegisterInfo &MRI) const {
4114  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
4115  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
4116  unsigned DstReg = MRI.createVirtualRegister(SRC);
4117  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
4118 
4119  if (RI.hasAGPRs(VRC)) {
4120  VRC = RI.getEquivalentVGPRClass(VRC);
4121  unsigned NewSrcReg = MRI.createVirtualRegister(VRC);
4122  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4123  get(TargetOpcode::COPY), NewSrcReg)
4124  .addReg(SrcReg);
4125  SrcReg = NewSrcReg;
4126  }
4127 
4128  if (SubRegs == 1) {
4129  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4130  get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
4131  .addReg(SrcReg);
4132  return DstReg;
4133  }
4134 
4136  for (unsigned i = 0; i < SubRegs; ++i) {
4137  unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4138  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4139  get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
4140  .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
4141  SRegs.push_back(SGPR);
4142  }
4143 
4144  MachineInstrBuilder MIB =
4145  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
4146  get(AMDGPU::REG_SEQUENCE), DstReg);
4147  for (unsigned i = 0; i < SubRegs; ++i) {
4148  MIB.addReg(SRegs[i]);
4149  MIB.addImm(RI.getSubRegFromChannel(i));
4150  }
4151  return DstReg;
4152 }
4153 
4155  MachineInstr &MI) const {
4156 
4157  // If the pointer is store in VGPRs, then we need to move them to
4158  // SGPRs using v_readfirstlane. This is safe because we only select
4159  // loads with uniform pointers to SMRD instruction so we know the
4160  // pointer value is uniform.
4161  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
4162  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
4163  unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
4164  SBase->setReg(SGPR);
4165  }
4166  MachineOperand *SOff = getNamedOperand(MI, AMDGPU::OpName::soff);
4167  if (SOff && !RI.isSGPRClass(MRI.getRegClass(SOff->getReg()))) {
4168  unsigned SGPR = readlaneVGPRToSGPR(SOff->getReg(), MI, MRI);
4169  SOff->setReg(SGPR);
4170  }
4171 }
4172 
4175  const TargetRegisterClass *DstRC,
4176  MachineOperand &Op,
4178  const DebugLoc &DL) const {
4179  unsigned OpReg = Op.getReg();
4180  unsigned OpSubReg = Op.getSubReg();
4181 
4182  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
4183  RI.getRegClassForReg(MRI, OpReg), OpSubReg);
4184 
4185  // Check if operand is already the correct register class.
4186  if (DstRC == OpRC)
4187  return;
4188 
4189  unsigned DstReg = MRI.createVirtualRegister(DstRC);
4190  MachineInstr *Copy =
4191  BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
4192 
4193  Op.setReg(DstReg);
4194  Op.setSubReg(0);
4195 
4196  MachineInstr *Def = MRI.getVRegDef(OpReg);
4197  if (!Def)
4198  return;
4199 
4200  // Try to eliminate the copy if it is copying an immediate value.
4201  if (Def->isMoveImmediate())
4202  FoldImmediate(*Copy, *Def, OpReg, &MRI);
4203 }
4204 
4205 // Emit the actual waterfall loop, executing the wrapped instruction for each
4206 // unique value of \p Rsrc across all lanes. In the best case we execute 1
4207 // iteration, in the worst case we execute 64 (once per lane).
4208 static void
4210  MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
4211  const DebugLoc &DL, MachineOperand &Rsrc) {
4212  MachineFunction &MF = *OrigBB.getParent();
4213  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4214  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4215  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4216  unsigned SaveExecOpc =
4217  ST.isWave32() ? AMDGPU::S_AND_SAVEEXEC_B32 : AMDGPU::S_AND_SAVEEXEC_B64;
4218  unsigned XorTermOpc =
4219  ST.isWave32() ? AMDGPU::S_XOR_B32_term : AMDGPU::S_XOR_B64_term;
4220  unsigned AndOpc =
4221  ST.isWave32() ? AMDGPU::S_AND_B32 : AMDGPU::S_AND_B64;
4222  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4223 
4224  MachineBasicBlock::iterator I = LoopBB.begin();
4225 
4226  unsigned VRsrc = Rsrc.getReg();
4227  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
4228 
4229  unsigned SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4230  unsigned CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4231  unsigned CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4232  unsigned AndCond = MRI.createVirtualRegister(BoolXExecRC);
4233  unsigned SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4234  unsigned SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4235  unsigned SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4236  unsigned SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4237  unsigned SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4238 
4239  // Beginning of the loop, read the next Rsrc variant.
4240  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
4241  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
4242  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
4243  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
4244  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
4245  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
4246  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
4247  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
4248 
4249  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
4250  .addReg(SRsrcSub0)
4251  .addImm(AMDGPU::sub0)
4252  .addReg(SRsrcSub1)
4253  .addImm(AMDGPU::sub1)
4254  .addReg(SRsrcSub2)
4255  .addImm(AMDGPU::sub2)
4256  .addReg(SRsrcSub3)
4257  .addImm(AMDGPU::sub3);
4258 
4259  // Update Rsrc operand to use the SGPR Rsrc.
4260  Rsrc.setReg(SRsrc);
4261  Rsrc.setIsKill(true);
4262 
4263  // Identify all lanes with identical Rsrc operands in their VGPRs.
4264  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
4265  .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
4266  .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
4267  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
4268  .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
4269  .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
4270  BuildMI(LoopBB, I, DL, TII.get(AndOpc), AndCond)
4271  .addReg(CondReg0)
4272  .addReg(CondReg1);
4273 
4274  MRI.setSimpleHint(SaveExec, AndCond);
4275 
4276  // Update EXEC to matching lanes, saving original to SaveExec.
4277  BuildMI(LoopBB, I, DL, TII.get(SaveExecOpc), SaveExec)
4278  .addReg(AndCond, RegState::Kill);
4279 
4280  // The original instruction is here; we insert the terminators after it.
4281  I = LoopBB.end();
4282 
4283  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
4284  BuildMI(LoopBB, I, DL, TII.get(XorTermOpc), Exec)
4285  .addReg(Exec)
4286  .addReg(SaveExec);
4287  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
4288 }
4289 
4290 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
4291 // with SGPRs by iterating over all unique values across all lanes.
4293  MachineOperand &Rsrc, MachineDominatorTree *MDT) {
4294  MachineBasicBlock &MBB = *MI.getParent();
4295  MachineFunction &MF = *MBB.getParent();
4296  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
4297  const SIRegisterInfo *TRI = ST.getRegisterInfo();
4300  const DebugLoc &DL = MI.getDebugLoc();
4301  unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
4302  unsigned MovExecOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
4303  const auto *BoolXExecRC = TRI->getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4304 
4305  unsigned SaveExec = MRI.createVirtualRegister(BoolXExecRC);
4306 
4307  // Save the EXEC mask
4308  BuildMI(MBB, I, DL, TII.get(MovExecOpc), SaveExec).addReg(Exec);
4309 
4310  // Killed uses in the instruction we are waterfalling around will be
4311  // incorrect due to the added control-flow.
4312  for (auto &MO : MI.uses()) {
4313  if (MO.isReg() && MO.isUse()) {
4314  MRI.clearKillFlags(MO.getReg());
4315  }
4316  }
4317 
4318  // To insert the loop we need to split the block. Move everything after this
4319  // point to a new block, and insert a new empty block between the two.
4321  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
4322  MachineFunction::iterator MBBI(MBB);
4323  ++MBBI;
4324 
4325  MF.insert(MBBI, LoopBB);
4326  MF.insert(MBBI, RemainderBB);
4327 
4328  LoopBB->addSuccessor(LoopBB);
4329  LoopBB->addSuccessor(RemainderBB);
4330 
4331  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
4333  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
4334  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
4335  LoopBB->splice(LoopBB->begin(), &MBB, J);
4336 
4337  MBB.addSuccessor(LoopBB);
4338 
4339  // Update dominators. We know that MBB immediately dominates LoopBB, that
4340  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
4341  // dominates all of the successors transferred to it from MBB that MBB used
4342  // to dominate.
4343  if (MDT) {
4344  MDT->addNewBlock(LoopBB, &MBB);
4345  MDT->addNewBlock(RemainderBB, LoopBB);
4346  for (auto &Succ : RemainderBB->successors()) {
4347  if (MDT->dominates(&MBB, Succ)) {
4348  MDT->changeImmediateDominator(Succ, RemainderBB);
4349  }
4350  }
4351  }
4352 
4353  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
4354 
4355  // Restore the EXEC mask
4356  MachineBasicBlock::iterator First = RemainderBB->begin();
4357  BuildMI(*RemainderBB, First, DL, TII.get(MovExecOpc), Exec).addReg(SaveExec);
4358 }
4359 
4360 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
4361 static std::tuple<unsigned, unsigned>
4363  MachineBasicBlock &MBB = *MI.getParent();
4364  MachineFunction &MF = *MBB.getParent();
4366 
4367  // Extract the ptr from the resource descriptor.
4368  unsigned RsrcPtr =
4369  TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
4370  AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
4371 
4372  // Create an empty resource descriptor
4373  unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
4374  unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4375  unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
4376  unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
4377  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
4378 
4379  // Zero64 = 0
4380  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
4381  .addImm(0);
4382 
4383  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
4384  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
4385  .addImm(RsrcDataFormat & 0xFFFFFFFF);
4386 
4387  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
4388  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
4389  .addImm(RsrcDataFormat >> 32);
4390 
4391  // NewSRsrc = {Zero64, SRsrcFormat}
4392  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
4393  .addReg(Zero64)
4394  .addImm(AMDGPU::sub0_sub1)
4395  .addReg(SRsrcFormatLo)
4396  .addImm(AMDGPU::sub2)
4397  .addReg(SRsrcFormatHi)
4398  .addImm(AMDGPU::sub3);
4399 
4400  return std::make_tuple(RsrcPtr, NewSRsrc);
4401 }
4402 
4404  MachineDominatorTree *MDT) const {
4405  MachineFunction &MF = *MI.getParent()->getParent();
4407 
4408  // Legalize VOP2
4409  if (isVOP2(MI) || isVOPC(MI)) {
4410  legalizeOperandsVOP2(MRI, MI);
4411  return;
4412  }
4413 
4414  // Legalize VOP3
4415  if (isVOP3(MI)) {
4416  legalizeOperandsVOP3(MRI, MI);
4417  return;
4418  }
4419 
4420  // Legalize SMRD
4421  if (isSMRD(MI)) {
4422  legalizeOperandsSMRD(MRI, MI);
4423  return;
4424  }
4425 
4426  // Legalize REG_SEQUENCE and PHI
4427  // The register class of the operands much be the same type as the register
4428  // class of the output.
4429  if (MI.getOpcode() == AMDGPU::PHI) {
4430  const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
4431  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
4432  if (!MI.getOperand(i).isReg() ||
4434  continue;
4435  const TargetRegisterClass *OpRC =
4436  MRI.getRegClass(MI.getOperand(i).getReg());
4437  if (RI.hasVectorRegisters(OpRC)) {
4438  VRC = OpRC;
4439  } else {
4440  SRC = OpRC;
4441  }
4442  }
4443 
4444  // If any of the operands are VGPR registers, then they all most be
4445  // otherwise we will create illegal VGPR->SGPR copies when legalizing
4446  // them.
4447  if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
4448  if (!VRC) {
4449  assert(SRC);
4450  VRC = RI.hasAGPRs(getOpRegClass(MI, 0)) ? RI.getEquivalentAGPRClass(SRC)
4451  : RI.getEquivalentVGPRClass(SRC);
4452  }
4453  RC = VRC;
4454  } else {
4455  RC = SRC;
4456  }
4457 
4458  // Update all the operands so they have the same type.
4459  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4460  MachineOperand &Op = MI.getOperand(I);
4462  continue;
4463 
4464  // MI is a PHI instruction.
4465  MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
4466  MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
4467 
4468  // Avoid creating no-op copies with the same src and dst reg class. These
4469  // confuse some of the machine passes.
4470  legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
4471  }
4472  }
4473 
4474  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
4475  // VGPR dest type and SGPR sources, insert copies so all operands are
4476  // VGPRs. This seems to help operand folding / the register coalescer.
4477  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
4478  MachineBasicBlock *MBB = MI.getParent();
4479  const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
4480  if (RI.hasVGPRs(DstRC)) {
4481  // Update all the operands so they are VGPR register classes. These may
4482  // not be the same register class because REG_SEQUENCE supports mixing
4483  // subregister index types e.g. sub0_sub1 + sub2 + sub3
4484  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
4485  MachineOperand &Op = MI.getOperand(I);
4487  continue;
4488 
4489  const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
4490  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
4491  if (VRC == OpRC)
4492  continue;
4493 
4494  legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
4495  Op.setIsKill();
4496  }
4497  }
4498 
4499  return;
4500  }
4501 
4502  // Legalize INSERT_SUBREG
4503  // src0 must have the same register class as dst
4504  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
4505  unsigned Dst = MI.getOperand(0).getReg();
4506  unsigned Src0 = MI.getOperand(1).getReg();
4507  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
4508  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
4509  if (DstRC != Src0RC) {
4510  MachineBasicBlock *MBB = MI.getParent();
4511  MachineOperand &Op = MI.getOperand(1);
4512  legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
4513  }
4514  return;
4515  }
4516 
4517  // Legalize SI_INIT_M0
4518  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
4519  MachineOperand &Src = MI.getOperand(0);
4520  if (Src.isReg() && RI.hasVectorRegisters(MRI.getRegClass(Src.getReg())))
4521  Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
4522  return;
4523  }
4524 
4525  // Legalize MIMG and MUBUF/MTBUF for shaders.
4526  //
4527  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
4528  // scratch memory access. In both cases, the legalization never involves
4529  // conversion to the addr64 form.
4530  if (isMIMG(MI) ||
4532  (isMUBUF(MI) || isMTBUF(MI)))) {
4533  MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
4534  if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
4535  unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
4536  SRsrc->setReg(SGPR);
4537  }
4538 
4539  MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
4540  if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
4541  unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
4542  SSamp->setReg(SGPR);
4543  }
4544  return;
4545  }
4546 
4547  // Legalize MUBUF* instructions.
4548  int RsrcIdx =
4549  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
4550  if (RsrcIdx != -1) {
4551  // We have an MUBUF instruction
4552  MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
4553  unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
4554  if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
4555  RI.getRegClass(RsrcRC))) {
4556  // The operands are legal.
4557  // FIXME: We may need to legalize operands besided srsrc.
4558  return;
4559  }
4560 
4561  // Legalize a VGPR Rsrc.
4562  //
4563  // If the instruction is _ADDR64, we can avoid a waterfall by extracting
4564  // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
4565  // a zero-value SRsrc.
4566  //
4567  // If the instruction is _OFFSET (both idxen and offen disabled), and we
4568  // support ADDR64 instructions, we can convert to ADDR64 and do the same as
4569  // above.
4570  //
4571  // Otherwise we are on non-ADDR64 hardware, and/or we have
4572  // idxen/offen/bothen and we fall back to a waterfall loop.
4573 
4574  MachineBasicBlock &MBB = *MI.getParent();
4575 
4576  MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
4577  if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
4578  // This is already an ADDR64 instruction so we need to add the pointer
4579  // extracted from the resource descriptor to the current value of VAddr.
4580  unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4581  unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4582  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4583 
4584  const auto *BoolXExecRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
4585  unsigned CondReg0 = MRI.createVirtualRegister(BoolXExecRC);
4586  unsigned CondReg1 = MRI.createVirtualRegister(BoolXExecRC);
4587 
4588  unsigned RsrcPtr, NewSRsrc;
4589  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4590 
4591  // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
4592  const DebugLoc &DL = MI.getDebugLoc();
4593  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e64), NewVAddrLo)
4594  .addDef(CondReg0)
4595  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4596  .addReg(VAddr->getReg(), 0, AMDGPU::sub0)
4597  .addImm(0);
4598 
4599  // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
4600  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e64), NewVAddrHi)
4601  .addDef(CondReg1, RegState::Dead)
4602  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4603  .addReg(VAddr->getReg(), 0, AMDGPU::sub1)
4604  .addReg(CondReg0, RegState::Kill)
4605  .addImm(0);
4606 
4607  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4608  BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4609  .addReg(NewVAddrLo)
4610  .addImm(AMDGPU::sub0)
4611  .addReg(NewVAddrHi)
4612  .addImm(AMDGPU::sub1);
4613 
4614  VAddr->setReg(NewVAddr);
4615  Rsrc->setReg(NewSRsrc);
4616  } else if (!VAddr && ST.hasAddr64()) {
4617  // This instructions is the _OFFSET variant, so we need to convert it to
4618  // ADDR64.
4619  assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4621  "FIXME: Need to emit flat atomics here");
4622 
4623  unsigned RsrcPtr, NewSRsrc;
4624  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4625 
4626  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4627  MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4628  MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4629  MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4630  unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4631 
4632  // Atomics rith return have have an additional tied operand and are
4633  // missing some of the special bits.
4634  MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4635  MachineInstr *Addr64;
4636 
4637  if (!VDataIn) {
4638  // Regular buffer load / store.
4639  MachineInstrBuilder MIB =
4640  BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4641  .add(*VData)
4642  .addReg(NewVAddr)
4643  .addReg(NewSRsrc)
4644  .add(*SOffset)
4645  .add(*Offset);
4646 
4647  // Atomics do not have this operand.
4648  if (const MachineOperand *GLC =
4649  getNamedOperand(MI, AMDGPU::OpName::glc)) {
4650  MIB.addImm(GLC->getImm());
4651  }
4652  if (const MachineOperand *DLC =
4653  getNamedOperand(MI, AMDGPU::OpName::dlc)) {
4654  MIB.addImm(DLC->getImm());
4655  }
4656 
4657  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4658 
4659  if (const MachineOperand *TFE =
4660  getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4661  MIB.addImm(TFE->getImm());
4662  }
4663 
4664  MIB.cloneMemRefs(MI);
4665  Addr64 = MIB;
4666  } else {
4667  // Atomics with return.
4668  Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4669  .add(*VData)
4670  .add(*VDataIn)
4671  .addReg(NewVAddr)
4672  .addReg(NewSRsrc)
4673  .add(*SOffset)
4674  .add(*Offset)
4675  .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4676  .cloneMemRefs(MI);
4677  }
4678 
4679  MI.removeFromParent();
4680 
4681  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4682  BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4683  NewVAddr)
4684  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4685  .addImm(AMDGPU::sub0)
4686  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4687  .addImm(AMDGPU::sub1);
4688  } else {
4689  // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4690  // to SGPRs.
4691  loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4692  }
4693  }
4694 }
4695 
4697  MachineDominatorTree *MDT) const {
4698  SetVectorType Worklist;
4699  Worklist.insert(&TopInst);
4700 
4701  while (!Worklist.empty()) {
4702  MachineInstr &Inst = *Worklist.pop_back_val();
4703  MachineBasicBlock *MBB = Inst.getParent();
4705 
4706  unsigned Opcode = Inst.getOpcode();
4707  unsigned NewOpcode = getVALUOp(Inst);
4708 
4709  // Handle some special cases
4710  switch (Opcode) {
4711  default:
4712  break;
4713  case AMDGPU::S_ADD_U64_PSEUDO:
4714  case AMDGPU::S_SUB_U64_PSEUDO:
4715  splitScalar64BitAddSub(Worklist, Inst, MDT);
4716  Inst.eraseFromParent();
4717  continue;
4718  case AMDGPU::S_ADD_I32:
4719  case AMDGPU::S_SUB_I32:
4720  // FIXME: The u32 versions currently selected use the carry.
4721  if (moveScalarAddSub(Worklist, Inst, MDT))
4722  continue;
4723 
4724  // Default handling
4725  break;
4726  case AMDGPU::S_AND_B64:
4727  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_AND_B32, MDT);
4728  Inst.eraseFromParent();
4729  continue;
4730 
4731  case AMDGPU::S_OR_B64:
4732  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_OR_B32, MDT);
4733  Inst.eraseFromParent();
4734  continue;
4735 
4736  case AMDGPU::S_XOR_B64:
4737  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XOR_B32, MDT);
4738  Inst.eraseFromParent();
4739  continue;
4740 
4741  case AMDGPU::S_NAND_B64:
4742  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NAND_B32, MDT);
4743  Inst.eraseFromParent();
4744  continue;
4745 
4746  case AMDGPU::S_NOR_B64:
4747  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_NOR_B32, MDT);
4748  Inst.eraseFromParent();
4749  continue;
4750 
4751  case AMDGPU::S_XNOR_B64:
4752  if (ST.hasDLInsts())
4753  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4754  else
4755  splitScalar64BitXnor(Worklist, Inst, MDT);
4756  Inst.eraseFromParent();
4757  continue;
4758 
4759  case AMDGPU::S_ANDN2_B64:
4760  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ANDN2_B32, MDT);
4761  Inst.eraseFromParent();
4762  continue;
4763 
4764  case AMDGPU::S_ORN2_B64:
4765  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_ORN2_B32, MDT);
4766  Inst.eraseFromParent();
4767  continue;
4768 
4769  case AMDGPU::S_NOT_B64:
4770  splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::S_NOT_B32);
4771  Inst.eraseFromParent();
4772  continue;
4773 
4774  case AMDGPU::S_BCNT1_I32_B64:
4775  splitScalar64BitBCNT(Worklist, Inst);
4776  Inst.eraseFromParent();
4777  continue;
4778 
4779  case AMDGPU::S_BFE_I64:
4780  splitScalar64BitBFE(Worklist, Inst);
4781  Inst.eraseFromParent();
4782  continue;
4783 
4784  case AMDGPU::S_LSHL_B32:
4785  if (ST.hasOnlyRevVALUShifts()) {
4786  NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4787  swapOperands(Inst);
4788  }
4789  break;
4790  case AMDGPU::S_ASHR_I32:
4791  if (ST.hasOnlyRevVALUShifts()) {
4792  NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4793  swapOperands(Inst);
4794  }
4795  break;
4796  case AMDGPU::S_LSHR_B32:
4797  if (ST.hasOnlyRevVALUShifts()) {
4798  NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4799  swapOperands(Inst);
4800  }
4801  break;
4802  case AMDGPU::S_LSHL_B64:
4803  if (ST.hasOnlyRevVALUShifts()) {
4804  NewOpcode = AMDGPU::V_LSHLREV_B64;
4805  swapOperands(Inst);
4806  }
4807  break;
4808  case AMDGPU::S_ASHR_I64:
4809  if (ST.hasOnlyRevVALUShifts()) {
4810  NewOpcode = AMDGPU::V_ASHRREV_I64;
4811  swapOperands(Inst);
4812  }
4813  break;
4814  case AMDGPU::S_LSHR_B64:
4815  if (ST.hasOnlyRevVALUShifts()) {
4816  NewOpcode = AMDGPU::V_LSHRREV_B64;
4817  swapOperands(Inst);
4818  }
4819  break;
4820 
4821  case AMDGPU::S_ABS_I32:
4822  lowerScalarAbs(Worklist, Inst);
4823  Inst.eraseFromParent();
4824  continue;
4825 
4826  case AMDGPU::S_CBRANCH_SCC0:
4827  case AMDGPU::S_CBRANCH_SCC1:
4828  // Clear unused bits of vcc
4829  if (ST.isWave32())
4830  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B32),
4831  AMDGPU::VCC_LO)
4832  .addReg(AMDGPU::EXEC_LO)
4833  .addReg(AMDGPU::VCC_LO);
4834  else
4835  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4836  AMDGPU::VCC)
4837  .addReg(AMDGPU::EXEC)
4838  .addReg(AMDGPU::VCC);
4839  break;
4840 
4841  case AMDGPU::S_BFE_U64:
4842  case AMDGPU::S_BFM_B64:
4843  llvm_unreachable("Moving this op to VALU not implemented");
4844 
4845  case AMDGPU::S_PACK_LL_B32_B16:
4846  case AMDGPU::S_PACK_LH_B32_B16:
4847  case AMDGPU::S_PACK_HH_B32_B16:
4848  movePackToVALU(Worklist, MRI, Inst);
4849  Inst.eraseFromParent();
4850  continue;
4851 
4852  case AMDGPU::S_XNOR_B32:
4853  lowerScalarXnor(Worklist, Inst);
4854  Inst.eraseFromParent();
4855  continue;
4856 
4857  case AMDGPU::S_NAND_B32:
4858  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_AND_B32);
4859  Inst.eraseFromParent();
4860  continue;
4861 
4862  case AMDGPU::S_NOR_B32:
4863  splitScalarNotBinop(Worklist, Inst, AMDGPU::S_OR_B32);
4864  Inst.eraseFromParent();
4865  continue;
4866 
4867  case AMDGPU::S_ANDN2_B32:
4868  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_AND_B32);
4869  Inst.eraseFromParent();
4870  continue;
4871 
4872  case AMDGPU::S_ORN2_B32:
4873  splitScalarBinOpN2(Worklist, Inst, AMDGPU::S_OR_B32);
4874  Inst.eraseFromParent();
4875  continue;
4876  }
4877 
4878  if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4879  // We cannot move this instruction to the VALU, so we should try to
4880  // legalize its operands instead.
4881  legalizeOperands(Inst, MDT);
4882  continue;
4883  }
4884 
4885  // Use the new VALU Opcode.
4886  const MCInstrDesc &NewDesc = get(NewOpcode);
4887  Inst.setDesc(NewDesc);
4888 
4889  // Remove any references to SCC. Vector instructions can't read from it, and
4890  // We're just about to add the implicit use / defs of VCC, and we don't want
4891  // both.
4892  for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
4893  MachineOperand &Op = Inst.getOperand(i);
4894  if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
4895  // Only propagate through live-def of SCC.
4896  if (Op.isDef() && !Op.isDead())
4897  addSCCDefUsersToVALUWorklist(Op, Inst, Worklist);
4898  Inst.RemoveOperand(i);
4899  }
4900  }
4901 
4902  if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4903  // We are converting these to a BFE, so we need to add the missing
4904  // operands for the size and offset.
4905  unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4908 
4909  } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4910  // The VALU version adds the second operand to the result, so insert an
4911  // extra 0 operand.
4913  }
4914 
4916  fixImplicitOperands(Inst);
4917 
4918  if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4919  const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4920  // If we need to move this to VGPRs, we need to unpack the second operand
4921  // back into the 2 separate ones for bit offset and width.
4922  assert(OffsetWidthOp.isImm() &&
4923  "Scalar BFE is only implemented for constant width and offset");
4924  uint32_t Imm = OffsetWidthOp.getImm();
4925 
4926  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4927  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4928  Inst.RemoveOperand(2); // Remove old immediate.
4929  Inst.addOperand(MachineOperand::CreateImm(Offset));
4930  Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4931  }
4932 
4933  bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4934  unsigned NewDstReg = AMDGPU::NoRegister;
4935  if (HasDst) {
4936  unsigned DstReg = Inst.getOperand(0).getReg();
4938  continue;
4939 
4940  // Update the destination register class.
4941  const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4942  if (!NewDstRC)
4943  continue;
4944 
4945  if (Inst.isCopy() &&
4947  NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4948  // Instead of creating a copy where src and dst are the same register
4949  // class, we just replace all uses of dst with src. These kinds of
4950  // copies interfere with the heuristics MachineSink uses to decide
4951  // whether or not to split a critical edge. Since the pass assumes
4952  // that copies will end up as machine instructions and not be
4953  // eliminated.
4954  addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4955  MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4956  MRI.clearKillFlags(Inst.getOperand(1).getReg());
4957  Inst.getOperand(0).setReg(DstReg);
4958 
4959  // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4960  // these are deleted later, but at -O0 it would leave a suspicious
4961  // looking illegal copy of an undef register.
4962  for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4963  Inst.RemoveOperand(I);
4964  Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4965  continue;
4966  }
4967 
4968  NewDstReg = MRI.createVirtualRegister(NewDstRC);
4969  MRI.replaceRegWith(DstReg, NewDstReg);
4970  }
4971 
4972  // Legalize the operands
4973  legalizeOperands(Inst, MDT);
4974 
4975  if (HasDst)
4976  addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
4977  }
4978 }
4979 
4980 // Add/sub require special handling to deal with carry outs.
4981 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
4982  MachineDominatorTree *MDT) const {
4983  if (ST.hasAddNoCarry()) {
4984  // Assume there is no user of scc since we don't select this in that case.
4985  // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
4986  // is used.
4987 
4988  MachineBasicBlock &MBB = *Inst.getParent();
4990 
4991  unsigned OldDstReg = Inst.getOperand(0).getReg();
4992  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4993 
4994  unsigned Opc = Inst.getOpcode();
4995  assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
4996 
4997  unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
4998  AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
4999 
5000  assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
5001  Inst.RemoveOperand(3);
5002 
5003  Inst.setDesc(get(NewOpc));
5004  Inst.addOperand(MachineOperand::CreateImm(0)); // clamp bit
5005  Inst.addImplicitDefUseOperands(*MBB.getParent());
5006  MRI.replaceRegWith(OldDstReg, ResultReg);
5007  legalizeOperands(Inst, MDT);
5008 
5009  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5010  return true;
5011  }
5012 
5013  return false;
5014 }
5015 
5016 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
5017  MachineInstr &Inst) const {
5018  MachineBasicBlock &MBB = *Inst.getParent();
5020  MachineBasicBlock::iterator MII = Inst;
5021  DebugLoc DL = Inst.getDebugLoc();
5022 
5023  MachineOperand &Dest = Inst.getOperand(0);
5024  MachineOperand &Src = Inst.getOperand(1);
5025  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5026  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5027 
5028  unsigned SubOp = ST.hasAddNoCarry() ?
5029  AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
5030 
5031  BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
5032  .addImm(0)
5033  .addReg(Src.getReg());
5034 
5035  BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
5036  .addReg(Src.getReg())
5037  .addReg(TmpReg);
5038 
5039  MRI.replaceRegWith(Dest.getReg(), ResultReg);
5040  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
5041 }
5042 
5043 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
5044  MachineInstr &Inst) const {
5045  MachineBasicBlock &MBB = *Inst.getParent();
5047  MachineBasicBlock::iterator MII = Inst;
5048  const DebugLoc &DL = Inst.getDebugLoc();
5049 
5050  MachineOperand &Dest = Inst.getOperand(0);
5051  MachineOperand &Src0 = Inst.getOperand(1);
5052  MachineOperand &Src1 = Inst.getOperand(2);
5053 
5054  if (ST.hasDLInsts()) {
5055  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5056  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
5057  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
5058 
5059  BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
5060  .add(Src0)
5061  .add(Src1);
5062 
5063  MRI.replaceRegWith(Dest.getReg(), NewDest);
5064  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5065  } else {
5066  // Using the identity !(x ^ y) == (!x ^ y) == (x ^ !y), we can
5067  // invert either source and then perform the XOR. If either source is a
5068  // scalar register, then we can leave the inversion on the scalar unit to
5069  // acheive a better distrubution of scalar and vector instructions.
5070  bool Src0IsSGPR = Src0.isReg() &&
5071  RI.isSGPRClass(MRI.getRegClass(Src0.getReg()));
5072  bool Src1IsSGPR = Src1.isReg() &&
5073  RI.isSGPRClass(MRI.getRegClass(Src1.getReg()));
5074  MachineInstr *Xor;
5075  unsigned Temp = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5076  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5077 
5078  // Build a pair of scalar instructions and add them to the work list.
5079  // The next iteration over the work list will lower these to the vector
5080  // unit as necessary.
5081  if (Src0IsSGPR) {
5082  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src0);
5083  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5084  .addReg(Temp)
5085  .add(Src1);
5086  } else if (Src1IsSGPR) {
5087  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Temp).add(Src1);
5088  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), NewDest)
5089  .add(Src0)
5090  .addReg(Temp);
5091  } else {
5092  Xor = BuildMI(MBB, MII, DL, get(AMDGPU::S_XOR_B32), Temp)
5093  .add(Src0)
5094  .add(Src1);
5095  MachineInstr *Not =
5096  BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest).addReg(Temp);
5097  Worklist.insert(Not);
5098  }
5099 
5100  MRI.replaceRegWith(Dest.getReg(), NewDest);
5101 
5102  Worklist.insert(Xor);
5103 
5104  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5105  }
5106 }
5107 
5108 void SIInstrInfo::splitScalarNotBinop(SetVectorType &Worklist,
5109  MachineInstr &Inst,
5110  unsigned Opcode) const {
5111  MachineBasicBlock &MBB = *Inst.getParent();
5113  MachineBasicBlock::iterator MII = Inst;
5114  const DebugLoc &DL = Inst.getDebugLoc();
5115 
5116  MachineOperand &Dest = Inst.getOperand(0);
5117  MachineOperand &Src0 = Inst.getOperand(1);
5118  MachineOperand &Src1 = Inst.getOperand(2);
5119 
5120  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5121  unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5122 
5123  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), Interm)
5124  .add(Src0)
5125  .add(Src1);
5126 
5127  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), NewDest)
5128  .addReg(Interm);
5129 
5130  Worklist.insert(&Op);
5131  Worklist.insert(&Not);
5132 
5133  MRI.replaceRegWith(Dest.getReg(), NewDest);
5134  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5135 }
5136 
5137 void SIInstrInfo::splitScalarBinOpN2(SetVectorType& Worklist,
5138  MachineInstr &Inst,
5139  unsigned Opcode) const {
5140  MachineBasicBlock &MBB = *Inst.getParent();
5142  MachineBasicBlock::iterator MII = Inst;
5143  const DebugLoc &DL = Inst.getDebugLoc();
5144 
5145  MachineOperand &Dest = Inst.getOperand(0);
5146  MachineOperand &Src0 = Inst.getOperand(1);
5147  MachineOperand &Src1 = Inst.getOperand(2);
5148 
5149  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5150  unsigned Interm = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
5151 
5152  MachineInstr &Not = *BuildMI(MBB, MII, DL, get(AMDGPU::S_NOT_B32), Interm)
5153  .add(Src1);
5154 
5155  MachineInstr &Op = *BuildMI(MBB, MII, DL, get(Opcode), NewDest)
5156  .add(Src0)
5157  .addReg(Interm);
5158 
5159  Worklist.insert(&Not);
5160  Worklist.insert(&Op);
5161 
5162  MRI.replaceRegWith(Dest.getReg(), NewDest);
5163  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
5164 }
5165 
5166 void SIInstrInfo::splitScalar64BitUnaryOp(
5167  SetVectorType &Worklist, MachineInstr &Inst,
5168  unsigned Opcode) const {
5169  MachineBasicBlock &MBB = *Inst.getParent();
5171 
5172  MachineOperand &Dest = Inst.getOperand(0);
5173  MachineOperand &Src0 = Inst.getOperand(1);
5174  DebugLoc DL = Inst.getDebugLoc();
5175 
5176  MachineBasicBlock::iterator MII = Inst;
5177 
5178  const MCInstrDesc &InstDesc = get(Opcode);
5179  const TargetRegisterClass *Src0RC = Src0.isReg() ?
5180  MRI.getRegClass(Src0.getReg()) :
5181  &AMDGPU::SGPR_32RegClass;
5182 
5183  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5184 
5185  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5186  AMDGPU::sub0, Src0SubRC);
5187 
5188  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
5189  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
5190  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
5191 
5192  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
5193  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
5194 
5195  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5196  AMDGPU::sub1, Src0SubRC);
5197 
5198  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
5199  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
5200 
5201  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
5202  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5203  .addReg(DestSub0)
5204  .addImm(AMDGPU::sub0)
5205  .addReg(DestSub1)
5206  .addImm(AMDGPU::sub1);
5207 
5208  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5209 
5210  Worklist.insert(&LoHalf);
5211  Worklist.insert(&HiHalf);
5212 
5213  // We don't need to legalizeOperands here because for a single operand, src0
5214  // will support any kind of input.
5215 
5216  // Move all users of this moved value.
5217  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5218 }
5219 
5220 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
5221  MachineInstr &Inst,
5222  MachineDominatorTree *MDT) const {
5223  bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
5224 
5225  MachineBasicBlock &MBB = *Inst.getParent();
5227  const auto *CarryRC = RI.getRegClass(AMDGPU::SReg_1_XEXECRegClassID);
5228 
5229  unsigned FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
5230  unsigned DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5231  unsigned DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
5232 
5233  unsigned CarryReg = MRI.createVirtualRegister(CarryRC);
5234  unsigned DeadCarryReg = MRI.createVirtualRegister(CarryRC);
5235 
5236  MachineOperand &Dest = Inst.getOperand(0);
5237  MachineOperand &Src0 = Inst.getOperand(1);
5238  MachineOperand &Src1 = Inst.getOperand(2);
5239  const DebugLoc &DL = Inst.getDebugLoc();
5240  MachineBasicBlock::iterator MII = Inst;
5241 
5242  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
5243  const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
5244  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
5245  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
5246 
5247  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5248  AMDGPU::sub0, Src0SubRC);
5249  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5250  AMDGPU::sub0, Src1SubRC);
5251 
5252 
5253  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
5254  AMDGPU::sub1, Src0SubRC);
5255  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
5256  AMDGPU::sub1, Src1SubRC);
5257 
5258  unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
5259  MachineInstr *LoHalf =
5260  BuildMI(MBB, MII, DL, get(LoOpc), DestSub0)
5261  .addReg(CarryReg, RegState::Define)
5262  .add(SrcReg0Sub0)
5263  .add(SrcReg1Sub0)
5264  .addImm(0); // clamp bit
5265 
5266  unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
5267  MachineInstr *HiHalf =
5268  BuildMI(MBB, MII, DL, get(HiOpc), DestSub1)
5269  .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
5270  .add(SrcReg0Sub1)
5271  .add(SrcReg1Sub1)
5272  .addReg(CarryReg, RegState::Kill)
5273  .addImm(0); // clamp bit
5274 
5275  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
5276  .addReg(DestSub0)
5277  .addImm(AMDGPU::sub0)
5278  .addReg(DestSub1)
5279  .addImm(AMDGPU::sub1);
5280 
5281  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
5282 
5283  // Try to legalize the operands in case we need to swap the order to keep it
5284  // valid.
5285  legalizeOperands(*LoHalf, MDT);
5286  legalizeOperands(*HiHalf, MDT);
5287 
5288  // Move all users of this moved vlaue.
5289  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
5290 }
5291 
5292 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist,
5293  MachineInstr &Inst, unsigned Opcode,
5294  MachineDominatorTree *MDT) const {
5295  MachineB