LLVM  8.0.0svn
SIInstrInfo.cpp
Go to the documentation of this file.
1 //===- SIInstrInfo.cpp - SI Instruction Information ----------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 /// \file
11 /// SI Implementation of TargetInstrInfo.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "SIInstrInfo.h"
16 #include "AMDGPU.h"
17 #include "AMDGPUIntrinsicInfo.h"
18 #include "AMDGPUSubtarget.h"
19 #include "GCNHazardRecognizer.h"
20 #include "SIDefines.h"
21 #include "SIMachineFunctionInfo.h"
22 #include "SIRegisterInfo.h"
24 #include "Utils/AMDGPUBaseInfo.h"
25 #include "llvm/ADT/APInt.h"
26 #include "llvm/ADT/ArrayRef.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/StringRef.h"
48 #include "llvm/IR/DebugLoc.h"
49 #include "llvm/IR/DiagnosticInfo.h"
50 #include "llvm/IR/Function.h"
51 #include "llvm/IR/InlineAsm.h"
52 #include "llvm/IR/LLVMContext.h"
53 #include "llvm/MC/MCInstrDesc.h"
54 #include "llvm/Support/Casting.h"
56 #include "llvm/Support/Compiler.h"
61 #include <cassert>
62 #include <cstdint>
63 #include <iterator>
64 #include <utility>
65 
66 using namespace llvm;
67 
68 #define GET_INSTRINFO_CTOR_DTOR
69 #include "AMDGPUGenInstrInfo.inc"
70 
71 namespace llvm {
72 namespace AMDGPU {
73 #define GET_D16ImageDimIntrinsics_IMPL
74 #define GET_ImageDimIntrinsicTable_IMPL
75 #define GET_RsrcIntrinsics_IMPL
76 #include "AMDGPUGenSearchableTables.inc"
77 }
78 }
79 
80 
81 // Must be at least 4 to be able to branch over minimum unconditional branch
82 // code. This is only for making it possible to write reasonably small tests for
83 // long branches.
84 static cl::opt<unsigned>
85 BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16),
86  cl::desc("Restrict range of branch instructions (DEBUG)"));
87 
89  : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN),
90  RI(ST), ST(ST) {}
91 
92 //===----------------------------------------------------------------------===//
93 // TargetInstrInfo callbacks
94 //===----------------------------------------------------------------------===//
95 
96 static unsigned getNumOperandsNoGlue(SDNode *Node) {
97  unsigned N = Node->getNumOperands();
98  while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
99  --N;
100  return N;
101 }
102 
104  SDValue LastOp = Load->getOperand(getNumOperandsNoGlue(Load) - 1);
105  assert(LastOp.getValueType() == MVT::Other && "Chain missing from load node");
106  return LastOp;
107 }
108 
109 /// Returns true if both nodes have the same value for the given
110 /// operand \p Op, or if both nodes do not have this operand.
111 static bool nodesHaveSameOperandValue(SDNode *N0, SDNode* N1, unsigned OpName) {
112  unsigned Opc0 = N0->getMachineOpcode();
113  unsigned Opc1 = N1->getMachineOpcode();
114 
115  int Op0Idx = AMDGPU::getNamedOperandIdx(Opc0, OpName);
116  int Op1Idx = AMDGPU::getNamedOperandIdx(Opc1, OpName);
117 
118  if (Op0Idx == -1 && Op1Idx == -1)
119  return true;
120 
121 
122  if ((Op0Idx == -1 && Op1Idx != -1) ||
123  (Op1Idx == -1 && Op0Idx != -1))
124  return false;
125 
126  // getNamedOperandIdx returns the index for the MachineInstr's operands,
127  // which includes the result as the first operand. We are indexing into the
128  // MachineSDNode's operands, so we need to skip the result operand to get
129  // the real index.
130  --Op0Idx;
131  --Op1Idx;
132 
133  return N0->getOperand(Op0Idx) == N1->getOperand(Op1Idx);
134 }
135 
137  AliasAnalysis *AA) const {
138  // TODO: The generic check fails for VALU instructions that should be
139  // rematerializable due to implicit reads of exec. We really want all of the
140  // generic logic for this except for this.
141  switch (MI.getOpcode()) {
142  case AMDGPU::V_MOV_B32_e32:
143  case AMDGPU::V_MOV_B32_e64:
144  case AMDGPU::V_MOV_B64_PSEUDO:
145  return true;
146  default:
147  return false;
148  }
149 }
150 
152  int64_t &Offset0,
153  int64_t &Offset1) const {
154  if (!Load0->isMachineOpcode() || !Load1->isMachineOpcode())
155  return false;
156 
157  unsigned Opc0 = Load0->getMachineOpcode();
158  unsigned Opc1 = Load1->getMachineOpcode();
159 
160  // Make sure both are actually loads.
161  if (!get(Opc0).mayLoad() || !get(Opc1).mayLoad())
162  return false;
163 
164  if (isDS(Opc0) && isDS(Opc1)) {
165 
166  // FIXME: Handle this case:
167  if (getNumOperandsNoGlue(Load0) != getNumOperandsNoGlue(Load1))
168  return false;
169 
170  // Check base reg.
171  if (Load0->getOperand(1) != Load1->getOperand(1))
172  return false;
173 
174  // Check chain.
175  if (findChainOperand(Load0) != findChainOperand(Load1))
176  return false;
177 
178  // Skip read2 / write2 variants for simplicity.
179  // TODO: We should report true if the used offsets are adjacent (excluded
180  // st64 versions).
181  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::data1) != -1 ||
182  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::data1) != -1)
183  return false;
184 
185  Offset0 = cast<ConstantSDNode>(Load0->getOperand(2))->getZExtValue();
186  Offset1 = cast<ConstantSDNode>(Load1->getOperand(2))->getZExtValue();
187  return true;
188  }
189 
190  if (isSMRD(Opc0) && isSMRD(Opc1)) {
191  // Skip time and cache invalidation instructions.
192  if (AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::sbase) == -1 ||
193  AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::sbase) == -1)
194  return false;
195 
197 
198  // Check base reg.
199  if (Load0->getOperand(0) != Load1->getOperand(0))
200  return false;
201 
202  const ConstantSDNode *Load0Offset =
203  dyn_cast<ConstantSDNode>(Load0->getOperand(1));
204  const ConstantSDNode *Load1Offset =
205  dyn_cast<ConstantSDNode>(Load1->getOperand(1));
206 
207  if (!Load0Offset || !Load1Offset)
208  return false;
209 
210  // Check chain.
211  if (findChainOperand(Load0) != findChainOperand(Load1))
212  return false;
213 
214  Offset0 = Load0Offset->getZExtValue();
215  Offset1 = Load1Offset->getZExtValue();
216  return true;
217  }
218 
219  // MUBUF and MTBUF can access the same addresses.
220  if ((isMUBUF(Opc0) || isMTBUF(Opc0)) && (isMUBUF(Opc1) || isMTBUF(Opc1))) {
221 
222  // MUBUF and MTBUF have vaddr at different indices.
223  if (!nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::soffset) ||
224  findChainOperand(Load0) != findChainOperand(Load1) ||
225  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::vaddr) ||
226  !nodesHaveSameOperandValue(Load0, Load1, AMDGPU::OpName::srsrc))
227  return false;
228 
229  int OffIdx0 = AMDGPU::getNamedOperandIdx(Opc0, AMDGPU::OpName::offset);
230  int OffIdx1 = AMDGPU::getNamedOperandIdx(Opc1, AMDGPU::OpName::offset);
231 
232  if (OffIdx0 == -1 || OffIdx1 == -1)
233  return false;
234 
235  // getNamedOperandIdx returns the index for MachineInstrs. Since they
236  // inlcude the output in the operand list, but SDNodes don't, we need to
237  // subtract the index by one.
238  --OffIdx0;
239  --OffIdx1;
240 
241  SDValue Off0 = Load0->getOperand(OffIdx0);
242  SDValue Off1 = Load1->getOperand(OffIdx1);
243 
244  // The offset might be a FrameIndexSDNode.
245  if (!isa<ConstantSDNode>(Off0) || !isa<ConstantSDNode>(Off1))
246  return false;
247 
248  Offset0 = cast<ConstantSDNode>(Off0)->getZExtValue();
249  Offset1 = cast<ConstantSDNode>(Off1)->getZExtValue();
250  return true;
251  }
252 
253  return false;
254 }
255 
256 static bool isStride64(unsigned Opc) {
257  switch (Opc) {
258  case AMDGPU::DS_READ2ST64_B32:
259  case AMDGPU::DS_READ2ST64_B64:
260  case AMDGPU::DS_WRITE2ST64_B32:
261  case AMDGPU::DS_WRITE2ST64_B64:
262  return true;
263  default:
264  return false;
265  }
266 }
267 
268 bool SIInstrInfo::getMemOpBaseRegImmOfs(MachineInstr &LdSt, unsigned &BaseReg,
269  int64_t &Offset,
270  const TargetRegisterInfo *TRI) const {
271  unsigned Opc = LdSt.getOpcode();
272 
273  if (isDS(LdSt)) {
274  const MachineOperand *OffsetImm =
275  getNamedOperand(LdSt, AMDGPU::OpName::offset);
276  if (OffsetImm) {
277  // Normal, single offset LDS instruction.
278  const MachineOperand *AddrReg =
279  getNamedOperand(LdSt, AMDGPU::OpName::addr);
280 
281  BaseReg = AddrReg->getReg();
282  Offset = OffsetImm->getImm();
283  return true;
284  }
285 
286  // The 2 offset instructions use offset0 and offset1 instead. We can treat
287  // these as a load with a single offset if the 2 offsets are consecutive. We
288  // will use this for some partially aligned loads.
289  const MachineOperand *Offset0Imm =
290  getNamedOperand(LdSt, AMDGPU::OpName::offset0);
291  const MachineOperand *Offset1Imm =
292  getNamedOperand(LdSt, AMDGPU::OpName::offset1);
293 
294  uint8_t Offset0 = Offset0Imm->getImm();
295  uint8_t Offset1 = Offset1Imm->getImm();
296 
297  if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
298  // Each of these offsets is in element sized units, so we need to convert
299  // to bytes of the individual reads.
300 
301  unsigned EltSize;
302  if (LdSt.mayLoad())
303  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, 0)) / 16;
304  else {
305  assert(LdSt.mayStore());
306  int Data0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::data0);
307  EltSize = TRI->getRegSizeInBits(*getOpRegClass(LdSt, Data0Idx)) / 8;
308  }
309 
310  if (isStride64(Opc))
311  EltSize *= 64;
312 
313  const MachineOperand *AddrReg =
314  getNamedOperand(LdSt, AMDGPU::OpName::addr);
315  BaseReg = AddrReg->getReg();
316  Offset = EltSize * Offset0;
317  return true;
318  }
319 
320  return false;
321  }
322 
323  if (isMUBUF(LdSt) || isMTBUF(LdSt)) {
324  const MachineOperand *SOffset = getNamedOperand(LdSt, AMDGPU::OpName::soffset);
325  if (SOffset && SOffset->isReg())
326  return false;
327 
328  const MachineOperand *AddrReg =
329  getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
330  if (!AddrReg)
331  return false;
332 
333  const MachineOperand *OffsetImm =
334  getNamedOperand(LdSt, AMDGPU::OpName::offset);
335  BaseReg = AddrReg->getReg();
336  Offset = OffsetImm->getImm();
337 
338  if (SOffset) // soffset can be an inline immediate.
339  Offset += SOffset->getImm();
340 
341  return true;
342  }
343 
344  if (isSMRD(LdSt)) {
345  const MachineOperand *OffsetImm =
346  getNamedOperand(LdSt, AMDGPU::OpName::offset);
347  if (!OffsetImm)
348  return false;
349 
350  const MachineOperand *SBaseReg =
351  getNamedOperand(LdSt, AMDGPU::OpName::sbase);
352  BaseReg = SBaseReg->getReg();
353  Offset = OffsetImm->getImm();
354  return true;
355  }
356 
357  if (isFLAT(LdSt)) {
358  const MachineOperand *VAddr = getNamedOperand(LdSt, AMDGPU::OpName::vaddr);
359  if (VAddr) {
360  // Can't analyze 2 offsets.
361  if (getNamedOperand(LdSt, AMDGPU::OpName::saddr))
362  return false;
363 
364  BaseReg = VAddr->getReg();
365  } else {
366  // scratch instructions have either vaddr or saddr.
367  BaseReg = getNamedOperand(LdSt, AMDGPU::OpName::saddr)->getReg();
368  }
369 
370  Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm();
371  return true;
372  }
373 
374  return false;
375 }
376 
377 static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, unsigned BaseReg1,
378  const MachineInstr &MI2, unsigned BaseReg2) {
379  if (BaseReg1 == BaseReg2)
380  return true;
381 
382  if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
383  return false;
384 
385  auto MO1 = *MI1.memoperands_begin();
386  auto MO2 = *MI2.memoperands_begin();
387  if (MO1->getAddrSpace() != MO2->getAddrSpace())
388  return false;
389 
390  auto Base1 = MO1->getValue();
391  auto Base2 = MO2->getValue();
392  if (!Base1 || !Base2)
393  return false;
394  const MachineFunction &MF = *MI1.getParent()->getParent();
395  const DataLayout &DL = MF.getFunction().getParent()->getDataLayout();
396  Base1 = GetUnderlyingObject(Base1, DL);
397  Base2 = GetUnderlyingObject(Base1, DL);
398 
399  if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
400  return false;
401 
402  return Base1 == Base2;
403 }
404 
406  unsigned BaseReg1,
407  MachineInstr &SecondLdSt,
408  unsigned BaseReg2,
409  unsigned NumLoads) const {
410  if (!memOpsHaveSameBasePtr(FirstLdSt, BaseReg1, SecondLdSt, BaseReg2))
411  return false;
412 
413  const MachineOperand *FirstDst = nullptr;
414  const MachineOperand *SecondDst = nullptr;
415 
416  if ((isMUBUF(FirstLdSt) && isMUBUF(SecondLdSt)) ||
417  (isMTBUF(FirstLdSt) && isMTBUF(SecondLdSt)) ||
418  (isFLAT(FirstLdSt) && isFLAT(SecondLdSt))) {
419  const unsigned MaxGlobalLoadCluster = 6;
420  if (NumLoads > MaxGlobalLoadCluster)
421  return false;
422 
423  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdata);
424  if (!FirstDst)
425  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
426  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdata);
427  if (!SecondDst)
428  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
429  } else if (isSMRD(FirstLdSt) && isSMRD(SecondLdSt)) {
430  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::sdst);
431  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::sdst);
432  } else if (isDS(FirstLdSt) && isDS(SecondLdSt)) {
433  FirstDst = getNamedOperand(FirstLdSt, AMDGPU::OpName::vdst);
434  SecondDst = getNamedOperand(SecondLdSt, AMDGPU::OpName::vdst);
435  }
436 
437  if (!FirstDst || !SecondDst)
438  return false;
439 
440  // Try to limit clustering based on the total number of bytes loaded
441  // rather than the number of instructions. This is done to help reduce
442  // register pressure. The method used is somewhat inexact, though,
443  // because it assumes that all loads in the cluster will load the
444  // same number of bytes as FirstLdSt.
445 
446  // The unit of this value is bytes.
447  // FIXME: This needs finer tuning.
448  unsigned LoadClusterThreshold = 16;
449 
450  const MachineRegisterInfo &MRI =
451  FirstLdSt.getParent()->getParent()->getRegInfo();
452  const TargetRegisterClass *DstRC = MRI.getRegClass(FirstDst->getReg());
453 
454  return (NumLoads * (RI.getRegSizeInBits(*DstRC) / 8)) <= LoadClusterThreshold;
455 }
456 
457 // FIXME: This behaves strangely. If, for example, you have 32 load + stores,
458 // the first 16 loads will be interleaved with the stores, and the next 16 will
459 // be clustered as expected. It should really split into 2 16 store batches.
460 //
461 // Loads are clustered until this returns false, rather than trying to schedule
462 // groups of stores. This also means we have to deal with saying different
463 // address space loads should be clustered, and ones which might cause bank
464 // conflicts.
465 //
466 // This might be deprecated so it might not be worth that much effort to fix.
468  int64_t Offset0, int64_t Offset1,
469  unsigned NumLoads) const {
470  assert(Offset1 > Offset0 &&
471  "Second offset should be larger than first offset!");
472  // If we have less than 16 loads in a row, and the offsets are within 64
473  // bytes, then schedule together.
474 
475  // A cacheline is 64 bytes (for global memory).
476  return (NumLoads <= 16 && (Offset1 - Offset0) < 64);
477 }
478 
481  const DebugLoc &DL, unsigned DestReg,
482  unsigned SrcReg, bool KillSrc) {
483  MachineFunction *MF = MBB.getParent();
484  DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(),
485  "illegal SGPR to VGPR copy",
486  DL, DS_Error);
487  LLVMContext &C = MF->getFunction().getContext();
488  C.diagnose(IllegalCopy);
489 
490  BuildMI(MBB, MI, DL, TII->get(AMDGPU::SI_ILLEGAL_COPY), DestReg)
491  .addReg(SrcReg, getKillRegState(KillSrc));
492 }
493 
496  const DebugLoc &DL, unsigned DestReg,
497  unsigned SrcReg, bool KillSrc) const {
498  const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg);
499 
500  if (RC == &AMDGPU::VGPR_32RegClass) {
501  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg) ||
502  AMDGPU::SReg_32RegClass.contains(SrcReg));
503  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
504  .addReg(SrcReg, getKillRegState(KillSrc));
505  return;
506  }
507 
508  if (RC == &AMDGPU::SReg_32_XM0RegClass ||
509  RC == &AMDGPU::SReg_32RegClass) {
510  if (SrcReg == AMDGPU::SCC) {
511  BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg)
512  .addImm(-1)
513  .addImm(0);
514  return;
515  }
516 
517  if (!AMDGPU::SReg_32RegClass.contains(SrcReg)) {
518  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
519  return;
520  }
521 
522  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
523  .addReg(SrcReg, getKillRegState(KillSrc));
524  return;
525  }
526 
527  if (RC == &AMDGPU::SReg_64RegClass) {
528  if (DestReg == AMDGPU::VCC) {
529  if (AMDGPU::SReg_64RegClass.contains(SrcReg)) {
530  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), AMDGPU::VCC)
531  .addReg(SrcReg, getKillRegState(KillSrc));
532  } else {
533  // FIXME: Hack until VReg_1 removed.
534  assert(AMDGPU::VGPR_32RegClass.contains(SrcReg));
535  BuildMI(MBB, MI, DL, get(AMDGPU::V_CMP_NE_U32_e32))
536  .addImm(0)
537  .addReg(SrcReg, getKillRegState(KillSrc));
538  }
539 
540  return;
541  }
542 
543  if (!AMDGPU::SReg_64RegClass.contains(SrcReg)) {
544  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
545  return;
546  }
547 
548  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
549  .addReg(SrcReg, getKillRegState(KillSrc));
550  return;
551  }
552 
553  if (DestReg == AMDGPU::SCC) {
554  assert(AMDGPU::SReg_32RegClass.contains(SrcReg));
555  BuildMI(MBB, MI, DL, get(AMDGPU::S_CMP_LG_U32))
556  .addReg(SrcReg, getKillRegState(KillSrc))
557  .addImm(0);
558  return;
559  }
560 
561  unsigned EltSize = 4;
562  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
563  if (RI.isSGPRClass(RC)) {
564  if (RI.getRegSizeInBits(*RC) > 32) {
565  Opcode = AMDGPU::S_MOV_B64;
566  EltSize = 8;
567  } else {
568  Opcode = AMDGPU::S_MOV_B32;
569  EltSize = 4;
570  }
571 
572  if (!RI.isSGPRClass(RI.getPhysRegClass(SrcReg))) {
573  reportIllegalCopy(this, MBB, MI, DL, DestReg, SrcReg, KillSrc);
574  return;
575  }
576  }
577 
578  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RC, EltSize);
579  bool Forward = RI.getHWRegIndex(DestReg) <= RI.getHWRegIndex(SrcReg);
580 
581  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
582  unsigned SubIdx;
583  if (Forward)
584  SubIdx = SubIndices[Idx];
585  else
586  SubIdx = SubIndices[SubIndices.size() - Idx - 1];
587 
588  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
589  get(Opcode), RI.getSubReg(DestReg, SubIdx));
590 
591  Builder.addReg(RI.getSubReg(SrcReg, SubIdx));
592 
593  if (Idx == 0)
594  Builder.addReg(DestReg, RegState::Define | RegState::Implicit);
595 
596  bool UseKill = KillSrc && Idx == SubIndices.size() - 1;
597  Builder.addReg(SrcReg, getKillRegState(UseKill) | RegState::Implicit);
598  }
599 }
600 
601 int SIInstrInfo::commuteOpcode(unsigned Opcode) const {
602  int NewOpc;
603 
604  // Try to map original to commuted opcode
605  NewOpc = AMDGPU::getCommuteRev(Opcode);
606  if (NewOpc != -1)
607  // Check if the commuted (REV) opcode exists on the target.
608  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
609 
610  // Try to map commuted to original opcode
611  NewOpc = AMDGPU::getCommuteOrig(Opcode);
612  if (NewOpc != -1)
613  // Check if the original (non-REV) opcode exists on the target.
614  return pseudoToMCOpcode(NewOpc) != -1 ? NewOpc : -1;
615 
616  return Opcode;
617 }
618 
621  const DebugLoc &DL, unsigned DestReg,
622  int64_t Value) const {
624  const TargetRegisterClass *RegClass = MRI.getRegClass(DestReg);
625  if (RegClass == &AMDGPU::SReg_32RegClass ||
626  RegClass == &AMDGPU::SGPR_32RegClass ||
627  RegClass == &AMDGPU::SReg_32_XM0RegClass ||
628  RegClass == &AMDGPU::SReg_32_XM0_XEXECRegClass) {
629  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B32), DestReg)
630  .addImm(Value);
631  return;
632  }
633 
634  if (RegClass == &AMDGPU::SReg_64RegClass ||
635  RegClass == &AMDGPU::SGPR_64RegClass ||
636  RegClass == &AMDGPU::SReg_64_XEXECRegClass) {
637  BuildMI(MBB, MI, DL, get(AMDGPU::S_MOV_B64), DestReg)
638  .addImm(Value);
639  return;
640  }
641 
642  if (RegClass == &AMDGPU::VGPR_32RegClass) {
643  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DestReg)
644  .addImm(Value);
645  return;
646  }
647  if (RegClass == &AMDGPU::VReg_64RegClass) {
648  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO), DestReg)
649  .addImm(Value);
650  return;
651  }
652 
653  unsigned EltSize = 4;
654  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
655  if (RI.isSGPRClass(RegClass)) {
656  if (RI.getRegSizeInBits(*RegClass) > 32) {
657  Opcode = AMDGPU::S_MOV_B64;
658  EltSize = 8;
659  } else {
660  Opcode = AMDGPU::S_MOV_B32;
661  EltSize = 4;
662  }
663  }
664 
665  ArrayRef<int16_t> SubIndices = RI.getRegSplitParts(RegClass, EltSize);
666  for (unsigned Idx = 0; Idx < SubIndices.size(); ++Idx) {
667  int64_t IdxValue = Idx == 0 ? Value : 0;
668 
669  MachineInstrBuilder Builder = BuildMI(MBB, MI, DL,
670  get(Opcode), RI.getSubReg(DestReg, Idx));
671  Builder.addImm(IdxValue);
672  }
673 }
674 
675 const TargetRegisterClass *
677  return &AMDGPU::VGPR_32RegClass;
678 }
679 
682  const DebugLoc &DL, unsigned DstReg,
684  unsigned TrueReg,
685  unsigned FalseReg) const {
687  assert(MRI.getRegClass(DstReg) == &AMDGPU::VGPR_32RegClass &&
688  "Not a VGPR32 reg");
689 
690  if (Cond.size() == 1) {
691  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
692  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
693  .add(Cond[0]);
694  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
695  .addReg(FalseReg)
696  .addReg(TrueReg)
697  .addReg(SReg);
698  } else if (Cond.size() == 2) {
699  assert(Cond[0].isImm() && "Cond[0] is not an immediate");
700  switch (Cond[0].getImm()) {
701  case SIInstrInfo::SCC_TRUE: {
702  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
703  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
704  .addImm(-1)
705  .addImm(0);
706  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
707  .addReg(FalseReg)
708  .addReg(TrueReg)
709  .addReg(SReg);
710  break;
711  }
712  case SIInstrInfo::SCC_FALSE: {
713  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
714  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
715  .addImm(0)
716  .addImm(-1);
717  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
718  .addReg(FalseReg)
719  .addReg(TrueReg)
720  .addReg(SReg);
721  break;
722  }
723  case SIInstrInfo::VCCNZ: {
724  MachineOperand RegOp = Cond[1];
725  RegOp.setImplicit(false);
726  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
727  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
728  .add(RegOp);
729  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
730  .addReg(FalseReg)
731  .addReg(TrueReg)
732  .addReg(SReg);
733  break;
734  }
735  case SIInstrInfo::VCCZ: {
736  MachineOperand RegOp = Cond[1];
737  RegOp.setImplicit(false);
738  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
739  BuildMI(MBB, I, DL, get(AMDGPU::COPY), SReg)
740  .add(RegOp);
741  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
742  .addReg(TrueReg)
743  .addReg(FalseReg)
744  .addReg(SReg);
745  break;
746  }
747  case SIInstrInfo::EXECNZ: {
748  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
749  unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
750  BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
751  .addImm(0);
752  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
753  .addImm(-1)
754  .addImm(0);
755  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
756  .addReg(FalseReg)
757  .addReg(TrueReg)
758  .addReg(SReg);
759  break;
760  }
761  case SIInstrInfo::EXECZ: {
762  unsigned SReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
763  unsigned SReg2 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
764  BuildMI(MBB, I, DL, get(AMDGPU::S_OR_SAVEEXEC_B64), SReg2)
765  .addImm(0);
766  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), SReg)
767  .addImm(0)
768  .addImm(-1);
769  BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg)
770  .addReg(FalseReg)
771  .addReg(TrueReg)
772  .addReg(SReg);
773  llvm_unreachable("Unhandled branch predicate EXECZ");
774  break;
775  }
776  default:
777  llvm_unreachable("invalid branch predicate");
778  }
779  } else {
780  llvm_unreachable("Can only handle Cond size 1 or 2");
781  }
782 }
783 
786  const DebugLoc &DL,
787  unsigned SrcReg, int Value) const {
789  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
790  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_EQ_I32_e64), Reg)
791  .addImm(Value)
792  .addReg(SrcReg);
793 
794  return Reg;
795 }
796 
799  const DebugLoc &DL,
800  unsigned SrcReg, int Value) const {
802  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
803  BuildMI(*MBB, I, DL, get(AMDGPU::V_CMP_NE_I32_e64), Reg)
804  .addImm(Value)
805  .addReg(SrcReg);
806 
807  return Reg;
808 }
809 
810 unsigned SIInstrInfo::getMovOpcode(const TargetRegisterClass *DstRC) const {
811 
812  if (RI.getRegSizeInBits(*DstRC) == 32) {
813  return RI.isSGPRClass(DstRC) ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
814  } else if (RI.getRegSizeInBits(*DstRC) == 64 && RI.isSGPRClass(DstRC)) {
815  return AMDGPU::S_MOV_B64;
816  } else if (RI.getRegSizeInBits(*DstRC) == 64 && !RI.isSGPRClass(DstRC)) {
817  return AMDGPU::V_MOV_B64_PSEUDO;
818  }
819  return AMDGPU::COPY;
820 }
821 
822 static unsigned getSGPRSpillSaveOpcode(unsigned Size) {
823  switch (Size) {
824  case 4:
825  return AMDGPU::SI_SPILL_S32_SAVE;
826  case 8:
827  return AMDGPU::SI_SPILL_S64_SAVE;
828  case 16:
829  return AMDGPU::SI_SPILL_S128_SAVE;
830  case 32:
831  return AMDGPU::SI_SPILL_S256_SAVE;
832  case 64:
833  return AMDGPU::SI_SPILL_S512_SAVE;
834  default:
835  llvm_unreachable("unknown register size");
836  }
837 }
838 
839 static unsigned getVGPRSpillSaveOpcode(unsigned Size) {
840  switch (Size) {
841  case 4:
842  return AMDGPU::SI_SPILL_V32_SAVE;
843  case 8:
844  return AMDGPU::SI_SPILL_V64_SAVE;
845  case 12:
846  return AMDGPU::SI_SPILL_V96_SAVE;
847  case 16:
848  return AMDGPU::SI_SPILL_V128_SAVE;
849  case 32:
850  return AMDGPU::SI_SPILL_V256_SAVE;
851  case 64:
852  return AMDGPU::SI_SPILL_V512_SAVE;
853  default:
854  llvm_unreachable("unknown register size");
855  }
856 }
857 
860  unsigned SrcReg, bool isKill,
861  int FrameIndex,
862  const TargetRegisterClass *RC,
863  const TargetRegisterInfo *TRI) const {
864  MachineFunction *MF = MBB.getParent();
866  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
867  DebugLoc DL = MBB.findDebugLoc(MI);
868 
869  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
870  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
871  MachinePointerInfo PtrInfo
872  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
873  MachineMemOperand *MMO
875  Size, Align);
876  unsigned SpillSize = TRI->getSpillSize(*RC);
877 
878  if (RI.isSGPRClass(RC)) {
879  MFI->setHasSpilledSGPRs();
880 
881  // We are only allowed to create one new instruction when spilling
882  // registers, so we need to use pseudo instruction for spilling SGPRs.
883  const MCInstrDesc &OpDesc = get(getSGPRSpillSaveOpcode(SpillSize));
884 
885  // The SGPR spill/restore instructions only work on number sgprs, so we need
886  // to make sure we are using the correct register class.
887  if (TargetRegisterInfo::isVirtualRegister(SrcReg) && SpillSize == 4) {
889  MRI.constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass);
890  }
891 
892  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc)
893  .addReg(SrcReg, getKillRegState(isKill)) // data
894  .addFrameIndex(FrameIndex) // addr
895  .addMemOperand(MMO)
897  .addReg(MFI->getFrameOffsetReg(), RegState::Implicit);
898  // Add the scratch resource registers as implicit uses because we may end up
899  // needing them, and need to ensure that the reserved registers are
900  // correctly handled.
901 
902  FrameInfo.setStackID(FrameIndex, SIStackID::SGPR_SPILL);
903  if (ST.hasScalarStores()) {
904  // m0 is used for offset to scalar stores if used to spill.
905  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
906  }
907 
908  return;
909  }
910 
911  assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
912 
913  unsigned Opcode = getVGPRSpillSaveOpcode(SpillSize);
914  MFI->setHasSpilledVGPRs();
915  BuildMI(MBB, MI, DL, get(Opcode))
916  .addReg(SrcReg, getKillRegState(isKill)) // data
917  .addFrameIndex(FrameIndex) // addr
918  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
919  .addReg(MFI->getFrameOffsetReg()) // scratch_offset
920  .addImm(0) // offset
921  .addMemOperand(MMO);
922 }
923 
924 static unsigned getSGPRSpillRestoreOpcode(unsigned Size) {
925  switch (Size) {
926  case 4:
927  return AMDGPU::SI_SPILL_S32_RESTORE;
928  case 8:
929  return AMDGPU::SI_SPILL_S64_RESTORE;
930  case 16:
931  return AMDGPU::SI_SPILL_S128_RESTORE;
932  case 32:
933  return AMDGPU::SI_SPILL_S256_RESTORE;
934  case 64:
935  return AMDGPU::SI_SPILL_S512_RESTORE;
936  default:
937  llvm_unreachable("unknown register size");
938  }
939 }
940 
941 static unsigned getVGPRSpillRestoreOpcode(unsigned Size) {
942  switch (Size) {
943  case 4:
944  return AMDGPU::SI_SPILL_V32_RESTORE;
945  case 8:
946  return AMDGPU::SI_SPILL_V64_RESTORE;
947  case 12:
948  return AMDGPU::SI_SPILL_V96_RESTORE;
949  case 16:
950  return AMDGPU::SI_SPILL_V128_RESTORE;
951  case 32:
952  return AMDGPU::SI_SPILL_V256_RESTORE;
953  case 64:
954  return AMDGPU::SI_SPILL_V512_RESTORE;
955  default:
956  llvm_unreachable("unknown register size");
957  }
958 }
959 
962  unsigned DestReg, int FrameIndex,
963  const TargetRegisterClass *RC,
964  const TargetRegisterInfo *TRI) const {
965  MachineFunction *MF = MBB.getParent();
967  MachineFrameInfo &FrameInfo = MF->getFrameInfo();
968  DebugLoc DL = MBB.findDebugLoc(MI);
969  unsigned Align = FrameInfo.getObjectAlignment(FrameIndex);
970  unsigned Size = FrameInfo.getObjectSize(FrameIndex);
971  unsigned SpillSize = TRI->getSpillSize(*RC);
972 
973  MachinePointerInfo PtrInfo
974  = MachinePointerInfo::getFixedStack(*MF, FrameIndex);
975 
977  PtrInfo, MachineMemOperand::MOLoad, Size, Align);
978 
979  if (RI.isSGPRClass(RC)) {
980  // FIXME: Maybe this should not include a memoperand because it will be
981  // lowered to non-memory instructions.
982  const MCInstrDesc &OpDesc = get(getSGPRSpillRestoreOpcode(SpillSize));
983  if (TargetRegisterInfo::isVirtualRegister(DestReg) && SpillSize == 4) {
985  MRI.constrainRegClass(DestReg, &AMDGPU::SReg_32_XM0RegClass);
986  }
987 
988  FrameInfo.setStackID(FrameIndex, SIStackID::SGPR_SPILL);
989  MachineInstrBuilder Spill = BuildMI(MBB, MI, DL, OpDesc, DestReg)
990  .addFrameIndex(FrameIndex) // addr
991  .addMemOperand(MMO)
993  .addReg(MFI->getFrameOffsetReg(), RegState::Implicit);
994 
995  if (ST.hasScalarStores()) {
996  // m0 is used for offset to scalar stores if used to spill.
997  Spill.addReg(AMDGPU::M0, RegState::ImplicitDefine | RegState::Dead);
998  }
999 
1000  return;
1001  }
1002 
1003  assert(RI.hasVGPRs(RC) && "Only VGPR spilling expected");
1004 
1005  unsigned Opcode = getVGPRSpillRestoreOpcode(SpillSize);
1006  BuildMI(MBB, MI, DL, get(Opcode), DestReg)
1007  .addFrameIndex(FrameIndex) // vaddr
1008  .addReg(MFI->getScratchRSrcReg()) // scratch_rsrc
1009  .addReg(MFI->getFrameOffsetReg()) // scratch_offset
1010  .addImm(0) // offset
1011  .addMemOperand(MMO);
1012 }
1013 
1014 /// \param @Offset Offset in bytes of the FrameIndex being spilled
1016  MachineBasicBlock &MBB, MachineInstr &MI, RegScavenger *RS, unsigned TmpReg,
1017  unsigned FrameOffset, unsigned Size) const {
1018  MachineFunction *MF = MBB.getParent();
1020  const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
1021  DebugLoc DL = MBB.findDebugLoc(MI);
1022  unsigned WorkGroupSize = MFI->getMaxFlatWorkGroupSize();
1023  unsigned WavefrontSize = ST.getWavefrontSize();
1024 
1025  unsigned TIDReg = MFI->getTIDReg();
1026  if (!MFI->hasCalculatedTID()) {
1027  MachineBasicBlock &Entry = MBB.getParent()->front();
1028  MachineBasicBlock::iterator Insert = Entry.front();
1029  DebugLoc DL = Insert->getDebugLoc();
1030 
1031  TIDReg = RI.findUnusedRegister(MF->getRegInfo(), &AMDGPU::VGPR_32RegClass,
1032  *MF);
1033  if (TIDReg == AMDGPU::NoRegister)
1034  return TIDReg;
1035 
1037  WorkGroupSize > WavefrontSize) {
1038  unsigned TIDIGXReg
1040  unsigned TIDIGYReg
1042  unsigned TIDIGZReg
1044  unsigned InputPtrReg =
1046  for (unsigned Reg : {TIDIGXReg, TIDIGYReg, TIDIGZReg}) {
1047  if (!Entry.isLiveIn(Reg))
1048  Entry.addLiveIn(Reg);
1049  }
1050 
1051  RS->enterBasicBlock(Entry);
1052  // FIXME: Can we scavenge an SReg_64 and access the subregs?
1053  unsigned STmp0 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1054  unsigned STmp1 = RS->scavengeRegister(&AMDGPU::SGPR_32RegClass, 0);
1055  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp0)
1056  .addReg(InputPtrReg)
1058  BuildMI(Entry, Insert, DL, get(AMDGPU::S_LOAD_DWORD_IMM), STmp1)
1059  .addReg(InputPtrReg)
1061 
1062  // NGROUPS.X * NGROUPS.Y
1063  BuildMI(Entry, Insert, DL, get(AMDGPU::S_MUL_I32), STmp1)
1064  .addReg(STmp1)
1065  .addReg(STmp0);
1066  // (NGROUPS.X * NGROUPS.Y) * TIDIG.X
1067  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MUL_U32_U24_e32), TIDReg)
1068  .addReg(STmp1)
1069  .addReg(TIDIGXReg);
1070  // NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)
1071  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MAD_U32_U24), TIDReg)
1072  .addReg(STmp0)
1073  .addReg(TIDIGYReg)
1074  .addReg(TIDReg);
1075  // (NGROUPS.Z * TIDIG.Y + (NGROUPS.X * NGROPUS.Y * TIDIG.X)) + TIDIG.Z
1076  getAddNoCarry(Entry, Insert, DL, TIDReg)
1077  .addReg(TIDReg)
1078  .addReg(TIDIGZReg);
1079  } else {
1080  // Get the wave id
1081  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_LO_U32_B32_e64),
1082  TIDReg)
1083  .addImm(-1)
1084  .addImm(0);
1085 
1086  BuildMI(Entry, Insert, DL, get(AMDGPU::V_MBCNT_HI_U32_B32_e64),
1087  TIDReg)
1088  .addImm(-1)
1089  .addReg(TIDReg);
1090  }
1091 
1092  BuildMI(Entry, Insert, DL, get(AMDGPU::V_LSHLREV_B32_e32),
1093  TIDReg)
1094  .addImm(2)
1095  .addReg(TIDReg);
1096  MFI->setTIDReg(TIDReg);
1097  }
1098 
1099  // Add FrameIndex to LDS offset
1100  unsigned LDSOffset = MFI->getLDSSize() + (FrameOffset * WorkGroupSize);
1101  getAddNoCarry(MBB, MI, DL, TmpReg)
1102  .addImm(LDSOffset)
1103  .addReg(TIDReg);
1104 
1105  return TmpReg;
1106 }
1107 
1110  int Count) const {
1111  DebugLoc DL = MBB.findDebugLoc(MI);
1112  while (Count > 0) {
1113  int Arg;
1114  if (Count >= 8)
1115  Arg = 7;
1116  else
1117  Arg = Count - 1;
1118  Count -= 8;
1119  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOP))
1120  .addImm(Arg);
1121  }
1122 }
1123 
1126  insertWaitStates(MBB, MI, 1);
1127 }
1128 
1130  auto MF = MBB.getParent();
1131  SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
1132 
1133  assert(Info->isEntryFunction());
1134 
1135  if (MBB.succ_empty()) {
1136  bool HasNoTerminator = MBB.getFirstTerminator() == MBB.end();
1137  if (HasNoTerminator)
1138  BuildMI(MBB, MBB.end(), DebugLoc(),
1139  get(Info->returnsVoid() ? AMDGPU::S_ENDPGM : AMDGPU::SI_RETURN_TO_EPILOG));
1140  }
1141 }
1142 
1144  switch (MI.getOpcode()) {
1145  default: return 1; // FIXME: Do wait states equal cycles?
1146 
1147  case AMDGPU::S_NOP:
1148  return MI.getOperand(0).getImm() + 1;
1149  }
1150 }
1151 
1153  MachineBasicBlock &MBB = *MI.getParent();
1154  DebugLoc DL = MBB.findDebugLoc(MI);
1155  switch (MI.getOpcode()) {
1156  default: return TargetInstrInfo::expandPostRAPseudo(MI);
1157  case AMDGPU::S_MOV_B64_term:
1158  // This is only a terminator to get the correct spill code placement during
1159  // register allocation.
1160  MI.setDesc(get(AMDGPU::S_MOV_B64));
1161  break;
1162 
1163  case AMDGPU::S_XOR_B64_term:
1164  // This is only a terminator to get the correct spill code placement during
1165  // register allocation.
1166  MI.setDesc(get(AMDGPU::S_XOR_B64));
1167  break;
1168 
1169  case AMDGPU::S_ANDN2_B64_term:
1170  // This is only a terminator to get the correct spill code placement during
1171  // register allocation.
1172  MI.setDesc(get(AMDGPU::S_ANDN2_B64));
1173  break;
1174 
1175  case AMDGPU::V_MOV_B64_PSEUDO: {
1176  unsigned Dst = MI.getOperand(0).getReg();
1177  unsigned DstLo = RI.getSubReg(Dst, AMDGPU::sub0);
1178  unsigned DstHi = RI.getSubReg(Dst, AMDGPU::sub1);
1179 
1180  const MachineOperand &SrcOp = MI.getOperand(1);
1181  // FIXME: Will this work for 64-bit floating point immediates?
1182  assert(!SrcOp.isFPImm());
1183  if (SrcOp.isImm()) {
1184  APInt Imm(64, SrcOp.getImm());
1185  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1186  .addImm(Imm.getLoBits(32).getZExtValue())
1187  .addReg(Dst, RegState::Implicit | RegState::Define);
1188  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1189  .addImm(Imm.getHiBits(32).getZExtValue())
1190  .addReg(Dst, RegState::Implicit | RegState::Define);
1191  } else {
1192  assert(SrcOp.isReg());
1193  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstLo)
1194  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub0))
1196  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), DstHi)
1197  .addReg(RI.getSubReg(SrcOp.getReg(), AMDGPU::sub1))
1199  }
1200  MI.eraseFromParent();
1201  break;
1202  }
1203  case AMDGPU::V_SET_INACTIVE_B32: {
1204  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1205  .addReg(AMDGPU::EXEC);
1206  BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B32_e32), MI.getOperand(0).getReg())
1207  .add(MI.getOperand(2));
1208  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1209  .addReg(AMDGPU::EXEC);
1210  MI.eraseFromParent();
1211  break;
1212  }
1213  case AMDGPU::V_SET_INACTIVE_B64: {
1214  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1215  .addReg(AMDGPU::EXEC);
1216  MachineInstr *Copy = BuildMI(MBB, MI, DL, get(AMDGPU::V_MOV_B64_PSEUDO),
1217  MI.getOperand(0).getReg())
1218  .add(MI.getOperand(2));
1219  expandPostRAPseudo(*Copy);
1220  BuildMI(MBB, MI, DL, get(AMDGPU::S_NOT_B64), AMDGPU::EXEC)
1221  .addReg(AMDGPU::EXEC);
1222  MI.eraseFromParent();
1223  break;
1224  }
1225  case AMDGPU::V_MOVRELD_B32_V1:
1226  case AMDGPU::V_MOVRELD_B32_V2:
1227  case AMDGPU::V_MOVRELD_B32_V4:
1228  case AMDGPU::V_MOVRELD_B32_V8:
1229  case AMDGPU::V_MOVRELD_B32_V16: {
1230  const MCInstrDesc &MovRelDesc = get(AMDGPU::V_MOVRELD_B32_e32);
1231  unsigned VecReg = MI.getOperand(0).getReg();
1232  bool IsUndef = MI.getOperand(1).isUndef();
1233  unsigned SubReg = AMDGPU::sub0 + MI.getOperand(3).getImm();
1234  assert(VecReg == MI.getOperand(1).getReg());
1235 
1236  MachineInstr *MovRel =
1237  BuildMI(MBB, MI, DL, MovRelDesc)
1238  .addReg(RI.getSubReg(VecReg, SubReg), RegState::Undef)
1239  .add(MI.getOperand(2))
1240  .addReg(VecReg, RegState::ImplicitDefine)
1241  .addReg(VecReg,
1242  RegState::Implicit | (IsUndef ? RegState::Undef : 0));
1243 
1244  const int ImpDefIdx =
1245  MovRelDesc.getNumOperands() + MovRelDesc.getNumImplicitUses();
1246  const int ImpUseIdx = ImpDefIdx + 1;
1247  MovRel->tieOperands(ImpDefIdx, ImpUseIdx);
1248 
1249  MI.eraseFromParent();
1250  break;
1251  }
1252  case AMDGPU::SI_PC_ADD_REL_OFFSET: {
1253  MachineFunction &MF = *MBB.getParent();
1254  unsigned Reg = MI.getOperand(0).getReg();
1255  unsigned RegLo = RI.getSubReg(Reg, AMDGPU::sub0);
1256  unsigned RegHi = RI.getSubReg(Reg, AMDGPU::sub1);
1257 
1258  // Create a bundle so these instructions won't be re-ordered by the
1259  // post-RA scheduler.
1260  MIBundleBuilder Bundler(MBB, MI);
1261  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_GETPC_B64), Reg));
1262 
1263  // Add 32-bit offset from this instruction to the start of the
1264  // constant data.
1265  Bundler.append(BuildMI(MF, DL, get(AMDGPU::S_ADD_U32), RegLo)
1266  .addReg(RegLo)
1267  .add(MI.getOperand(1)));
1268 
1269  MachineInstrBuilder MIB = BuildMI(MF, DL, get(AMDGPU::S_ADDC_U32), RegHi)
1270  .addReg(RegHi);
1272  MIB.addImm(0);
1273  else
1274  MIB.add(MI.getOperand(2));
1275 
1276  Bundler.append(MIB);
1277  finalizeBundle(MBB, Bundler.begin());
1278 
1279  MI.eraseFromParent();
1280  break;
1281  }
1282  case AMDGPU::EXIT_WWM: {
1283  // This only gets its own opcode so that SIFixWWMLiveness can tell when WWM
1284  // is exited.
1285  MI.setDesc(get(AMDGPU::S_MOV_B64));
1286  break;
1287  }
1288  case TargetOpcode::BUNDLE: {
1289  if (!MI.mayLoad())
1290  return false;
1291 
1292  // If it is a load it must be a memory clause
1294  I->isBundledWithSucc(); ++I) {
1295  I->unbundleFromSucc();
1296  for (MachineOperand &MO : I->operands())
1297  if (MO.isReg())
1298  MO.setIsInternalRead(false);
1299  }
1300 
1301  MI.eraseFromParent();
1302  break;
1303  }
1304  }
1305  return true;
1306 }
1307 
1309  MachineOperand &Src0,
1310  unsigned Src0OpName,
1311  MachineOperand &Src1,
1312  unsigned Src1OpName) const {
1313  MachineOperand *Src0Mods = getNamedOperand(MI, Src0OpName);
1314  if (!Src0Mods)
1315  return false;
1316 
1317  MachineOperand *Src1Mods = getNamedOperand(MI, Src1OpName);
1318  assert(Src1Mods &&
1319  "All commutable instructions have both src0 and src1 modifiers");
1320 
1321  int Src0ModsVal = Src0Mods->getImm();
1322  int Src1ModsVal = Src1Mods->getImm();
1323 
1324  Src1Mods->setImm(Src0ModsVal);
1325  Src0Mods->setImm(Src1ModsVal);
1326  return true;
1327 }
1328 
1330  MachineOperand &RegOp,
1331  MachineOperand &NonRegOp) {
1332  unsigned Reg = RegOp.getReg();
1333  unsigned SubReg = RegOp.getSubReg();
1334  bool IsKill = RegOp.isKill();
1335  bool IsDead = RegOp.isDead();
1336  bool IsUndef = RegOp.isUndef();
1337  bool IsDebug = RegOp.isDebug();
1338 
1339  if (NonRegOp.isImm())
1340  RegOp.ChangeToImmediate(NonRegOp.getImm());
1341  else if (NonRegOp.isFI())
1342  RegOp.ChangeToFrameIndex(NonRegOp.getIndex());
1343  else
1344  return nullptr;
1345 
1346  NonRegOp.ChangeToRegister(Reg, false, false, IsKill, IsDead, IsUndef, IsDebug);
1347  NonRegOp.setSubReg(SubReg);
1348 
1349  return &MI;
1350 }
1351 
1353  unsigned Src0Idx,
1354  unsigned Src1Idx) const {
1355  assert(!NewMI && "this should never be used");
1356 
1357  unsigned Opc = MI.getOpcode();
1358  int CommutedOpcode = commuteOpcode(Opc);
1359  if (CommutedOpcode == -1)
1360  return nullptr;
1361 
1362  assert(AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0) ==
1363  static_cast<int>(Src0Idx) &&
1364  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1) ==
1365  static_cast<int>(Src1Idx) &&
1366  "inconsistency with findCommutedOpIndices");
1367 
1368  MachineOperand &Src0 = MI.getOperand(Src0Idx);
1369  MachineOperand &Src1 = MI.getOperand(Src1Idx);
1370 
1371  MachineInstr *CommutedMI = nullptr;
1372  if (Src0.isReg() && Src1.isReg()) {
1373  if (isOperandLegal(MI, Src1Idx, &Src0)) {
1374  // Be sure to copy the source modifiers to the right place.
1375  CommutedMI
1376  = TargetInstrInfo::commuteInstructionImpl(MI, NewMI, Src0Idx, Src1Idx);
1377  }
1378 
1379  } else if (Src0.isReg() && !Src1.isReg()) {
1380  // src0 should always be able to support any operand type, so no need to
1381  // check operand legality.
1382  CommutedMI = swapRegAndNonRegOperand(MI, Src0, Src1);
1383  } else if (!Src0.isReg() && Src1.isReg()) {
1384  if (isOperandLegal(MI, Src1Idx, &Src0))
1385  CommutedMI = swapRegAndNonRegOperand(MI, Src1, Src0);
1386  } else {
1387  // FIXME: Found two non registers to commute. This does happen.
1388  return nullptr;
1389  }
1390 
1391  if (CommutedMI) {
1392  swapSourceModifiers(MI, Src0, AMDGPU::OpName::src0_modifiers,
1393  Src1, AMDGPU::OpName::src1_modifiers);
1394 
1395  CommutedMI->setDesc(get(CommutedOpcode));
1396  }
1397 
1398  return CommutedMI;
1399 }
1400 
1401 // This needs to be implemented because the source modifiers may be inserted
1402 // between the true commutable operands, and the base
1403 // TargetInstrInfo::commuteInstruction uses it.
1405  unsigned &SrcOpIdx1) const {
1406  return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
1407 }
1408 
1409 bool SIInstrInfo::findCommutedOpIndices(MCInstrDesc Desc, unsigned &SrcOpIdx0,
1410  unsigned &SrcOpIdx1) const {
1411  if (!Desc.isCommutable())
1412  return false;
1413 
1414  unsigned Opc = Desc.getOpcode();
1415  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
1416  if (Src0Idx == -1)
1417  return false;
1418 
1419  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
1420  if (Src1Idx == -1)
1421  return false;
1422 
1423  return fixCommutedOpIndices(SrcOpIdx0, SrcOpIdx1, Src0Idx, Src1Idx);
1424 }
1425 
1426 bool SIInstrInfo::isBranchOffsetInRange(unsigned BranchOp,
1427  int64_t BrOffset) const {
1428  // BranchRelaxation should never have to check s_setpc_b64 because its dest
1429  // block is unanalyzable.
1430  assert(BranchOp != AMDGPU::S_SETPC_B64);
1431 
1432  // Convert to dwords.
1433  BrOffset /= 4;
1434 
1435  // The branch instructions do PC += signext(SIMM16 * 4) + 4, so the offset is
1436  // from the next instruction.
1437  BrOffset -= 1;
1438 
1439  return isIntN(BranchOffsetBits, BrOffset);
1440 }
1441 
1443  const MachineInstr &MI) const {
1444  if (MI.getOpcode() == AMDGPU::S_SETPC_B64) {
1445  // This would be a difficult analysis to perform, but can always be legal so
1446  // there's no need to analyze it.
1447  return nullptr;
1448  }
1449 
1450  return MI.getOperand(0).getMBB();
1451 }
1452 
1454  MachineBasicBlock &DestBB,
1455  const DebugLoc &DL,
1456  int64_t BrOffset,
1457  RegScavenger *RS) const {
1458  assert(RS && "RegScavenger required for long branching");
1459  assert(MBB.empty() &&
1460  "new block should be inserted for expanding unconditional branch");
1461  assert(MBB.pred_size() == 1);
1462 
1463  MachineFunction *MF = MBB.getParent();
1464  MachineRegisterInfo &MRI = MF->getRegInfo();
1465 
1466  // FIXME: Virtual register workaround for RegScavenger not working with empty
1467  // blocks.
1468  unsigned PCReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
1469 
1470  auto I = MBB.end();
1471 
1472  // We need to compute the offset relative to the instruction immediately after
1473  // s_getpc_b64. Insert pc arithmetic code before last terminator.
1474  MachineInstr *GetPC = BuildMI(MBB, I, DL, get(AMDGPU::S_GETPC_B64), PCReg);
1475 
1476  // TODO: Handle > 32-bit block address.
1477  if (BrOffset >= 0) {
1478  BuildMI(MBB, I, DL, get(AMDGPU::S_ADD_U32))
1479  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1480  .addReg(PCReg, 0, AMDGPU::sub0)
1482  BuildMI(MBB, I, DL, get(AMDGPU::S_ADDC_U32))
1483  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1484  .addReg(PCReg, 0, AMDGPU::sub1)
1485  .addImm(0);
1486  } else {
1487  // Backwards branch.
1488  BuildMI(MBB, I, DL, get(AMDGPU::S_SUB_U32))
1489  .addReg(PCReg, RegState::Define, AMDGPU::sub0)
1490  .addReg(PCReg, 0, AMDGPU::sub0)
1492  BuildMI(MBB, I, DL, get(AMDGPU::S_SUBB_U32))
1493  .addReg(PCReg, RegState::Define, AMDGPU::sub1)
1494  .addReg(PCReg, 0, AMDGPU::sub1)
1495  .addImm(0);
1496  }
1497 
1498  // Insert the indirect branch after the other terminator.
1499  BuildMI(&MBB, DL, get(AMDGPU::S_SETPC_B64))
1500  .addReg(PCReg);
1501 
1502  // FIXME: If spilling is necessary, this will fail because this scavenger has
1503  // no emergency stack slots. It is non-trivial to spill in this situation,
1504  // because the restore code needs to be specially placed after the
1505  // jump. BranchRelaxation then needs to be made aware of the newly inserted
1506  // block.
1507  //
1508  // If a spill is needed for the pc register pair, we need to insert a spill
1509  // restore block right before the destination block, and insert a short branch
1510  // into the old destination block's fallthrough predecessor.
1511  // e.g.:
1512  //
1513  // s_cbranch_scc0 skip_long_branch:
1514  //
1515  // long_branch_bb:
1516  // spill s[8:9]
1517  // s_getpc_b64 s[8:9]
1518  // s_add_u32 s8, s8, restore_bb
1519  // s_addc_u32 s9, s9, 0
1520  // s_setpc_b64 s[8:9]
1521  //
1522  // skip_long_branch:
1523  // foo;
1524  //
1525  // .....
1526  //
1527  // dest_bb_fallthrough_predecessor:
1528  // bar;
1529  // s_branch dest_bb
1530  //
1531  // restore_bb:
1532  // restore s[8:9]
1533  // fallthrough dest_bb
1534  ///
1535  // dest_bb:
1536  // buzz;
1537 
1538  RS->enterBasicBlockEnd(MBB);
1539  unsigned Scav = RS->scavengeRegisterBackwards(
1540  AMDGPU::SReg_64RegClass,
1541  MachineBasicBlock::iterator(GetPC), false, 0);
1542  MRI.replaceRegWith(PCReg, Scav);
1543  MRI.clearVirtRegs();
1544  RS->setRegUsed(Scav);
1545 
1546  return 4 + 8 + 4 + 4;
1547 }
1548 
1549 unsigned SIInstrInfo::getBranchOpcode(SIInstrInfo::BranchPredicate Cond) {
1550  switch (Cond) {
1551  case SIInstrInfo::SCC_TRUE:
1552  return AMDGPU::S_CBRANCH_SCC1;
1553  case SIInstrInfo::SCC_FALSE:
1554  return AMDGPU::S_CBRANCH_SCC0;
1555  case SIInstrInfo::VCCNZ:
1556  return AMDGPU::S_CBRANCH_VCCNZ;
1557  case SIInstrInfo::VCCZ:
1558  return AMDGPU::S_CBRANCH_VCCZ;
1559  case SIInstrInfo::EXECNZ:
1560  return AMDGPU::S_CBRANCH_EXECNZ;
1561  case SIInstrInfo::EXECZ:
1562  return AMDGPU::S_CBRANCH_EXECZ;
1563  default:
1564  llvm_unreachable("invalid branch predicate");
1565  }
1566 }
1567 
1568 SIInstrInfo::BranchPredicate SIInstrInfo::getBranchPredicate(unsigned Opcode) {
1569  switch (Opcode) {
1570  case AMDGPU::S_CBRANCH_SCC0:
1571  return SCC_FALSE;
1572  case AMDGPU::S_CBRANCH_SCC1:
1573  return SCC_TRUE;
1574  case AMDGPU::S_CBRANCH_VCCNZ:
1575  return VCCNZ;
1576  case AMDGPU::S_CBRANCH_VCCZ:
1577  return VCCZ;
1578  case AMDGPU::S_CBRANCH_EXECNZ:
1579  return EXECNZ;
1580  case AMDGPU::S_CBRANCH_EXECZ:
1581  return EXECZ;
1582  default:
1583  return INVALID_BR;
1584  }
1585 }
1586 
1589  MachineBasicBlock *&TBB,
1590  MachineBasicBlock *&FBB,
1592  bool AllowModify) const {
1593  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1594  // Unconditional Branch
1595  TBB = I->getOperand(0).getMBB();
1596  return false;
1597  }
1598 
1599  MachineBasicBlock *CondBB = nullptr;
1600 
1601  if (I->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
1602  CondBB = I->getOperand(1).getMBB();
1603  Cond.push_back(I->getOperand(0));
1604  } else {
1605  BranchPredicate Pred = getBranchPredicate(I->getOpcode());
1606  if (Pred == INVALID_BR)
1607  return true;
1608 
1609  CondBB = I->getOperand(0).getMBB();
1611  Cond.push_back(I->getOperand(1)); // Save the branch register.
1612  }
1613  ++I;
1614 
1615  if (I == MBB.end()) {
1616  // Conditional branch followed by fall-through.
1617  TBB = CondBB;
1618  return false;
1619  }
1620 
1621  if (I->getOpcode() == AMDGPU::S_BRANCH) {
1622  TBB = CondBB;
1623  FBB = I->getOperand(0).getMBB();
1624  return false;
1625  }
1626 
1627  return true;
1628 }
1629 
1631  MachineBasicBlock *&FBB,
1633  bool AllowModify) const {
1635  auto E = MBB.end();
1636  if (I == E)
1637  return false;
1638 
1639  // Skip over the instructions that are artificially terminators for special
1640  // exec management.
1641  while (I != E && !I->isBranch() && !I->isReturn() &&
1642  I->getOpcode() != AMDGPU::SI_MASK_BRANCH) {
1643  switch (I->getOpcode()) {
1644  case AMDGPU::SI_MASK_BRANCH:
1645  case AMDGPU::S_MOV_B64_term:
1646  case AMDGPU::S_XOR_B64_term:
1647  case AMDGPU::S_ANDN2_B64_term:
1648  break;
1649  case AMDGPU::SI_IF:
1650  case AMDGPU::SI_ELSE:
1651  case AMDGPU::SI_KILL_I1_TERMINATOR:
1652  case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1653  // FIXME: It's messy that these need to be considered here at all.
1654  return true;
1655  default:
1656  llvm_unreachable("unexpected non-branch terminator inst");
1657  }
1658 
1659  ++I;
1660  }
1661 
1662  if (I == E)
1663  return false;
1664 
1665  if (I->getOpcode() != AMDGPU::SI_MASK_BRANCH)
1666  return analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify);
1667 
1668  ++I;
1669 
1670  // TODO: Should be able to treat as fallthrough?
1671  if (I == MBB.end())
1672  return true;
1673 
1674  if (analyzeBranchImpl(MBB, I, TBB, FBB, Cond, AllowModify))
1675  return true;
1676 
1677  MachineBasicBlock *MaskBrDest = I->getOperand(0).getMBB();
1678 
1679  // Specifically handle the case where the conditional branch is to the same
1680  // destination as the mask branch. e.g.
1681  //
1682  // si_mask_branch BB8
1683  // s_cbranch_execz BB8
1684  // s_cbranch BB9
1685  //
1686  // This is required to understand divergent loops which may need the branches
1687  // to be relaxed.
1688  if (TBB != MaskBrDest || Cond.empty())
1689  return true;
1690 
1691  auto Pred = Cond[0].getImm();
1692  return (Pred != EXECZ && Pred != EXECNZ);
1693 }
1694 
1696  int *BytesRemoved) const {
1698 
1699  unsigned Count = 0;
1700  unsigned RemovedSize = 0;
1701  while (I != MBB.end()) {
1702  MachineBasicBlock::iterator Next = std::next(I);
1703  if (I->getOpcode() == AMDGPU::SI_MASK_BRANCH) {
1704  I = Next;
1705  continue;
1706  }
1707 
1708  RemovedSize += getInstSizeInBytes(*I);
1709  I->eraseFromParent();
1710  ++Count;
1711  I = Next;
1712  }
1713 
1714  if (BytesRemoved)
1715  *BytesRemoved = RemovedSize;
1716 
1717  return Count;
1718 }
1719 
1720 // Copy the flags onto the implicit condition register operand.
1722  const MachineOperand &OrigCond) {
1723  CondReg.setIsUndef(OrigCond.isUndef());
1724  CondReg.setIsKill(OrigCond.isKill());
1725 }
1726 
1728  MachineBasicBlock *TBB,
1729  MachineBasicBlock *FBB,
1731  const DebugLoc &DL,
1732  int *BytesAdded) const {
1733  if (!FBB && Cond.empty()) {
1734  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1735  .addMBB(TBB);
1736  if (BytesAdded)
1737  *BytesAdded = 4;
1738  return 1;
1739  }
1740 
1741  if(Cond.size() == 1 && Cond[0].isReg()) {
1742  BuildMI(&MBB, DL, get(AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO))
1743  .add(Cond[0])
1744  .addMBB(TBB);
1745  return 1;
1746  }
1747 
1748  assert(TBB && Cond[0].isImm());
1749 
1750  unsigned Opcode
1751  = getBranchOpcode(static_cast<BranchPredicate>(Cond[0].getImm()));
1752 
1753  if (!FBB) {
1754  Cond[1].isUndef();
1755  MachineInstr *CondBr =
1756  BuildMI(&MBB, DL, get(Opcode))
1757  .addMBB(TBB);
1758 
1759  // Copy the flags onto the implicit condition register operand.
1760  preserveCondRegFlags(CondBr->getOperand(1), Cond[1]);
1761 
1762  if (BytesAdded)
1763  *BytesAdded = 4;
1764  return 1;
1765  }
1766 
1767  assert(TBB && FBB);
1768 
1769  MachineInstr *CondBr =
1770  BuildMI(&MBB, DL, get(Opcode))
1771  .addMBB(TBB);
1772  BuildMI(&MBB, DL, get(AMDGPU::S_BRANCH))
1773  .addMBB(FBB);
1774 
1775  MachineOperand &CondReg = CondBr->getOperand(1);
1776  CondReg.setIsUndef(Cond[1].isUndef());
1777  CondReg.setIsKill(Cond[1].isKill());
1778 
1779  if (BytesAdded)
1780  *BytesAdded = 8;
1781 
1782  return 2;
1783 }
1784 
1786  SmallVectorImpl<MachineOperand> &Cond) const {
1787  if (Cond.size() != 2) {
1788  return true;
1789  }
1790 
1791  if (Cond[0].isImm()) {
1792  Cond[0].setImm(-Cond[0].getImm());
1793  return false;
1794  }
1795 
1796  return true;
1797 }
1798 
1801  unsigned TrueReg, unsigned FalseReg,
1802  int &CondCycles,
1803  int &TrueCycles, int &FalseCycles) const {
1804  switch (Cond[0].getImm()) {
1805  case VCCNZ:
1806  case VCCZ: {
1807  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1808  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1809  assert(MRI.getRegClass(FalseReg) == RC);
1810 
1811  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1812  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1813 
1814  // Limit to equal cost for branch vs. N v_cndmask_b32s.
1815  return !RI.isSGPRClass(RC) && NumInsts <= 6;
1816  }
1817  case SCC_TRUE:
1818  case SCC_FALSE: {
1819  // FIXME: We could insert for VGPRs if we could replace the original compare
1820  // with a vector one.
1821  const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1822  const TargetRegisterClass *RC = MRI.getRegClass(TrueReg);
1823  assert(MRI.getRegClass(FalseReg) == RC);
1824 
1825  int NumInsts = AMDGPU::getRegBitWidth(RC->getID()) / 32;
1826 
1827  // Multiples of 8 can do s_cselect_b64
1828  if (NumInsts % 2 == 0)
1829  NumInsts /= 2;
1830 
1831  CondCycles = TrueCycles = FalseCycles = NumInsts; // ???
1832  return RI.isSGPRClass(RC);
1833  }
1834  default:
1835  return false;
1836  }
1837 }
1838 
1841  unsigned DstReg, ArrayRef<MachineOperand> Cond,
1842  unsigned TrueReg, unsigned FalseReg) const {
1843  BranchPredicate Pred = static_cast<BranchPredicate>(Cond[0].getImm());
1844  if (Pred == VCCZ || Pred == SCC_FALSE) {
1845  Pred = static_cast<BranchPredicate>(-Pred);
1846  std::swap(TrueReg, FalseReg);
1847  }
1848 
1850  const TargetRegisterClass *DstRC = MRI.getRegClass(DstReg);
1851  unsigned DstSize = RI.getRegSizeInBits(*DstRC);
1852 
1853  if (DstSize == 32) {
1854  unsigned SelOp = Pred == SCC_TRUE ?
1855  AMDGPU::S_CSELECT_B32 : AMDGPU::V_CNDMASK_B32_e32;
1856 
1857  // Instruction's operands are backwards from what is expected.
1858  MachineInstr *Select =
1859  BuildMI(MBB, I, DL, get(SelOp), DstReg)
1860  .addReg(FalseReg)
1861  .addReg(TrueReg);
1862 
1863  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1864  return;
1865  }
1866 
1867  if (DstSize == 64 && Pred == SCC_TRUE) {
1868  MachineInstr *Select =
1869  BuildMI(MBB, I, DL, get(AMDGPU::S_CSELECT_B64), DstReg)
1870  .addReg(FalseReg)
1871  .addReg(TrueReg);
1872 
1873  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1874  return;
1875  }
1876 
1877  static const int16_t Sub0_15[] = {
1878  AMDGPU::sub0, AMDGPU::sub1, AMDGPU::sub2, AMDGPU::sub3,
1879  AMDGPU::sub4, AMDGPU::sub5, AMDGPU::sub6, AMDGPU::sub7,
1880  AMDGPU::sub8, AMDGPU::sub9, AMDGPU::sub10, AMDGPU::sub11,
1881  AMDGPU::sub12, AMDGPU::sub13, AMDGPU::sub14, AMDGPU::sub15,
1882  };
1883 
1884  static const int16_t Sub0_15_64[] = {
1885  AMDGPU::sub0_sub1, AMDGPU::sub2_sub3,
1886  AMDGPU::sub4_sub5, AMDGPU::sub6_sub7,
1887  AMDGPU::sub8_sub9, AMDGPU::sub10_sub11,
1888  AMDGPU::sub12_sub13, AMDGPU::sub14_sub15,
1889  };
1890 
1891  unsigned SelOp = AMDGPU::V_CNDMASK_B32_e32;
1892  const TargetRegisterClass *EltRC = &AMDGPU::VGPR_32RegClass;
1893  const int16_t *SubIndices = Sub0_15;
1894  int NElts = DstSize / 32;
1895 
1896  // 64-bit select is only avaialble for SALU.
1897  if (Pred == SCC_TRUE) {
1898  SelOp = AMDGPU::S_CSELECT_B64;
1899  EltRC = &AMDGPU::SGPR_64RegClass;
1900  SubIndices = Sub0_15_64;
1901 
1902  assert(NElts % 2 == 0);
1903  NElts /= 2;
1904  }
1905 
1907  MBB, I, DL, get(AMDGPU::REG_SEQUENCE), DstReg);
1908 
1909  I = MIB->getIterator();
1910 
1912  for (int Idx = 0; Idx != NElts; ++Idx) {
1913  unsigned DstElt = MRI.createVirtualRegister(EltRC);
1914  Regs.push_back(DstElt);
1915 
1916  unsigned SubIdx = SubIndices[Idx];
1917 
1918  MachineInstr *Select =
1919  BuildMI(MBB, I, DL, get(SelOp), DstElt)
1920  .addReg(FalseReg, 0, SubIdx)
1921  .addReg(TrueReg, 0, SubIdx);
1922  preserveCondRegFlags(Select->getOperand(3), Cond[1]);
1923 
1924  MIB.addReg(DstElt)
1925  .addImm(SubIdx);
1926  }
1927 }
1928 
1930  switch (MI.getOpcode()) {
1931  case AMDGPU::V_MOV_B32_e32:
1932  case AMDGPU::V_MOV_B32_e64:
1933  case AMDGPU::V_MOV_B64_PSEUDO: {
1934  // If there are additional implicit register operands, this may be used for
1935  // register indexing so the source register operand isn't simply copied.
1936  unsigned NumOps = MI.getDesc().getNumOperands() +
1937  MI.getDesc().getNumImplicitUses();
1938 
1939  return MI.getNumOperands() == NumOps;
1940  }
1941  case AMDGPU::S_MOV_B32:
1942  case AMDGPU::S_MOV_B64:
1943  case AMDGPU::COPY:
1944  return true;
1945  default:
1946  return false;
1947  }
1948 }
1949 
1951  unsigned Kind) const {
1952  switch(Kind) {
1963  }
1964  return AMDGPUAS::FLAT_ADDRESS;
1965 }
1966 
1968  unsigned Opc = MI.getOpcode();
1969  int Src0ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1970  AMDGPU::OpName::src0_modifiers);
1971  int Src1ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1972  AMDGPU::OpName::src1_modifiers);
1973  int Src2ModIdx = AMDGPU::getNamedOperandIdx(Opc,
1974  AMDGPU::OpName::src2_modifiers);
1975 
1976  MI.RemoveOperand(Src2ModIdx);
1977  MI.RemoveOperand(Src1ModIdx);
1978  MI.RemoveOperand(Src0ModIdx);
1979 }
1980 
1982  unsigned Reg, MachineRegisterInfo *MRI) const {
1983  if (!MRI->hasOneNonDBGUse(Reg))
1984  return false;
1985 
1986  switch (DefMI.getOpcode()) {
1987  default:
1988  return false;
1989  case AMDGPU::S_MOV_B64:
1990  // TODO: We could fold 64-bit immediates, but this get compilicated
1991  // when there are sub-registers.
1992  return false;
1993 
1994  case AMDGPU::V_MOV_B32_e32:
1995  case AMDGPU::S_MOV_B32:
1996  break;
1997  }
1998 
1999  const MachineOperand *ImmOp = getNamedOperand(DefMI, AMDGPU::OpName::src0);
2000  assert(ImmOp);
2001  // FIXME: We could handle FrameIndex values here.
2002  if (!ImmOp->isImm())
2003  return false;
2004 
2005  unsigned Opc = UseMI.getOpcode();
2006  if (Opc == AMDGPU::COPY) {
2007  bool isVGPRCopy = RI.isVGPR(*MRI, UseMI.getOperand(0).getReg());
2008  unsigned NewOpc = isVGPRCopy ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
2009  UseMI.setDesc(get(NewOpc));
2010  UseMI.getOperand(1).ChangeToImmediate(ImmOp->getImm());
2011  UseMI.addImplicitDefUseOperands(*UseMI.getParent()->getParent());
2012  return true;
2013  }
2014 
2015  if (Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64 ||
2016  Opc == AMDGPU::V_MAD_F16 || Opc == AMDGPU::V_MAC_F16_e64) {
2017  // Don't fold if we are using source or output modifiers. The new VOP2
2018  // instructions don't have them.
2019  if (hasAnyModifiersSet(UseMI))
2020  return false;
2021 
2022  // If this is a free constant, there's no reason to do this.
2023  // TODO: We could fold this here instead of letting SIFoldOperands do it
2024  // later.
2025  MachineOperand *Src0 = getNamedOperand(UseMI, AMDGPU::OpName::src0);
2026 
2027  // Any src operand can be used for the legality check.
2028  if (isInlineConstant(UseMI, *Src0, *ImmOp))
2029  return false;
2030 
2031  bool IsF32 = Opc == AMDGPU::V_MAD_F32 || Opc == AMDGPU::V_MAC_F32_e64;
2032  MachineOperand *Src1 = getNamedOperand(UseMI, AMDGPU::OpName::src1);
2033  MachineOperand *Src2 = getNamedOperand(UseMI, AMDGPU::OpName::src2);
2034 
2035  // Multiplied part is the constant: Use v_madmk_{f16, f32}.
2036  // We should only expect these to be on src0 due to canonicalizations.
2037  if (Src0->isReg() && Src0->getReg() == Reg) {
2038  if (!Src1->isReg() || RI.isSGPRClass(MRI->getRegClass(Src1->getReg())))
2039  return false;
2040 
2041  if (!Src2->isReg() || RI.isSGPRClass(MRI->getRegClass(Src2->getReg())))
2042  return false;
2043 
2044  // We need to swap operands 0 and 1 since madmk constant is at operand 1.
2045 
2046  const int64_t Imm = ImmOp->getImm();
2047 
2048  // FIXME: This would be a lot easier if we could return a new instruction
2049  // instead of having to modify in place.
2050 
2051  // Remove these first since they are at the end.
2052  UseMI.RemoveOperand(
2053  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2054  UseMI.RemoveOperand(
2055  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2056 
2057  unsigned Src1Reg = Src1->getReg();
2058  unsigned Src1SubReg = Src1->getSubReg();
2059  Src0->setReg(Src1Reg);
2060  Src0->setSubReg(Src1SubReg);
2061  Src0->setIsKill(Src1->isKill());
2062 
2063  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2064  Opc == AMDGPU::V_MAC_F16_e64)
2065  UseMI.untieRegOperand(
2066  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2067 
2068  Src1->ChangeToImmediate(Imm);
2069 
2070  removeModOperands(UseMI);
2071  UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADMK_F32 : AMDGPU::V_MADMK_F16));
2072 
2073  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2074  if (DeleteDef)
2075  DefMI.eraseFromParent();
2076 
2077  return true;
2078  }
2079 
2080  // Added part is the constant: Use v_madak_{f16, f32}.
2081  if (Src2->isReg() && Src2->getReg() == Reg) {
2082  // Not allowed to use constant bus for another operand.
2083  // We can however allow an inline immediate as src0.
2084  bool Src0Inlined = false;
2085  if (Src0->isReg()) {
2086  // Try to inline constant if possible.
2087  // If the Def moves immediate and the use is single
2088  // We are saving VGPR here.
2089  MachineInstr *Def = MRI->getUniqueVRegDef(Src0->getReg());
2090  if (Def && Def->isMoveImmediate() &&
2091  isInlineConstant(Def->getOperand(1)) &&
2092  MRI->hasOneUse(Src0->getReg())) {
2093  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2094  Src0Inlined = true;
2095  } else if ((RI.isPhysicalRegister(Src0->getReg()) &&
2096  RI.isSGPRClass(RI.getPhysRegClass(Src0->getReg()))) ||
2097  (RI.isVirtualRegister(Src0->getReg()) &&
2098  RI.isSGPRClass(MRI->getRegClass(Src0->getReg()))))
2099  return false;
2100  // VGPR is okay as Src0 - fallthrough
2101  }
2102 
2103  if (Src1->isReg() && !Src0Inlined ) {
2104  // We have one slot for inlinable constant so far - try to fill it
2105  MachineInstr *Def = MRI->getUniqueVRegDef(Src1->getReg());
2106  if (Def && Def->isMoveImmediate() &&
2107  isInlineConstant(Def->getOperand(1)) &&
2108  MRI->hasOneUse(Src1->getReg()) &&
2109  commuteInstruction(UseMI)) {
2110  Src0->ChangeToImmediate(Def->getOperand(1).getImm());
2111  } else if ((RI.isPhysicalRegister(Src1->getReg()) &&
2112  RI.isSGPRClass(RI.getPhysRegClass(Src1->getReg()))) ||
2113  (RI.isVirtualRegister(Src1->getReg()) &&
2114  RI.isSGPRClass(MRI->getRegClass(Src1->getReg()))))
2115  return false;
2116  // VGPR is okay as Src1 - fallthrough
2117  }
2118 
2119  const int64_t Imm = ImmOp->getImm();
2120 
2121  // FIXME: This would be a lot easier if we could return a new instruction
2122  // instead of having to modify in place.
2123 
2124  // Remove these first since they are at the end.
2125  UseMI.RemoveOperand(
2126  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod));
2127  UseMI.RemoveOperand(
2128  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp));
2129 
2130  if (Opc == AMDGPU::V_MAC_F32_e64 ||
2131  Opc == AMDGPU::V_MAC_F16_e64)
2132  UseMI.untieRegOperand(
2133  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2));
2134 
2135  // ChangingToImmediate adds Src2 back to the instruction.
2136  Src2->ChangeToImmediate(Imm);
2137 
2138  // These come before src2.
2139  removeModOperands(UseMI);
2140  UseMI.setDesc(get(IsF32 ? AMDGPU::V_MADAK_F32 : AMDGPU::V_MADAK_F16));
2141 
2142  bool DeleteDef = MRI->hasOneNonDBGUse(Reg);
2143  if (DeleteDef)
2144  DefMI.eraseFromParent();
2145 
2146  return true;
2147  }
2148  }
2149 
2150  return false;
2151 }
2152 
2153 static bool offsetsDoNotOverlap(int WidthA, int OffsetA,
2154  int WidthB, int OffsetB) {
2155  int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
2156  int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
2157  int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
2158  return LowOffset + LowWidth <= HighOffset;
2159 }
2160 
2161 bool SIInstrInfo::checkInstOffsetsDoNotOverlap(MachineInstr &MIa,
2162  MachineInstr &MIb) const {
2163  unsigned BaseReg0, BaseReg1;
2164  int64_t Offset0, Offset1;
2165 
2166  if (getMemOpBaseRegImmOfs(MIa, BaseReg0, Offset0, &RI) &&
2167  getMemOpBaseRegImmOfs(MIb, BaseReg1, Offset1, &RI)) {
2168 
2169  if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand()) {
2170  // FIXME: Handle ds_read2 / ds_write2.
2171  return false;
2172  }
2173  unsigned Width0 = (*MIa.memoperands_begin())->getSize();
2174  unsigned Width1 = (*MIb.memoperands_begin())->getSize();
2175  if (BaseReg0 == BaseReg1 &&
2176  offsetsDoNotOverlap(Width0, Offset0, Width1, Offset1)) {
2177  return true;
2178  }
2179  }
2180 
2181  return false;
2182 }
2183 
2185  MachineInstr &MIb,
2186  AliasAnalysis *AA) const {
2187  assert((MIa.mayLoad() || MIa.mayStore()) &&
2188  "MIa must load from or modify a memory location");
2189  assert((MIb.mayLoad() || MIb.mayStore()) &&
2190  "MIb must load from or modify a memory location");
2191 
2193  return false;
2194 
2195  // XXX - Can we relax this between address spaces?
2196  if (MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
2197  return false;
2198 
2199  if (AA && MIa.hasOneMemOperand() && MIb.hasOneMemOperand()) {
2200  const MachineMemOperand *MMOa = *MIa.memoperands_begin();
2201  const MachineMemOperand *MMOb = *MIb.memoperands_begin();
2202  if (MMOa->getValue() && MMOb->getValue()) {
2203  MemoryLocation LocA(MMOa->getValue(), MMOa->getSize(), MMOa->getAAInfo());
2204  MemoryLocation LocB(MMOb->getValue(), MMOb->getSize(), MMOb->getAAInfo());
2205  if (!AA->alias(LocA, LocB))
2206  return true;
2207  }
2208  }
2209 
2210  // TODO: Should we check the address space from the MachineMemOperand? That
2211  // would allow us to distinguish objects we know don't alias based on the
2212  // underlying address space, even if it was lowered to a different one,
2213  // e.g. private accesses lowered to use MUBUF instructions on a scratch
2214  // buffer.
2215  if (isDS(MIa)) {
2216  if (isDS(MIb))
2217  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2218 
2219  return !isFLAT(MIb) || isSegmentSpecificFLAT(MIb);
2220  }
2221 
2222  if (isMUBUF(MIa) || isMTBUF(MIa)) {
2223  if (isMUBUF(MIb) || isMTBUF(MIb))
2224  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2225 
2226  return !isFLAT(MIb) && !isSMRD(MIb);
2227  }
2228 
2229  if (isSMRD(MIa)) {
2230  if (isSMRD(MIb))
2231  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2232 
2233  return !isFLAT(MIb) && !isMUBUF(MIa) && !isMTBUF(MIa);
2234  }
2235 
2236  if (isFLAT(MIa)) {
2237  if (isFLAT(MIb))
2238  return checkInstOffsetsDoNotOverlap(MIa, MIb);
2239 
2240  return false;
2241  }
2242 
2243  return false;
2244 }
2245 
2246 static int64_t getFoldableImm(const MachineOperand* MO) {
2247  if (!MO->isReg())
2248  return false;
2249  const MachineFunction *MF = MO->getParent()->getParent()->getParent();
2250  const MachineRegisterInfo &MRI = MF->getRegInfo();
2251  auto Def = MRI.getUniqueVRegDef(MO->getReg());
2252  if (Def && Def->getOpcode() == AMDGPU::V_MOV_B32_e32 &&
2253  Def->getOperand(1).isImm())
2254  return Def->getOperand(1).getImm();
2255  return AMDGPU::NoRegister;
2256 }
2257 
2259  MachineInstr &MI,
2260  LiveVariables *LV) const {
2261  unsigned Opc = MI.getOpcode();
2262  bool IsF16 = false;
2263  bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e32 || Opc == AMDGPU::V_FMAC_F32_e64;
2264 
2265  switch (Opc) {
2266  default:
2267  return nullptr;
2268  case AMDGPU::V_MAC_F16_e64:
2269  IsF16 = true;
2271  case AMDGPU::V_MAC_F32_e64:
2272  case AMDGPU::V_FMAC_F32_e64:
2273  break;
2274  case AMDGPU::V_MAC_F16_e32:
2275  IsF16 = true;
2277  case AMDGPU::V_MAC_F32_e32:
2278  case AMDGPU::V_FMAC_F32_e32: {
2279  int Src0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(),
2280  AMDGPU::OpName::src0);
2281  const MachineOperand *Src0 = &MI.getOperand(Src0Idx);
2282  if (!Src0->isReg() && !Src0->isImm())
2283  return nullptr;
2284 
2285  if (Src0->isImm() && !isInlineConstant(MI, Src0Idx, *Src0))
2286  return nullptr;
2287 
2288  break;
2289  }
2290  }
2291 
2292  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
2293  const MachineOperand *Src0 = getNamedOperand(MI, AMDGPU::OpName::src0);
2294  const MachineOperand *Src0Mods =
2295  getNamedOperand(MI, AMDGPU::OpName::src0_modifiers);
2296  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2297  const MachineOperand *Src1Mods =
2298  getNamedOperand(MI, AMDGPU::OpName::src1_modifiers);
2299  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2300  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2301  const MachineOperand *Omod = getNamedOperand(MI, AMDGPU::OpName::omod);
2302 
2303  if (!IsFMA && !Src0Mods && !Src1Mods && !Clamp && !Omod &&
2304  // If we have an SGPR input, we will violate the constant bus restriction.
2305  (!Src0->isReg() || !RI.isSGPRReg(MBB->getParent()->getRegInfo(), Src0->getReg()))) {
2306  if (auto Imm = getFoldableImm(Src2)) {
2307  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2308  get(IsF16 ? AMDGPU::V_MADAK_F16 : AMDGPU::V_MADAK_F32))
2309  .add(*Dst)
2310  .add(*Src0)
2311  .add(*Src1)
2312  .addImm(Imm);
2313  }
2314  if (auto Imm = getFoldableImm(Src1)) {
2315  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2316  get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32))
2317  .add(*Dst)
2318  .add(*Src0)
2319  .addImm(Imm)
2320  .add(*Src2);
2321  }
2322  if (auto Imm = getFoldableImm(Src0)) {
2323  if (isOperandLegal(MI, AMDGPU::getNamedOperandIdx(AMDGPU::V_MADMK_F32,
2324  AMDGPU::OpName::src0), Src1))
2325  return BuildMI(*MBB, MI, MI.getDebugLoc(),
2326  get(IsF16 ? AMDGPU::V_MADMK_F16 : AMDGPU::V_MADMK_F32))
2327  .add(*Dst)
2328  .add(*Src1)
2329  .addImm(Imm)
2330  .add(*Src2);
2331  }
2332  }
2333 
2334  assert((!IsFMA || !IsF16) && "fmac only expected with f32");
2335  unsigned NewOpc = IsFMA ? AMDGPU::V_FMA_F32 :
2336  (IsF16 ? AMDGPU::V_MAD_F16 : AMDGPU::V_MAD_F32);
2337  return BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
2338  .add(*Dst)
2339  .addImm(Src0Mods ? Src0Mods->getImm() : 0)
2340  .add(*Src0)
2341  .addImm(Src1Mods ? Src1Mods->getImm() : 0)
2342  .add(*Src1)
2343  .addImm(0) // Src mods
2344  .add(*Src2)
2345  .addImm(Clamp ? Clamp->getImm() : 0)
2346  .addImm(Omod ? Omod->getImm() : 0);
2347 }
2348 
2349 // It's not generally safe to move VALU instructions across these since it will
2350 // start using the register as a base index rather than directly.
2351 // XXX - Why isn't hasSideEffects sufficient for these?
2353  switch (MI.getOpcode()) {
2354  case AMDGPU::S_SET_GPR_IDX_ON:
2355  case AMDGPU::S_SET_GPR_IDX_MODE:
2356  case AMDGPU::S_SET_GPR_IDX_OFF:
2357  return true;
2358  default:
2359  return false;
2360  }
2361 }
2362 
2364  const MachineBasicBlock *MBB,
2365  const MachineFunction &MF) const {
2366  // XXX - Do we want the SP check in the base implementation?
2367 
2368  // Target-independent instructions do not have an implicit-use of EXEC, even
2369  // when they operate on VGPRs. Treating EXEC modifications as scheduling
2370  // boundaries prevents incorrect movements of such instructions.
2371  return TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF) ||
2372  MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
2373  MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
2374  MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
2376 }
2377 
2379  unsigned Opcode = MI.getOpcode();
2380 
2381  if (MI.mayStore() && isSMRD(MI))
2382  return true; // scalar store or atomic
2383 
2384  // These instructions cause shader I/O that may cause hardware lockups
2385  // when executed with an empty EXEC mask.
2386  //
2387  // Note: exp with VM = DONE = 0 is automatically skipped by hardware when
2388  // EXEC = 0, but checking for that case here seems not worth it
2389  // given the typical code patterns.
2390  if (Opcode == AMDGPU::S_SENDMSG || Opcode == AMDGPU::S_SENDMSGHALT ||
2391  Opcode == AMDGPU::EXP || Opcode == AMDGPU::EXP_DONE)
2392  return true;
2393 
2394  if (MI.isInlineAsm())
2395  return true; // conservative assumption
2396 
2397  // These are like SALU instructions in terms of effects, so it's questionable
2398  // whether we should return true for those.
2399  //
2400  // However, executing them with EXEC = 0 causes them to operate on undefined
2401  // data, which we avoid by returning true here.
2402  if (Opcode == AMDGPU::V_READFIRSTLANE_B32 || Opcode == AMDGPU::V_READLANE_B32)
2403  return true;
2404 
2405  return false;
2406 }
2407 
2408 bool SIInstrInfo::isInlineConstant(const APInt &Imm) const {
2409  switch (Imm.getBitWidth()) {
2410  case 32:
2412  ST.hasInv2PiInlineImm());
2413  case 64:
2415  ST.hasInv2PiInlineImm());
2416  case 16:
2417  return ST.has16BitInsts() &&
2419  ST.hasInv2PiInlineImm());
2420  default:
2421  llvm_unreachable("invalid bitwidth");
2422  }
2423 }
2424 
2426  uint8_t OperandType) const {
2427  if (!MO.isImm() ||
2428  OperandType < AMDGPU::OPERAND_SRC_FIRST ||
2429  OperandType > AMDGPU::OPERAND_SRC_LAST)
2430  return false;
2431 
2432  // MachineOperand provides no way to tell the true operand size, since it only
2433  // records a 64-bit value. We need to know the size to determine if a 32-bit
2434  // floating point immediate bit pattern is legal for an integer immediate. It
2435  // would be for any 32-bit integer operand, but would not be for a 64-bit one.
2436 
2437  int64_t Imm = MO.getImm();
2438  switch (OperandType) {
2443  int32_t Trunc = static_cast<int32_t>(Imm);
2445  }
2451  ST.hasInv2PiInlineImm());
2456  if (isInt<16>(Imm) || isUInt<16>(Imm)) {
2457  // A few special case instructions have 16-bit operands on subtargets
2458  // where 16-bit instructions are not legal.
2459  // TODO: Do the 32-bit immediates work? We shouldn't really need to handle
2460  // constants in these cases
2461  int16_t Trunc = static_cast<int16_t>(Imm);
2462  return ST.has16BitInsts() &&
2464  }
2465 
2466  return false;
2467  }
2470  if (isUInt<16>(Imm)) {
2471  int16_t Trunc = static_cast<int16_t>(Imm);
2472  return ST.has16BitInsts() &&
2474  }
2475  if (!(Imm & 0xffff)) {
2476  return ST.has16BitInsts() &&
2478  }
2479  uint32_t Trunc = static_cast<uint32_t>(Imm);
2481  }
2482  default:
2483  llvm_unreachable("invalid bitwidth");
2484  }
2485 }
2486 
2488  const MCOperandInfo &OpInfo) const {
2489  switch (MO.getType()) {
2491  return false;
2493  return !isInlineConstant(MO, OpInfo);
2499  return true;
2500  default:
2501  llvm_unreachable("unexpected operand type");
2502  }
2503 }
2504 
2505 static bool compareMachineOp(const MachineOperand &Op0,
2506  const MachineOperand &Op1) {
2507  if (Op0.getType() != Op1.getType())
2508  return false;
2509 
2510  switch (Op0.getType()) {
2512  return Op0.getReg() == Op1.getReg();
2514  return Op0.getImm() == Op1.getImm();
2515  default:
2516  llvm_unreachable("Didn't expect to be comparing these operand types");
2517  }
2518 }
2519 
2521  const MachineOperand &MO) const {
2522  const MCOperandInfo &OpInfo = get(MI.getOpcode()).OpInfo[OpNo];
2523 
2524  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
2525 
2526  if (OpInfo.OperandType == MCOI::OPERAND_IMMEDIATE)
2527  return true;
2528 
2529  if (OpInfo.RegClass < 0)
2530  return false;
2531 
2532  if (MO.isImm() && isInlineConstant(MO, OpInfo))
2533  return RI.opCanUseInlineConstant(OpInfo.OperandType);
2534 
2535  return RI.opCanUseLiteralConstant(OpInfo.OperandType);
2536 }
2537 
2538 bool SIInstrInfo::hasVALU32BitEncoding(unsigned Opcode) const {
2539  int Op32 = AMDGPU::getVOPe32(Opcode);
2540  if (Op32 == -1)
2541  return false;
2542 
2543  return pseudoToMCOpcode(Op32) != -1;
2544 }
2545 
2546 bool SIInstrInfo::hasModifiers(unsigned Opcode) const {
2547  // The src0_modifier operand is present on all instructions
2548  // that have modifiers.
2549 
2550  return AMDGPU::getNamedOperandIdx(Opcode,
2551  AMDGPU::OpName::src0_modifiers) != -1;
2552 }
2553 
2555  unsigned OpName) const {
2556  const MachineOperand *Mods = getNamedOperand(MI, OpName);
2557  return Mods && Mods->getImm();
2558 }
2559 
2561  return hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
2562  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
2563  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers) ||
2564  hasModifiersSet(MI, AMDGPU::OpName::clamp) ||
2565  hasModifiersSet(MI, AMDGPU::OpName::omod);
2566 }
2567 
2569  const MachineRegisterInfo &MRI) const {
2570  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2571  // Can't shrink instruction with three operands.
2572  // FIXME: v_cndmask_b32 has 3 operands and is shrinkable, but we need to add
2573  // a special case for it. It can only be shrunk if the third operand
2574  // is vcc. We should handle this the same way we handle vopc, by addding
2575  // a register allocation hint pre-regalloc and then do the shrinking
2576  // post-regalloc.
2577  if (Src2) {
2578  switch (MI.getOpcode()) {
2579  default: return false;
2580 
2581  case AMDGPU::V_ADDC_U32_e64:
2582  case AMDGPU::V_SUBB_U32_e64:
2583  case AMDGPU::V_SUBBREV_U32_e64: {
2584  const MachineOperand *Src1
2585  = getNamedOperand(MI, AMDGPU::OpName::src1);
2586  if (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()))
2587  return false;
2588  // Additional verification is needed for sdst/src2.
2589  return true;
2590  }
2591  case AMDGPU::V_MAC_F32_e64:
2592  case AMDGPU::V_MAC_F16_e64:
2593  case AMDGPU::V_FMAC_F32_e64:
2594  if (!Src2->isReg() || !RI.isVGPR(MRI, Src2->getReg()) ||
2595  hasModifiersSet(MI, AMDGPU::OpName::src2_modifiers))
2596  return false;
2597  break;
2598 
2599  case AMDGPU::V_CNDMASK_B32_e64:
2600  break;
2601  }
2602  }
2603 
2604  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2605  if (Src1 && (!Src1->isReg() || !RI.isVGPR(MRI, Src1->getReg()) ||
2606  hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers)))
2607  return false;
2608 
2609  // We don't need to check src0, all input types are legal, so just make sure
2610  // src0 isn't using any modifiers.
2611  if (hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers))
2612  return false;
2613 
2614  // Check output modifiers
2615  return !hasModifiersSet(MI, AMDGPU::OpName::omod) &&
2616  !hasModifiersSet(MI, AMDGPU::OpName::clamp);
2617 }
2618 
2619 // Set VCC operand with all flags from \p Orig, except for setting it as
2620 // implicit.
2622  const MachineOperand &Orig) {
2623 
2624  for (MachineOperand &Use : MI.implicit_operands()) {
2625  if (Use.isUse() && Use.getReg() == AMDGPU::VCC) {
2626  Use.setIsUndef(Orig.isUndef());
2627  Use.setIsKill(Orig.isKill());
2628  return;
2629  }
2630  }
2631 }
2632 
2634  unsigned Op32) const {
2635  MachineBasicBlock *MBB = MI.getParent();;
2636  MachineInstrBuilder Inst32 =
2637  BuildMI(*MBB, MI, MI.getDebugLoc(), get(Op32));
2638 
2639  // Add the dst operand if the 32-bit encoding also has an explicit $vdst.
2640  // For VOPC instructions, this is replaced by an implicit def of vcc.
2641  int Op32DstIdx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::vdst);
2642  if (Op32DstIdx != -1) {
2643  // dst
2644  Inst32.add(MI.getOperand(0));
2645  } else {
2646  assert(MI.getOperand(0).getReg() == AMDGPU::VCC &&
2647  "Unexpected case");
2648  }
2649 
2650  Inst32.add(*getNamedOperand(MI, AMDGPU::OpName::src0));
2651 
2652  const MachineOperand *Src1 = getNamedOperand(MI, AMDGPU::OpName::src1);
2653  if (Src1)
2654  Inst32.add(*Src1);
2655 
2656  const MachineOperand *Src2 = getNamedOperand(MI, AMDGPU::OpName::src2);
2657 
2658  if (Src2) {
2659  int Op32Src2Idx = AMDGPU::getNamedOperandIdx(Op32, AMDGPU::OpName::src2);
2660  if (Op32Src2Idx != -1) {
2661  Inst32.add(*Src2);
2662  } else {
2663  // In the case of V_CNDMASK_B32_e32, the explicit operand src2 is
2664  // replaced with an implicit read of vcc. This was already added
2665  // during the initial BuildMI, so find it to preserve the flags.
2666  copyFlagsToImplicitVCC(*Inst32, *Src2);
2667  }
2668  }
2669 
2670  return Inst32;
2671 }
2672 
2674  const MachineOperand &MO,
2675  const MCOperandInfo &OpInfo) const {
2676  // Literal constants use the constant bus.
2677  //if (isLiteralConstantLike(MO, OpInfo))
2678  // return true;
2679  if (MO.isImm())
2680  return !isInlineConstant(MO, OpInfo);
2681 
2682  if (!MO.isReg())
2683  return true; // Misc other operands like FrameIndex
2684 
2685  if (!MO.isUse())
2686  return false;
2687 
2689  return RI.isSGPRClass(MRI.getRegClass(MO.getReg()));
2690 
2691  // FLAT_SCR is just an SGPR pair.
2692  if (!MO.isImplicit() && (MO.getReg() == AMDGPU::FLAT_SCR))
2693  return true;
2694 
2695  // EXEC register uses the constant bus.
2696  if (!MO.isImplicit() && MO.getReg() == AMDGPU::EXEC)
2697  return true;
2698 
2699  // SGPRs use the constant bus
2700  return (MO.getReg() == AMDGPU::VCC || MO.getReg() == AMDGPU::M0 ||
2701  (!MO.isImplicit() &&
2702  (AMDGPU::SGPR_32RegClass.contains(MO.getReg()) ||
2703  AMDGPU::SGPR_64RegClass.contains(MO.getReg()))));
2704 }
2705 
2706 static unsigned findImplicitSGPRRead(const MachineInstr &MI) {
2707  for (const MachineOperand &MO : MI.implicit_operands()) {
2708  // We only care about reads.
2709  if (MO.isDef())
2710  continue;
2711 
2712  switch (MO.getReg()) {
2713  case AMDGPU::VCC:
2714  case AMDGPU::M0:
2715  case AMDGPU::FLAT_SCR:
2716  return MO.getReg();
2717 
2718  default:
2719  break;
2720  }
2721  }
2722 
2723  return AMDGPU::NoRegister;
2724 }
2725 
2726 static bool shouldReadExec(const MachineInstr &MI) {
2727  if (SIInstrInfo::isVALU(MI)) {
2728  switch (MI.getOpcode()) {
2729  case AMDGPU::V_READLANE_B32:
2730  case AMDGPU::V_READLANE_B32_si:
2731  case AMDGPU::V_READLANE_B32_vi:
2732  case AMDGPU::V_WRITELANE_B32:
2733  case AMDGPU::V_WRITELANE_B32_si:
2734  case AMDGPU::V_WRITELANE_B32_vi:
2735  return false;
2736  }
2737 
2738  return true;
2739  }
2740 
2741  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()) ||
2742  SIInstrInfo::isSALU(MI) ||
2743  SIInstrInfo::isSMRD(MI))
2744  return false;
2745 
2746  return true;
2747 }
2748 
2749 static bool isSubRegOf(const SIRegisterInfo &TRI,
2750  const MachineOperand &SuperVec,
2751  const MachineOperand &SubReg) {
2753  return TRI.isSubRegister(SuperVec.getReg(), SubReg.getReg());
2754 
2755  return SubReg.getSubReg() != AMDGPU::NoSubRegister &&
2756  SubReg.getReg() == SuperVec.getReg();
2757 }
2758 
2760  StringRef &ErrInfo) const {
2761  uint16_t Opcode = MI.getOpcode();
2762  if (SIInstrInfo::isGenericOpcode(MI.getOpcode()))
2763  return true;
2764 
2765  const MachineFunction *MF = MI.getParent()->getParent();
2766  const MachineRegisterInfo &MRI = MF->getRegInfo();
2767 
2768  int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2769  int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2770  int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2771 
2772  // Make sure the number of operands is correct.
2773  const MCInstrDesc &Desc = get(Opcode);
2774  if (!Desc.isVariadic() &&
2775  Desc.getNumOperands() != MI.getNumExplicitOperands()) {
2776  ErrInfo = "Instruction has wrong number of operands.";
2777  return false;
2778  }
2779 
2780  if (MI.isInlineAsm()) {
2781  // Verify register classes for inlineasm constraints.
2782  for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
2783  I != E; ++I) {
2784  const TargetRegisterClass *RC = MI.getRegClassConstraint(I, this, &RI);
2785  if (!RC)
2786  continue;
2787 
2788  const MachineOperand &Op = MI.getOperand(I);
2789  if (!Op.isReg())
2790  continue;
2791 
2792  unsigned Reg = Op.getReg();
2793  if (!TargetRegisterInfo::isVirtualRegister(Reg) && !RC->contains(Reg)) {
2794  ErrInfo = "inlineasm operand has incorrect register class.";
2795  return false;
2796  }
2797  }
2798 
2799  return true;
2800  }
2801 
2802  // Make sure the register classes are correct.
2803  for (int i = 0, e = Desc.getNumOperands(); i != e; ++i) {
2804  if (MI.getOperand(i).isFPImm()) {
2805  ErrInfo = "FPImm Machine Operands are not supported. ISel should bitcast "
2806  "all fp values to integers.";
2807  return false;
2808  }
2809 
2810  int RegClass = Desc.OpInfo[i].RegClass;
2811 
2812  switch (Desc.OpInfo[i].OperandType) {
2814  if (MI.getOperand(i).isImm()) {
2815  ErrInfo = "Illegal immediate value for operand.";
2816  return false;
2817  }
2818  break;
2821  break;
2828  const MachineOperand &MO = MI.getOperand(i);
2829  if (!MO.isReg() && (!MO.isImm() || !isInlineConstant(MI, i))) {
2830  ErrInfo = "Illegal immediate value for operand.";
2831  return false;
2832  }
2833  break;
2834  }
2837  // Check if this operand is an immediate.
2838  // FrameIndex operands will be replaced by immediates, so they are
2839  // allowed.
2840  if (!MI.getOperand(i).isImm() && !MI.getOperand(i).isFI()) {
2841  ErrInfo = "Expected immediate, but got non-immediate";
2842  return false;
2843  }
2845  default:
2846  continue;
2847  }
2848 
2849  if (!MI.getOperand(i).isReg())
2850  continue;
2851 
2852  if (RegClass != -1) {
2853  unsigned Reg = MI.getOperand(i).getReg();
2854  if (Reg == AMDGPU::NoRegister ||
2856  continue;
2857 
2858  const TargetRegisterClass *RC = RI.getRegClass(RegClass);
2859  if (!RC->contains(Reg)) {
2860  ErrInfo = "Operand has incorrect register class.";
2861  return false;
2862  }
2863  }
2864  }
2865 
2866  // Verify SDWA
2867  if (isSDWA(MI)) {
2868  if (!ST.hasSDWA()) {
2869  ErrInfo = "SDWA is not supported on this target";
2870  return false;
2871  }
2872 
2873  int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2874 
2875  const int OpIndicies[] = { DstIdx, Src0Idx, Src1Idx, Src2Idx };
2876 
2877  for (int OpIdx: OpIndicies) {
2878  if (OpIdx == -1)
2879  continue;
2880  const MachineOperand &MO = MI.getOperand(OpIdx);
2881 
2882  if (!ST.hasSDWAScalar()) {
2883  // Only VGPRS on VI
2884  if (!MO.isReg() || !RI.hasVGPRs(RI.getRegClassForReg(MRI, MO.getReg()))) {
2885  ErrInfo = "Only VGPRs allowed as operands in SDWA instructions on VI";
2886  return false;
2887  }
2888  } else {
2889  // No immediates on GFX9
2890  if (!MO.isReg()) {
2891  ErrInfo = "Only reg allowed as operands in SDWA instructions on GFX9";
2892  return false;
2893  }
2894  }
2895  }
2896 
2897  if (!ST.hasSDWAOmod()) {
2898  // No omod allowed on VI
2899  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
2900  if (OMod != nullptr &&
2901  (!OMod->isImm() || OMod->getImm() != 0)) {
2902  ErrInfo = "OMod not allowed in SDWA instructions on VI";
2903  return false;
2904  }
2905  }
2906 
2907  uint16_t BasicOpcode = AMDGPU::getBasicFromSDWAOp(Opcode);
2908  if (isVOPC(BasicOpcode)) {
2909  if (!ST.hasSDWASdst() && DstIdx != -1) {
2910  // Only vcc allowed as dst on VI for VOPC
2911  const MachineOperand &Dst = MI.getOperand(DstIdx);
2912  if (!Dst.isReg() || Dst.getReg() != AMDGPU::VCC) {
2913  ErrInfo = "Only VCC allowed as dst in SDWA instructions on VI";
2914  return false;
2915  }
2916  } else if (!ST.hasSDWAOutModsVOPC()) {
2917  // No clamp allowed on GFX9 for VOPC
2918  const MachineOperand *Clamp = getNamedOperand(MI, AMDGPU::OpName::clamp);
2919  if (Clamp && (!Clamp->isImm() || Clamp->getImm() != 0)) {
2920  ErrInfo = "Clamp not allowed in VOPC SDWA instructions on VI";
2921  return false;
2922  }
2923 
2924  // No omod allowed on GFX9 for VOPC
2925  const MachineOperand *OMod = getNamedOperand(MI, AMDGPU::OpName::omod);
2926  if (OMod && (!OMod->isImm() || OMod->getImm() != 0)) {
2927  ErrInfo = "OMod not allowed in VOPC SDWA instructions on VI";
2928  return false;
2929  }
2930  }
2931  }
2932 
2933  const MachineOperand *DstUnused = getNamedOperand(MI, AMDGPU::OpName::dst_unused);
2934  if (DstUnused && DstUnused->isImm() &&
2935  DstUnused->getImm() == AMDGPU::SDWA::UNUSED_PRESERVE) {
2936  const MachineOperand &Dst = MI.getOperand(DstIdx);
2937  if (!Dst.isReg() || !Dst.isTied()) {
2938  ErrInfo = "Dst register should have tied register";
2939  return false;
2940  }
2941 
2942  const MachineOperand &TiedMO =
2943  MI.getOperand(MI.findTiedOperandIdx(DstIdx));
2944  if (!TiedMO.isReg() || !TiedMO.isImplicit() || !TiedMO.isUse()) {
2945  ErrInfo =
2946  "Dst register should be tied to implicit use of preserved register";
2947  return false;
2948  } else if (TargetRegisterInfo::isPhysicalRegister(TiedMO.getReg()) &&
2949  Dst.getReg() != TiedMO.getReg()) {
2950  ErrInfo = "Dst register should use same physical register as preserved";
2951  return false;
2952  }
2953  }
2954  }
2955 
2956  // Verify VOP*. Ignore multiple sgpr operands on writelane.
2957  if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32
2958  && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) {
2959  // Only look at the true operands. Only a real operand can use the constant
2960  // bus, and we don't want to check pseudo-operands like the source modifier
2961  // flags.
2962  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2963 
2964  unsigned ConstantBusCount = 0;
2965  unsigned LiteralCount = 0;
2966 
2967  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1)
2968  ++ConstantBusCount;
2969 
2970  unsigned SGPRUsed = findImplicitSGPRRead(MI);
2971  if (SGPRUsed != AMDGPU::NoRegister)
2972  ++ConstantBusCount;
2973 
2974  for (int OpIdx : OpIndices) {
2975  if (OpIdx == -1)
2976  break;
2977  const MachineOperand &MO = MI.getOperand(OpIdx);
2978  if (usesConstantBus(MRI, MO, MI.getDesc().OpInfo[OpIdx])) {
2979  if (MO.isReg()) {
2980  if (MO.getReg() != SGPRUsed)
2981  ++ConstantBusCount;
2982  SGPRUsed = MO.getReg();
2983  } else {
2984  ++ConstantBusCount;
2985  ++LiteralCount;
2986  }
2987  }
2988  }
2989  if (ConstantBusCount > 1) {
2990  ErrInfo = "VOP* instruction uses the constant bus more than once";
2991  return false;
2992  }
2993 
2994  if (isVOP3(MI) && LiteralCount) {
2995  ErrInfo = "VOP3 instruction uses literal";
2996  return false;
2997  }
2998  }
2999 
3000  // Verify misc. restrictions on specific instructions.
3001  if (Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F32 ||
3002  Desc.getOpcode() == AMDGPU::V_DIV_SCALE_F64) {
3003  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3004  const MachineOperand &Src1 = MI.getOperand(Src1Idx);
3005  const MachineOperand &Src2 = MI.getOperand(Src2Idx);
3006  if (Src0.isReg() && Src1.isReg() && Src2.isReg()) {
3007  if (!compareMachineOp(Src0, Src1) &&
3008  !compareMachineOp(Src0, Src2)) {
3009  ErrInfo = "v_div_scale_{f32|f64} require src0 = src1 or src2";
3010  return false;
3011  }
3012  }
3013  }
3014 
3015  if (isSOPK(MI)) {
3016  int64_t Imm = getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
3017  if (sopkIsZext(MI)) {
3018  if (!isUInt<16>(Imm)) {
3019  ErrInfo = "invalid immediate for SOPK instruction";
3020  return false;
3021  }
3022  } else {
3023  if (!isInt<16>(Imm)) {
3024  ErrInfo = "invalid immediate for SOPK instruction";
3025  return false;
3026  }
3027  }
3028  }
3029 
3030  if (Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e32 ||
3031  Desc.getOpcode() == AMDGPU::V_MOVRELS_B32_e64 ||
3032  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3033  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64) {
3034  const bool IsDst = Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e32 ||
3035  Desc.getOpcode() == AMDGPU::V_MOVRELD_B32_e64;
3036 
3037  const unsigned StaticNumOps = Desc.getNumOperands() +
3038  Desc.getNumImplicitUses();
3039  const unsigned NumImplicitOps = IsDst ? 2 : 1;
3040 
3041  // Allow additional implicit operands. This allows a fixup done by the post
3042  // RA scheduler where the main implicit operand is killed and implicit-defs
3043  // are added for sub-registers that remain live after this instruction.
3044  if (MI.getNumOperands() < StaticNumOps + NumImplicitOps) {
3045  ErrInfo = "missing implicit register operands";
3046  return false;
3047  }
3048 
3049  const MachineOperand *Dst = getNamedOperand(MI, AMDGPU::OpName::vdst);
3050  if (IsDst) {
3051  if (!Dst->isUse()) {
3052  ErrInfo = "v_movreld_b32 vdst should be a use operand";
3053  return false;
3054  }
3055 
3056  unsigned UseOpIdx;
3057  if (!MI.isRegTiedToUseOperand(StaticNumOps, &UseOpIdx) ||
3058  UseOpIdx != StaticNumOps + 1) {
3059  ErrInfo = "movrel implicit operands should be tied";
3060  return false;
3061  }
3062  }
3063 
3064  const MachineOperand &Src0 = MI.getOperand(Src0Idx);
3065  const MachineOperand &ImpUse
3066  = MI.getOperand(StaticNumOps + NumImplicitOps - 1);
3067  if (!ImpUse.isReg() || !ImpUse.isUse() ||
3068  !isSubRegOf(RI, ImpUse, IsDst ? *Dst : Src0)) {
3069  ErrInfo = "src0 should be subreg of implicit vector use";
3070  return false;
3071  }
3072  }
3073 
3074  // Make sure we aren't losing exec uses in the td files. This mostly requires
3075  // being careful when using let Uses to try to add other use registers.
3076  if (shouldReadExec(MI)) {
3077  if (!MI.hasRegisterImplicitUseOperand(AMDGPU::EXEC)) {
3078  ErrInfo = "VALU instruction does not implicitly read exec mask";
3079  return false;
3080  }
3081  }
3082 
3083  if (isSMRD(MI)) {
3084  if (MI.mayStore()) {
3085  // The register offset form of scalar stores may only use m0 as the
3086  // soffset register.
3087  const MachineOperand *Soff = getNamedOperand(MI, AMDGPU::OpName::soff);
3088  if (Soff && Soff->getReg() != AMDGPU::M0) {
3089  ErrInfo = "scalar stores must use m0 as offset register";
3090  return false;
3091  }
3092  }
3093  }
3094 
3095  if (isFLAT(MI) && !MF->getSubtarget<GCNSubtarget>().hasFlatInstOffsets()) {
3096  const MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
3097  if (Offset->getImm() != 0) {
3098  ErrInfo = "subtarget does not support offsets in flat instructions";
3099  return false;
3100  }
3101  }
3102 
3103  const MachineOperand *DppCt = getNamedOperand(MI, AMDGPU::OpName::dpp_ctrl);
3104  if (DppCt) {
3105  using namespace AMDGPU::DPP;
3106 
3107  unsigned DC = DppCt->getImm();
3108  if (DC == DppCtrl::DPP_UNUSED1 || DC == DppCtrl::DPP_UNUSED2 ||
3109  DC == DppCtrl::DPP_UNUSED3 || DC > DppCtrl::DPP_LAST ||
3114  ErrInfo = "Invalid dpp_ctrl value";
3115  return false;
3116  }
3117  }
3118 
3119  return true;
3120 }
3121 
3122 unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
3123  switch (MI.getOpcode()) {
3124  default: return AMDGPU::INSTRUCTION_LIST_END;
3125  case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
3126  case AMDGPU::COPY: return AMDGPU::COPY;
3127  case AMDGPU::PHI: return AMDGPU::PHI;
3128  case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
3129  case AMDGPU::WQM: return AMDGPU::WQM;
3130  case AMDGPU::WWM: return AMDGPU::WWM;
3131  case AMDGPU::S_MOV_B32:
3132  return MI.getOperand(1).isReg() ?
3133  AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
3134  case AMDGPU::S_ADD_I32:
3135  return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_I32_e32;
3136  case AMDGPU::S_ADDC_U32:
3137  return AMDGPU::V_ADDC_U32_e32;
3138  case AMDGPU::S_SUB_I32:
3139  return ST.hasAddNoCarry() ? AMDGPU::V_SUB_U32_e64 : AMDGPU::V_SUB_I32_e32;
3140  // FIXME: These are not consistently handled, and selected when the carry is
3141  // used.
3142  case AMDGPU::S_ADD_U32:
3143  return AMDGPU::V_ADD_I32_e32;
3144  case AMDGPU::S_SUB_U32:
3145  return AMDGPU::V_SUB_I32_e32;
3146  case AMDGPU::S_SUBB_U32: return AMDGPU::V_SUBB_U32_e32;
3147  case AMDGPU::S_MUL_I32: return AMDGPU::V_MUL_LO_I32;
3148  case AMDGPU::S_AND_B32: return AMDGPU::V_AND_B32_e64;
3149  case AMDGPU::S_OR_B32: return AMDGPU::V_OR_B32_e64;
3150  case AMDGPU::S_XOR_B32: return AMDGPU::V_XOR_B32_e64;
3151  case AMDGPU::S_MIN_I32: return AMDGPU::V_MIN_I32_e64;
3152  case AMDGPU::S_MIN_U32: return AMDGPU::V_MIN_U32_e64;
3153  case AMDGPU::S_MAX_I32: return AMDGPU::V_MAX_I32_e64;
3154  case AMDGPU::S_MAX_U32: return AMDGPU::V_MAX_U32_e64;
3155  case AMDGPU::S_ASHR_I32: return AMDGPU::V_ASHR_I32_e32;
3156  case AMDGPU::S_ASHR_I64: return AMDGPU::V_ASHR_I64;
3157  case AMDGPU::S_LSHL_B32: return AMDGPU::V_LSHL_B32_e32;
3158  case AMDGPU::S_LSHL_B64: return AMDGPU::V_LSHL_B64;
3159  case AMDGPU::S_LSHR_B32: return AMDGPU::V_LSHR_B32_e32;
3160  case AMDGPU::S_LSHR_B64: return AMDGPU::V_LSHR_B64;
3161  case AMDGPU::S_SEXT_I32_I8: return AMDGPU::V_BFE_I32;
3162  case AMDGPU::S_SEXT_I32_I16: return AMDGPU::V_BFE_I32;
3163  case AMDGPU::S_BFE_U32: return AMDGPU::V_BFE_U32;
3164  case AMDGPU::S_BFE_I32: return AMDGPU::V_BFE_I32;
3165  case AMDGPU::S_BFM_B32: return AMDGPU::V_BFM_B32_e64;
3166  case AMDGPU::S_BREV_B32: return AMDGPU::V_BFREV_B32_e32;
3167  case AMDGPU::S_NOT_B32: return AMDGPU::V_NOT_B32_e32;
3168  case AMDGPU::S_NOT_B64: return AMDGPU::V_NOT_B32_e32;
3169  case AMDGPU::S_CMP_EQ_I32: return AMDGPU::V_CMP_EQ_I32_e32;
3170  case AMDGPU::S_CMP_LG_I32: return AMDGPU::V_CMP_NE_I32_e32;
3171  case AMDGPU::S_CMP_GT_I32: return AMDGPU::V_CMP_GT_I32_e32;
3172  case AMDGPU::S_CMP_GE_I32: return AMDGPU::V_CMP_GE_I32_e32;
3173  case AMDGPU::S_CMP_LT_I32: return AMDGPU::V_CMP_LT_I32_e32;
3174  case AMDGPU::S_CMP_LE_I32: return AMDGPU::V_CMP_LE_I32_e32;
3175  case AMDGPU::S_CMP_EQ_U32: return AMDGPU::V_CMP_EQ_U32_e32;
3176  case AMDGPU::S_CMP_LG_U32: return AMDGPU::V_CMP_NE_U32_e32;
3177  case AMDGPU::S_CMP_GT_U32: return AMDGPU::V_CMP_GT_U32_e32;
3178  case AMDGPU::S_CMP_GE_U32: return AMDGPU::V_CMP_GE_U32_e32;
3179  case AMDGPU::S_CMP_LT_U32: return AMDGPU::V_CMP_LT_U32_e32;
3180  case AMDGPU::S_CMP_LE_U32: return AMDGPU::V_CMP_LE_U32_e32;
3181  case AMDGPU::S_CMP_EQ_U64: return AMDGPU::V_CMP_EQ_U64_e32;
3182  case AMDGPU::S_CMP_LG_U64: return AMDGPU::V_CMP_NE_U64_e32;
3183  case AMDGPU::S_BCNT1_I32_B32: return AMDGPU::V_BCNT_U32_B32_e64;
3184  case AMDGPU::S_FF1_I32_B32: return AMDGPU::V_FFBL_B32_e32;
3185  case AMDGPU::S_FLBIT_I32_B32: return AMDGPU::V_FFBH_U32_e32;
3186  case AMDGPU::S_FLBIT_I32: return AMDGPU::V_FFBH_I32_e64;
3187  case AMDGPU::S_CBRANCH_SCC0: return AMDGPU::S_CBRANCH_VCCZ;
3188  case AMDGPU::S_CBRANCH_SCC1: return AMDGPU::S_CBRANCH_VCCNZ;
3189  }
3190 }
3191 
3193  unsigned OpNo) const {
3195  const MCInstrDesc &Desc = get(MI.getOpcode());
3196  if (MI.isVariadic() || OpNo >= Desc.getNumOperands() ||
3197  Desc.OpInfo[OpNo].RegClass == -1) {
3198  unsigned Reg = MI.getOperand(OpNo).getReg();
3199 
3201  return MRI.getRegClass(Reg);
3202  return RI.getPhysRegClass(Reg);
3203  }
3204 
3205  unsigned RCID = Desc.OpInfo[OpNo].RegClass;
3206  return RI.getRegClass(RCID);
3207 }
3208 
3209 bool SIInstrInfo::canReadVGPR(const MachineInstr &MI, unsigned OpNo) const {
3210  switch (MI.getOpcode()) {
3211  case AMDGPU::COPY:
3212  case AMDGPU::REG_SEQUENCE:
3213  case AMDGPU::PHI:
3214  case AMDGPU::INSERT_SUBREG:
3215  return RI.hasVGPRs(getOpRegClass(MI, 0));
3216  default:
3217  return RI.hasVGPRs(getOpRegClass(MI, OpNo));
3218  }
3219 }
3220 
3221 void SIInstrInfo::legalizeOpWithMove(MachineInstr &MI, unsigned OpIdx) const {
3223  MachineBasicBlock *MBB = MI.getParent();
3224  MachineOperand &MO = MI.getOperand(OpIdx);
3226  unsigned RCID = get(MI.getOpcode()).OpInfo[OpIdx].RegClass;
3227  const TargetRegisterClass *RC = RI.getRegClass(RCID);
3228  unsigned Opcode = AMDGPU::V_MOV_B32_e32;
3229  if (MO.isReg())
3230  Opcode = AMDGPU::COPY;
3231  else if (RI.isSGPRClass(RC))
3232  Opcode = AMDGPU::S_MOV_B32;
3233 
3234  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(RC);
3235  if (RI.getCommonSubClass(&AMDGPU::VReg_64RegClass, VRC))
3236  VRC = &AMDGPU::VReg_64RegClass;
3237  else
3238  VRC = &AMDGPU::VGPR_32RegClass;
3239 
3240  unsigned Reg = MRI.createVirtualRegister(VRC);
3241  DebugLoc DL = MBB->findDebugLoc(I);
3242  BuildMI(*MI.getParent(), I, DL, get(Opcode), Reg).add(MO);
3243  MO.ChangeToRegister(Reg, false);
3244 }
3245 
3248  MachineOperand &SuperReg,
3249  const TargetRegisterClass *SuperRC,
3250  unsigned SubIdx,
3251  const TargetRegisterClass *SubRC)
3252  const {
3253  MachineBasicBlock *MBB = MI->getParent();
3254  DebugLoc DL = MI->getDebugLoc();
3255  unsigned SubReg = MRI.createVirtualRegister(SubRC);
3256 
3257  if (SuperReg.getSubReg() == AMDGPU::NoSubRegister) {
3258  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3259  .addReg(SuperReg.getReg(), 0, SubIdx);
3260  return SubReg;
3261  }
3262 
3263  // Just in case the super register is itself a sub-register, copy it to a new
3264  // value so we don't need to worry about merging its subreg index with the
3265  // SubIdx passed to this function. The register coalescer should be able to
3266  // eliminate this extra copy.
3267  unsigned NewSuperReg = MRI.createVirtualRegister(SuperRC);
3268 
3269  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), NewSuperReg)
3270  .addReg(SuperReg.getReg(), 0, SuperReg.getSubReg());
3271 
3272  BuildMI(*MBB, MI, DL, get(TargetOpcode::COPY), SubReg)
3273  .addReg(NewSuperReg, 0, SubIdx);
3274 
3275  return SubReg;
3276 }
3277 
3281  MachineOperand &Op,
3282  const TargetRegisterClass *SuperRC,
3283  unsigned SubIdx,
3284  const TargetRegisterClass *SubRC) const {
3285  if (Op.isImm()) {
3286  if (SubIdx == AMDGPU::sub0)
3287  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm()));
3288  if (SubIdx == AMDGPU::sub1)
3289  return MachineOperand::CreateImm(static_cast<int32_t>(Op.getImm() >> 32));
3290 
3291  llvm_unreachable("Unhandled register index for immediate");
3292  }
3293 
3294  unsigned SubReg = buildExtractSubReg(MII, MRI, Op, SuperRC,
3295  SubIdx, SubRC);
3296  return MachineOperand::CreateReg(SubReg, false);
3297 }
3298 
3299 // Change the order of operands from (0, 1, 2) to (0, 2, 1)
3300 void SIInstrInfo::swapOperands(MachineInstr &Inst) const {
3301  assert(Inst.getNumExplicitOperands() == 3);
3302  MachineOperand Op1 = Inst.getOperand(1);
3303  Inst.RemoveOperand(1);
3304  Inst.addOperand(Op1);
3305 }
3306 
3308  const MCOperandInfo &OpInfo,
3309  const MachineOperand &MO) const {
3310  if (!MO.isReg())
3311  return false;
3312 
3313  unsigned Reg = MO.getReg();
3314  const TargetRegisterClass *RC =
3316  MRI.getRegClass(Reg) :
3317  RI.getPhysRegClass(Reg);
3318 
3319  const SIRegisterInfo *TRI =
3320  static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo());
3321  RC = TRI->getSubRegClass(RC, MO.getSubReg());
3322 
3323  // In order to be legal, the common sub-class must be equal to the
3324  // class of the current operand. For example:
3325  //
3326  // v_mov_b32 s0 ; Operand defined as vsrc_b32
3327  // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL
3328  //
3329  // s_sendmsg 0, s0 ; Operand defined as m0reg
3330  // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL
3331 
3332  return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC;
3333 }
3334 
3336  const MCOperandInfo &OpInfo,
3337  const MachineOperand &MO) const {
3338  if (MO.isReg())
3339  return isLegalRegOperand(MRI, OpInfo, MO);
3340 
3341  // Handle non-register types that are treated like immediates.
3342  assert(MO.isImm() || MO.isTargetIndex() || MO.isFI());
3343  return true;
3344 }
3345 
3346 bool SIInstrInfo::isOperandLegal(const MachineInstr &MI, unsigned OpIdx,
3347  const MachineOperand *MO) const {
3349  const MCInstrDesc &InstDesc = MI.getDesc();
3350  const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpIdx];
3351  const TargetRegisterClass *DefinedRC =
3352  OpInfo.RegClass != -1 ? RI.getRegClass(OpInfo.RegClass) : nullptr;
3353  if (!MO)
3354  MO = &MI.getOperand(OpIdx);
3355 
3356  if (isVALU(MI) && usesConstantBus(MRI, *MO, OpInfo)) {
3357 
3358  RegSubRegPair SGPRUsed;
3359  if (MO->isReg())
3360  SGPRUsed = RegSubRegPair(MO->getReg(), MO->getSubReg());
3361 
3362  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
3363  if (i == OpIdx)
3364  continue;
3365  const MachineOperand &Op = MI.getOperand(i);
3366  if (Op.isReg()) {
3367  if ((Op.getReg() != SGPRUsed.Reg || Op.getSubReg() != SGPRUsed.SubReg) &&
3368  usesConstantBus(MRI, Op, InstDesc.OpInfo[i])) {
3369  return false;
3370  }
3371  } else if (InstDesc.OpInfo[i].OperandType == AMDGPU::OPERAND_KIMM32) {
3372  return false;
3373  }
3374  }
3375  }
3376 
3377  if (MO->isReg()) {
3378  assert(DefinedRC);
3379  return isLegalRegOperand(MRI, OpInfo, *MO);
3380  }
3381 
3382  // Handle non-register types that are treated like immediates.
3383  assert(MO->isImm() || MO->isTargetIndex() || MO->isFI());
3384 
3385  if (!DefinedRC) {
3386  // This operand expects an immediate.
3387  return true;
3388  }
3389 
3390  return isImmOperandLegal(MI, OpIdx, *MO);
3391 }
3392 
3394  MachineInstr &MI) const {
3395  unsigned Opc = MI.getOpcode();
3396  const MCInstrDesc &InstrDesc = get(Opc);
3397 
3398  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
3399  MachineOperand &Src1 = MI.getOperand(Src1Idx);
3400 
3401  // If there is an implicit SGPR use such as VCC use for v_addc_u32/v_subb_u32
3402  // we need to only have one constant bus use.
3403  //
3404  // Note we do not need to worry about literal constants here. They are
3405  // disabled for the operand type for instructions because they will always
3406  // violate the one constant bus use rule.
3407  bool HasImplicitSGPR = findImplicitSGPRRead(MI) != AMDGPU::NoRegister;
3408  if (HasImplicitSGPR) {
3409  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3410  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3411 
3412  if (Src0.isReg() && RI.isSGPRReg(MRI, Src0.getReg()))
3413  legalizeOpWithMove(MI, Src0Idx);
3414  }
3415 
3416  // Special case: V_WRITELANE_B32 accepts only immediate or SGPR operands for
3417  // both the value to write (src0) and lane select (src1). Fix up non-SGPR
3418  // src0/src1 with V_READFIRSTLANE.
3419  if (Opc == AMDGPU::V_WRITELANE_B32) {
3420  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3421  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3422  const DebugLoc &DL = MI.getDebugLoc();
3423  if (Src0.isReg() && RI.isVGPR(MRI, Src0.getReg())) {
3424  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3425  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3426  .add(Src0);
3427  Src0.ChangeToRegister(Reg, false);
3428  }
3429  if (Src1.isReg() && RI.isVGPR(MRI, Src1.getReg())) {
3430  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3431  const DebugLoc &DL = MI.getDebugLoc();
3432  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3433  .add(Src1);
3434  Src1.ChangeToRegister(Reg, false);
3435  }
3436  return;
3437  }
3438 
3439  // VOP2 src0 instructions support all operand types, so we don't need to check
3440  // their legality. If src1 is already legal, we don't need to do anything.
3441  if (isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src1))
3442  return;
3443 
3444  // Special case: V_READLANE_B32 accepts only immediate or SGPR operands for
3445  // lane select. Fix up using V_READFIRSTLANE, since we assume that the lane
3446  // select is uniform.
3447  if (Opc == AMDGPU::V_READLANE_B32 && Src1.isReg() &&
3448  RI.isVGPR(MRI, Src1.getReg())) {
3449  unsigned Reg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
3450  const DebugLoc &DL = MI.getDebugLoc();
3451  BuildMI(*MI.getParent(), MI, DL, get(AMDGPU::V_READFIRSTLANE_B32), Reg)
3452  .add(Src1);
3453  Src1.ChangeToRegister(Reg, false);
3454  return;
3455  }
3456 
3457  // We do not use commuteInstruction here because it is too aggressive and will
3458  // commute if it is possible. We only want to commute here if it improves
3459  // legality. This can be called a fairly large number of times so don't waste
3460  // compile time pointlessly swapping and checking legality again.
3461  if (HasImplicitSGPR || !MI.isCommutable()) {
3462  legalizeOpWithMove(MI, Src1Idx);
3463  return;
3464  }
3465 
3466  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
3467  MachineOperand &Src0 = MI.getOperand(Src0Idx);
3468 
3469  // If src0 can be used as src1, commuting will make the operands legal.
3470  // Otherwise we have to give up and insert a move.
3471  //
3472  // TODO: Other immediate-like operand kinds could be commuted if there was a
3473  // MachineOperand::ChangeTo* for them.
3474  if ((!Src1.isImm() && !Src1.isReg()) ||
3475  !isLegalRegOperand(MRI, InstrDesc.OpInfo[Src1Idx], Src0)) {
3476  legalizeOpWithMove(MI, Src1Idx);
3477  return;
3478  }
3479 
3480  int CommutedOpc = commuteOpcode(MI);
3481  if (CommutedOpc == -1) {
3482  legalizeOpWithMove(MI, Src1Idx);
3483  return;
3484  }
3485 
3486  MI.setDesc(get(CommutedOpc));
3487 
3488  unsigned Src0Reg = Src0.getReg();
3489  unsigned Src0SubReg = Src0.getSubReg();
3490  bool Src0Kill = Src0.isKill();
3491 
3492  if (Src1.isImm())
3493  Src0.ChangeToImmediate(Src1.getImm());
3494  else if (Src1.isReg()) {
3495  Src0.ChangeToRegister(Src1.getReg(), false, false, Src1.isKill());
3496  Src0.setSubReg(Src1.getSubReg());
3497  } else
3498  llvm_unreachable("Should only have register or immediate operands");
3499 
3500  Src1.ChangeToRegister(Src0Reg, false, false, Src0Kill);
3501  Src1.setSubReg(Src0SubReg);
3502 }
3503 
3504 // Legalize VOP3 operands. Because all operand types are supported for any
3505 // operand, and since literal constants are not allowed and should never be
3506 // seen, we only need to worry about inserting copies if we use multiple SGPR
3507 // operands.
3509  MachineInstr &MI) const {
3510  unsigned Opc = MI.getOpcode();
3511 
3512  int VOP3Idx[3] = {
3513  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0),
3514  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1),
3515  AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)
3516  };
3517 
3518  // Find the one SGPR operand we are allowed to use.
3519  unsigned SGPRReg = findUsedSGPR(MI, VOP3Idx);
3520 
3521  for (unsigned i = 0; i < 3; ++i) {
3522  int Idx = VOP3Idx[i];
3523  if (Idx == -1)
3524  break;
3525  MachineOperand &MO = MI.getOperand(Idx);
3526 
3527  // We should never see a VOP3 instruction with an illegal immediate operand.
3528  if (!MO.isReg())
3529  continue;
3530 
3531  if (!RI.isSGPRClass(MRI.getRegClass(MO.getReg())))
3532  continue; // VGPRs are legal
3533 
3534  if (SGPRReg == AMDGPU::NoRegister || SGPRReg == MO.getReg()) {
3535  SGPRReg = MO.getReg();
3536  // We can use one SGPR in each VOP3 instruction.
3537  continue;
3538  }
3539 
3540  // If we make it this far, then the operand is not legal and we must
3541  // legalize it.
3542  legalizeOpWithMove(MI, Idx);
3543  }
3544 }
3545 
3547  MachineRegisterInfo &MRI) const {
3548  const TargetRegisterClass *VRC = MRI.getRegClass(SrcReg);
3549  const TargetRegisterClass *SRC = RI.getEquivalentSGPRClass(VRC);
3550  unsigned DstReg = MRI.createVirtualRegister(SRC);
3551  unsigned SubRegs = RI.getRegSizeInBits(*VRC) / 32;
3552 
3553  if (SubRegs == 1) {
3554  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3555  get(AMDGPU::V_READFIRSTLANE_B32), DstReg)
3556  .addReg(SrcReg);
3557  return DstReg;
3558  }
3559 
3561  for (unsigned i = 0; i < SubRegs; ++i) {
3562  unsigned SGPR = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3563  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3564  get(AMDGPU::V_READFIRSTLANE_B32), SGPR)
3565  .addReg(SrcReg, 0, RI.getSubRegFromChannel(i));
3566  SRegs.push_back(SGPR);
3567  }
3568 
3569  MachineInstrBuilder MIB =
3570  BuildMI(*UseMI.getParent(), UseMI, UseMI.getDebugLoc(),
3571  get(AMDGPU::REG_SEQUENCE), DstReg);
3572  for (unsigned i = 0; i < SubRegs; ++i) {
3573  MIB.addReg(SRegs[i]);
3574  MIB.addImm(RI.getSubRegFromChannel(i));
3575  }
3576  return DstReg;
3577 }
3578 
3580  MachineInstr &MI) const {
3581 
3582  // If the pointer is store in VGPRs, then we need to move them to
3583  // SGPRs using v_readfirstlane. This is safe because we only select
3584  // loads with uniform pointers to SMRD instruction so we know the
3585  // pointer value is uniform.
3586  MachineOperand *SBase = getNamedOperand(MI, AMDGPU::OpName::sbase);
3587  if (SBase && !RI.isSGPRClass(MRI.getRegClass(SBase->getReg()))) {
3588  unsigned SGPR = readlaneVGPRToSGPR(SBase->getReg(), MI, MRI);
3589  SBase->setReg(SGPR);
3590  }
3591 }
3592 
3595  const TargetRegisterClass *DstRC,
3596  MachineOperand &Op,
3598  const DebugLoc &DL) const {
3599  unsigned OpReg = Op.getReg();
3600  unsigned OpSubReg = Op.getSubReg();
3601 
3602  const TargetRegisterClass *OpRC = RI.getSubClassWithSubReg(
3603  RI.getRegClassForReg(MRI, OpReg), OpSubReg);
3604 
3605  // Check if operand is already the correct register class.
3606  if (DstRC == OpRC)
3607  return;
3608 
3609  unsigned DstReg = MRI.createVirtualRegister(DstRC);
3610  MachineInstr *Copy =
3611  BuildMI(InsertMBB, I, DL, get(AMDGPU::COPY), DstReg).add(Op);
3612 
3613  Op.setReg(DstReg);
3614  Op.setSubReg(0);
3615 
3616  MachineInstr *Def = MRI.getVRegDef(OpReg);
3617  if (!Def)
3618  return;
3619 
3620  // Try to eliminate the copy if it is copying an immediate value.
3621  if (Def->isMoveImmediate())
3622  FoldImmediate(*Copy, *Def, OpReg, &MRI);
3623 }
3624 
3625 // Emit the actual waterfall loop, executing the wrapped instruction for each
3626 // unique value of \p Rsrc across all lanes. In the best case we execute 1
3627 // iteration, in the worst case we execute 64 (once per lane).
3628 static void
3630  MachineBasicBlock &OrigBB, MachineBasicBlock &LoopBB,
3631  const DebugLoc &DL, MachineOperand &Rsrc) {
3632  MachineBasicBlock::iterator I = LoopBB.begin();
3633 
3634  unsigned VRsrc = Rsrc.getReg();
3635  unsigned VRsrcUndef = getUndefRegState(Rsrc.isUndef());
3636 
3637  unsigned SaveExec = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3638  unsigned CondReg0 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3639  unsigned CondReg1 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3640  unsigned AndCond = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3641  unsigned SRsrcSub0 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3642  unsigned SRsrcSub1 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3643  unsigned SRsrcSub2 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3644  unsigned SRsrcSub3 = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3645  unsigned SRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
3646 
3647  // Beginning of the loop, read the next Rsrc variant.
3648  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub0)
3649  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub0);
3650  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub1)
3651  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub1);
3652  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub2)
3653  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub2);
3654  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_READFIRSTLANE_B32), SRsrcSub3)
3655  .addReg(VRsrc, VRsrcUndef, AMDGPU::sub3);
3656 
3657  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), SRsrc)
3658  .addReg(SRsrcSub0)
3659  .addImm(AMDGPU::sub0)
3660  .addReg(SRsrcSub1)
3661  .addImm(AMDGPU::sub1)
3662  .addReg(SRsrcSub2)
3663  .addImm(AMDGPU::sub2)
3664  .addReg(SRsrcSub3)
3665  .addImm(AMDGPU::sub3);
3666 
3667  // Update Rsrc operand to use the SGPR Rsrc.
3668  Rsrc.setReg(SRsrc);
3669  Rsrc.setIsKill(true);
3670 
3671  // Identify all lanes with identical Rsrc operands in their VGPRs.
3672  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg0)
3673  .addReg(SRsrc, 0, AMDGPU::sub0_sub1)
3674  .addReg(VRsrc, 0, AMDGPU::sub0_sub1);
3675  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::V_CMP_EQ_U64_e64), CondReg1)
3676  .addReg(SRsrc, 0, AMDGPU::sub2_sub3)
3677  .addReg(VRsrc, 0, AMDGPU::sub2_sub3);
3678  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_AND_B64), AndCond)
3679  .addReg(CondReg0)
3680  .addReg(CondReg1);
3681 
3682  MRI.setSimpleHint(SaveExec, AndCond);
3683 
3684  // Update EXEC to matching lanes, saving original to SaveExec.
3685  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_AND_SAVEEXEC_B64), SaveExec)
3686  .addReg(AndCond, RegState::Kill);
3687 
3688  // The original instruction is here; we insert the terminators after it.
3689  I = LoopBB.end();
3690 
3691  // Update EXEC, switch all done bits to 0 and all todo bits to 1.
3692  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_XOR_B64_term), AMDGPU::EXEC)
3693  .addReg(AMDGPU::EXEC)
3694  .addReg(SaveExec);
3695  BuildMI(LoopBB, I, DL, TII.get(AMDGPU::S_CBRANCH_EXECNZ)).addMBB(&LoopBB);
3696 }
3697 
3698 // Build a waterfall loop around \p MI, replacing the VGPR \p Rsrc register
3699 // with SGPRs by iterating over all unique values across all lanes.
3701  MachineOperand &Rsrc, MachineDominatorTree *MDT) {
3702  MachineBasicBlock &MBB = *MI.getParent();
3703  MachineFunction &MF = *MBB.getParent();
3706  const DebugLoc &DL = MI.getDebugLoc();
3707 
3708  unsigned SaveExec = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
3709 
3710  // Save the EXEC mask
3711  BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B64), SaveExec)
3712  .addReg(AMDGPU::EXEC);
3713 
3714  // Killed uses in the instruction we are waterfalling around will be
3715  // incorrect due to the added control-flow.
3716  for (auto &MO : MI.uses()) {
3717  if (MO.isReg() && MO.isUse()) {
3718  MRI.clearKillFlags(MO.getReg());
3719  }
3720  }
3721 
3722  // To insert the loop we need to split the block. Move everything after this
3723  // point to a new block, and insert a new empty block between the two.
3725  MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
3726  MachineFunction::iterator MBBI(MBB);
3727  ++MBBI;
3728 
3729  MF.insert(MBBI, LoopBB);
3730  MF.insert(MBBI, RemainderBB);
3731 
3732  LoopBB->addSuccessor(LoopBB);
3733  LoopBB->addSuccessor(RemainderBB);
3734 
3735  // Move MI to the LoopBB, and the remainder of the block to RemainderBB.
3737  RemainderBB->transferSuccessorsAndUpdatePHIs(&MBB);
3738  RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
3739  LoopBB->splice(LoopBB->begin(), &MBB, J);
3740 
3741  MBB.addSuccessor(LoopBB);
3742 
3743  // Update dominators. We know that MBB immediately dominates LoopBB, that
3744  // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately
3745  // dominates all of the successors transferred to it from MBB that MBB used
3746  // to dominate.
3747  if (MDT) {
3748  MDT->addNewBlock(LoopBB, &MBB);
3749  MDT->addNewBlock(RemainderBB, LoopBB);
3750  for (auto &Succ : RemainderBB->successors()) {
3751  if (MDT->dominates(&MBB, Succ)) {
3752  MDT->changeImmediateDominator(Succ, RemainderBB);
3753  }
3754  }
3755  }
3756 
3757  emitLoadSRsrcFromVGPRLoop(TII, MRI, MBB, *LoopBB, DL, Rsrc);
3758 
3759  // Restore the EXEC mask
3760  MachineBasicBlock::iterator First = RemainderBB->begin();
3761  BuildMI(*RemainderBB, First, DL, TII.get(AMDGPU::S_MOV_B64), AMDGPU::EXEC)
3762  .addReg(SaveExec);
3763 }
3764 
3765 // Extract pointer from Rsrc and return a zero-value Rsrc replacement.
3766 static std::tuple<unsigned, unsigned>
3768  MachineBasicBlock &MBB = *MI.getParent();
3769  MachineFunction &MF = *MBB.getParent();
3771 
3772  // Extract the ptr from the resource descriptor.
3773  unsigned RsrcPtr =
3774  TII.buildExtractSubReg(MI, MRI, Rsrc, &AMDGPU::VReg_128RegClass,
3775  AMDGPU::sub0_sub1, &AMDGPU::VReg_64RegClass);
3776 
3777  // Create an empty resource descriptor
3778  unsigned Zero64 = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
3779  unsigned SRsrcFormatLo = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3780  unsigned SRsrcFormatHi = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass);
3781  unsigned NewSRsrc = MRI.createVirtualRegister(&AMDGPU::SReg_128RegClass);
3782  uint64_t RsrcDataFormat = TII.getDefaultRsrcDataFormat();
3783 
3784  // Zero64 = 0
3785  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B64), Zero64)
3786  .addImm(0);
3787 
3788  // SRsrcFormatLo = RSRC_DATA_FORMAT{31-0}
3789  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatLo)
3790  .addImm(RsrcDataFormat & 0xFFFFFFFF);
3791 
3792  // SRsrcFormatHi = RSRC_DATA_FORMAT{63-32}
3793  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), SRsrcFormatHi)
3794  .addImm(RsrcDataFormat >> 32);
3795 
3796  // NewSRsrc = {Zero64, SRsrcFormat}
3797  BuildMI(MBB, MI, MI.getDebugLoc(), TII.get(AMDGPU::REG_SEQUENCE), NewSRsrc)
3798  .addReg(Zero64)
3799  .addImm(AMDGPU::sub0_sub1)
3800  .addReg(SRsrcFormatLo)
3801  .addImm(AMDGPU::sub2)
3802  .addReg(SRsrcFormatHi)
3803  .addImm(AMDGPU::sub3);
3804 
3805  return std::make_tuple(RsrcPtr, NewSRsrc);
3806 }
3807 
3809  MachineDominatorTree *MDT) const {
3810  MachineFunction &MF = *MI.getParent()->getParent();
3812 
3813  // Legalize VOP2
3814  if (isVOP2(MI) || isVOPC(MI)) {
3815  legalizeOperandsVOP2(MRI, MI);
3816  return;
3817  }
3818 
3819  // Legalize VOP3
3820  if (isVOP3(MI)) {
3821  legalizeOperandsVOP3(MRI, MI);
3822  return;
3823  }
3824 
3825  // Legalize SMRD
3826  if (isSMRD(MI)) {
3827  legalizeOperandsSMRD(MRI, MI);
3828  return;
3829  }
3830 
3831  // Legalize REG_SEQUENCE and PHI
3832  // The register class of the operands much be the same type as the register
3833  // class of the output.
3834  if (MI.getOpcode() == AMDGPU::PHI) {
3835  const TargetRegisterClass *RC = nullptr, *SRC = nullptr, *VRC = nullptr;
3836  for (unsigned i = 1, e = MI.getNumOperands(); i != e; i += 2) {
3837  if (!MI.getOperand(i).isReg() ||
3839  continue;
3840  const TargetRegisterClass *OpRC =
3841  MRI.getRegClass(MI.getOperand(i).getReg());
3842  if (RI.hasVGPRs(OpRC)) {
3843  VRC = OpRC;
3844  } else {
3845  SRC = OpRC;
3846  }
3847  }
3848 
3849  // If any of the operands are VGPR registers, then they all most be
3850  // otherwise we will create illegal VGPR->SGPR copies when legalizing
3851  // them.
3852  if (VRC || !RI.isSGPRClass(getOpRegClass(MI, 0))) {
3853  if (!VRC) {
3854  assert(SRC);
3855  VRC = RI.getEquivalentVGPRClass(SRC);
3856  }
3857  RC = VRC;
3858  } else {
3859  RC = SRC;
3860  }
3861 
3862  // Update all the operands so they have the same type.
3863  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3864  MachineOperand &Op = MI.getOperand(I);
3866  continue;
3867 
3868  // MI is a PHI instruction.
3869  MachineBasicBlock *InsertBB = MI.getOperand(I + 1).getMBB();
3870  MachineBasicBlock::iterator Insert = InsertBB->getFirstTerminator();
3871 
3872  // Avoid creating no-op copies with the same src and dst reg class. These
3873  // confuse some of the machine passes.
3874  legalizeGenericOperand(*InsertBB, Insert, RC, Op, MRI, MI.getDebugLoc());
3875  }
3876  }
3877 
3878  // REG_SEQUENCE doesn't really require operand legalization, but if one has a
3879  // VGPR dest type and SGPR sources, insert copies so all operands are
3880  // VGPRs. This seems to help operand folding / the register coalescer.
3881  if (MI.getOpcode() == AMDGPU::REG_SEQUENCE) {
3882  MachineBasicBlock *MBB = MI.getParent();
3883  const TargetRegisterClass *DstRC = getOpRegClass(MI, 0);
3884  if (RI.hasVGPRs(DstRC)) {
3885  // Update all the operands so they are VGPR register classes. These may
3886  // not be the same register class because REG_SEQUENCE supports mixing
3887  // subregister index types e.g. sub0_sub1 + sub2 + sub3
3888  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
3889  MachineOperand &Op = MI.getOperand(I);
3891  continue;
3892 
3893  const TargetRegisterClass *OpRC = MRI.getRegClass(Op.getReg());
3894  const TargetRegisterClass *VRC = RI.getEquivalentVGPRClass(OpRC);
3895  if (VRC == OpRC)
3896  continue;
3897 
3898  legalizeGenericOperand(*MBB, MI, VRC, Op, MRI, MI.getDebugLoc());
3899  Op.setIsKill();
3900  }
3901  }
3902 
3903  return;
3904  }
3905 
3906  // Legalize INSERT_SUBREG
3907  // src0 must have the same register class as dst
3908  if (MI.getOpcode() == AMDGPU::INSERT_SUBREG) {
3909  unsigned Dst = MI.getOperand(0).getReg();
3910  unsigned Src0 = MI.getOperand(1).getReg();
3911  const TargetRegisterClass *DstRC = MRI.getRegClass(Dst);
3912  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0);
3913  if (DstRC != Src0RC) {
3914  MachineBasicBlock *MBB = MI.getParent();
3915  MachineOperand &Op = MI.getOperand(1);
3916  legalizeGenericOperand(*MBB, MI, DstRC, Op, MRI, MI.getDebugLoc());
3917  }
3918  return;
3919  }
3920 
3921  // Legalize SI_INIT_M0
3922  if (MI.getOpcode() == AMDGPU::SI_INIT_M0) {
3923  MachineOperand &Src = MI.getOperand(0);
3924  if (Src.isReg() && RI.hasVGPRs(MRI.getRegClass(Src.getReg())))
3925  Src.setReg(readlaneVGPRToSGPR(Src.getReg(), MI, MRI));
3926  return;
3927  }
3928 
3929  // Legalize MIMG and MUBUF/MTBUF for shaders.
3930  //
3931  // Shaders only generate MUBUF/MTBUF instructions via intrinsics or via
3932  // scratch memory access. In both cases, the legalization never involves
3933  // conversion to the addr64 form.
3934  if (isMIMG(MI) ||
3936  (isMUBUF(MI) || isMTBUF(MI)))) {
3937  MachineOperand *SRsrc = getNamedOperand(MI, AMDGPU::OpName::srsrc);
3938  if (SRsrc && !RI.isSGPRClass(MRI.getRegClass(SRsrc->getReg()))) {
3939  unsigned SGPR = readlaneVGPRToSGPR(SRsrc->getReg(), MI, MRI);
3940  SRsrc->setReg(SGPR);
3941  }
3942 
3943  MachineOperand *SSamp = getNamedOperand(MI, AMDGPU::OpName::ssamp);
3944  if (SSamp && !RI.isSGPRClass(MRI.getRegClass(SSamp->getReg()))) {
3945  unsigned SGPR = readlaneVGPRToSGPR(SSamp->getReg(), MI, MRI);
3946  SSamp->setReg(SGPR);
3947  }
3948  return;
3949  }
3950 
3951  // Legalize MUBUF* instructions.
3952  int RsrcIdx =
3953  AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::srsrc);
3954  if (RsrcIdx != -1) {
3955  // We have an MUBUF instruction
3956  MachineOperand *Rsrc = &MI.getOperand(RsrcIdx);
3957  unsigned RsrcRC = get(MI.getOpcode()).OpInfo[RsrcIdx].RegClass;
3958  if (RI.getCommonSubClass(MRI.getRegClass(Rsrc->getReg()),
3959  RI.getRegClass(RsrcRC))) {
3960  // The operands are legal.
3961  // FIXME: We may need to legalize operands besided srsrc.
3962  return;
3963  }
3964 
3965  // Legalize a VGPR Rsrc.
3966  //
3967  // If the instruction is _ADDR64, we can avoid a waterfall by extracting
3968  // the base pointer from the VGPR Rsrc, adding it to the VAddr, then using
3969  // a zero-value SRsrc.
3970  //
3971  // If the instruction is _OFFSET (both idxen and offen disabled), and we
3972  // support ADDR64 instructions, we can convert to ADDR64 and do the same as
3973  // above.
3974  //
3975  // Otherwise we are on non-ADDR64 hardware, and/or we have
3976  // idxen/offen/bothen and we fall back to a waterfall loop.
3977 
3978  MachineBasicBlock &MBB = *MI.getParent();
3979 
3980  MachineOperand *VAddr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
3981  if (VAddr && AMDGPU::getIfAddr64Inst(MI.getOpcode()) != -1) {
3982  // This is already an ADDR64 instruction so we need to add the pointer
3983  // extracted from the resource descriptor to the current value of VAddr.
3984  unsigned NewVAddrLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3985  unsigned NewVAddrHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
3986  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
3987 
3988  unsigned RsrcPtr, NewSRsrc;
3989  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
3990 
3991  // NewVaddrLo = RsrcPtr:sub0 + VAddr:sub0
3992  DebugLoc DL = MI.getDebugLoc();
3993  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADD_I32_e32), NewVAddrLo)
3994  .addReg(RsrcPtr, 0, AMDGPU::sub0)
3995  .addReg(VAddr->getReg(), 0, AMDGPU::sub0);
3996 
3997  // NewVaddrHi = RsrcPtr:sub1 + VAddr:sub1
3998  BuildMI(MBB, MI, DL, get(AMDGPU::V_ADDC_U32_e32), NewVAddrHi)
3999  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4000  .addReg(VAddr->getReg(), 0, AMDGPU::sub1);
4001 
4002  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4003  BuildMI(MBB, MI, MI.getDebugLoc(), get(AMDGPU::REG_SEQUENCE), NewVAddr)
4004  .addReg(NewVAddrLo)
4005  .addImm(AMDGPU::sub0)
4006  .addReg(NewVAddrHi)
4007  .addImm(AMDGPU::sub1);
4008 
4009  VAddr->setReg(NewVAddr);
4010  Rsrc->setReg(NewSRsrc);
4011  } else if (!VAddr && ST.hasAddr64()) {
4012  // This instructions is the _OFFSET variant, so we need to convert it to
4013  // ADDR64.
4014  assert(MBB.getParent()->getSubtarget<GCNSubtarget>().getGeneration()
4016  "FIXME: Need to emit flat atomics here");
4017 
4018  unsigned RsrcPtr, NewSRsrc;
4019  std::tie(RsrcPtr, NewSRsrc) = extractRsrcPtr(*this, MI, *Rsrc);
4020 
4021  unsigned NewVAddr = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4022  MachineOperand *VData = getNamedOperand(MI, AMDGPU::OpName::vdata);
4023  MachineOperand *Offset = getNamedOperand(MI, AMDGPU::OpName::offset);
4024  MachineOperand *SOffset = getNamedOperand(MI, AMDGPU::OpName::soffset);
4025  unsigned Addr64Opcode = AMDGPU::getAddr64Inst(MI.getOpcode());
4026 
4027  // Atomics rith return have have an additional tied operand and are
4028  // missing some of the special bits.
4029  MachineOperand *VDataIn = getNamedOperand(MI, AMDGPU::OpName::vdata_in);
4030  MachineInstr *Addr64;
4031 
4032  if (!VDataIn) {
4033  // Regular buffer load / store.
4034  MachineInstrBuilder MIB =
4035  BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4036  .add(*VData)
4037  .addReg(NewVAddr)
4038  .addReg(NewSRsrc)
4039  .add(*SOffset)
4040  .add(*Offset);
4041 
4042  // Atomics do not have this operand.
4043  if (const MachineOperand *GLC =
4044  getNamedOperand(MI, AMDGPU::OpName::glc)) {
4045  MIB.addImm(GLC->getImm());
4046  }
4047 
4048  MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc));
4049 
4050  if (const MachineOperand *TFE =
4051  getNamedOperand(MI, AMDGPU::OpName::tfe)) {
4052  MIB.addImm(TFE->getImm());
4053  }
4054 
4055  MIB.cloneMemRefs(MI);
4056  Addr64 = MIB;
4057  } else {
4058  // Atomics with return.
4059  Addr64 = BuildMI(MBB, MI, MI.getDebugLoc(), get(Addr64Opcode))
4060  .add(*VData)
4061  .add(*VDataIn)
4062  .addReg(NewVAddr)
4063  .addReg(NewSRsrc)
4064  .add(*SOffset)
4065  .add(*Offset)
4066  .addImm(getNamedImmOperand(MI, AMDGPU::OpName::slc))
4067  .cloneMemRefs(MI);
4068  }
4069 
4070  MI.removeFromParent();
4071 
4072  // NewVaddr = {NewVaddrHi, NewVaddrLo}
4073  BuildMI(MBB, Addr64, Addr64->getDebugLoc(), get(AMDGPU::REG_SEQUENCE),
4074  NewVAddr)
4075  .addReg(RsrcPtr, 0, AMDGPU::sub0)
4076  .addImm(AMDGPU::sub0)
4077  .addReg(RsrcPtr, 0, AMDGPU::sub1)
4078  .addImm(AMDGPU::sub1);
4079  } else {
4080  // This is another variant; legalize Rsrc with waterfall loop from VGPRs
4081  // to SGPRs.
4082  loadSRsrcFromVGPR(*this, MI, *Rsrc, MDT);
4083  }
4084  }
4085 }
4086 
4088  MachineDominatorTree *MDT) const {
4089  SetVectorType Worklist;
4090  Worklist.insert(&TopInst);
4091 
4092  while (!Worklist.empty()) {
4093  MachineInstr &Inst = *Worklist.pop_back_val();
4094  MachineBasicBlock *MBB = Inst.getParent();
4096 
4097  unsigned Opcode = Inst.getOpcode();
4098  unsigned NewOpcode = getVALUOp(Inst);
4099 
4100  // Handle some special cases
4101  switch (Opcode) {
4102  default:
4103  break;
4104  case AMDGPU::S_ADD_U64_PSEUDO:
4105  case AMDGPU::S_SUB_U64_PSEUDO:
4106  splitScalar64BitAddSub(Worklist, Inst, MDT);
4107  Inst.eraseFromParent();
4108  continue;
4109  case AMDGPU::S_ADD_I32:
4110  case AMDGPU::S_SUB_I32:
4111  // FIXME: The u32 versions currently selected use the carry.
4112  if (moveScalarAddSub(Worklist, Inst, MDT))
4113  continue;
4114 
4115  // Default handling
4116  break;
4117  case AMDGPU::S_AND_B64:
4118  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_AND_B32_e64, MDT);
4119  Inst.eraseFromParent();
4120  continue;
4121 
4122  case AMDGPU::S_OR_B64:
4123  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_OR_B32_e64, MDT);
4124  Inst.eraseFromParent();
4125  continue;
4126 
4127  case AMDGPU::S_XOR_B64:
4128  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::V_XOR_B32_e64, MDT);
4129  Inst.eraseFromParent();
4130  continue;
4131 
4132  case AMDGPU::S_NOT_B64:
4133  splitScalar64BitUnaryOp(Worklist, Inst, AMDGPU::V_NOT_B32_e32);
4134  Inst.eraseFromParent();
4135  continue;
4136 
4137  case AMDGPU::S_BCNT1_I32_B64:
4138  splitScalar64BitBCNT(Worklist, Inst);
4139  Inst.eraseFromParent();
4140  continue;
4141 
4142  case AMDGPU::S_BFE_I64:
4143  splitScalar64BitBFE(Worklist, Inst);
4144  Inst.eraseFromParent();
4145  continue;
4146 
4147  case AMDGPU::S_LSHL_B32:
4148  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4149  NewOpcode = AMDGPU::V_LSHLREV_B32_e64;
4150  swapOperands(Inst);
4151  }
4152  break;
4153  case AMDGPU::S_ASHR_I32:
4154  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4155  NewOpcode = AMDGPU::V_ASHRREV_I32_e64;
4156  swapOperands(Inst);
4157  }
4158  break;
4159  case AMDGPU::S_LSHR_B32:
4160  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4161  NewOpcode = AMDGPU::V_LSHRREV_B32_e64;
4162  swapOperands(Inst);
4163  }
4164  break;
4165  case AMDGPU::S_LSHL_B64:
4166  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4167  NewOpcode = AMDGPU::V_LSHLREV_B64;
4168  swapOperands(Inst);
4169  }
4170  break;
4171  case AMDGPU::S_ASHR_I64:
4172  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4173  NewOpcode = AMDGPU::V_ASHRREV_I64;
4174  swapOperands(Inst);
4175  }
4176  break;
4177  case AMDGPU::S_LSHR_B64:
4178  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
4179  NewOpcode = AMDGPU::V_LSHRREV_B64;
4180  swapOperands(Inst);
4181  }
4182  break;
4183 
4184  case AMDGPU::S_ABS_I32:
4185  lowerScalarAbs(Worklist, Inst);
4186  Inst.eraseFromParent();
4187  continue;
4188 
4189  case AMDGPU::S_CBRANCH_SCC0:
4190  case AMDGPU::S_CBRANCH_SCC1:
4191  // Clear unused bits of vcc
4192  BuildMI(*MBB, Inst, Inst.getDebugLoc(), get(AMDGPU::S_AND_B64),
4193  AMDGPU::VCC)
4194  .addReg(AMDGPU::EXEC)
4195  .addReg(AMDGPU::VCC);
4196  break;
4197 
4198  case AMDGPU::S_BFE_U64:
4199  case AMDGPU::S_BFM_B64:
4200  llvm_unreachable("Moving this op to VALU not implemented");
4201 
4202  case AMDGPU::S_PACK_LL_B32_B16:
4203  case AMDGPU::S_PACK_LH_B32_B16:
4204  case AMDGPU::S_PACK_HH_B32_B16:
4205  movePackToVALU(Worklist, MRI, Inst);
4206  Inst.eraseFromParent();
4207  continue;
4208 
4209  case AMDGPU::S_XNOR_B32:
4210  lowerScalarXnor(Worklist, Inst);
4211  Inst.eraseFromParent();
4212  continue;
4213 
4214  case AMDGPU::S_XNOR_B64:
4215  splitScalar64BitBinaryOp(Worklist, Inst, AMDGPU::S_XNOR_B32, MDT);
4216  Inst.eraseFromParent();
4217  continue;
4218 
4219  case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR:
4220  case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR:
4221  case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR:
4222  case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR:
4223  case AMDGPU::S_BUFFER_LOAD_DWORDX16_SGPR: {
4224  unsigned VDst;
4225  unsigned NewOpcode;
4226 
4227  switch(Opcode) {
4228  case AMDGPU::S_BUFFER_LOAD_DWORD_SGPR:
4229  NewOpcode = AMDGPU::BUFFER_LOAD_DWORD_OFFEN;
4230  VDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4231  break;
4232  case AMDGPU::S_BUFFER_LOAD_DWORDX2_SGPR:
4233  NewOpcode = AMDGPU::BUFFER_LOAD_DWORDX2_OFFEN;
4234  VDst = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4235  break;
4236  case AMDGPU::S_BUFFER_LOAD_DWORDX4_SGPR:
4237  NewOpcode = AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN;
4238  VDst = MRI.createVirtualRegister(&AMDGPU::VReg_128RegClass);
4239  break;
4240  case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR:
4241  case AMDGPU::S_BUFFER_LOAD_DWORDX16_SGPR:
4242  splitScalarBuffer(Worklist, Inst);
4243  Inst.eraseFromParent();
4244  continue;
4245  }
4246 
4247  const MachineOperand *VAddr = getNamedOperand(Inst, AMDGPU::OpName::soff);
4248  auto Add = MRI.getUniqueVRegDef(VAddr->getReg());
4249  unsigned Offset = 0;
4250 
4251  // FIXME: This isn't safe because the addressing mode doesn't work
4252  // correctly if vaddr is negative.
4253  //
4254  // FIXME: Should probably be done somewhere else, maybe SIFoldOperands.
4255  //
4256  // See if we can extract an immediate offset by recognizing one of these:
4257  // V_ADD_I32_e32 dst, imm, src1
4258  // V_ADD_I32_e32 dst, (S_MOV_B32 imm), src1
4259  // V_ADD will be removed by "Remove dead machine instructions".
4260  if (Add &&
4261  (Add->getOpcode() == AMDGPU::V_ADD_I32_e32 ||
4262  Add->getOpcode() == AMDGPU::V_ADD_U32_e32 ||
4263  Add->getOpcode() == AMDGPU::V_ADD_U32_e64)) {
4264  static const unsigned SrcNames[2] = {
4265  AMDGPU::OpName::src0,
4266  AMDGPU::OpName::src1,
4267  };
4268 
4269  // Find a literal offset in one of source operands.
4270  for (int i = 0; i < 2; i++) {
4271  const MachineOperand *Src =
4272  getNamedOperand(*Add, SrcNames[i]);
4273 
4274  if (Src->isReg()) {
4275  MachineInstr *Def = MRI.getUniqueVRegDef(Src->getReg());
4276  if (Def) {
4277  if (Def->isMoveImmediate())
4278  Src = &Def->getOperand(1);
4279  else if (Def->isCopy()) {
4280  auto Mov = MRI.getUniqueVRegDef(Def->getOperand(1).getReg());
4281  if (Mov && Mov->isMoveImmediate()) {
4282  Src = &Mov->getOperand(1);
4283  }
4284  }
4285  }
4286  }
4287 
4288  if (Src) {
4289  if (Src->isImm())
4290  Offset = Src->getImm();
4291  else if (Src->isCImm())
4292  Offset = Src->getCImm()->getZExtValue();
4293  }
4294 
4295  if (Offset && isLegalMUBUFImmOffset(Offset)) {
4296  VAddr = getNamedOperand(*Add, SrcNames[!i]);
4297  break;
4298  }
4299 
4300  Offset = 0;
4301  }
4302  }
4303 
4304  MachineInstr *NewInstr =
4305  BuildMI(*MBB, Inst, Inst.getDebugLoc(),
4306  get(NewOpcode), VDst)
4307  .add(*VAddr) // vaddr
4308  .add(*getNamedOperand(Inst, AMDGPU::OpName::sbase)) // srsrc
4309  .addImm(0) // soffset
4310  .addImm(Offset) // offset
4311  .addImm(getNamedOperand(Inst, AMDGPU::OpName::glc)->getImm())
4312  .addImm(0) // slc
4313  .addImm(0) // tfe
4314  .cloneMemRefs(Inst)
4315  .getInstr();
4316 
4317  MRI.replaceRegWith(getNamedOperand(Inst, AMDGPU::OpName::sdst)->getReg(),
4318  VDst);
4319  addUsersToMoveToVALUWorklist(VDst, MRI, Worklist);
4320  Inst.eraseFromParent();
4321 
4322  // Legalize all operands other than the offset. Notably, convert the srsrc
4323  // into SGPRs using v_readfirstlane if needed.
4324  legalizeOperands(*NewInstr, MDT);
4325  continue;
4326  }
4327  }
4328 
4329  if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
4330  // We cannot move this instruction to the VALU, so we should try to
4331  // legalize its operands instead.
4332  legalizeOperands(Inst, MDT);
4333  continue;
4334  }
4335 
4336  // Use the new VALU Opcode.
4337  const MCInstrDesc &NewDesc = get(NewOpcode);
4338  Inst.setDesc(NewDesc);
4339 
4340  // Remove any references to SCC. Vector instructions can't read from it, and
4341  // We're just about to add the implicit use / defs of VCC, and we don't want
4342  // both.
4343  for (unsigned i = Inst.getNumOperands() - 1; i > 0; --i) {
4344  MachineOperand &Op = Inst.getOperand(i);
4345  if (Op.isReg() && Op.getReg() == AMDGPU::SCC) {
4346  Inst.RemoveOperand(i);
4347  addSCCDefUsersToVALUWorklist(Inst, Worklist);
4348  }
4349  }
4350 
4351  if (Opcode == AMDGPU::S_SEXT_I32_I8 || Opcode == AMDGPU::S_SEXT_I32_I16) {
4352  // We are converting these to a BFE, so we need to add the missing
4353  // operands for the size and offset.
4354  unsigned Size = (Opcode == AMDGPU::S_SEXT_I32_I8) ? 8 : 16;
4357 
4358  } else if (Opcode == AMDGPU::S_BCNT1_I32_B32) {
4359  // The VALU version adds the second operand to the result, so insert an
4360  // extra 0 operand.
4362  }
4363 
4365 
4366  if (Opcode == AMDGPU::S_BFE_I32 || Opcode == AMDGPU::S_BFE_U32) {
4367  const MachineOperand &OffsetWidthOp = Inst.getOperand(2);
4368  // If we need to move this to VGPRs, we need to unpack the second operand
4369  // back into the 2 separate ones for bit offset and width.
4370  assert(OffsetWidthOp.isImm() &&
4371  "Scalar BFE is only implemented for constant width and offset");
4372  uint32_t Imm = OffsetWidthOp.getImm();
4373 
4374  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4375  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4376  Inst.RemoveOperand(2); // Remove old immediate.
4377  Inst.addOperand(MachineOperand::CreateImm(Offset));
4378  Inst.addOperand(MachineOperand::CreateImm(BitWidth));
4379  }
4380 
4381  bool HasDst = Inst.getOperand(0).isReg() && Inst.getOperand(0).isDef();
4382  unsigned NewDstReg = AMDGPU::NoRegister;
4383  if (HasDst) {
4384  unsigned DstReg = Inst.getOperand(0).getReg();
4386  continue;
4387 
4388  // Update the destination register class.
4389  const TargetRegisterClass *NewDstRC = getDestEquivalentVGPRClass(Inst);
4390  if (!NewDstRC)
4391  continue;
4392 
4393  if (Inst.isCopy() &&
4395  NewDstRC == RI.getRegClassForReg(MRI, Inst.getOperand(1).getReg())) {
4396  // Instead of creating a copy where src and dst are the same register
4397  // class, we just replace all uses of dst with src. These kinds of
4398  // copies interfere with the heuristics MachineSink uses to decide
4399  // whether or not to split a critical edge. Since the pass assumes
4400  // that copies will end up as machine instructions and not be
4401  // eliminated.
4402  addUsersToMoveToVALUWorklist(DstReg, MRI, Worklist);
4403  MRI.replaceRegWith(DstReg, Inst.getOperand(1).getReg());
4404  MRI.clearKillFlags(Inst.getOperand(1).getReg());
4405  Inst.getOperand(0).setReg(DstReg);
4406 
4407  // Make sure we don't leave around a dead VGPR->SGPR copy. Normally
4408  // these are deleted later, but at -O0 it would leave a suspicious
4409  // looking illegal copy of an undef register.
4410  for (unsigned I = Inst.getNumOperands() - 1; I != 0; --I)
4411  Inst.RemoveOperand(I);
4412  Inst.setDesc(get(AMDGPU::IMPLICIT_DEF));
4413  continue;
4414  }
4415 
4416  NewDstReg = MRI.createVirtualRegister(NewDstRC);
4417  MRI.replaceRegWith(DstReg, NewDstReg);
4418  }
4419 
4420  // Legalize the operands
4421  legalizeOperands(Inst, MDT);
4422 
4423  if (HasDst)
4424  addUsersToMoveToVALUWorklist(NewDstReg, MRI, Worklist);
4425  }
4426 }
4427 
4428 // Add/sub require special handling to deal with carry outs.
4429 bool SIInstrInfo::moveScalarAddSub(SetVectorType &Worklist, MachineInstr &Inst,
4430  MachineDominatorTree *MDT) const {
4431  if (ST.hasAddNoCarry()) {
4432  // Assume there is no user of scc since we don't select this in that case.
4433  // Since scc isn't used, it doesn't really matter if the i32 or u32 variant
4434  // is used.
4435 
4436  MachineBasicBlock &MBB = *Inst.getParent();
4438 
4439  unsigned OldDstReg = Inst.getOperand(0).getReg();
4440  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4441 
4442  unsigned Opc = Inst.getOpcode();
4443  assert(Opc == AMDGPU::S_ADD_I32 || Opc == AMDGPU::S_SUB_I32);
4444 
4445  unsigned NewOpc = Opc == AMDGPU::S_ADD_I32 ?
4446  AMDGPU::V_ADD_U32_e64 : AMDGPU::V_SUB_U32_e64;
4447 
4448  assert(Inst.getOperand(3).getReg() == AMDGPU::SCC);
4449  Inst.RemoveOperand(3);
4450 
4451  Inst.setDesc(get(NewOpc));
4452  Inst.addImplicitDefUseOperands(*MBB.getParent());
4453  MRI.replaceRegWith(OldDstReg, ResultReg);
4454  legalizeOperands(Inst, MDT);
4455 
4456  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4457  return true;
4458  }
4459 
4460  return false;
4461 }
4462 
4463 void SIInstrInfo::lowerScalarAbs(SetVectorType &Worklist,
4464  MachineInstr &Inst) const {
4465  MachineBasicBlock &MBB = *Inst.getParent();
4467  MachineBasicBlock::iterator MII = Inst;
4468  DebugLoc DL = Inst.getDebugLoc();
4469 
4470  MachineOperand &Dest = Inst.getOperand(0);
4471  MachineOperand &Src = Inst.getOperand(1);
4472  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4473  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4474 
4475  unsigned SubOp = ST.hasAddNoCarry() ?
4476  AMDGPU::V_SUB_U32_e32 : AMDGPU::V_SUB_I32_e32;
4477 
4478  BuildMI(MBB, MII, DL, get(SubOp), TmpReg)
4479  .addImm(0)
4480  .addReg(Src.getReg());
4481 
4482  BuildMI(MBB, MII, DL, get(AMDGPU::V_MAX_I32_e64), ResultReg)
4483  .addReg(Src.getReg())
4484  .addReg(TmpReg);
4485 
4486  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4487  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4488 }
4489 
4490 void SIInstrInfo::lowerScalarXnor(SetVectorType &Worklist,
4491  MachineInstr &Inst) const {
4492  MachineBasicBlock &MBB = *Inst.getParent();
4494  MachineBasicBlock::iterator MII = Inst;
4495  const DebugLoc &DL = Inst.getDebugLoc();
4496 
4497  MachineOperand &Dest = Inst.getOperand(0);
4498  MachineOperand &Src0 = Inst.getOperand(1);
4499  MachineOperand &Src1 = Inst.getOperand(2);
4500 
4501  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src0, MRI, DL);
4502  legalizeGenericOperand(MBB, MII, &AMDGPU::VGPR_32RegClass, Src1, MRI, DL);
4503 
4504  unsigned NewDest = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4505  if (ST.hasDLInsts()) {
4506  BuildMI(MBB, MII, DL, get(AMDGPU::V_XNOR_B32_e64), NewDest)
4507  .add(Src0)
4508  .add(Src1);
4509  } else {
4510  unsigned Xor = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4511  BuildMI(MBB, MII, DL, get(AMDGPU::V_XOR_B32_e64), Xor)
4512  .add(Src0)
4513  .add(Src1);
4514 
4515  BuildMI(MBB, MII, DL, get(AMDGPU::V_NOT_B32_e64), NewDest)
4516  .addReg(Xor);
4517  }
4518 
4519  MRI.replaceRegWith(Dest.getReg(), NewDest);
4520  addUsersToMoveToVALUWorklist(NewDest, MRI, Worklist);
4521 }
4522 
4523 void SIInstrInfo::splitScalar64BitUnaryOp(
4524  SetVectorType &Worklist, MachineInstr &Inst,
4525  unsigned Opcode) const {
4526  MachineBasicBlock &MBB = *Inst.getParent();
4528 
4529  MachineOperand &Dest = Inst.getOperand(0);
4530  MachineOperand &Src0 = Inst.getOperand(1);
4531  DebugLoc DL = Inst.getDebugLoc();
4532 
4533  MachineBasicBlock::iterator MII = Inst;
4534 
4535  const MCInstrDesc &InstDesc = get(Opcode);
4536  const TargetRegisterClass *Src0RC = Src0.isReg() ?
4537  MRI.getRegClass(Src0.getReg()) :
4538  &AMDGPU::SGPR_32RegClass;
4539 
4540  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4541 
4542  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4543  AMDGPU::sub0, Src0SubRC);
4544 
4545  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4546  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4547  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
4548 
4549  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
4550  BuildMI(MBB, MII, DL, InstDesc, DestSub0).add(SrcReg0Sub0);
4551 
4552  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4553  AMDGPU::sub1, Src0SubRC);
4554 
4555  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
4556  BuildMI(MBB, MII, DL, InstDesc, DestSub1).add(SrcReg0Sub1);
4557 
4558  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4559  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4560  .addReg(DestSub0)
4561  .addImm(AMDGPU::sub0)
4562  .addReg(DestSub1)
4563  .addImm(AMDGPU::sub1);
4564 
4565  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4566 
4567  // We don't need to legalizeOperands here because for a single operand, src0
4568  // will support any kind of input.
4569 
4570  // Move all users of this moved value.
4571  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4572 }
4573 
4574 void SIInstrInfo::splitScalar64BitAddSub(SetVectorType &Worklist,
4575  MachineInstr &Inst,
4576  MachineDominatorTree *MDT) const {
4577  bool IsAdd = (Inst.getOpcode() == AMDGPU::S_ADD_U64_PSEUDO);
4578 
4579  MachineBasicBlock &MBB = *Inst.getParent();
4581 
4582  unsigned FullDestReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4583  unsigned DestSub0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4584  unsigned DestSub1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4585 
4586  unsigned CarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
4587  unsigned DeadCarryReg = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
4588 
4589  MachineOperand &Dest = Inst.getOperand(0);
4590  MachineOperand &Src0 = Inst.getOperand(1);
4591  MachineOperand &Src1 = Inst.getOperand(2);
4592  const DebugLoc &DL = Inst.getDebugLoc();
4593  MachineBasicBlock::iterator MII = Inst;
4594 
4595  const TargetRegisterClass *Src0RC = MRI.getRegClass(Src0.getReg());
4596  const TargetRegisterClass *Src1RC = MRI.getRegClass(Src1.getReg());
4597  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4598  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
4599 
4600  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4601  AMDGPU::sub0, Src0SubRC);
4602  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4603  AMDGPU::sub0, Src1SubRC);
4604 
4605 
4606  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4607  AMDGPU::sub1, Src0SubRC);
4608  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4609  AMDGPU::sub1, Src1SubRC);
4610 
4611  unsigned LoOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
4612  MachineInstr *LoHalf =
4613  BuildMI(MBB, MII, DL, get(LoOpc), DestSub0)
4614  .addReg(CarryReg, RegState::Define)
4615  .add(SrcReg0Sub0)
4616  .add(SrcReg1Sub0);
4617 
4618  unsigned HiOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64;
4619  MachineInstr *HiHalf =
4620  BuildMI(MBB, MII, DL, get(HiOpc), DestSub1)
4621  .addReg(DeadCarryReg, RegState::Define | RegState::Dead)
4622  .add(SrcReg0Sub1)
4623  .add(SrcReg1Sub1)
4624  .addReg(CarryReg, RegState::Kill);
4625 
4626  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4627  .addReg(DestSub0)
4628  .addImm(AMDGPU::sub0)
4629  .addReg(DestSub1)
4630  .addImm(AMDGPU::sub1);
4631 
4632  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4633 
4634  // Try to legalize the operands in case we need to swap the order to keep it
4635  // valid.
4636  legalizeOperands(*LoHalf, MDT);
4637  legalizeOperands(*HiHalf, MDT);
4638 
4639  // Move all users of this moved vlaue.
4640  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4641 }
4642 
4643 void SIInstrInfo::splitScalar64BitBinaryOp(SetVectorType &Worklist,
4644  MachineInstr &Inst, unsigned Opcode,
4645  MachineDominatorTree *MDT) const {
4646  MachineBasicBlock &MBB = *Inst.getParent();
4648 
4649  MachineOperand &Dest = Inst.getOperand(0);
4650  MachineOperand &Src0 = Inst.getOperand(1);
4651  MachineOperand &Src1 = Inst.getOperand(2);
4652  DebugLoc DL = Inst.getDebugLoc();
4653 
4654  MachineBasicBlock::iterator MII = Inst;
4655 
4656  const MCInstrDesc &InstDesc = get(Opcode);
4657  const TargetRegisterClass *Src0RC = Src0.isReg() ?
4658  MRI.getRegClass(Src0.getReg()) :
4659  &AMDGPU::SGPR_32RegClass;
4660 
4661  const TargetRegisterClass *Src0SubRC = RI.getSubRegClass(Src0RC, AMDGPU::sub0);
4662  const TargetRegisterClass *Src1RC = Src1.isReg() ?
4663  MRI.getRegClass(Src1.getReg()) :
4664  &AMDGPU::SGPR_32RegClass;
4665 
4666  const TargetRegisterClass *Src1SubRC = RI.getSubRegClass(Src1RC, AMDGPU::sub0);
4667 
4668  MachineOperand SrcReg0Sub0 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4669  AMDGPU::sub0, Src0SubRC);
4670  MachineOperand SrcReg1Sub0 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4671  AMDGPU::sub0, Src1SubRC);
4672 
4673  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4674  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4675  const TargetRegisterClass *NewDestSubRC = RI.getSubRegClass(NewDestRC, AMDGPU::sub0);
4676 
4677  unsigned DestSub0 = MRI.createVirtualRegister(NewDestSubRC);
4678  MachineInstr &LoHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub0)
4679  .add(SrcReg0Sub0)
4680  .add(SrcReg1Sub0);
4681 
4682  MachineOperand SrcReg0Sub1 = buildExtractSubRegOrImm(MII, MRI, Src0, Src0RC,
4683  AMDGPU::sub1, Src0SubRC);
4684  MachineOperand SrcReg1Sub1 = buildExtractSubRegOrImm(MII, MRI, Src1, Src1RC,
4685  AMDGPU::sub1, Src1SubRC);
4686 
4687  unsigned DestSub1 = MRI.createVirtualRegister(NewDestSubRC);
4688  MachineInstr &HiHalf = *BuildMI(MBB, MII, DL, InstDesc, DestSub1)
4689  .add(SrcReg0Sub1)
4690  .add(SrcReg1Sub1);
4691 
4692  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4693  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), FullDestReg)
4694  .addReg(DestSub0)
4695  .addImm(AMDGPU::sub0)
4696  .addReg(DestSub1)
4697  .addImm(AMDGPU::sub1);
4698 
4699  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4700 
4701  // Try to legalize the operands in case we need to swap the order to keep it
4702  // valid.
4703  legalizeOperands(LoHalf, MDT);
4704  legalizeOperands(HiHalf, MDT);
4705 
4706  // Move all users of this moved vlaue.
4707  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4708 }
4709 
4710 void SIInstrInfo::splitScalar64BitBCNT(
4711  SetVectorType &Worklist, MachineInstr &Inst) const {
4712  MachineBasicBlock &MBB = *Inst.getParent();
4714 
4715  MachineBasicBlock::iterator MII = Inst;
4716  DebugLoc DL = Inst.getDebugLoc();
4717 
4718  MachineOperand &Dest = Inst.getOperand(0);
4719  MachineOperand &Src = Inst.getOperand(1);
4720 
4721  const MCInstrDesc &InstDesc = get(AMDGPU::V_BCNT_U32_B32_e64);
4722  const TargetRegisterClass *SrcRC = Src.isReg() ?
4723  MRI.getRegClass(Src.getReg()) :
4724  &AMDGPU::SGPR_32RegClass;
4725 
4726  unsigned MidReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4727  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4728 
4729  const TargetRegisterClass *SrcSubRC = RI.getSubRegClass(SrcRC, AMDGPU::sub0);
4730 
4731  MachineOperand SrcRegSub0 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
4732  AMDGPU::sub0, SrcSubRC);
4733  MachineOperand SrcRegSub1 = buildExtractSubRegOrImm(MII, MRI, Src, SrcRC,
4734  AMDGPU::sub1, SrcSubRC);
4735 
4736  BuildMI(MBB, MII, DL, InstDesc, MidReg).add(SrcRegSub0).addImm(0);
4737 
4738  BuildMI(MBB, MII, DL, InstDesc, ResultReg).add(SrcRegSub1).addReg(MidReg);
4739 
4740  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4741 
4742  // We don't need to legalize operands here. src0 for etiher instruction can be
4743  // an SGPR, and the second input is unused or determined here.
4744  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4745 }
4746 
4747 void SIInstrInfo::splitScalar64BitBFE(SetVectorType &Worklist,
4748  MachineInstr &Inst) const {
4749  MachineBasicBlock &MBB = *Inst.getParent();
4751  MachineBasicBlock::iterator MII = Inst;
4752  DebugLoc DL = Inst.getDebugLoc();
4753 
4754  MachineOperand &Dest = Inst.getOperand(0);
4755  uint32_t Imm = Inst.getOperand(2).getImm();
4756  uint32_t Offset = Imm & 0x3f; // Extract bits [5:0].
4757  uint32_t BitWidth = (Imm & 0x7f0000) >> 16; // Extract bits [22:16].
4758 
4759  (void) Offset;
4760 
4761  // Only sext_inreg cases handled.
4762  assert(Inst.getOpcode() == AMDGPU::S_BFE_I64 && BitWidth <= 32 &&
4763  Offset == 0 && "Not implemented");
4764 
4765  if (BitWidth < 32) {
4766  unsigned MidRegLo = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4767  unsigned MidRegHi = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4768  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4769 
4770  BuildMI(MBB, MII, DL, get(AMDGPU::V_BFE_I32), MidRegLo)
4771  .addReg(Inst.getOperand(1).getReg(), 0, AMDGPU::sub0)
4772  .addImm(0)
4773  .addImm(BitWidth);
4774 
4775  BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e32), MidRegHi)
4776  .addImm(31)
4777  .addReg(MidRegLo);
4778 
4779  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
4780  .addReg(MidRegLo)
4781  .addImm(AMDGPU::sub0)
4782  .addReg(MidRegHi)
4783  .addImm(AMDGPU::sub1);
4784 
4785  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4786  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4787  return;
4788  }
4789 
4790  MachineOperand &Src = Inst.getOperand(1);
4791  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4792  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_64RegClass);
4793 
4794  BuildMI(MBB, MII, DL, get(AMDGPU::V_ASHRREV_I32_e64), TmpReg)
4795  .addImm(31)
4796  .addReg(Src.getReg(), 0, AMDGPU::sub0);
4797 
4798  BuildMI(MBB, MII, DL, get(TargetOpcode::REG_SEQUENCE), ResultReg)
4799  .addReg(Src.getReg(), 0, AMDGPU::sub0)
4800  .addImm(AMDGPU::sub0)
4801  .addReg(TmpReg)
4802  .addImm(AMDGPU::sub1);
4803 
4804  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4805  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4806 }
4807 
4808 void SIInstrInfo::splitScalarBuffer(SetVectorType &Worklist,
4809  MachineInstr &Inst) const {
4810  MachineBasicBlock &MBB = *Inst.getParent();
4812 
4813  MachineBasicBlock::iterator MII = Inst;
4814  auto &DL = Inst.getDebugLoc();
4815 
4816  MachineOperand &Dest = *getNamedOperand(Inst, AMDGPU::OpName::sdst);;
4817  MachineOperand &Rsrc = *getNamedOperand(Inst, AMDGPU::OpName::sbase);
4818  MachineOperand &Offset = *getNamedOperand(Inst, AMDGPU::OpName::soff);
4819  MachineOperand &Glc = *getNamedOperand(Inst, AMDGPU::OpName::glc);
4820 
4821  unsigned Opcode = Inst.getOpcode();
4822  unsigned NewOpcode = AMDGPU::BUFFER_LOAD_DWORDX4_OFFEN;
4823  unsigned Count = 0;
4824  const TargetRegisterClass *DestRC = MRI.getRegClass(Dest.getReg());
4825  const TargetRegisterClass *NewDestRC = RI.getEquivalentVGPRClass(DestRC);
4826 
4827  switch(Opcode) {
4828  default:
4829  return;
4830  case AMDGPU::S_BUFFER_LOAD_DWORDX8_SGPR:
4831  Count = 2;
4832  break;
4833  case AMDGPU::S_BUFFER_LOAD_DWORDX16_SGPR:
4834  Count = 4;
4835  break;
4836  }
4837 
4838  // FIXME: Should also attempt to build VAddr and Offset like the non-split
4839  // case (see call site for this function)
4840 
4841  // Create a vector of result registers
4842  SmallVector<unsigned, 8> ResultRegs;
4843  for (unsigned i = 0; i < Count ; ++i) {
4844  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VReg_128RegClass);
4845  MachineInstr &NewMI = *BuildMI(MBB, MII, DL, get(NewOpcode), ResultReg)
4846  .addReg(Offset.getReg()) // offset
4847  .addReg(Rsrc.getReg()) // rsrc
4848  .addImm(0) // soffset
4849  .addImm(i << 4) // inst_offset
4850  .addImm(Glc.getImm()) // glc
4851  .addImm(0) // slc
4852  .addImm(0) // tfe
4853  .addMemOperand(*Inst.memoperands_begin());
4854  // Extract the 4 32 bit sub-registers from the result to add into the final REG_SEQUENCE
4855  auto &NewDestOp = NewMI.getOperand(0);
4856  for (unsigned i = 0 ; i < 4 ; i++)
4857  ResultRegs.push_back(buildExtractSubReg(MII, MRI, NewDestOp, &AMDGPU::VReg_128RegClass,
4858  RI.getSubRegFromChannel(i), &AMDGPU::VGPR_32RegClass));
4859  }
4860  // Create a new combined result to replace original with
4861  unsigned FullDestReg = MRI.createVirtualRegister(NewDestRC);
4862  MachineInstrBuilder CombinedResBuilder = BuildMI(MBB, MII, DL,
4863  get(TargetOpcode::REG_SEQUENCE), FullDestReg);
4864 
4865  for (unsigned i = 0 ; i < Count * 4 ; ++i) {
4866  CombinedResBuilder
4867  .addReg(ResultRegs[i])
4868  .addImm(RI.getSubRegFromChannel(i));
4869  }
4870 
4871  MRI.replaceRegWith(Dest.getReg(), FullDestReg);
4872  addUsersToMoveToVALUWorklist(FullDestReg, MRI, Worklist);
4873 }
4874 
4875 void SIInstrInfo::addUsersToMoveToVALUWorklist(
4876  unsigned DstReg,
4878  SetVectorType &Worklist) const {
4879  for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
4880  E = MRI.use_end(); I != E;) {
4881  MachineInstr &UseMI = *I->getParent();
4882  if (!canReadVGPR(UseMI, I.getOperandNo())) {
4883  Worklist.insert(&UseMI);
4884 
4885  do {
4886  ++I;
4887  } while (I != E && I->getParent() == &UseMI);
4888  } else {
4889  ++I;
4890  }
4891  }
4892 }
4893 
4894 void SIInstrInfo::movePackToVALU(SetVectorType &Worklist,
4895  MachineRegisterInfo &MRI,
4896  MachineInstr &Inst) const {
4897  unsigned ResultReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4898  MachineBasicBlock *MBB = Inst.getParent();
4899  MachineOperand &Src0 = Inst.getOperand(1);
4900  MachineOperand &Src1 = Inst.getOperand(2);
4901  const DebugLoc &DL = Inst.getDebugLoc();
4902 
4903  switch (Inst.getOpcode()) {
4904  case AMDGPU::S_PACK_LL_B32_B16: {
4905  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4906  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4907 
4908  // FIXME: Can do a lot better if we know the high bits of src0 or src1 are
4909  // 0.
4910  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4911  .addImm(0xffff);
4912 
4913  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_B32_e64), TmpReg)
4914  .addReg(ImmReg, RegState::Kill)
4915  .add(Src0);
4916 
4917  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHL_OR_B32), ResultReg)
4918  .add(Src1)
4919  .addImm(16)
4920  .addReg(TmpReg, RegState::Kill);
4921  break;
4922  }
4923  case AMDGPU::S_PACK_LH_B32_B16: {
4924  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4925  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4926  .addImm(0xffff);
4927  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_BFI_B32), ResultReg)
4928  .addReg(ImmReg, RegState::Kill)
4929  .add(Src0)
4930  .add(Src1);
4931  break;
4932  }
4933  case AMDGPU::S_PACK_HH_B32_B16: {
4934  unsigned ImmReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4935  unsigned TmpReg = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
4936  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_LSHRREV_B32_e64), TmpReg)
4937  .addImm(16)
4938  .add(Src0);
4939  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_MOV_B32_e32), ImmReg)
4940  .addImm(0xffff0000);
4941  BuildMI(*MBB, Inst, DL, get(AMDGPU::V_AND_OR_B32), ResultReg)
4942  .add(Src1)
4943  .addReg(ImmReg, RegState::Kill)
4944  .addReg(TmpReg, RegState::Kill);
4945  break;
4946  }
4947  default:
4948  llvm_unreachable("unhandled s_pack_* instruction");
4949  }
4950 
4951  MachineOperand &Dest = Inst.getOperand(0);
4952  MRI.replaceRegWith(Dest.getReg(), ResultReg);
4953  addUsersToMoveToVALUWorklist(ResultReg, MRI, Worklist);
4954 }
4955 
4956 void SIInstrInfo::addSCCDefUsersToVALUWorklist(
4957  MachineInstr &SCCDefInst, SetVectorType &Worklist) const {
4958  // This assumes that all the users of SCC are in the same block
4959  // as the SCC def.
4960  for (MachineInstr &MI :
4962  SCCDefInst.getParent()->end())) {
4963  // Exit if we find another SCC def.
4964  if (MI.findRegisterDefOperandIdx(AMDGPU::SCC, false, false, &RI) != -1)
4965  return;
4966 
4967  if (MI.findRegisterUseOperandIdx(AMDGPU::SCC, false, &RI) != -1)
4968  Worklist.insert(&MI);
4969  }
4970 }
4971 
4972 const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
4973  const MachineInstr &Inst) const {
4974  const TargetRegisterClass *NewDstRC = getOpRegClass(Inst, 0);
4975 
4976  switch (Inst.getOpcode()) {
4977  // For target instructions, getOpRegClass just returns the virtual register
4978  // class associated with the operand, so we need to find an equivalent VGPR
4979  // register class in order to move the instruction to the VALU.
4980  case AMDGPU::COPY:
4981  case AMDGPU::PHI:
4982  case AMDGPU::REG_SEQUENCE:
4983  case AMDGPU::INSERT_SUBREG:
4984  case AMDGPU::WQM:
4985  case AMDGPU::WWM:
4986  if (RI.hasVGPRs(NewDstRC))
4987  return nullptr;
4988 
4989  NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
4990  if (!NewDstRC)
4991  return nullptr;
4992  return NewDstRC;
4993  default:
4994  return NewDstRC;
4995  }
4996 }
4997 
4998 // Find the one SGPR operand we are allowed to use.
4999 unsigned SIInstrInfo::findUsedSGPR(const MachineInstr &MI,
5000  int OpIndices[3]) const {
5001  const MCInstrDesc &Desc = MI.getDesc();
5002 
5003  // Find the one SGPR operand we are allowed to use.
5004  //
5005  // First we need to consider the instruction's operand requirements before
5006  // legalizing. Some operands are required to be SGPRs, such as implicit uses
5007  // of VCC, but we are still bound by the constant bus requirement to only use
5008  // one.
5009  //
5010  // If the operand's class is an SGPR, we can never move it.
5011 
5012  unsigned SGPRReg = findImplicitSGPRRead(MI);
5013  if (SGPRReg != AMDGPU::NoRegister)
5014  return SGPRReg;
5015 
5016  unsigned UsedSGPRs[3] = { AMDGPU::NoRegister };
5017  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
5018 
5019  for (unsigned i = 0; i < 3; ++i) {
5020  int Idx = OpIndices[i];
5021  if (Idx == -1)
5022  break;
5023 
5024  const MachineOperand &MO = MI.getOperand(Idx);
5025  if (!MO.isReg())
5026  continue;
5027 
5028  // Is this operand statically required to be an SGPR based on the operand
5029  // constraints?
5030  const TargetRegisterClass *OpRC = RI.getRegClass(Desc.OpInfo[Idx].RegClass);
5031  bool IsRequiredSGPR = RI.isSGPRClass(OpRC);
5032  if (IsRequiredSGPR)
5033  return MO.getReg();
5034 
5035  // If this could be a VGPR or an SGPR, Check the dynamic register class.
5036  unsigned Reg = MO.getReg();
5037  const TargetRegisterClass *RegRC = MRI.getRegClass(Reg);
5038  if (RI.isSGPRClass(RegRC))
5039  UsedSGPRs[i] = Reg;
5040  }
5041 
5042  // We don't have a required SGPR operand, so we have a bit more freedom in
5043  // selecting operands to move.
5044 
5045  // Try to select the most used SGPR. If an SGPR is equal to one of the
5046  // others, we choose that.
5047  //
5048  // e.g.
5049  // V_FMA_F32 v0, s0, s0, s0 -> No moves
5050  // V_FMA_F32 v0, s0, s1, s0 -> Move s1
5051 
5052  // TODO: If some of the operands are 64-bit SGPRs and some 32, we should
5053  // prefer those.
5054 
5055  if (UsedSGPRs[0] != AMDGPU::NoRegister) {
5056  if (UsedSGPRs[0] == UsedSGPRs[1] || UsedSGPRs[0] == UsedSGPRs[2])
5057  SGPRReg = UsedSGPRs[0];
5058  }
5059 
5060  if (SGPRReg == AMDGPU::NoRegister && UsedSGPRs[1] != AMDGPU::NoRegister) {
5061  if (UsedSGPRs[1] == UsedSGPRs[2])
5062  SGPRReg = UsedSGPRs[1];
5063  }
5064 
5065  return SGPRReg;
5066 }
5067 
5069  unsigned OperandName) const {
5070  int Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), OperandName);
5071  if (Idx == -1)
5072  return nullptr;
5073 
5074  return &MI.getOperand(Idx);
5075 }
5076 
5078  uint64_t RsrcDataFormat = AMDGPU::RSRC_DATA_FORMAT;
5079  if (ST.isAmdHsaOS()) {
5080  // Set ATC = 1. GFX9 doesn't have this bit.
5081  if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS)
5082  RsrcDataFormat |= (1ULL << 56);
5083 
5084  // Set MTYPE = 2 (MTYPE_UC = uncached). GFX9 doesn't have this.
5085  // BTW, it disables TC L2 and therefore decreases performance.
5086  if (ST.getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS)
5087  RsrcDataFormat |= (2ULL << 59);
5088  }
5089 
5090  return RsrcDataFormat;
5091 }
5092 
5094  uint64_t Rsrc23 = getDefaultRsrcDataFormat() |
5096  0xffffffff; // Size;
5097 
5098  // GFX9 doesn't have ELEMENT_SIZE.
5099  if (ST.getGeneration() <= AMDGPUSubtarget::VOLCANIC_ISLANDS) {
5100  uint64_t EltSizeValue = Log2_32(ST.getMaxPrivateElementSize()) - 1;
5101  Rsrc23 |= EltSizeValue << AMDGPU::RSRC_ELEMENT_SIZE_SHIFT;
5102  }
5103 
5104  // IndexStride = 64.
5105  Rsrc23 |= UINT64_C(3) << AMDGPU::RSRC_INDEX_STRIDE_SHIFT;
5106 
5107  // If TID_ENABLE is set, DATA_FORMAT specifies stride bits [14:17].
5108  // Clear them unless we want a huge stride.
5109  if (ST.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
5110  Rsrc23 &= ~AMDGPU::RSRC_DATA_FORMAT;
5111 
5112  return Rsrc23;
5113 }
5114 
5116  unsigned Opc = MI.getOpcode();
5117 
5118  return isSMRD(Opc);
5119 }
5120 
5122  unsigned Opc = MI.getOpcode();
5123 
5124  return isMUBUF(Opc) || isMTBUF(Opc) || isMIMG(Opc);
5125 }
5126 
5128  int &FrameIndex) const {
5129  const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::vaddr);
5130  if (!Addr || !Addr->isFI())
5131  return AMDGPU::NoRegister;
5132 
5133  assert(!MI.memoperands_empty() &&
5135 
5136  FrameIndex = Addr->getIndex();
5137  return getNamedOperand(MI, AMDGPU::OpName::vdata)->getReg();
5138 }
5139 
5141  int &FrameIndex) const {
5142  const MachineOperand *Addr = getNamedOperand(MI, AMDGPU::OpName::addr);
5143  assert(Addr && Addr->isFI());
5144  FrameIndex = Addr->getIndex();
5145  return getNamedOperand(MI, AMDGPU::OpName::data)->getReg();
5146 }
5147 
5149  int &FrameIndex) const {
5150  if (!MI.mayLoad())
5151  return AMDGPU::NoRegister;
5152 
5153  if (isMUBUF(MI) || isVGPRSpill(MI))
5154  return isStackAccess(MI, FrameIndex);
5155 
5156  if (isSGPRSpill(MI))
5157  return isSGPRStackAccess(MI, FrameIndex);
5158 
5159  return AMDGPU::NoRegister;
5160 }
5161 
5163  int &FrameIndex) const {
5164  if (!MI.mayStore())
5165  return AMDGPU::NoRegister;
5166 
5167  if (isMUBUF(MI) || isVGPRSpill(MI))
5168  return isStackAccess(MI, FrameIndex);
5169 
5170  if (isSGPRSpill(MI))
5171  return isSGPRStackAccess(MI, FrameIndex);
5172 
5173  return AMDGPU::NoRegister;
5174 }
5175 
5177  unsigned Size = 0;
5180  while (++I != E && I->isInsideBundle()) {
5181  assert(!I->isBundle() && "No nested bundle!");
5182  Size += getInstSizeInBytes(*I);
5183  }
5184 
5185  return Size;
5186 }
5187 
5189  unsigned Opc = MI.getOpcode();
5190  const MCInstrDesc &Desc = getMCOpcodeFromPseudo(Opc);
5191  unsigned DescSize = Desc.getSize();
5192 
5193  // If we have a definitive size, we can use it. Otherwise we need to inspect
5194  // the operands to know the size.
5195  if (isFixedSize(MI))
5196  return DescSize;
5197 
5198  // 4-byte instructions may have a 32-bit literal encoded after them. Check
5199  // operands that coud ever be literals.
5200  if (isVALU(MI) || isSALU(MI)) {
5201  int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
5202  if (Src0Idx == -1)
5203  return DescSize; // No operands.
5204 
5205  if (isLiteralConstantLike(MI.getOperand(Src0Idx), Desc.OpInfo[Src0Idx]))
5206  return DescSize + 4;
5207 
5208  int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
5209  if (Src1Idx == -1)
5210  return DescSize;
5211 
5212  if (isLiteralConstantLike(MI.getOperand(Src1Idx), Desc.OpInfo[Src1Idx]))
5213  return DescSize + 4;
5214 
5215  int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
5216  if (Src2Idx == -1)
5217  return DescSize;
5218 
5219  if (isLiteralConstantLike(MI.getOperand(Src2Idx), Desc.OpInfo[Src2Idx]))
5220  return DescSize + 4;
5221 
5222  return DescSize;
5223  }
5224 
5225  switch (Opc) {
5226  case TargetOpcode::IMPLICIT_DEF:
5227  case TargetOpcode::KILL:
5228  case TargetOpcode::DBG_VALUE:
5230  return 0;
5231  case TargetOpcode::BUNDLE:
5232  return getInstBundleSize(MI);
5233  case TargetOpcode::INLINEASM: {
5234  const MachineFunction *MF = MI.getParent()->getParent();
5235  const char *AsmStr = MI.getOperand(0).getSymbolName();
5236  return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
5237  }
5238  default:
5239  return DescSize;
5240  }
5241 }
5242 
5244  if (!isFLAT(MI))
5245  return false;
5246 
5247  if (MI.memoperands_empty())
5248  return true;
5249 
5250  for (const MachineMemOperand *MMO : MI.memoperands()) {
5251  if (MMO->getAddrSpace() == AMDGPUAS::FLAT_ADDRESS)
5252  return true;
5253  }
5254  return false;
5255 }
5256 
5258  return Branch.getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO;
5259 }
5260 
5262  MachineBasicBlock *IfEnd) const {
5264  assert(TI != IfEntry->end());
5265 
5266  MachineInstr *Branch = &(*TI);
5267  MachineFunction *MF = IfEntry->getParent();
5268  MachineRegisterInfo &MRI = IfEntry->getParent()->getRegInfo();
5269 
5270  if (Branch->getOpcode() == AMDGPU::SI_NON_UNIFORM_BRCOND_PSEUDO) {
5271  unsigned DstReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
5272  MachineInstr *SIIF =
5273  BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_IF), DstReg)
5274  .add(Branch->getOperand(0))
5275  .add(Branch->getOperand(1));
5276  MachineInstr *SIEND =
5277  BuildMI(*MF, Branch->getDebugLoc(), get(AMDGPU::SI_END_CF))
5278  .addReg(DstReg);
5279 
5280  IfEntry->erase(TI);
5281  IfEntry->insert(IfEntry->end(), SIIF);
5282  IfEnd->insert(IfEnd->getFirstNonPHI(), SIEND);
5283  }
5284 }
5285 
5287  MachineBasicBlock *LoopEntry, MachineBasicBlock *LoopEnd) const {
5289  // We expect 2 terminators, one conditional and one unconditional.
5290