LLVM  9.0.0svn
AMDGPUAsmParser.cpp
Go to the documentation of this file.
1 //===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "AMDGPU.h"
10 #include "AMDKernelCodeT.h"
13 #include "SIDefines.h"
14 #include "SIInstrInfo.h"
16 #include "Utils/AMDGPUAsmUtils.h"
17 #include "Utils/AMDGPUBaseInfo.h"
19 #include "llvm/ADT/APFloat.h"
20 #include "llvm/ADT/APInt.h"
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallString.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/ADT/StringSwitch.h"
27 #include "llvm/ADT/Twine.h"
28 #include "llvm/BinaryFormat/ELF.h"
29 #include "llvm/MC/MCAsmInfo.h"
30 #include "llvm/MC/MCContext.h"
31 #include "llvm/MC/MCExpr.h"
32 #include "llvm/MC/MCInst.h"
33 #include "llvm/MC/MCInstrDesc.h"
34 #include "llvm/MC/MCInstrInfo.h"
40 #include "llvm/MC/MCRegisterInfo.h"
41 #include "llvm/MC/MCStreamer.h"
43 #include "llvm/MC/MCSymbol.h"
46 #include "llvm/Support/Casting.h"
47 #include "llvm/Support/Compiler.h"
51 #include "llvm/Support/SMLoc.h"
55 #include <algorithm>
56 #include <cassert>
57 #include <cstdint>
58 #include <cstring>
59 #include <iterator>
60 #include <map>
61 #include <memory>
62 #include <string>
63 
64 using namespace llvm;
65 using namespace llvm::AMDGPU;
66 using namespace llvm::amdhsa;
67 
68 namespace {
69 
70 class AMDGPUAsmParser;
71 
72 enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_TTMP, IS_SPECIAL };
73 
74 //===----------------------------------------------------------------------===//
75 // Operand
76 //===----------------------------------------------------------------------===//
77 
78 class AMDGPUOperand : public MCParsedAsmOperand {
79  enum KindTy {
80  Token,
81  Immediate,
82  Register,
83  Expression
84  } Kind;
85 
86  SMLoc StartLoc, EndLoc;
87  const AMDGPUAsmParser *AsmParser;
88 
89 public:
90  AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
91  : MCParsedAsmOperand(), Kind(Kind_), AsmParser(AsmParser_) {}
92 
93  using Ptr = std::unique_ptr<AMDGPUOperand>;
94 
95  struct Modifiers {
96  bool Abs = false;
97  bool Neg = false;
98  bool Sext = false;
99 
100  bool hasFPModifiers() const { return Abs || Neg; }
101  bool hasIntModifiers() const { return Sext; }
102  bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
103 
104  int64_t getFPModifiersOperand() const {
105  int64_t Operand = 0;
106  Operand |= Abs ? SISrcMods::ABS : 0u;
107  Operand |= Neg ? SISrcMods::NEG : 0u;
108  return Operand;
109  }
110 
111  int64_t getIntModifiersOperand() const {
112  int64_t Operand = 0;
113  Operand |= Sext ? SISrcMods::SEXT : 0u;
114  return Operand;
115  }
116 
117  int64_t getModifiersOperand() const {
118  assert(!(hasFPModifiers() && hasIntModifiers())
119  && "fp and int modifiers should not be used simultaneously");
120  if (hasFPModifiers()) {
121  return getFPModifiersOperand();
122  } else if (hasIntModifiers()) {
123  return getIntModifiersOperand();
124  } else {
125  return 0;
126  }
127  }
128 
129  friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
130  };
131 
132  enum ImmTy {
133  ImmTyNone,
134  ImmTyGDS,
135  ImmTyLDS,
136  ImmTyOffen,
137  ImmTyIdxen,
138  ImmTyAddr64,
139  ImmTyOffset,
140  ImmTyInstOffset,
141  ImmTyOffset0,
142  ImmTyOffset1,
143  ImmTyDLC,
144  ImmTyGLC,
145  ImmTySLC,
146  ImmTyTFE,
147  ImmTyD16,
148  ImmTyClampSI,
149  ImmTyOModSI,
150  ImmTyDppCtrl,
151  ImmTyDppRowMask,
152  ImmTyDppBankMask,
153  ImmTyDppBoundCtrl,
154  ImmTySdwaDstSel,
155  ImmTySdwaSrc0Sel,
156  ImmTySdwaSrc1Sel,
157  ImmTySdwaDstUnused,
158  ImmTyDMask,
159  ImmTyDim,
160  ImmTyUNorm,
161  ImmTyDA,
162  ImmTyR128A16,
163  ImmTyLWE,
164  ImmTyExpTgt,
165  ImmTyExpCompr,
166  ImmTyExpVM,
167  ImmTyFORMAT,
168  ImmTyHwreg,
169  ImmTyOff,
170  ImmTySendMsg,
171  ImmTyInterpSlot,
172  ImmTyInterpAttr,
173  ImmTyAttrChan,
174  ImmTyOpSel,
175  ImmTyOpSelHi,
176  ImmTyNegLo,
177  ImmTyNegHi,
178  ImmTySwizzle,
179  ImmTyGprIdxMode,
180  ImmTyEndpgm,
181  ImmTyHigh
182  };
183 
184 private:
185  struct TokOp {
186  const char *Data;
187  unsigned Length;
188  };
189 
190  struct ImmOp {
191  int64_t Val;
192  ImmTy Type;
193  bool IsFPImm;
194  Modifiers Mods;
195  };
196 
197  struct RegOp {
198  unsigned RegNo;
199  Modifiers Mods;
200  };
201 
202  union {
203  TokOp Tok;
204  ImmOp Imm;
205  RegOp Reg;
206  const MCExpr *Expr;
207  };
208 
209 public:
210  bool isToken() const override {
211  if (Kind == Token)
212  return true;
213 
214  if (Kind != Expression || !Expr)
215  return false;
216 
217  // When parsing operands, we can't always tell if something was meant to be
218  // a token, like 'gds', or an expression that references a global variable.
219  // In this case, we assume the string is an expression, and if we need to
220  // interpret is a token, then we treat the symbol name as the token.
221  return isa<MCSymbolRefExpr>(Expr);
222  }
223 
224  bool isImm() const override {
225  return Kind == Immediate;
226  }
227 
228  bool isInlinableImm(MVT type) const;
229  bool isLiteralImm(MVT type) const;
230 
231  bool isRegKind() const {
232  return Kind == Register;
233  }
234 
235  bool isReg() const override {
236  return isRegKind() && !hasModifiers();
237  }
238 
239  bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const {
240  return isRegClass(RCID) || isInlinableImm(type) || isLiteralImm(type);
241  }
242 
243  bool isRegOrImmWithInt16InputMods() const {
244  return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i16);
245  }
246 
247  bool isRegOrImmWithInt32InputMods() const {
248  return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i32);
249  }
250 
251  bool isRegOrImmWithInt64InputMods() const {
252  return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::i64);
253  }
254 
255  bool isRegOrImmWithFP16InputMods() const {
256  return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f16);
257  }
258 
259  bool isRegOrImmWithFP32InputMods() const {
260  return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f32);
261  }
262 
263  bool isRegOrImmWithFP64InputMods() const {
264  return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::f64);
265  }
266 
267  bool isVReg() const {
268  return isRegClass(AMDGPU::VGPR_32RegClassID) ||
269  isRegClass(AMDGPU::VReg_64RegClassID) ||
270  isRegClass(AMDGPU::VReg_96RegClassID) ||
271  isRegClass(AMDGPU::VReg_128RegClassID) ||
272  isRegClass(AMDGPU::VReg_256RegClassID) ||
273  isRegClass(AMDGPU::VReg_512RegClassID);
274  }
275 
276  bool isVReg32() const {
277  return isRegClass(AMDGPU::VGPR_32RegClassID);
278  }
279 
280  bool isVReg32OrOff() const {
281  return isOff() || isVReg32();
282  }
283 
284  bool isSDWAOperand(MVT type) const;
285  bool isSDWAFP16Operand() const;
286  bool isSDWAFP32Operand() const;
287  bool isSDWAInt16Operand() const;
288  bool isSDWAInt32Operand() const;
289 
290  bool isImmTy(ImmTy ImmT) const {
291  return isImm() && Imm.Type == ImmT;
292  }
293 
294  bool isImmModifier() const {
295  return isImm() && Imm.Type != ImmTyNone;
296  }
297 
298  bool isClampSI() const { return isImmTy(ImmTyClampSI); }
299  bool isOModSI() const { return isImmTy(ImmTyOModSI); }
300  bool isDMask() const { return isImmTy(ImmTyDMask); }
301  bool isDim() const { return isImmTy(ImmTyDim); }
302  bool isUNorm() const { return isImmTy(ImmTyUNorm); }
303  bool isDA() const { return isImmTy(ImmTyDA); }
304  bool isR128A16() const { return isImmTy(ImmTyR128A16); }
305  bool isLWE() const { return isImmTy(ImmTyLWE); }
306  bool isOff() const { return isImmTy(ImmTyOff); }
307  bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
308  bool isExpVM() const { return isImmTy(ImmTyExpVM); }
309  bool isExpCompr() const { return isImmTy(ImmTyExpCompr); }
310  bool isOffen() const { return isImmTy(ImmTyOffen); }
311  bool isIdxen() const { return isImmTy(ImmTyIdxen); }
312  bool isAddr64() const { return isImmTy(ImmTyAddr64); }
313  bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
314  bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<8>(getImm()); }
315  bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
316 
317  bool isOffsetU12() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isUInt<12>(getImm()); }
318  bool isOffsetS13() const { return (isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset)) && isInt<13>(getImm()); }
319  bool isGDS() const { return isImmTy(ImmTyGDS); }
320  bool isLDS() const { return isImmTy(ImmTyLDS); }
321  bool isDLC() const { return isImmTy(ImmTyDLC); }
322  bool isGLC() const { return isImmTy(ImmTyGLC); }
323  bool isSLC() const { return isImmTy(ImmTySLC); }
324  bool isTFE() const { return isImmTy(ImmTyTFE); }
325  bool isD16() const { return isImmTy(ImmTyD16); }
326  bool isFORMAT() const { return isImmTy(ImmTyFORMAT) && isUInt<8>(getImm()); }
327  bool isBankMask() const { return isImmTy(ImmTyDppBankMask); }
328  bool isRowMask() const { return isImmTy(ImmTyDppRowMask); }
329  bool isBoundCtrl() const { return isImmTy(ImmTyDppBoundCtrl); }
330  bool isSDWADstSel() const { return isImmTy(ImmTySdwaDstSel); }
331  bool isSDWASrc0Sel() const { return isImmTy(ImmTySdwaSrc0Sel); }
332  bool isSDWASrc1Sel() const { return isImmTy(ImmTySdwaSrc1Sel); }
333  bool isSDWADstUnused() const { return isImmTy(ImmTySdwaDstUnused); }
334  bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
335  bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
336  bool isAttrChan() const { return isImmTy(ImmTyAttrChan); }
337  bool isOpSel() const { return isImmTy(ImmTyOpSel); }
338  bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
339  bool isNegLo() const { return isImmTy(ImmTyNegLo); }
340  bool isNegHi() const { return isImmTy(ImmTyNegHi); }
341  bool isHigh() const { return isImmTy(ImmTyHigh); }
342 
343  bool isMod() const {
344  return isClampSI() || isOModSI();
345  }
346 
347  bool isRegOrImm() const {
348  return isReg() || isImm();
349  }
350 
351  bool isRegClass(unsigned RCID) const;
352 
353  bool isInlineValue() const;
354 
355  bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
356  return (isRegClass(RCID) || isInlinableImm(type)) && !hasModifiers();
357  }
358 
359  bool isSCSrcB16() const {
360  return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
361  }
362 
363  bool isSCSrcV2B16() const {
364  return isSCSrcB16();
365  }
366 
367  bool isSCSrcB32() const {
368  return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
369  }
370 
371  bool isSCSrcB64() const {
372  return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
373  }
374 
375  bool isSCSrcF16() const {
376  return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
377  }
378 
379  bool isSCSrcV2F16() const {
380  return isSCSrcF16();
381  }
382 
383  bool isSCSrcF32() const {
384  return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
385  }
386 
387  bool isSCSrcF64() const {
388  return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
389  }
390 
391  bool isSSrcB32() const {
392  return isSCSrcB32() || isLiteralImm(MVT::i32) || isExpr();
393  }
394 
395  bool isSSrcB16() const {
396  return isSCSrcB16() || isLiteralImm(MVT::i16);
397  }
398 
399  bool isSSrcV2B16() const {
400  llvm_unreachable("cannot happen");
401  return isSSrcB16();
402  }
403 
404  bool isSSrcB64() const {
405  // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
406  // See isVSrc64().
407  return isSCSrcB64() || isLiteralImm(MVT::i64);
408  }
409 
410  bool isSSrcF32() const {
411  return isSCSrcB32() || isLiteralImm(MVT::f32) || isExpr();
412  }
413 
414  bool isSSrcF64() const {
415  return isSCSrcB64() || isLiteralImm(MVT::f64);
416  }
417 
418  bool isSSrcF16() const {
419  return isSCSrcB16() || isLiteralImm(MVT::f16);
420  }
421 
422  bool isSSrcV2F16() const {
423  llvm_unreachable("cannot happen");
424  return isSSrcF16();
425  }
426 
427  bool isSSrcOrLdsB32() const {
428  return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
429  isLiteralImm(MVT::i32) || isExpr();
430  }
431 
432  bool isVCSrcB32() const {
433  return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
434  }
435 
436  bool isVCSrcB64() const {
437  return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
438  }
439 
440  bool isVCSrcB16() const {
441  return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
442  }
443 
444  bool isVCSrcV2B16() const {
445  return isVCSrcB16();
446  }
447 
448  bool isVCSrcF32() const {
449  return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
450  }
451 
452  bool isVCSrcF64() const {
453  return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
454  }
455 
456  bool isVCSrcF16() const {
457  return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
458  }
459 
460  bool isVCSrcV2F16() const {
461  return isVCSrcF16();
462  }
463 
464  bool isVSrcB32() const {
465  return isVCSrcF32() || isLiteralImm(MVT::i32) || isExpr();
466  }
467 
468  bool isVSrcB64() const {
469  return isVCSrcF64() || isLiteralImm(MVT::i64);
470  }
471 
472  bool isVSrcB16() const {
473  return isVCSrcF16() || isLiteralImm(MVT::i16);
474  }
475 
476  bool isVSrcV2B16() const {
477  return isVSrcB16() || isLiteralImm(MVT::v2i16);
478  }
479 
480  bool isVSrcF32() const {
481  return isVCSrcF32() || isLiteralImm(MVT::f32) || isExpr();
482  }
483 
484  bool isVSrcF64() const {
485  return isVCSrcF64() || isLiteralImm(MVT::f64);
486  }
487 
488  bool isVSrcF16() const {
489  return isVCSrcF16() || isLiteralImm(MVT::f16);
490  }
491 
492  bool isVSrcV2F16() const {
493  return isVSrcF16() || isLiteralImm(MVT::v2f16);
494  }
495 
496  bool isKImmFP32() const {
497  return isLiteralImm(MVT::f32);
498  }
499 
500  bool isKImmFP16() const {
501  return isLiteralImm(MVT::f16);
502  }
503 
504  bool isMem() const override {
505  return false;
506  }
507 
508  bool isExpr() const {
509  return Kind == Expression;
510  }
511 
512  bool isSoppBrTarget() const {
513  return isExpr() || isImm();
514  }
515 
516  bool isSWaitCnt() const;
517  bool isHwreg() const;
518  bool isSendMsg() const;
519  bool isSwizzle() const;
520  bool isSMRDOffset8() const;
521  bool isSMRDOffset20() const;
522  bool isSMRDLiteralOffset() const;
523  bool isDPPCtrl() const;
524  bool isGPRIdxMode() const;
525  bool isS16Imm() const;
526  bool isU16Imm() const;
527  bool isEndpgm() const;
528 
529  StringRef getExpressionAsToken() const {
530  assert(isExpr());
531  const MCSymbolRefExpr *S = cast<MCSymbolRefExpr>(Expr);
532  return S->getSymbol().getName();
533  }
534 
535  StringRef getToken() const {
536  assert(isToken());
537 
538  if (Kind == Expression)
539  return getExpressionAsToken();
540 
541  return StringRef(Tok.Data, Tok.Length);
542  }
543 
544  int64_t getImm() const {
545  assert(isImm());
546  return Imm.Val;
547  }
548 
549  ImmTy getImmTy() const {
550  assert(isImm());
551  return Imm.Type;
552  }
553 
554  unsigned getReg() const override {
555  assert(isRegKind());
556  return Reg.RegNo;
557  }
558 
559  SMLoc getStartLoc() const override {
560  return StartLoc;
561  }
562 
563  SMLoc getEndLoc() const override {
564  return EndLoc;
565  }
566 
567  SMRange getLocRange() const {
568  return SMRange(StartLoc, EndLoc);
569  }
570 
571  Modifiers getModifiers() const {
572  assert(isRegKind() || isImmTy(ImmTyNone));
573  return isRegKind() ? Reg.Mods : Imm.Mods;
574  }
575 
576  void setModifiers(Modifiers Mods) {
577  assert(isRegKind() || isImmTy(ImmTyNone));
578  if (isRegKind())
579  Reg.Mods = Mods;
580  else
581  Imm.Mods = Mods;
582  }
583 
584  bool hasModifiers() const {
585  return getModifiers().hasModifiers();
586  }
587 
588  bool hasFPModifiers() const {
589  return getModifiers().hasFPModifiers();
590  }
591 
592  bool hasIntModifiers() const {
593  return getModifiers().hasIntModifiers();
594  }
595 
596  uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
597 
598  void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
599 
600  void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
601 
602  template <unsigned Bitwidth>
603  void addKImmFPOperands(MCInst &Inst, unsigned N) const;
604 
605  void addKImmFP16Operands(MCInst &Inst, unsigned N) const {
606  addKImmFPOperands<16>(Inst, N);
607  }
608 
609  void addKImmFP32Operands(MCInst &Inst, unsigned N) const {
610  addKImmFPOperands<32>(Inst, N);
611  }
612 
613  void addRegOperands(MCInst &Inst, unsigned N) const;
614 
615  void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
616  if (isRegKind())
617  addRegOperands(Inst, N);
618  else if (isExpr())
619  Inst.addOperand(MCOperand::createExpr(Expr));
620  else
621  addImmOperands(Inst, N);
622  }
623 
624  void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
625  Modifiers Mods = getModifiers();
626  Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
627  if (isRegKind()) {
628  addRegOperands(Inst, N);
629  } else {
630  addImmOperands(Inst, N, false);
631  }
632  }
633 
634  void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
635  assert(!hasIntModifiers());
636  addRegOrImmWithInputModsOperands(Inst, N);
637  }
638 
639  void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
640  assert(!hasFPModifiers());
641  addRegOrImmWithInputModsOperands(Inst, N);
642  }
643 
644  void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
645  Modifiers Mods = getModifiers();
646  Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
647  assert(isRegKind());
648  addRegOperands(Inst, N);
649  }
650 
651  void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
652  assert(!hasIntModifiers());
653  addRegWithInputModsOperands(Inst, N);
654  }
655 
656  void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
657  assert(!hasFPModifiers());
658  addRegWithInputModsOperands(Inst, N);
659  }
660 
661  void addSoppBrTargetOperands(MCInst &Inst, unsigned N) const {
662  if (isImm())
663  addImmOperands(Inst, N);
664  else {
665  assert(isExpr());
666  Inst.addOperand(MCOperand::createExpr(Expr));
667  }
668  }
669 
670  static void printImmTy(raw_ostream& OS, ImmTy Type) {
671  switch (Type) {
672  case ImmTyNone: OS << "None"; break;
673  case ImmTyGDS: OS << "GDS"; break;
674  case ImmTyLDS: OS << "LDS"; break;
675  case ImmTyOffen: OS << "Offen"; break;
676  case ImmTyIdxen: OS << "Idxen"; break;
677  case ImmTyAddr64: OS << "Addr64"; break;
678  case ImmTyOffset: OS << "Offset"; break;
679  case ImmTyInstOffset: OS << "InstOffset"; break;
680  case ImmTyOffset0: OS << "Offset0"; break;
681  case ImmTyOffset1: OS << "Offset1"; break;
682  case ImmTyDLC: OS << "DLC"; break;
683  case ImmTyGLC: OS << "GLC"; break;
684  case ImmTySLC: OS << "SLC"; break;
685  case ImmTyTFE: OS << "TFE"; break;
686  case ImmTyD16: OS << "D16"; break;
687  case ImmTyFORMAT: OS << "FORMAT"; break;
688  case ImmTyClampSI: OS << "ClampSI"; break;
689  case ImmTyOModSI: OS << "OModSI"; break;
690  case ImmTyDppCtrl: OS << "DppCtrl"; break;
691  case ImmTyDppRowMask: OS << "DppRowMask"; break;
692  case ImmTyDppBankMask: OS << "DppBankMask"; break;
693  case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
694  case ImmTySdwaDstSel: OS << "SdwaDstSel"; break;
695  case ImmTySdwaSrc0Sel: OS << "SdwaSrc0Sel"; break;
696  case ImmTySdwaSrc1Sel: OS << "SdwaSrc1Sel"; break;
697  case ImmTySdwaDstUnused: OS << "SdwaDstUnused"; break;
698  case ImmTyDMask: OS << "DMask"; break;
699  case ImmTyDim: OS << "Dim"; break;
700  case ImmTyUNorm: OS << "UNorm"; break;
701  case ImmTyDA: OS << "DA"; break;
702  case ImmTyR128A16: OS << "R128A16"; break;
703  case ImmTyLWE: OS << "LWE"; break;
704  case ImmTyOff: OS << "Off"; break;
705  case ImmTyExpTgt: OS << "ExpTgt"; break;
706  case ImmTyExpCompr: OS << "ExpCompr"; break;
707  case ImmTyExpVM: OS << "ExpVM"; break;
708  case ImmTyHwreg: OS << "Hwreg"; break;
709  case ImmTySendMsg: OS << "SendMsg"; break;
710  case ImmTyInterpSlot: OS << "InterpSlot"; break;
711  case ImmTyInterpAttr: OS << "InterpAttr"; break;
712  case ImmTyAttrChan: OS << "AttrChan"; break;
713  case ImmTyOpSel: OS << "OpSel"; break;
714  case ImmTyOpSelHi: OS << "OpSelHi"; break;
715  case ImmTyNegLo: OS << "NegLo"; break;
716  case ImmTyNegHi: OS << "NegHi"; break;
717  case ImmTySwizzle: OS << "Swizzle"; break;
718  case ImmTyGprIdxMode: OS << "GprIdxMode"; break;
719  case ImmTyHigh: OS << "High"; break;
720  case ImmTyEndpgm:
721  OS << "Endpgm";
722  break;
723  }
724  }
725 
726  void print(raw_ostream &OS) const override {
727  switch (Kind) {
728  case Register:
729  OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
730  break;
731  case Immediate:
732  OS << '<' << getImm();
733  if (getImmTy() != ImmTyNone) {
734  OS << " type: "; printImmTy(OS, getImmTy());
735  }
736  OS << " mods: " << Imm.Mods << '>';
737  break;
738  case Token:
739  OS << '\'' << getToken() << '\'';
740  break;
741  case Expression:
742  OS << "<expr " << *Expr << '>';
743  break;
744  }
745  }
746 
747  static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
748  int64_t Val, SMLoc Loc,
749  ImmTy Type = ImmTyNone,
750  bool IsFPImm = false) {
751  auto Op = llvm::make_unique<AMDGPUOperand>(Immediate, AsmParser);
752  Op->Imm.Val = Val;
753  Op->Imm.IsFPImm = IsFPImm;
754  Op->Imm.Type = Type;
755  Op->Imm.Mods = Modifiers();
756  Op->StartLoc = Loc;
757  Op->EndLoc = Loc;
758  return Op;
759  }
760 
761  static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
762  StringRef Str, SMLoc Loc,
763  bool HasExplicitEncodingSize = true) {
764  auto Res = llvm::make_unique<AMDGPUOperand>(Token, AsmParser);
765  Res->Tok.Data = Str.data();
766  Res->Tok.Length = Str.size();
767  Res->StartLoc = Loc;
768  Res->EndLoc = Loc;
769  return Res;
770  }
771 
772  static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
773  unsigned RegNo, SMLoc S,
774  SMLoc E) {
775  auto Op = llvm::make_unique<AMDGPUOperand>(Register, AsmParser);
776  Op->Reg.RegNo = RegNo;
777  Op->Reg.Mods = Modifiers();
778  Op->StartLoc = S;
779  Op->EndLoc = E;
780  return Op;
781  }
782 
783  static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
784  const class MCExpr *Expr, SMLoc S) {
785  auto Op = llvm::make_unique<AMDGPUOperand>(Expression, AsmParser);
786  Op->Expr = Expr;
787  Op->StartLoc = S;
788  Op->EndLoc = S;
789  return Op;
790  }
791 };
792 
793 raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
794  OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
795  return OS;
796 }
797 
798 //===----------------------------------------------------------------------===//
799 // AsmParser
800 //===----------------------------------------------------------------------===//
801 
802 // Holds info related to the current kernel, e.g. count of SGPRs used.
803 // Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
804 // .amdgpu_hsa_kernel or at EOF.
805 class KernelScopeInfo {
806  int SgprIndexUnusedMin = -1;
807  int VgprIndexUnusedMin = -1;
808  MCContext *Ctx = nullptr;
809 
810  void usesSgprAt(int i) {
811  if (i >= SgprIndexUnusedMin) {
812  SgprIndexUnusedMin = ++i;
813  if (Ctx) {
814  MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
815  Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
816  }
817  }
818  }
819 
820  void usesVgprAt(int i) {
821  if (i >= VgprIndexUnusedMin) {
822  VgprIndexUnusedMin = ++i;
823  if (Ctx) {
824  MCSymbol * const Sym = Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
825  Sym->setVariableValue(MCConstantExpr::create(VgprIndexUnusedMin, *Ctx));
826  }
827  }
828  }
829 
830 public:
831  KernelScopeInfo() = default;
832 
833  void initialize(MCContext &Context) {
834  Ctx = &Context;
835  usesSgprAt(SgprIndexUnusedMin = -1);
836  usesVgprAt(VgprIndexUnusedMin = -1);
837  }
838 
839  void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex, unsigned RegWidth) {
840  switch (RegKind) {
841  case IS_SGPR: usesSgprAt(DwordRegIndex + RegWidth - 1); break;
842  case IS_VGPR: usesVgprAt(DwordRegIndex + RegWidth - 1); break;
843  default: break;
844  }
845  }
846 };
847 
848 class AMDGPUAsmParser : public MCTargetAsmParser {
849  MCAsmParser &Parser;
850 
851  // Number of extra operands parsed after the first optional operand.
852  // This may be necessary to skip hardcoded mandatory operands.
853  static const unsigned MAX_OPR_LOOKAHEAD = 8;
854 
855  unsigned ForcedEncodingSize = 0;
856  bool ForcedDPP = false;
857  bool ForcedSDWA = false;
858  KernelScopeInfo KernelScope;
859 
860  /// @name Auto-generated Match Functions
861  /// {
862 
863 #define GET_ASSEMBLER_HEADER
864 #include "AMDGPUGenAsmMatcher.inc"
865 
866  /// }
867 
868 private:
869  bool ParseAsAbsoluteExpression(uint32_t &Ret);
870  bool OutOfRangeError(SMRange Range);
871  /// Calculate VGPR/SGPR blocks required for given target, reserved
872  /// registers, and user-specified NextFreeXGPR values.
873  ///
874  /// \param Features [in] Target features, used for bug corrections.
875  /// \param VCCUsed [in] Whether VCC special SGPR is reserved.
876  /// \param FlatScrUsed [in] Whether FLAT_SCRATCH special SGPR is reserved.
877  /// \param XNACKUsed [in] Whether XNACK_MASK special SGPR is reserved.
878  /// \param NextFreeVGPR [in] Max VGPR number referenced, plus one.
879  /// \param VGPRRange [in] Token range, used for VGPR diagnostics.
880  /// \param NextFreeSGPR [in] Max SGPR number referenced, plus one.
881  /// \param SGPRRange [in] Token range, used for SGPR diagnostics.
882  /// \param VGPRBlocks [out] Result VGPR block count.
883  /// \param SGPRBlocks [out] Result SGPR block count.
884  bool calculateGPRBlocks(const FeatureBitset &Features, bool VCCUsed,
885  bool FlatScrUsed, bool XNACKUsed,
886  unsigned NextFreeVGPR, SMRange VGPRRange,
887  unsigned NextFreeSGPR, SMRange SGPRRange,
888  unsigned &VGPRBlocks, unsigned &SGPRBlocks);
889  bool ParseDirectiveAMDGCNTarget();
890  bool ParseDirectiveAMDHSAKernel();
891  bool ParseDirectiveMajorMinor(uint32_t &Major, uint32_t &Minor);
892  bool ParseDirectiveHSACodeObjectVersion();
893  bool ParseDirectiveHSACodeObjectISA();
894  bool ParseAMDKernelCodeTValue(StringRef ID, amd_kernel_code_t &Header);
895  bool ParseDirectiveAMDKernelCodeT();
896  bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo) const;
897  bool ParseDirectiveAMDGPUHsaKernel();
898 
899  bool ParseDirectiveISAVersion();
900  bool ParseDirectiveHSAMetadata();
901  bool ParseDirectivePALMetadataBegin();
902  bool ParseDirectivePALMetadata();
903 
904  /// Common code to parse out a block of text (typically YAML) between start and
905  /// end directives.
906  bool ParseToEndDirective(const char *AssemblerDirectiveBegin,
907  const char *AssemblerDirectiveEnd,
908  std::string &CollectString);
909 
910  bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
911  RegisterKind RegKind, unsigned Reg1,
912  unsigned RegNum);
913  bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
914  unsigned& RegNum, unsigned& RegWidth,
915  unsigned *DwordRegIndex);
916  bool isRegister();
917  bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const;
918  Optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
919  void initializeGprCountSymbol(RegisterKind RegKind);
920  bool updateGprCountSymbols(RegisterKind RegKind, unsigned DwordRegIndex,
921  unsigned RegWidth);
922  void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
923  bool IsAtomic, bool IsAtomicReturn, bool IsLds = false);
924  void cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
925  bool IsGdsHardcoded);
926 
927 public:
928  enum AMDGPUMatchResultTy {
929  Match_PreferE32 = FIRST_TARGET_MATCH_RESULT_TY
930  };
931  enum OperandMode {
932  OperandMode_Default,
933  OperandMode_NSA,
934  };
935 
936  using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
937 
938  AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
939  const MCInstrInfo &MII,
940  const MCTargetOptions &Options)
941  : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
942  MCAsmParserExtension::Initialize(Parser);
943 
944  if (getFeatureBits().none()) {
945  // Set default features.
946  copySTI().ToggleFeature("southern-islands");
947  }
948 
949  setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
950 
951  {
952  // TODO: make those pre-defined variables read-only.
953  // Currently there is none suitable machinery in the core llvm-mc for this.
954  // MCSymbol::isRedefinable is intended for another purpose, and
955  // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
956  AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
957  MCContext &Ctx = getContext();
958  if (ISA.Major >= 6 && AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())) {
959  MCSymbol *Sym =
960  Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_number"));
961  Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
962  Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_minor"));
963  Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
964  Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_stepping"));
965  Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
966  } else {
967  MCSymbol *Sym =
968  Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
969  Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
970  Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
971  Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
972  Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
973  Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
974  }
975  if (ISA.Major >= 6 && AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())) {
976  initializeGprCountSymbol(IS_VGPR);
977  initializeGprCountSymbol(IS_SGPR);
978  } else
979  KernelScope.initialize(getContext());
980  }
981  }
982 
983  bool hasXNACK() const {
984  return AMDGPU::hasXNACK(getSTI());
985  }
986 
987  bool hasMIMG_R128() const {
988  return AMDGPU::hasMIMG_R128(getSTI());
989  }
990 
991  bool hasPackedD16() const {
992  return AMDGPU::hasPackedD16(getSTI());
993  }
994 
995  bool isSI() const {
996  return AMDGPU::isSI(getSTI());
997  }
998 
999  bool isCI() const {
1000  return AMDGPU::isCI(getSTI());
1001  }
1002 
1003  bool isVI() const {
1004  return AMDGPU::isVI(getSTI());
1005  }
1006 
1007  bool isGFX9() const {
1008  return AMDGPU::isGFX9(getSTI());
1009  }
1010 
1011  bool isGFX10() const {
1012  return AMDGPU::isGFX10(getSTI());
1013  }
1014 
1015  bool hasInv2PiInlineImm() const {
1016  return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1017  }
1018 
1019  bool hasFlatOffsets() const {
1020  return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1021  }
1022 
1023  bool hasSGPR102_SGPR103() const {
1024  return !isVI() && !isGFX9();
1025  }
1026 
1027  bool hasSGPR104_SGPR105() const {
1028  return isGFX10();
1029  }
1030 
1031  bool hasIntClamp() const {
1032  return getFeatureBits()[AMDGPU::FeatureIntClamp];
1033  }
1034 
1035  AMDGPUTargetStreamer &getTargetStreamer() {
1036  MCTargetStreamer &TS = *getParser().getStreamer().getTargetStreamer();
1037  return static_cast<AMDGPUTargetStreamer &>(TS);
1038  }
1039 
1040  const MCRegisterInfo *getMRI() const {
1041  // We need this const_cast because for some reason getContext() is not const
1042  // in MCAsmParser.
1043  return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
1044  }
1045 
1046  const MCInstrInfo *getMII() const {
1047  return &MII;
1048  }
1049 
1050  const FeatureBitset &getFeatureBits() const {
1051  return getSTI().getFeatureBits();
1052  }
1053 
1054  void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
1055  void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1056  void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1057 
1058  unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
1059  bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
1060  bool isForcedDPP() const { return ForcedDPP; }
1061  bool isForcedSDWA() const { return ForcedSDWA; }
1062  ArrayRef<unsigned> getMatchedVariants() const;
1063 
1064  std::unique_ptr<AMDGPUOperand> parseRegister();
1065  bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) override;
1066  unsigned checkTargetMatchPredicate(MCInst &Inst) override;
1067  unsigned validateTargetOperandClass(MCParsedAsmOperand &Op,
1068  unsigned Kind) override;
1069  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1070  OperandVector &Operands, MCStreamer &Out,
1071  uint64_t &ErrorInfo,
1072  bool MatchingInlineAsm) override;
1073  bool ParseDirective(AsmToken DirectiveID) override;
1074  OperandMatchResultTy parseOperand(OperandVector &Operands, StringRef Mnemonic,
1075  OperandMode Mode = OperandMode_Default);
1076  StringRef parseMnemonicSuffix(StringRef Name);
1077  bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
1078  SMLoc NameLoc, OperandVector &Operands) override;
1079  //bool ProcessInstruction(MCInst &Inst);
1080 
1081  OperandMatchResultTy parseIntWithPrefix(const char *Prefix, int64_t &Int);
1082 
1083  OperandMatchResultTy
1084  parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1085  AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1086  bool (*ConvertResult)(int64_t &) = nullptr);
1087 
1088  OperandMatchResultTy parseOperandArrayWithPrefix(
1089  const char *Prefix,
1090  OperandVector &Operands,
1091  AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1092  bool (*ConvertResult)(int64_t&) = nullptr);
1093 
1094  OperandMatchResultTy
1095  parseNamedBit(const char *Name, OperandVector &Operands,
1096  AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1097  OperandMatchResultTy parseStringWithPrefix(StringRef Prefix,
1098  StringRef &Value);
1099 
1100  bool isModifier();
1101  bool isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1102  bool isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1103  bool isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1104  bool isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const;
1105  bool parseSP3NegModifier();
1106  OperandMatchResultTy parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false);
1107  OperandMatchResultTy parseReg(OperandVector &Operands);
1108  OperandMatchResultTy parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false);
1109  OperandMatchResultTy parseRegOrImmWithFPInputMods(OperandVector &Operands, bool AllowImm = true);
1110  OperandMatchResultTy parseRegOrImmWithIntInputMods(OperandVector &Operands, bool AllowImm = true);
1111  OperandMatchResultTy parseRegWithFPInputMods(OperandVector &Operands);
1112  OperandMatchResultTy parseRegWithIntInputMods(OperandVector &Operands);
1113  OperandMatchResultTy parseVReg32OrOff(OperandVector &Operands);
1114  OperandMatchResultTy parseDfmtNfmt(OperandVector &Operands);
1115 
1116  void cvtDSOffset01(MCInst &Inst, const OperandVector &Operands);
1117  void cvtDS(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, false); }
1118  void cvtDSGds(MCInst &Inst, const OperandVector &Operands) { cvtDSImpl(Inst, Operands, true); }
1119  void cvtExp(MCInst &Inst, const OperandVector &Operands);
1120 
1121  bool parseCnt(int64_t &IntVal);
1122  OperandMatchResultTy parseSWaitCntOps(OperandVector &Operands);
1123  OperandMatchResultTy parseHwreg(OperandVector &Operands);
1124 
1125 private:
1126  struct OperandInfoTy {
1127  int64_t Id;
1128  bool IsSymbolic = false;
1129 
1130  OperandInfoTy(int64_t Id_) : Id(Id_) {}
1131  };
1132 
1133  bool parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId);
1134  bool parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset, int64_t &Width);
1135 
1136  void errorExpTgt();
1137  OperandMatchResultTy parseExpTgtImpl(StringRef Str, uint8_t &Val);
1138 
1139  bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc);
1140  bool validateSOPLiteral(const MCInst &Inst) const;
1141  bool validateConstantBusLimitations(const MCInst &Inst);
1142  bool validateEarlyClobberLimitations(const MCInst &Inst);
1143  bool validateIntClampSupported(const MCInst &Inst);
1144  bool validateMIMGAtomicDMask(const MCInst &Inst);
1145  bool validateMIMGGatherDMask(const MCInst &Inst);
1146  bool validateMIMGDataSize(const MCInst &Inst);
1147  bool validateMIMGAddrSize(const MCInst &Inst);
1148  bool validateMIMGD16(const MCInst &Inst);
1149  bool validateMIMGDim(const MCInst &Inst);
1150  bool validateLdsDirect(const MCInst &Inst);
1151  bool validateVOP3Literal(const MCInst &Inst) const;
1152  bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1153  bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1154  unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1155 
1156  bool isId(const StringRef Id) const;
1157  bool isId(const AsmToken &Token, const StringRef Id) const;
1158  bool isToken(const AsmToken::TokenKind Kind) const;
1159  bool trySkipId(const StringRef Id);
1160  bool trySkipId(const StringRef Id, const AsmToken::TokenKind Kind);
1161  bool trySkipToken(const AsmToken::TokenKind Kind);
1162  bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1163  bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1164  void peekTokens(MutableArrayRef<AsmToken> Tokens);
1165  AsmToken::TokenKind getTokenKind() const;
1166  bool parseExpr(int64_t &Imm);
1167  StringRef getTokenStr() const;
1168  AsmToken peekToken();
1169  AsmToken getToken() const;
1170  SMLoc getLoc() const;
1171  void lex();
1172 
1173 public:
1174  OperandMatchResultTy parseOptionalOperand(OperandVector &Operands);
1175  OperandMatchResultTy parseOptionalOpr(OperandVector &Operands);
1176 
1177  OperandMatchResultTy parseExpTgt(OperandVector &Operands);
1178  OperandMatchResultTy parseSendMsgOp(OperandVector &Operands);
1179  OperandMatchResultTy parseInterpSlot(OperandVector &Operands);
1180  OperandMatchResultTy parseInterpAttr(OperandVector &Operands);
1181  OperandMatchResultTy parseSOppBrTarget(OperandVector &Operands);
1182 
1183  bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1184  const unsigned MinVal,
1185  const unsigned MaxVal,
1186  const StringRef ErrMsg);
1187  OperandMatchResultTy parseSwizzleOp(OperandVector &Operands);
1188  bool parseSwizzleOffset(int64_t &Imm);
1189  bool parseSwizzleMacro(int64_t &Imm);
1190  bool parseSwizzleQuadPerm(int64_t &Imm);
1191  bool parseSwizzleBitmaskPerm(int64_t &Imm);
1192  bool parseSwizzleBroadcast(int64_t &Imm);
1193  bool parseSwizzleSwap(int64_t &Imm);
1194  bool parseSwizzleReverse(int64_t &Imm);
1195 
1196  OperandMatchResultTy parseGPRIdxMode(OperandVector &Operands);
1197  int64_t parseGPRIdxMacro();
1198 
1199  void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false); }
1200  void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, false); }
1201  void cvtMubufAtomicReturn(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true, true); }
1202  void cvtMubufLds(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false, false, true); }
1203  void cvtMtbuf(MCInst &Inst, const OperandVector &Operands);
1204 
1205  AMDGPUOperand::Ptr defaultDLC() const;
1206  AMDGPUOperand::Ptr defaultGLC() const;
1207  AMDGPUOperand::Ptr defaultSLC() const;
1208 
1209  AMDGPUOperand::Ptr defaultSMRDOffset8() const;
1210  AMDGPUOperand::Ptr defaultSMRDOffset20() const;
1211  AMDGPUOperand::Ptr defaultSMRDLiteralOffset() const;
1212  AMDGPUOperand::Ptr defaultOffsetU12() const;
1213  AMDGPUOperand::Ptr defaultOffsetS13() const;
1214 
1215  OperandMatchResultTy parseOModOperand(OperandVector &Operands);
1216 
1217  void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1218  OptionalImmIndexMap &OptionalIdx);
1219  void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1220  void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1221  void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1222 
1223  void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1224 
1225  void cvtMIMG(MCInst &Inst, const OperandVector &Operands,
1226  bool IsAtomic = false);
1227  void cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands);
1228 
1229  OperandMatchResultTy parseDim(OperandVector &Operands);
1230  OperandMatchResultTy parseDPPCtrl(OperandVector &Operands);
1231  AMDGPUOperand::Ptr defaultRowMask() const;
1232  AMDGPUOperand::Ptr defaultBankMask() const;
1233  AMDGPUOperand::Ptr defaultBoundCtrl() const;
1234  void cvtDPP(MCInst &Inst, const OperandVector &Operands);
1235 
1236  OperandMatchResultTy parseSDWASel(OperandVector &Operands, StringRef Prefix,
1237  AMDGPUOperand::ImmTy Type);
1238  OperandMatchResultTy parseSDWADstUnused(OperandVector &Operands);
1239  void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1240  void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1241  void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1242  void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1243  void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1244  uint64_t BasicInstType, bool skipVcc = false);
1245 
1246  OperandMatchResultTy parseEndpgmOp(OperandVector &Operands);
1247  AMDGPUOperand::Ptr defaultEndpgmImmOperands() const;
1248 };
1249 
1250 struct OptionalOperand {
1251  const char *Name;
1252  AMDGPUOperand::ImmTy Type;
1253  bool IsBit;
1254  bool (*ConvertResult)(int64_t&);
1255 };
1256 
1257 } // end anonymous namespace
1258 
1259 // May be called with integer type with equivalent bitwidth.
1260 static const fltSemantics *getFltSemantics(unsigned Size) {
1261  switch (Size) {
1262  case 4:
1263  return &APFloat::IEEEsingle();
1264  case 8:
1265  return &APFloat::IEEEdouble();
1266  case 2:
1267  return &APFloat::IEEEhalf();
1268  default:
1269  llvm_unreachable("unsupported fp type");
1270  }
1271 }
1272 
1273 static const fltSemantics *getFltSemantics(MVT VT) {
1274  return getFltSemantics(VT.getSizeInBits() / 8);
1275 }
1276 
1277 static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
1278  switch (OperandType) {
1279  case AMDGPU::OPERAND_REG_IMM_INT32:
1280  case AMDGPU::OPERAND_REG_IMM_FP32:
1281  case AMDGPU::OPERAND_REG_INLINE_C_INT32:
1282  case AMDGPU::OPERAND_REG_INLINE_C_FP32:
1283  return &APFloat::IEEEsingle();
1284  case AMDGPU::OPERAND_REG_IMM_INT64:
1285  case AMDGPU::OPERAND_REG_IMM_FP64:
1286  case AMDGPU::OPERAND_REG_INLINE_C_INT64:
1287  case AMDGPU::OPERAND_REG_INLINE_C_FP64:
1288  return &APFloat::IEEEdouble();
1289  case AMDGPU::OPERAND_REG_IMM_INT16:
1290  case AMDGPU::OPERAND_REG_IMM_FP16:
1291  case AMDGPU::OPERAND_REG_INLINE_C_INT16:
1292  case AMDGPU::OPERAND_REG_INLINE_C_FP16:
1293  case AMDGPU::OPERAND_REG_INLINE_C_V2INT16:
1294  case AMDGPU::OPERAND_REG_INLINE_C_V2FP16:
1295  case AMDGPU::OPERAND_REG_IMM_V2INT16:
1296  case AMDGPU::OPERAND_REG_IMM_V2FP16:
1297  return &APFloat::IEEEhalf();
1298  default:
1299  llvm_unreachable("unsupported fp type");
1300  }
1301 }
1302 
1303 //===----------------------------------------------------------------------===//
1304 // Operand
1305 //===----------------------------------------------------------------------===//
1306 
1307 static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1308  bool Lost;
1309 
1310  // Convert literal to single precision
1311  APFloat::opStatus Status = FPLiteral.convert(*getFltSemantics(VT),
1312  APFloat::rmNearestTiesToEven,
1313  &Lost);
1314  // We allow precision lost but not overflow or underflow
1315  if (Status != APFloat::opOK &&
1316  Lost &&
1317  ((Status & APFloat::opOverflow) != 0 ||
1318  (Status & APFloat::opUnderflow) != 0)) {
1319  return false;
1320  }
1321 
1322  return true;
1323 }
1324 
1325 static bool isSafeTruncation(int64_t Val, unsigned Size) {
1326  return isUIntN(Size, Val) || isIntN(Size, Val);
1327 }
1328 
1329 bool AMDGPUOperand::isInlinableImm(MVT type) const {
1330 
1331  // This is a hack to enable named inline values like
1332  // shared_base with both 32-bit and 64-bit operands.
1333  // Note that these values are defined as
1334  // 32-bit operands only.
1335  if (isInlineValue()) {
1336  return true;
1337  }
1338 
1339  if (!isImmTy(ImmTyNone)) {
1340  // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
1341  return false;
1342  }
1343  // TODO: We should avoid using host float here. It would be better to
1344  // check the float bit values which is what a few other places do.
1345  // We've had bot failures before due to weird NaN support on mips hosts.
1346 
1347  APInt Literal(64, Imm.Val);
1348 
1349  if (Imm.IsFPImm) { // We got fp literal token
1350  if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1351  return AMDGPU::isInlinableLiteral64(Imm.Val,
1352  AsmParser->hasInv2PiInlineImm());
1353  }
1354 
1355  APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1356  if (!canLosslesslyConvertToFPType(FPLiteral, type))
1357  return false;
1358 
1359  if (type.getScalarSizeInBits() == 16) {
1361  static_cast<int16_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1362  AsmParser->hasInv2PiInlineImm());
1363  }
1364 
1365  // Check if single precision literal is inlinable
1367  static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
1368  AsmParser->hasInv2PiInlineImm());
1369  }
1370 
1371  // We got int literal token.
1372  if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
1373  return AMDGPU::isInlinableLiteral64(Imm.Val,
1374  AsmParser->hasInv2PiInlineImm());
1375  }
1376 
1377  if (!isSafeTruncation(Imm.Val, type.getScalarSizeInBits())) {
1378  return false;
1379  }
1380 
1381  if (type.getScalarSizeInBits() == 16) {
1383  static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
1384  AsmParser->hasInv2PiInlineImm());
1385  }
1386 
1388  static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
1389  AsmParser->hasInv2PiInlineImm());
1390 }
1391 
1392 bool AMDGPUOperand::isLiteralImm(MVT type) const {
1393  // Check that this immediate can be added as literal
1394  if (!isImmTy(ImmTyNone)) {
1395  return false;
1396  }
1397 
1398  if (!Imm.IsFPImm) {
1399  // We got int literal token.
1400 
1401  if (type == MVT::f64 && hasFPModifiers()) {
1402  // Cannot apply fp modifiers to int literals preserving the same semantics
1403  // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
1404  // disable these cases.
1405  return false;
1406  }
1407 
1408  unsigned Size = type.getSizeInBits();
1409  if (Size == 64)
1410  Size = 32;
1411 
1412  // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
1413  // types.
1414  return isSafeTruncation(Imm.Val, Size);
1415  }
1416 
1417  // We got fp literal token
1418  if (type == MVT::f64) { // Expected 64-bit fp operand
1419  // We would set low 64-bits of literal to zeroes but we accept this literals
1420  return true;
1421  }
1422 
1423  if (type == MVT::i64) { // Expected 64-bit int operand
1424  // We don't allow fp literals in 64-bit integer instructions. It is
1425  // unclear how we should encode them.
1426  return false;
1427  }
1428 
1429  // We allow fp literals with f16x2 operands assuming that the specified
1430  // literal goes into the lower half and the upper half is zero. We also
1431  // require that the literal may be losslesly converted to f16.
1432  MVT ExpectedType = (type == MVT::v2f16)? MVT::f16 :
1433  (type == MVT::v2i16)? MVT::i16 : type;
1434 
1435  APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
1436  return canLosslesslyConvertToFPType(FPLiteral, ExpectedType);
1437 }
1438 
1439 bool AMDGPUOperand::isRegClass(unsigned RCID) const {
1440  return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
1441 }
1442 
1443 bool AMDGPUOperand::isSDWAOperand(MVT type) const {
1444  if (AsmParser->isVI())
1445  return isVReg32();
1446  else if (AsmParser->isGFX9() || AsmParser->isGFX10())
1447  return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
1448  else
1449  return false;
1450 }
1451 
1452 bool AMDGPUOperand::isSDWAFP16Operand() const {
1453  return isSDWAOperand(MVT::f16);
1454 }
1455 
1456 bool AMDGPUOperand::isSDWAFP32Operand() const {
1457  return isSDWAOperand(MVT::f32);
1458 }
1459 
1460 bool AMDGPUOperand::isSDWAInt16Operand() const {
1461  return isSDWAOperand(MVT::i16);
1462 }
1463 
1464 bool AMDGPUOperand::isSDWAInt32Operand() const {
1465  return isSDWAOperand(MVT::i32);
1466 }
1467 
1468 uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
1469 {
1470  assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1471  assert(Size == 2 || Size == 4 || Size == 8);
1472 
1473  const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
1474 
1475  if (Imm.Mods.Abs) {
1476  Val &= ~FpSignMask;
1477  }
1478  if (Imm.Mods.Neg) {
1479  Val ^= FpSignMask;
1480  }
1481 
1482  return Val;
1483 }
1484 
1485 void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
1486  if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
1487  Inst.getNumOperands())) {
1488  addLiteralImmOperand(Inst, Imm.Val,
1489  ApplyModifiers &
1490  isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
1491  } else {
1492  assert(!isImmTy(ImmTyNone) || !hasModifiers());
1493  Inst.addOperand(MCOperand::createImm(Imm.Val));
1494  }
1495 }
1496 
1497 void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
1498  const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
1499  auto OpNum = Inst.getNumOperands();
1500  // Check that this operand accepts literals
1501  assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
1502 
1503  if (ApplyModifiers) {
1504  assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
1505  const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
1506  Val = applyInputFPModifiers(Val, Size);
1507  }
1508 
1509  APInt Literal(64, Val);
1510  uint8_t OpTy = InstDesc.OpInfo[OpNum].OperandType;
1511 
1512  if (Imm.IsFPImm) { // We got fp literal token
1513  switch (OpTy) {
1518  if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
1519  AsmParser->hasInv2PiInlineImm())) {
1520  Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
1521  return;
1522  }
1523 
1524  // Non-inlineable
1525  if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
1526  // For fp operands we check if low 32 bits are zeros
1527  if (Literal.getLoBits(32) != 0) {
1528  const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
1529  "Can't encode literal as exact 64-bit floating-point operand. "
1530  "Low 32-bits will be set to zero");
1531  }
1532 
1533  Inst.addOperand(MCOperand::createImm(Literal.lshr(32).getZExtValue()));
1534  return;
1535  }
1536 
1537  // We don't allow fp literals in 64-bit integer instructions. It is
1538  // unclear how we should encode them. This case should be checked earlier
1539  // in predicate methods (isLiteralImm())
1540  llvm_unreachable("fp literal in 64-bit integer instruction.");
1541 
1554  bool lost;
1555  APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1556  // Convert literal to single precision
1557  FPLiteral.convert(*getOpFltSemantics(OpTy),
1559  // We allow precision lost but not overflow or underflow. This should be
1560  // checked earlier in isLiteralImm()
1561 
1562  uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
1563  Inst.addOperand(MCOperand::createImm(ImmVal));
1564  return;
1565  }
1566  default:
1567  llvm_unreachable("invalid operand size");
1568  }
1569 
1570  return;
1571  }
1572 
1573  // We got int literal token.
1574  // Only sign extend inline immediates.
1575  switch (OpTy) {
1582  if (isSafeTruncation(Val, 32) &&
1583  AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
1584  AsmParser->hasInv2PiInlineImm())) {
1585  Inst.addOperand(MCOperand::createImm(Val));
1586  return;
1587  }
1588 
1589  Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
1590  return;
1591 
1596  if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
1597  Inst.addOperand(MCOperand::createImm(Val));
1598  return;
1599  }
1600 
1602  return;
1603 
1608  if (isSafeTruncation(Val, 16) &&
1609  AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1610  AsmParser->hasInv2PiInlineImm())) {
1611  Inst.addOperand(MCOperand::createImm(Val));
1612  return;
1613  }
1614 
1615  Inst.addOperand(MCOperand::createImm(Val & 0xffff));
1616  return;
1617 
1620  assert(isSafeTruncation(Val, 16));
1621  assert(AMDGPU::isInlinableLiteral16(static_cast<int16_t>(Val),
1622  AsmParser->hasInv2PiInlineImm()));
1623 
1624  Inst.addOperand(MCOperand::createImm(Val));
1625  return;
1626  }
1627  default:
1628  llvm_unreachable("invalid operand size");
1629  }
1630 }
1631 
1632 template <unsigned Bitwidth>
1633 void AMDGPUOperand::addKImmFPOperands(MCInst &Inst, unsigned N) const {
1634  APInt Literal(64, Imm.Val);
1635 
1636  if (!Imm.IsFPImm) {
1637  // We got int literal token.
1638  Inst.addOperand(MCOperand::createImm(Literal.getLoBits(Bitwidth).getZExtValue()));
1639  return;
1640  }
1641 
1642  bool Lost;
1643  APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
1644  FPLiteral.convert(*getFltSemantics(Bitwidth / 8),
1646  Inst.addOperand(MCOperand::createImm(FPLiteral.bitcastToAPInt().getZExtValue()));
1647 }
1648 
1649 void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
1650  Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
1651 }
1652 
1653 static bool isInlineValue(unsigned Reg) {
1654  switch (Reg) {
1655  case AMDGPU::SRC_SHARED_BASE:
1656  case AMDGPU::SRC_SHARED_LIMIT:
1657  case AMDGPU::SRC_PRIVATE_BASE:
1658  case AMDGPU::SRC_PRIVATE_LIMIT:
1659  case AMDGPU::SRC_POPS_EXITING_WAVE_ID:
1660  return true;
1661  default:
1662  return false;
1663  }
1664 }
1665 
1666 bool AMDGPUOperand::isInlineValue() const {
1667  return isRegKind() && ::isInlineValue(getReg());
1668 }
1669 
1670 //===----------------------------------------------------------------------===//
1671 // AsmParser
1672 //===----------------------------------------------------------------------===//
1673 
1674 static int getRegClass(RegisterKind Is, unsigned RegWidth) {
1675  if (Is == IS_VGPR) {
1676  switch (RegWidth) {
1677  default: return -1;
1678  case 1: return AMDGPU::VGPR_32RegClassID;
1679  case 2: return AMDGPU::VReg_64RegClassID;
1680  case 3: return AMDGPU::VReg_96RegClassID;
1681  case 4: return AMDGPU::VReg_128RegClassID;
1682  case 8: return AMDGPU::VReg_256RegClassID;
1683  case 16: return AMDGPU::VReg_512RegClassID;
1684  }
1685  } else if (Is == IS_TTMP) {
1686  switch (RegWidth) {
1687  default: return -1;
1688  case 1: return AMDGPU::TTMP_32RegClassID;
1689  case 2: return AMDGPU::TTMP_64RegClassID;
1690  case 4: return AMDGPU::TTMP_128RegClassID;
1691  case 8: return AMDGPU::TTMP_256RegClassID;
1692  case 16: return AMDGPU::TTMP_512RegClassID;
1693  }
1694  } else if (Is == IS_SGPR) {
1695  switch (RegWidth) {
1696  default: return -1;
1697  case 1: return AMDGPU::SGPR_32RegClassID;
1698  case 2: return AMDGPU::SGPR_64RegClassID;
1699  case 4: return AMDGPU::SGPR_128RegClassID;
1700  case 8: return AMDGPU::SGPR_256RegClassID;
1701  case 16: return AMDGPU::SGPR_512RegClassID;
1702  }
1703  }
1704  return -1;
1705 }
1706 
1707 static unsigned getSpecialRegForName(StringRef RegName) {
1708  return StringSwitch<unsigned>(RegName)
1709  .Case("exec", AMDGPU::EXEC)
1710  .Case("vcc", AMDGPU::VCC)
1711  .Case("flat_scratch", AMDGPU::FLAT_SCR)
1712  .Case("xnack_mask", AMDGPU::XNACK_MASK)
1713  .Case("shared_base", AMDGPU::SRC_SHARED_BASE)
1714  .Case("src_shared_base", AMDGPU::SRC_SHARED_BASE)
1715  .Case("shared_limit", AMDGPU::SRC_SHARED_LIMIT)
1716  .Case("src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
1717  .Case("private_base", AMDGPU::SRC_PRIVATE_BASE)
1718  .Case("src_private_base", AMDGPU::SRC_PRIVATE_BASE)
1719  .Case("private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
1720  .Case("src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
1721  .Case("pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
1722  .Case("src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
1723  .Case("lds_direct", AMDGPU::LDS_DIRECT)
1724  .Case("src_lds_direct", AMDGPU::LDS_DIRECT)
1725  .Case("m0", AMDGPU::M0)
1726  .Case("scc", AMDGPU::SCC)
1727  .Case("tba", AMDGPU::TBA)
1728  .Case("tma", AMDGPU::TMA)
1729  .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
1730  .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
1731  .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
1732  .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
1733  .Case("vcc_lo", AMDGPU::VCC_LO)
1734  .Case("vcc_hi", AMDGPU::VCC_HI)
1735  .Case("exec_lo", AMDGPU::EXEC_LO)
1736  .Case("exec_hi", AMDGPU::EXEC_HI)
1737  .Case("tma_lo", AMDGPU::TMA_LO)
1738  .Case("tma_hi", AMDGPU::TMA_HI)
1739  .Case("tba_lo", AMDGPU::TBA_LO)
1740  .Case("tba_hi", AMDGPU::TBA_HI)
1741  .Case("null", AMDGPU::SGPR_NULL)
1742  .Default(0);
1743 }
1744 
1745 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
1746  SMLoc &EndLoc) {
1747  auto R = parseRegister();
1748  if (!R) return true;
1749  assert(R->isReg());
1750  RegNo = R->getReg();
1751  StartLoc = R->getStartLoc();
1752  EndLoc = R->getEndLoc();
1753  return false;
1754 }
1755 
1756 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
1757  RegisterKind RegKind, unsigned Reg1,
1758  unsigned RegNum) {
1759  switch (RegKind) {
1760  case IS_SPECIAL:
1761  if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
1762  Reg = AMDGPU::EXEC;
1763  RegWidth = 2;
1764  return true;
1765  }
1766  if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
1767  Reg = AMDGPU::FLAT_SCR;
1768  RegWidth = 2;
1769  return true;
1770  }
1771  if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
1772  Reg = AMDGPU::XNACK_MASK;
1773  RegWidth = 2;
1774  return true;
1775  }
1776  if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
1777  Reg = AMDGPU::VCC;
1778  RegWidth = 2;
1779  return true;
1780  }
1781  if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
1782  Reg = AMDGPU::TBA;
1783  RegWidth = 2;
1784  return true;
1785  }
1786  if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
1787  Reg = AMDGPU::TMA;
1788  RegWidth = 2;
1789  return true;
1790  }
1791  return false;
1792  case IS_VGPR:
1793  case IS_SGPR:
1794  case IS_TTMP:
1795  if (Reg1 != Reg + RegWidth) {
1796  return false;
1797  }
1798  RegWidth++;
1799  return true;
1800  default:
1801  llvm_unreachable("unexpected register kind");
1802  }
1803 }
1804 
1805 static const StringRef Registers[] = {
1806  { "v" },
1807  { "s" },
1808  { "ttmp" },
1809 };
1810 
1811 bool
1812 AMDGPUAsmParser::isRegister(const AsmToken &Token,
1813  const AsmToken &NextToken) const {
1814 
1815  // A list of consecutive registers: [s0,s1,s2,s3]
1816  if (Token.is(AsmToken::LBrac))
1817  return true;
1818 
1819  if (!Token.is(AsmToken::Identifier))
1820  return false;
1821 
1822  // A single register like s0 or a range of registers like s[0:1]
1823 
1824  StringRef RegName = Token.getString();
1825 
1826  for (StringRef Reg : Registers) {
1827  if (RegName.startswith(Reg)) {
1828  if (Reg.size() < RegName.size()) {
1829  unsigned RegNum;
1830  // A single register with an index: rXX
1831  if (!RegName.substr(Reg.size()).getAsInteger(10, RegNum))
1832  return true;
1833  } else {
1834  // A range of registers: r[XX:YY].
1835  if (NextToken.is(AsmToken::LBrac))
1836  return true;
1837  }
1838  }
1839  }
1840 
1841  return getSpecialRegForName(RegName);
1842 }
1843 
1844 bool
1845 AMDGPUAsmParser::isRegister()
1846 {
1847  return isRegister(getToken(), peekToken());
1848 }
1849 
1850 bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1851  unsigned &RegNum, unsigned &RegWidth,
1852  unsigned *DwordRegIndex) {
1853  if (DwordRegIndex) { *DwordRegIndex = 0; }
1854  const MCRegisterInfo *TRI = getContext().getRegisterInfo();
1855  if (getLexer().is(AsmToken::Identifier)) {
1856  StringRef RegName = Parser.getTok().getString();
1857  if ((Reg = getSpecialRegForName(RegName))) {
1858  Parser.Lex();
1859  RegKind = IS_SPECIAL;
1860  } else {
1861  unsigned RegNumIndex = 0;
1862  if (RegName[0] == 'v') {
1863  RegNumIndex = 1;
1864  RegKind = IS_VGPR;
1865  } else if (RegName[0] == 's') {
1866  RegNumIndex = 1;
1867  RegKind = IS_SGPR;
1868  } else if (RegName.startswith("ttmp")) {
1869  RegNumIndex = strlen("ttmp");
1870  RegKind = IS_TTMP;
1871  } else {
1872  return false;
1873  }
1874  if (RegName.size() > RegNumIndex) {
1875  // Single 32-bit register: vXX.
1876  if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
1877  return false;
1878  Parser.Lex();
1879  RegWidth = 1;
1880  } else {
1881  // Range of registers: v[XX:YY]. ":YY" is optional.
1882  Parser.Lex();
1883  int64_t RegLo, RegHi;
1884  if (getLexer().isNot(AsmToken::LBrac))
1885  return false;
1886  Parser.Lex();
1887 
1888  if (getParser().parseAbsoluteExpression(RegLo))
1889  return false;
1890 
1891  const bool isRBrace = getLexer().is(AsmToken::RBrac);
1892  if (!isRBrace && getLexer().isNot(AsmToken::Colon))
1893  return false;
1894  Parser.Lex();
1895 
1896  if (isRBrace) {
1897  RegHi = RegLo;
1898  } else {
1899  if (getParser().parseAbsoluteExpression(RegHi))
1900  return false;
1901 
1902  if (getLexer().isNot(AsmToken::RBrac))
1903  return false;
1904  Parser.Lex();
1905  }
1906  RegNum = (unsigned) RegLo;
1907  RegWidth = (RegHi - RegLo) + 1;
1908  }
1909  }
1910  } else if (getLexer().is(AsmToken::LBrac)) {
1911  // List of consecutive registers: [s0,s1,s2,s3]
1912  Parser.Lex();
1913  if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
1914  return false;
1915  if (RegWidth != 1)
1916  return false;
1917  RegisterKind RegKind1;
1918  unsigned Reg1, RegNum1, RegWidth1;
1919  do {
1920  if (getLexer().is(AsmToken::Comma)) {
1921  Parser.Lex();
1922  } else if (getLexer().is(AsmToken::RBrac)) {
1923  Parser.Lex();
1924  break;
1925  } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
1926  if (RegWidth1 != 1) {
1927  return false;
1928  }
1929  if (RegKind1 != RegKind) {
1930  return false;
1931  }
1932  if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
1933  return false;
1934  }
1935  } else {
1936  return false;
1937  }
1938  } while (true);
1939  } else {
1940  return false;
1941  }
1942  switch (RegKind) {
1943  case IS_SPECIAL:
1944  RegNum = 0;
1945  RegWidth = 1;
1946  break;
1947  case IS_VGPR:
1948  case IS_SGPR:
1949  case IS_TTMP:
1950  {
1951  unsigned Size = 1;
1952  if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
1953  // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
1954  Size = std::min(RegWidth, 4u);
1955  }
1956  if (RegNum % Size != 0)
1957  return false;
1958  if (DwordRegIndex) { *DwordRegIndex = RegNum; }
1959  RegNum = RegNum / Size;
1960  int RCID = getRegClass(RegKind, RegWidth);
1961  if (RCID == -1)
1962  return false;
1963  const MCRegisterClass RC = TRI->getRegClass(RCID);
1964  if (RegNum >= RC.getNumRegs())
1965  return false;
1966  Reg = RC.getRegister(RegNum);
1967  break;
1968  }
1969 
1970  default:
1971  llvm_unreachable("unexpected register kind");
1972  }
1973 
1974  if (!subtargetHasRegister(*TRI, Reg))
1975  return false;
1976  return true;
1977 }
1978 
1980 AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
1981  switch (RegKind) {
1982  case IS_VGPR:
1983  return StringRef(".amdgcn.next_free_vgpr");
1984  case IS_SGPR:
1985  return StringRef(".amdgcn.next_free_sgpr");
1986  default:
1987  return None;
1988  }
1989 }
1990 
1991 void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
1992  auto SymbolName = getGprCountSymbolName(RegKind);
1993  assert(SymbolName && "initializing invalid register kind");
1994  MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
1995  Sym->setVariableValue(MCConstantExpr::create(0, getContext()));
1996 }
1997 
1998 bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
1999  unsigned DwordRegIndex,
2000  unsigned RegWidth) {
2001  // Symbols are only defined for GCN targets
2002  if (AMDGPU::getIsaVersion(getSTI().getCPU()).Major < 6)
2003  return true;
2004 
2005  auto SymbolName = getGprCountSymbolName(RegKind);
2006  if (!SymbolName)
2007  return true;
2008  MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
2009 
2010  int64_t NewMax = DwordRegIndex + RegWidth - 1;
2011  int64_t OldCount;
2012 
2013  if (!Sym->isVariable())
2014  return !Error(getParser().getTok().getLoc(),
2015  ".amdgcn.next_free_{v,s}gpr symbols must be variable");
2016  if (!Sym->getVariableValue(false)->evaluateAsAbsolute(OldCount))
2017  return !Error(
2018  getParser().getTok().getLoc(),
2019  ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
2020 
2021  if (OldCount <= NewMax)
2022  Sym->setVariableValue(MCConstantExpr::create(NewMax + 1, getContext()));
2023 
2024  return true;
2025 }
2026 
2027 std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
2028  const auto &Tok = Parser.getTok();
2029  SMLoc StartLoc = Tok.getLoc();
2030  SMLoc EndLoc = Tok.getEndLoc();
2032  unsigned Reg, RegNum, RegWidth, DwordRegIndex;
2033 
2034  if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
2035  //FIXME: improve error messages (bug 41303).
2036  Error(StartLoc, "not a valid operand.");
2037  return nullptr;
2038  }
2039  if (AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())) {
2040  if (!updateGprCountSymbols(RegKind, DwordRegIndex, RegWidth))
2041  return nullptr;
2042  } else
2043  KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
2044  return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc);
2045 }
2046 
2048 AMDGPUAsmParser::parseImm(OperandVector &Operands, bool HasSP3AbsModifier) {
2049  // TODO: add syntactic sugar for 1/(2*PI)
2050 
2051  assert(!isRegister());
2052  assert(!isModifier());
2053 
2054  const auto& Tok = getToken();
2055  const auto& NextTok = peekToken();
2056  bool IsReal = Tok.is(AsmToken::Real);
2057  SMLoc S = getLoc();
2058  bool Negate = false;
2059 
2060  if (!IsReal && Tok.is(AsmToken::Minus) && NextTok.is(AsmToken::Real)) {
2061  lex();
2062  IsReal = true;
2063  Negate = true;
2064  }
2065 
2066  if (IsReal) {
2067  // Floating-point expressions are not supported.
2068  // Can only allow floating-point literals with an
2069  // optional sign.
2070 
2071  StringRef Num = getTokenStr();
2072  lex();
2073 
2074  APFloat RealVal(APFloat::IEEEdouble());
2075  auto roundMode = APFloat::rmNearestTiesToEven;
2076  if (RealVal.convertFromString(Num, roundMode) == APFloat::opInvalidOp) {
2077  return MatchOperand_ParseFail;
2078  }
2079  if (Negate)
2080  RealVal.changeSign();
2081 
2082  Operands.push_back(
2083  AMDGPUOperand::CreateImm(this, RealVal.bitcastToAPInt().getZExtValue(), S,
2084  AMDGPUOperand::ImmTyNone, true));
2085 
2086  return MatchOperand_Success;
2087 
2088  } else {
2089  int64_t IntVal;
2090  const MCExpr *Expr;
2091  SMLoc S = getLoc();
2092 
2093  if (HasSP3AbsModifier) {
2094  // This is a workaround for handling expressions
2095  // as arguments of SP3 'abs' modifier, for example:
2096  // |1.0|
2097  // |-1|
2098  // |1+x|
2099  // This syntax is not compatible with syntax of standard
2100  // MC expressions (due to the trailing '|').
2101  SMLoc EndLoc;
2102  if (getParser().parsePrimaryExpr(Expr, EndLoc))
2103  return MatchOperand_ParseFail;
2104  } else {
2105  if (Parser.parseExpression(Expr))
2106  return MatchOperand_ParseFail;
2107  }
2108 
2109  if (Expr->evaluateAsAbsolute(IntVal)) {
2110  Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
2111  } else {
2112  Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
2113  }
2114 
2115  return MatchOperand_Success;
2116  }
2117 
2118  return MatchOperand_NoMatch;
2119 }
2120 
2122 AMDGPUAsmParser::parseReg(OperandVector &Operands) {
2123  if (!isRegister())
2124  return MatchOperand_NoMatch;
2125 
2126  if (auto R = parseRegister()) {
2127  assert(R->isReg());
2128  Operands.push_back(std::move(R));
2129  return MatchOperand_Success;
2130  }
2131  return MatchOperand_ParseFail;
2132 }
2133 
2135 AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod) {
2136  auto res = parseReg(Operands);
2137  if (res != MatchOperand_NoMatch) {
2138  return res;
2139  } else if (isModifier()) {
2140  return MatchOperand_NoMatch;
2141  } else {
2142  return parseImm(Operands, HasSP3AbsMod);
2143  }
2144 }
2145 
2146 bool
2147 AMDGPUAsmParser::isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
2148  if (Token.is(AsmToken::Identifier) && NextToken.is(AsmToken::LParen)) {
2149  const auto &str = Token.getString();
2150  return str == "abs" || str == "neg" || str == "sext";
2151  }
2152  return false;
2153 }
2154 
2155 bool
2156 AMDGPUAsmParser::isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const {
2157  return Token.is(AsmToken::Identifier) && NextToken.is(AsmToken::Colon);
2158 }
2159 
2160 bool
2161 AMDGPUAsmParser::isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
2162  return isNamedOperandModifier(Token, NextToken) || Token.is(AsmToken::Pipe);
2163 }
2164 
2165 bool
2166 AMDGPUAsmParser::isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
2167  return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
2168 }
2169 
2170 // Check if this is an operand modifier or an opcode modifier
2171 // which may look like an expression but it is not. We should
2172 // avoid parsing these modifiers as expressions. Currently
2173 // recognized sequences are:
2174 // |...|
2175 // abs(...)
2176 // neg(...)
2177 // sext(...)
2178 // -reg
2179 // -|...|
2180 // -abs(...)
2181 // name:...
2182 // Note that simple opcode modifiers like 'gds' may be parsed as
2183 // expressions; this is a special case. See getExpressionAsToken.
2184 //
2185 bool
2186 AMDGPUAsmParser::isModifier() {
2187 
2188  AsmToken Tok = getToken();
2189  AsmToken NextToken[2];
2190  peekTokens(NextToken);
2191 
2192  return isOperandModifier(Tok, NextToken[0]) ||
2193  (Tok.is(AsmToken::Minus) && isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
2194  isOpcodeModifierWithVal(Tok, NextToken[0]);
2195 }
2196 
2197 // Check if the current token is an SP3 'neg' modifier.
2198 // Currently this modifier is allowed in the following context:
2199 //
2200 // 1. Before a register, e.g. "-v0", "-v[...]" or "-[v0,v1]".
2201 // 2. Before an 'abs' modifier: -abs(...)
2202 // 3. Before an SP3 'abs' modifier: -|...|
2203 //
2204 // In all other cases "-" is handled as a part
2205 // of an expression that follows the sign.
2206 //
2207 // Note: When "-" is followed by an integer literal,
2208 // this is interpreted as integer negation rather
2209 // than a floating-point NEG modifier applied to N.
2210 // Beside being contr-intuitive, such use of floating-point
2211 // NEG modifier would have resulted in different meaning
2212 // of integer literals used with VOP1/2/C and VOP3,
2213 // for example:
2214 // v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
2215 // v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
2216 // Negative fp literals with preceding "-" are
2217 // handled likewise for unifomtity
2218 //
2219 bool
2220 AMDGPUAsmParser::parseSP3NegModifier() {
2221 
2222  AsmToken NextToken[2];
2223  peekTokens(NextToken);
2224 
2225  if (isToken(AsmToken::Minus) &&
2226  (isRegister(NextToken[0], NextToken[1]) ||
2227  NextToken[0].is(AsmToken::Pipe) ||
2228  isId(NextToken[0], "abs"))) {
2229  lex();
2230  return true;
2231  }
2232 
2233  return false;
2234 }
2235 
2237 AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
2238  bool AllowImm) {
2239  bool Neg, SP3Neg;
2240  bool Abs, SP3Abs;
2241  SMLoc Loc;
2242 
2243  // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
2244  if (isToken(AsmToken::Minus) && peekToken().is(AsmToken::Minus)) {
2245  Error(getLoc(), "invalid syntax, expected 'neg' modifier");
2246  return MatchOperand_ParseFail;
2247  }
2248 
2249  SP3Neg = parseSP3NegModifier();
2250 
2251  Loc = getLoc();
2252  Neg = trySkipId("neg");
2253  if (Neg && SP3Neg) {
2254  Error(Loc, "expected register or immediate");
2255  return MatchOperand_ParseFail;
2256  }
2257  if (Neg && !skipToken(AsmToken::LParen, "expected left paren after neg"))
2258  return MatchOperand_ParseFail;
2259 
2260  Abs = trySkipId("abs");
2261  if (Abs && !skipToken(AsmToken::LParen, "expected left paren after abs"))
2262  return MatchOperand_ParseFail;
2263 
2264  Loc = getLoc();
2265  SP3Abs = trySkipToken(AsmToken::Pipe);
2266  if (Abs && SP3Abs) {
2267  Error(Loc, "expected register or immediate");
2268  return MatchOperand_ParseFail;
2269  }
2270 
2272  if (AllowImm) {
2273  Res = parseRegOrImm(Operands, SP3Abs);
2274  } else {
2275  Res = parseReg(Operands);
2276  }
2277  if (Res != MatchOperand_Success) {
2278  return (SP3Neg || Neg || SP3Abs || Abs)? MatchOperand_ParseFail : Res;
2279  }
2280 
2281  if (SP3Abs && !skipToken(AsmToken::Pipe, "expected vertical bar"))
2282  return MatchOperand_ParseFail;
2283  if (Abs && !skipToken(AsmToken::RParen, "expected closing parentheses"))
2284  return MatchOperand_ParseFail;
2285  if (Neg && !skipToken(AsmToken::RParen, "expected closing parentheses"))
2286  return MatchOperand_ParseFail;
2287 
2288  AMDGPUOperand::Modifiers Mods;
2289  Mods.Abs = Abs || SP3Abs;
2290  Mods.Neg = Neg || SP3Neg;
2291 
2292  if (Mods.hasFPModifiers()) {
2293  AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2294  if (Op.isExpr()) {
2295  Error(Op.getStartLoc(), "expected an absolute expression");
2296  return MatchOperand_ParseFail;
2297  }
2298  Op.setModifiers(Mods);
2299  }
2300  return MatchOperand_Success;
2301 }
2302 
2304 AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
2305  bool AllowImm) {
2306  bool Sext = trySkipId("sext");
2307  if (Sext && !skipToken(AsmToken::LParen, "expected left paren after sext"))
2308  return MatchOperand_ParseFail;
2309 
2311  if (AllowImm) {
2312  Res = parseRegOrImm(Operands);
2313  } else {
2314  Res = parseReg(Operands);
2315  }
2316  if (Res != MatchOperand_Success) {
2317  return Sext? MatchOperand_ParseFail : Res;
2318  }
2319 
2320  if (Sext && !skipToken(AsmToken::RParen, "expected closing parentheses"))
2321  return MatchOperand_ParseFail;
2322 
2323  AMDGPUOperand::Modifiers Mods;
2324  Mods.Sext = Sext;
2325 
2326  if (Mods.hasIntModifiers()) {
2327  AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
2328  if (Op.isExpr()) {
2329  Error(Op.getStartLoc(), "expected an absolute expression");
2330  return MatchOperand_ParseFail;
2331  }
2332  Op.setModifiers(Mods);
2333  }
2334 
2335  return MatchOperand_Success;
2336 }
2337 
2339 AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
2340  return parseRegOrImmWithFPInputMods(Operands, false);
2341 }
2342 
2344 AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
2345  return parseRegOrImmWithIntInputMods(Operands, false);
2346 }
2347 
2348 OperandMatchResultTy AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
2349  auto Loc = getLoc();
2350  if (trySkipId("off")) {
2351  Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Loc,
2352  AMDGPUOperand::ImmTyOff, false));
2353  return MatchOperand_Success;
2354  }
2355 
2356  if (!isRegister())
2357  return MatchOperand_NoMatch;
2358 
2359  std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
2360  if (Reg) {
2361  Operands.push_back(std::move(Reg));
2362  return MatchOperand_Success;
2363  }
2364 
2365  return MatchOperand_ParseFail;
2366 
2367 }
2368 
2369 unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
2370  uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
2371 
2372  if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
2373  (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
2374  (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
2375  (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
2376  return Match_InvalidOperand;
2377 
2378  if ((TSFlags & SIInstrFlags::VOP3) &&
2379  (TSFlags & SIInstrFlags::VOPAsmPrefer32Bit) &&
2380  getForcedEncodingSize() != 64)
2381  return Match_PreferE32;
2382 
2383  if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
2384  Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
2385  // v_mac_f32/16 allow only dst_sel == DWORD;
2386  auto OpNum =
2387  AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
2388  const auto &Op = Inst.getOperand(OpNum);
2389  if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
2390  return Match_InvalidOperand;
2391  }
2392  }
2393 
2394  if (TSFlags & SIInstrFlags::FLAT) {
2395  // FIXME: Produces error without correct column reported.
2396  auto Opcode = Inst.getOpcode();
2397  auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
2398 
2399  const auto &Op = Inst.getOperand(OpNum);
2400  if (!hasFlatOffsets() && Op.getImm() != 0)
2401  return Match_InvalidOperand;
2402 
2403  // GFX10: Address offset is 12-bit signed byte offset. Must be positive for
2404  // FLAT segment. For FLAT segment MSB is ignored and forced to zero.
2405  if (isGFX10()) {
2406  if (TSFlags & SIInstrFlags::IsNonFlatSeg) {
2407  if (!isInt<12>(Op.getImm()))
2408  return Match_InvalidOperand;
2409  } else {
2410  if (!isUInt<11>(Op.getImm()))
2411  return Match_InvalidOperand;
2412  }
2413  }
2414  }
2415 
2416  return Match_Success;
2417 }
2418 
2419 // What asm variants we should check
2420 ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
2421  if (getForcedEncodingSize() == 32) {
2422  static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
2423  return makeArrayRef(Variants);
2424  }
2425 
2426  if (isForcedVOP3()) {
2427  static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
2428  return makeArrayRef(Variants);
2429  }
2430 
2431  if (isForcedSDWA()) {
2432  static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
2434  return makeArrayRef(Variants);
2435  }
2436 
2437  if (isForcedDPP()) {
2438  static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
2439  return makeArrayRef(Variants);
2440  }
2441 
2442  static const unsigned Variants[] = {
2445  };
2446 
2447  return makeArrayRef(Variants);
2448 }
2449 
2450 unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
2451  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2452  const unsigned Num = Desc.getNumImplicitUses();
2453  for (unsigned i = 0; i < Num; ++i) {
2454  unsigned Reg = Desc.ImplicitUses[i];
2455  switch (Reg) {
2456  case AMDGPU::FLAT_SCR:
2457  case AMDGPU::VCC:
2458  case AMDGPU::VCC_LO:
2459  case AMDGPU::VCC_HI:
2460  case AMDGPU::M0:
2461  case AMDGPU::SGPR_NULL:
2462  return Reg;
2463  default:
2464  break;
2465  }
2466  }
2467  return AMDGPU::NoRegister;
2468 }
2469 
2470 // NB: This code is correct only when used to check constant
2471 // bus limitations because GFX7 support no f16 inline constants.
2472 // Note that there are no cases when a GFX7 opcode violates
2473 // constant bus limitations due to the use of an f16 constant.
2474 bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
2475  unsigned OpIdx) const {
2476  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
2477 
2478  if (!AMDGPU::isSISrcOperand(Desc, OpIdx)) {
2479  return false;
2480  }
2481 
2482  const MCOperand &MO = Inst.getOperand(OpIdx);
2483 
2484  int64_t Val = MO.getImm();
2485  auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
2486 
2487  switch (OpSize) { // expected operand size
2488  case 8:
2489  return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
2490  case 4:
2491  return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
2492  case 2: {
2493  const unsigned OperandType = Desc.OpInfo[OpIdx].OperandType;
2494  if (OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2INT16 ||
2495  OperandType == AMDGPU::OPERAND_REG_INLINE_C_V2FP16 ||
2496  OperandType == AMDGPU::OPERAND_REG_IMM_V2INT16 ||
2497  OperandType == AMDGPU::OPERAND_REG_IMM_V2FP16) {
2498  return AMDGPU::isInlinableLiteralV216(Val, hasInv2PiInlineImm());
2499  } else {
2500  return AMDGPU::isInlinableLiteral16(Val, hasInv2PiInlineImm());
2501  }
2502  }
2503  default:
2504  llvm_unreachable("invalid operand size");
2505  }
2506 }
2507 
2508 bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
2509  const MCOperand &MO = Inst.getOperand(OpIdx);
2510  if (MO.isImm()) {
2511  return !isInlineConstant(Inst, OpIdx);
2512  }
2513  return !MO.isReg() ||
2514  isSGPR(mc2PseudoReg(MO.getReg()), getContext().getRegisterInfo());
2515 }
2516 
2517 bool AMDGPUAsmParser::validateConstantBusLimitations(const MCInst &Inst) {
2518  const unsigned Opcode = Inst.getOpcode();
2519  const MCInstrDesc &Desc = MII.get(Opcode);
2520  unsigned ConstantBusUseCount = 0;
2521  unsigned NumLiterals = 0;
2522  unsigned LiteralSize;
2523 
2524  if (Desc.TSFlags &
2528  SIInstrFlags::SDWA)) {
2529  // Check special imm operands (used by madmk, etc)
2530  if (AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::imm) != -1) {
2531  ++ConstantBusUseCount;
2532  }
2533 
2534  SmallDenseSet<unsigned> SGPRsUsed;
2535  unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
2536  if (SGPRUsed != AMDGPU::NoRegister) {
2537  SGPRsUsed.insert(SGPRUsed);
2538  ++ConstantBusUseCount;
2539  }
2540 
2541  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2542  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2543  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2544 
2545  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2546 
2547  for (int OpIdx : OpIndices) {
2548  if (OpIdx == -1) break;
2549 
2550  const MCOperand &MO = Inst.getOperand(OpIdx);
2551  if (usesConstantBus(Inst, OpIdx)) {
2552  if (MO.isReg()) {
2553  const unsigned Reg = mc2PseudoReg(MO.getReg());
2554  // Pairs of registers with a partial intersections like these
2555  // s0, s[0:1]
2556  // flat_scratch_lo, flat_scratch
2557  // flat_scratch_lo, flat_scratch_hi
2558  // are theoretically valid but they are disabled anyway.
2559  // Note that this code mimics SIInstrInfo::verifyInstruction
2560  if (!SGPRsUsed.count(Reg)) {
2561  SGPRsUsed.insert(Reg);
2562  ++ConstantBusUseCount;
2563  }
2564  SGPRUsed = Reg;
2565  } else { // Expression or a literal
2566 
2567  if (Desc.OpInfo[OpIdx].OperandType == MCOI::OPERAND_IMMEDIATE)
2568  continue; // special operand like VINTERP attr_chan
2569 
2570  // An instruction may use only one literal.
2571  // This has been validated on the previous step.
2572  // See validateVOP3Literal.
2573  // This literal may be used as more than one operand.
2574  // If all these operands are of the same size,
2575  // this literal counts as one scalar value.
2576  // Otherwise it counts as 2 scalar values.
2577  // See "GFX10 Shader Programming", section 3.6.2.3.
2578 
2579  unsigned Size = AMDGPU::getOperandSize(Desc, OpIdx);
2580  if (Size < 4) Size = 4;
2581 
2582  if (NumLiterals == 0) {
2583  NumLiterals = 1;
2584  LiteralSize = Size;
2585  } else if (LiteralSize != Size) {
2586  NumLiterals = 2;
2587  }
2588  }
2589  }
2590  }
2591  }
2592  ConstantBusUseCount += NumLiterals;
2593 
2594  if (isGFX10())
2595  return ConstantBusUseCount <= 2;
2596 
2597  return ConstantBusUseCount <= 1;
2598 }
2599 
2600 bool AMDGPUAsmParser::validateEarlyClobberLimitations(const MCInst &Inst) {
2601  const unsigned Opcode = Inst.getOpcode();
2602  const MCInstrDesc &Desc = MII.get(Opcode);
2603 
2604  const int DstIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::vdst);
2605  if (DstIdx == -1 ||
2606  Desc.getOperandConstraint(DstIdx, MCOI::EARLY_CLOBBER) == -1) {
2607  return true;
2608  }
2609 
2610  const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2611 
2612  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2613  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2614  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2615 
2616  assert(DstIdx != -1);
2617  const MCOperand &Dst = Inst.getOperand(DstIdx);
2618  assert(Dst.isReg());
2619  const unsigned DstReg = mc2PseudoReg(Dst.getReg());
2620 
2621  const int SrcIndices[] = { Src0Idx, Src1Idx, Src2Idx };
2622 
2623  for (int SrcIdx : SrcIndices) {
2624  if (SrcIdx == -1) break;
2625  const MCOperand &Src = Inst.getOperand(SrcIdx);
2626  if (Src.isReg()) {
2627  const unsigned SrcReg = mc2PseudoReg(Src.getReg());
2628  if (isRegIntersect(DstReg, SrcReg, TRI)) {
2629  return false;
2630  }
2631  }
2632  }
2633 
2634  return true;
2635 }
2636 
2637 bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
2638 
2639  const unsigned Opc = Inst.getOpcode();
2640  const MCInstrDesc &Desc = MII.get(Opc);
2641 
2642  if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
2643  int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
2644  assert(ClampIdx != -1);
2645  return Inst.getOperand(ClampIdx).getImm() == 0;
2646  }
2647 
2648  return true;
2649 }
2650 
2651 bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst) {
2652 
2653  const unsigned Opc = Inst.getOpcode();
2654  const MCInstrDesc &Desc = MII.get(Opc);
2655 
2656  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2657  return true;
2658 
2659  int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
2660  int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2661  int TFEIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
2662 
2663  assert(VDataIdx != -1);
2664  assert(DMaskIdx != -1);
2665  assert(TFEIdx != -1);
2666 
2667  unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
2668  unsigned TFESize = Inst.getOperand(TFEIdx).getImm()? 1 : 0;
2669  unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2670  if (DMask == 0)
2671  DMask = 1;
2672 
2673  unsigned DataSize =
2674  (Desc.TSFlags & SIInstrFlags::Gather4) ? 4 : countPopulation(DMask);
2675  if (hasPackedD16()) {
2676  int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
2677  if (D16Idx >= 0 && Inst.getOperand(D16Idx).getImm())
2678  DataSize = (DataSize + 1) / 2;
2679  }
2680 
2681  return (VDataSize / 4) == DataSize + TFESize;
2682 }
2683 
2684 bool AMDGPUAsmParser::validateMIMGAddrSize(const MCInst &Inst) {
2685  const unsigned Opc = Inst.getOpcode();
2686  const MCInstrDesc &Desc = MII.get(Opc);
2687 
2688  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0 || !isGFX10())
2689  return true;
2690 
2691  const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc);
2692  const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
2693  AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode);
2694  int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
2695  int SrsrcIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::srsrc);
2696  int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
2697 
2698  assert(VAddr0Idx != -1);
2699  assert(SrsrcIdx != -1);
2700  assert(DimIdx != -1);
2701  assert(SrsrcIdx > VAddr0Idx);
2702 
2703  unsigned Dim = Inst.getOperand(DimIdx).getImm();
2704  const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByEncoding(Dim);
2705  bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
2706  unsigned VAddrSize =
2707  IsNSA ? SrsrcIdx - VAddr0Idx
2708  : AMDGPU::getRegOperandSize(getMRI(), Desc, VAddr0Idx) / 4;
2709 
2710  unsigned AddrSize = BaseOpcode->NumExtraArgs +
2711  (BaseOpcode->Gradients ? DimInfo->NumGradients : 0) +
2712  (BaseOpcode->Coordinates ? DimInfo->NumCoords : 0) +
2713  (BaseOpcode->LodOrClampOrMip ? 1 : 0);
2714  if (!IsNSA) {
2715  if (AddrSize > 8)
2716  AddrSize = 16;
2717  else if (AddrSize > 4)
2718  AddrSize = 8;
2719  }
2720 
2721  return VAddrSize == AddrSize;
2722 }
2723 
2724 bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
2725 
2726  const unsigned Opc = Inst.getOpcode();
2727  const MCInstrDesc &Desc = MII.get(Opc);
2728 
2729  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2730  return true;
2731  if (!Desc.mayLoad() || !Desc.mayStore())
2732  return true; // Not atomic
2733 
2734  int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2735  unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2736 
2737  // This is an incomplete check because image_atomic_cmpswap
2738  // may only use 0x3 and 0xf while other atomic operations
2739  // may use 0x1 and 0x3. However these limitations are
2740  // verified when we check that dmask matches dst size.
2741  return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
2742 }
2743 
2744 bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {
2745 
2746  const unsigned Opc = Inst.getOpcode();
2747  const MCInstrDesc &Desc = MII.get(Opc);
2748 
2749  if ((Desc.TSFlags & SIInstrFlags::Gather4) == 0)
2750  return true;
2751 
2752  int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
2753  unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
2754 
2755  // GATHER4 instructions use dmask in a different fashion compared to
2756  // other MIMG instructions. The only useful DMASK values are
2757  // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
2758  // (red,red,red,red) etc.) The ISA document doesn't mention
2759  // this.
2760  return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
2761 }
2762 
2763 bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
2764 
2765  const unsigned Opc = Inst.getOpcode();
2766  const MCInstrDesc &Desc = MII.get(Opc);
2767 
2768  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2769  return true;
2770 
2771  int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
2772  if (D16Idx >= 0 && Inst.getOperand(D16Idx).getImm()) {
2773  if (isCI() || isSI())
2774  return false;
2775  }
2776 
2777  return true;
2778 }
2779 
2780 bool AMDGPUAsmParser::validateMIMGDim(const MCInst &Inst) {
2781  const unsigned Opc = Inst.getOpcode();
2782  const MCInstrDesc &Desc = MII.get(Opc);
2783 
2784  if ((Desc.TSFlags & SIInstrFlags::MIMG) == 0)
2785  return true;
2786 
2787  int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
2788  if (DimIdx < 0)
2789  return true;
2790 
2791  long Imm = Inst.getOperand(DimIdx).getImm();
2792  if (Imm < 0 || Imm >= 8)
2793  return false;
2794 
2795  return true;
2796 }
2797 
2798 static bool IsRevOpcode(const unsigned Opcode)
2799 {
2800  switch (Opcode) {
2801  case AMDGPU::V_SUBREV_F32_e32:
2802  case AMDGPU::V_SUBREV_F32_e64:
2803  case AMDGPU::V_SUBREV_F32_e32_gfx10:
2804  case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
2805  case AMDGPU::V_SUBREV_F32_e32_vi:
2806  case AMDGPU::V_SUBREV_F32_e64_gfx10:
2807  case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
2808  case AMDGPU::V_SUBREV_F32_e64_vi:
2809 
2810  case AMDGPU::V_SUBREV_I32_e32:
2811  case AMDGPU::V_SUBREV_I32_e64:
2812  case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
2813  case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
2814 
2815  case AMDGPU::V_SUBBREV_U32_e32:
2816  case AMDGPU::V_SUBBREV_U32_e64:
2817  case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
2818  case AMDGPU::V_SUBBREV_U32_e32_vi:
2819  case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
2820  case AMDGPU::V_SUBBREV_U32_e64_vi:
2821 
2822  case AMDGPU::V_SUBREV_U32_e32:
2823  case AMDGPU::V_SUBREV_U32_e64:
2824  case AMDGPU::V_SUBREV_U32_e32_gfx9:
2825  case AMDGPU::V_SUBREV_U32_e32_vi:
2826  case AMDGPU::V_SUBREV_U32_e64_gfx9:
2827  case AMDGPU::V_SUBREV_U32_e64_vi:
2828 
2829  case AMDGPU::V_SUBREV_F16_e32:
2830  case AMDGPU::V_SUBREV_F16_e64:
2831  case AMDGPU::V_SUBREV_F16_e32_gfx10:
2832  case AMDGPU::V_SUBREV_F16_e32_vi:
2833  case AMDGPU::V_SUBREV_F16_e64_gfx10:
2834  case AMDGPU::V_SUBREV_F16_e64_vi:
2835 
2836  case AMDGPU::V_SUBREV_U16_e32:
2837  case AMDGPU::V_SUBREV_U16_e64:
2838  case AMDGPU::V_SUBREV_U16_e32_vi:
2839  case AMDGPU::V_SUBREV_U16_e64_vi:
2840 
2841  case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
2842  case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
2843  case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
2844 
2845  case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
2846  case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
2847 
2848  case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
2849  case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
2850 
2851  case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
2852  case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
2853 
2854  case AMDGPU::V_LSHRREV_B32_e32:
2855  case AMDGPU::V_LSHRREV_B32_e64:
2856  case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
2857  case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
2858  case AMDGPU::V_LSHRREV_B32_e32_vi:
2859  case AMDGPU::V_LSHRREV_B32_e64_vi:
2860  case AMDGPU::V_LSHRREV_B32_e32_gfx10:
2861  case AMDGPU::V_LSHRREV_B32_e64_gfx10:
2862 
2863  case AMDGPU::V_ASHRREV_I32_e32:
2864  case AMDGPU::V_ASHRREV_I32_e64:
2865  case AMDGPU::V_ASHRREV_I32_e32_gfx10:
2866  case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
2867  case AMDGPU::V_ASHRREV_I32_e32_vi:
2868  case AMDGPU::V_ASHRREV_I32_e64_gfx10:
2869  case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
2870  case AMDGPU::V_ASHRREV_I32_e64_vi:
2871 
2872  case AMDGPU::V_LSHLREV_B32_e32:
2873  case AMDGPU::V_LSHLREV_B32_e64:
2874  case AMDGPU::V_LSHLREV_B32_e32_gfx10:
2875  case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
2876  case AMDGPU::V_LSHLREV_B32_e32_vi:
2877  case AMDGPU::V_LSHLREV_B32_e64_gfx10:
2878  case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
2879  case AMDGPU::V_LSHLREV_B32_e64_vi:
2880 
2881  case AMDGPU::V_LSHLREV_B16_e32:
2882  case AMDGPU::V_LSHLREV_B16_e64:
2883  case AMDGPU::V_LSHLREV_B16_e32_vi:
2884  case AMDGPU::V_LSHLREV_B16_e64_vi:
2885  case AMDGPU::V_LSHLREV_B16_gfx10:
2886 
2887  case AMDGPU::V_LSHRREV_B16_e32:
2888  case AMDGPU::V_LSHRREV_B16_e64:
2889  case AMDGPU::V_LSHRREV_B16_e32_vi:
2890  case AMDGPU::V_LSHRREV_B16_e64_vi:
2891  case AMDGPU::V_LSHRREV_B16_gfx10:
2892 
2893  case AMDGPU::V_ASHRREV_I16_e32:
2894  case AMDGPU::V_ASHRREV_I16_e64:
2895  case AMDGPU::V_ASHRREV_I16_e32_vi:
2896  case AMDGPU::V_ASHRREV_I16_e64_vi:
2897  case AMDGPU::V_ASHRREV_I16_gfx10:
2898 
2899  case AMDGPU::V_LSHLREV_B64:
2900  case AMDGPU::V_LSHLREV_B64_gfx10:
2901  case AMDGPU::V_LSHLREV_B64_vi:
2902 
2903  case AMDGPU::V_LSHRREV_B64:
2904  case AMDGPU::V_LSHRREV_B64_gfx10:
2905  case AMDGPU::V_LSHRREV_B64_vi:
2906 
2907  case AMDGPU::V_ASHRREV_I64:
2908  case AMDGPU::V_ASHRREV_I64_gfx10:
2909  case AMDGPU::V_ASHRREV_I64_vi:
2910 
2911  case AMDGPU::V_PK_LSHLREV_B16:
2912  case AMDGPU::V_PK_LSHLREV_B16_gfx10:
2913  case AMDGPU::V_PK_LSHLREV_B16_vi:
2914 
2915  case AMDGPU::V_PK_LSHRREV_B16:
2916  case AMDGPU::V_PK_LSHRREV_B16_gfx10:
2917  case AMDGPU::V_PK_LSHRREV_B16_vi:
2918  case AMDGPU::V_PK_ASHRREV_I16:
2919  case AMDGPU::V_PK_ASHRREV_I16_gfx10:
2920  case AMDGPU::V_PK_ASHRREV_I16_vi:
2921  return true;
2922  default:
2923  return false;
2924  }
2925 }
2926 
2927 bool AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
2928 
2929  using namespace SIInstrFlags;
2930  const unsigned Opcode = Inst.getOpcode();
2931  const MCInstrDesc &Desc = MII.get(Opcode);
2932 
2933  // lds_direct register is defined so that it can be used
2934  // with 9-bit operands only. Ignore encodings which do not accept these.
2935  if ((Desc.TSFlags & (VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA)) == 0)
2936  return true;
2937 
2938  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2939  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2940  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
2941 
2942  const int SrcIndices[] = { Src1Idx, Src2Idx };
2943 
2944  // lds_direct cannot be specified as either src1 or src2.
2945  for (int SrcIdx : SrcIndices) {
2946  if (SrcIdx == -1) break;
2947  const MCOperand &Src = Inst.getOperand(SrcIdx);
2948  if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
2949  return false;
2950  }
2951  }
2952 
2953  if (Src0Idx == -1)
2954  return true;
2955 
2956  const MCOperand &Src = Inst.getOperand(Src0Idx);
2957  if (!Src.isReg() || Src.getReg() != LDS_DIRECT)
2958  return true;
2959 
2960  // lds_direct is specified as src0. Check additional limitations.
2961  return (Desc.TSFlags & SIInstrFlags::SDWA) == 0 && !IsRevOpcode(Opcode);
2962 }
2963 
2964 bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
2965  unsigned Opcode = Inst.getOpcode();
2966  const MCInstrDesc &Desc = MII.get(Opcode);
2967  if (!(Desc.TSFlags & (SIInstrFlags::SOP2 | SIInstrFlags::SOPC)))
2968  return true;
2969 
2970  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
2971  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
2972 
2973  const int OpIndices[] = { Src0Idx, Src1Idx };
2974 
2975  unsigned NumLiterals = 0;
2976  uint32_t LiteralValue;
2977 
2978  for (int OpIdx : OpIndices) {
2979  if (OpIdx == -1) break;
2980 
2981  const MCOperand &MO = Inst.getOperand(OpIdx);
2982  if (MO.isImm() &&
2983  // Exclude special imm operands (like that used by s_set_gpr_idx_on)
2984  AMDGPU::isSISrcOperand(Desc, OpIdx) &&
2985  !isInlineConstant(Inst, OpIdx)) {
2986  uint32_t Value = static_cast<uint32_t>(MO.getImm());
2987  if (NumLiterals == 0 || LiteralValue != Value) {
2988  LiteralValue = Value;
2989  ++NumLiterals;
2990  }
2991  }
2992  }
2993 
2994  return NumLiterals <= 1;
2995 }
2996 
2997 // VOP3 literal is only allowed in GFX10+ and only one can be used
2998 bool AMDGPUAsmParser::validateVOP3Literal(const MCInst &Inst) const {
2999  unsigned Opcode = Inst.getOpcode();
3000  const MCInstrDesc &Desc = MII.get(Opcode);
3002  return true;
3003 
3004  const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
3005  const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
3006  const int Src2Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src2);
3007 
3008  const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
3009 
3010  unsigned NumLiterals = 0;
3011  uint32_t LiteralValue;
3012 
3013  for (int OpIdx : OpIndices) {
3014  if (OpIdx == -1) break;
3015 
3016  const MCOperand &MO = Inst.getOperand(OpIdx);
3017  if (!MO.isImm() || !AMDGPU::isSISrcOperand(Desc, OpIdx))
3018  continue;
3019 
3020  if (!isInlineConstant(Inst, OpIdx)) {
3021  uint32_t Value = static_cast<uint32_t>(MO.getImm());
3022  if (NumLiterals == 0 || LiteralValue != Value) {
3023  LiteralValue = Value;
3024  ++NumLiterals;
3025  }
3026  }
3027  }
3028 
3029  return !NumLiterals ||
3030  (NumLiterals == 1 && getFeatureBits()[AMDGPU::FeatureVOP3Literal]);
3031 }
3032 
3033 bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
3034  const SMLoc &IDLoc) {
3035  if (!validateLdsDirect(Inst)) {
3036  Error(IDLoc,
3037  "invalid use of lds_direct");
3038  return false;
3039  }
3040  if (!validateSOPLiteral(Inst)) {
3041  Error(IDLoc,
3042  "only one literal operand is allowed");
3043  return false;
3044  }
3045  if (!validateVOP3Literal(Inst)) {
3046  Error(IDLoc,
3047  "invalid literal operand");
3048  return false;
3049  }
3050  if (!validateConstantBusLimitations(Inst)) {
3051  Error(IDLoc,
3052  "invalid operand (violates constant bus restrictions)");
3053  return false;
3054  }
3055  if (!validateEarlyClobberLimitations(Inst)) {
3056  Error(IDLoc,
3057  "destination must be different than all sources");
3058  return false;
3059  }
3060  if (!validateIntClampSupported(Inst)) {
3061  Error(IDLoc,
3062  "integer clamping is not supported on this GPU");
3063  return false;
3064  }
3065  // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
3066  if (!validateMIMGD16(Inst)) {
3067  Error(IDLoc,
3068  "d16 modifier is not supported on this GPU");
3069  return false;
3070  }
3071  if (!validateMIMGDim(Inst)) {
3072  Error(IDLoc, "dim modifier is required on this GPU");
3073  return false;
3074  }
3075  if (!validateMIMGDataSize(Inst)) {
3076  Error(IDLoc,
3077  "image data size does not match dmask and tfe");
3078  return false;
3079  }
3080  if (!validateMIMGAddrSize(Inst)) {
3081  Error(IDLoc,
3082  "image address size does not match dim and a16");
3083  return false;
3084  }
3085  if (!validateMIMGAtomicDMask(Inst)) {
3086  Error(IDLoc,
3087  "invalid atomic image dmask");
3088  return false;
3089  }
3090  if (!validateMIMGGatherDMask(Inst)) {
3091  Error(IDLoc,
3092  "invalid image_gather dmask: only one bit must be set");
3093  return false;
3094  }
3095 
3096  return true;
3097 }
3098 
3099 static std::string AMDGPUMnemonicSpellCheck(StringRef S,
3100  const FeatureBitset &FBS,
3101  unsigned VariantID = 0);
3102 
3103 bool AMDGPUAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
3104  OperandVector &Operands,
3105  MCStreamer &Out,
3106  uint64_t &ErrorInfo,
3107  bool MatchingInlineAsm) {
3108  MCInst Inst;
3109  unsigned Result = Match_Success;
3110  for (auto Variant : getMatchedVariants()) {
3111  uint64_t EI;
3112  auto R = MatchInstructionImpl(Operands, Inst, EI, MatchingInlineAsm,
3113  Variant);
3114  // We order match statuses from least to most specific. We use most specific
3115  // status as resulting
3116  // Match_MnemonicFail < Match_InvalidOperand < Match_MissingFeature < Match_PreferE32
3117  if ((R == Match_Success) ||
3118  (R == Match_PreferE32) ||
3119  (R == Match_MissingFeature && Result != Match_PreferE32) ||
3120  (R == Match_InvalidOperand && Result != Match_MissingFeature
3121  && Result != Match_PreferE32) ||
3122  (R == Match_MnemonicFail && Result != Match_InvalidOperand
3123  && Result != Match_MissingFeature
3124  && Result != Match_PreferE32)) {
3125  Result = R;
3126  ErrorInfo = EI;
3127  }
3128  if (R == Match_Success)
3129  break;
3130  }
3131 
3132  switch (Result) {
3133  default: break;
3134  case Match_Success:
3135  if (!validateInstruction(Inst, IDLoc)) {
3136  return true;
3137  }
3138  Inst.setLoc(IDLoc);
3139  Out.EmitInstruction(Inst, getSTI());
3140  return false;
3141 
3142  case Match_MissingFeature:
3143  return Error(IDLoc, "instruction not supported on this GPU");
3144 
3145  case Match_MnemonicFail: {
3146  FeatureBitset FBS = ComputeAvailableFeatures(getSTI().getFeatureBits());
3147  std::string Suggestion = AMDGPUMnemonicSpellCheck(
3148  ((AMDGPUOperand &)*Operands[0]).getToken(), FBS);
3149  return Error(IDLoc, "invalid instruction" + Suggestion,
3150  ((AMDGPUOperand &)*Operands[0]).getLocRange());
3151  }
3152 
3153  case Match_InvalidOperand: {
3154  SMLoc ErrorLoc = IDLoc;
3155  if (ErrorInfo != ~0ULL) {
3156  if (ErrorInfo >= Operands.size()) {
3157  return Error(IDLoc, "too few operands for instruction");
3158  }
3159  ErrorLoc = ((AMDGPUOperand &)*Operands[ErrorInfo]).getStartLoc();
3160  if (ErrorLoc == SMLoc())
3161  ErrorLoc = IDLoc;
3162  }
3163  return Error(ErrorLoc, "invalid operand for instruction");
3164  }
3165 
3166  case Match_PreferE32:
3167  return Error(IDLoc, "internal error: instruction without _e64 suffix "
3168  "should be encoded as e32");
3169  }
3170  llvm_unreachable("Implement any new match types added!");
3171 }
3172 
3173 bool AMDGPUAsmParser::ParseAsAbsoluteExpression(uint32_t &Ret) {
3174  int64_t Tmp = -1;
3175  if (getLexer().isNot(AsmToken::Integer) && getLexer().isNot(AsmToken::Identifier)) {
3176  return true;
3177  }
3178  if (getParser().parseAbsoluteExpression(Tmp)) {
3179  return true;
3180  }
3181  Ret = static_cast<uint32_t>(Tmp);
3182  return false;
3183 }
3184 
3185 bool AMDGPUAsmParser::ParseDirectiveMajorMinor(uint32_t &Major,
3186  uint32_t &Minor) {
3187  if (ParseAsAbsoluteExpression(Major))
3188  return TokError("invalid major version");
3189 
3190  if (getLexer().isNot(AsmToken::Comma))
3191  return TokError("minor version number required, comma expected");
3192  Lex();
3193 
3194  if (ParseAsAbsoluteExpression(Minor))
3195  return TokError("invalid minor version");
3196 
3197  return false;
3198 }
3199 
3200 bool AMDGPUAsmParser::ParseDirectiveAMDGCNTarget() {
3201  if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
3202  return TokError("directive only supported for amdgcn architecture");
3203 
3204  std::string Target;
3205 
3206  SMLoc TargetStart = getTok().getLoc();
3207  if (getParser().parseEscapedString(Target))
3208  return true;
3209  SMRange TargetRange = SMRange(TargetStart, getTok().getLoc());
3210 
3211  std::string ExpectedTarget;
3212  raw_string_ostream ExpectedTargetOS(ExpectedTarget);
3213  IsaInfo::streamIsaVersion(&getSTI(), ExpectedTargetOS);
3214 
3215  if (Target != ExpectedTargetOS.str())
3216  return getParser().Error(TargetRange.Start, "target must match options",
3217  TargetRange);
3218 
3219  getTargetStreamer().EmitDirectiveAMDGCNTarget(Target);
3220  return false;
3221 }
3222 
3223 bool AMDGPUAsmParser::OutOfRangeError(SMRange Range) {
3224  return getParser().Error(Range.Start, "value out of range", Range);
3225 }
3226 
3227 bool AMDGPUAsmParser::calculateGPRBlocks(
3228  const FeatureBitset &Features, bool VCCUsed, bool FlatScrUsed,
3229  bool XNACKUsed, unsigned NextFreeVGPR, SMRange VGPRRange,
3230  unsigned NextFreeSGPR, SMRange SGPRRange, unsigned &VGPRBlocks,
3231  unsigned &SGPRBlocks) {
3232  // TODO(scott.linder): These calculations are duplicated from
3233  // AMDGPUAsmPrinter::getSIProgramInfo and could be unified.
3234  IsaVersion Version = getIsaVersion(getSTI().getCPU());
3235 
3236  unsigned NumVGPRs = NextFreeVGPR;
3237  unsigned NumSGPRs = NextFreeSGPR;
3238 
3239  if (Version.Major >= 10)
3240  NumSGPRs = 0;
3241  else {
3242  unsigned MaxAddressableNumSGPRs =
3244 
3245  if (Version.Major >= 8 && !Features.test(FeatureSGPRInitBug) &&
3246  NumSGPRs > MaxAddressableNumSGPRs)
3247  return OutOfRangeError(SGPRRange);
3248 
3249  NumSGPRs +=
3250  IsaInfo::getNumExtraSGPRs(&getSTI(), VCCUsed, FlatScrUsed, XNACKUsed);
3251 
3252  if ((Version.Major <= 7 || Features.test(FeatureSGPRInitBug)) &&
3253  NumSGPRs > MaxAddressableNumSGPRs)
3254  return OutOfRangeError(SGPRRange);
3255 
3256  if (Features.test(FeatureSGPRInitBug))
3258  }
3259 
3260  VGPRBlocks = IsaInfo::getNumVGPRBlocks(&getSTI(), NumVGPRs);
3261  SGPRBlocks = IsaInfo::getNumSGPRBlocks(&getSTI(), NumSGPRs);
3262 
3263  return false;
3264 }
3265 
3266 bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
3267  if (getSTI().getTargetTriple().getArch() != Triple::amdgcn)
3268  return TokError("directive only supported for amdgcn architecture");
3269 
3270  if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA)
3271  return TokError("directive only supported for amdhsa OS");
3272 
3273  StringRef KernelName;
3274  if (getParser().parseIdentifier(KernelName))
3275  return true;
3276 
3278 
3279  StringSet<> Seen;
3280 
3281  IsaVersion IVersion = getIsaVersion(getSTI().getCPU());
3282 
3283  SMRange VGPRRange;
3284  uint64_t NextFreeVGPR = 0;
3285  SMRange SGPRRange;
3286  uint64_t NextFreeSGPR = 0;
3287  unsigned UserSGPRCount = 0;
3288  bool ReserveVCC = true;
3289  bool ReserveFlatScr = true;
3290  bool ReserveXNACK = hasXNACK();
3291 
3292  while (true) {
3293  while (getLexer().is(AsmToken::EndOfStatement))
3294  Lex();
3295 
3296  if (getLexer().isNot(AsmToken::Identifier))
3297  return TokError("expected .amdhsa_ directive or .end_amdhsa_kernel");
3298 
3299  StringRef ID = getTok().getIdentifier();
3300  SMRange IDRange = getTok().getLocRange();
3301  Lex();
3302 
3303  if (ID == ".end_amdhsa_kernel")
3304  break;
3305 
3306  if (Seen.find(ID) != Seen.end())
3307  return TokError(".amdhsa_ directives cannot be repeated");
3308  Seen.insert(ID);
3309 
3310  SMLoc ValStart = getTok().getLoc();
3311  int64_t IVal;
3312  if (getParser().parseAbsoluteExpression(IVal))
3313  return true;
3314  SMLoc ValEnd = getTok().getLoc();
3315  SMRange ValRange = SMRange(ValStart, ValEnd);
3316 
3317  if (IVal < 0)
3318  return OutOfRangeError(ValRange);
3319 
3320  uint64_t Val = IVal;
3321 
3322 #define PARSE_BITS_ENTRY(FIELD, ENTRY, VALUE, RANGE) \
3323  if (!isUInt<ENTRY##_WIDTH>(VALUE)) \
3324  return OutOfRangeError(RANGE); \
3325  AMDHSA_BITS_SET(FIELD, ENTRY, VALUE);
3326 
3327  if (ID == ".amdhsa_group_segment_fixed_size") {
3328  if (!isUInt<sizeof(KD.group_segment_fixed_size) * CHAR_BIT>(Val))
3329  return OutOfRangeError(ValRange);
3330  KD.group_segment_fixed_size = Val;
3331  } else if (ID == ".amdhsa_private_segment_fixed_size") {
3332  if (!isUInt<sizeof(KD.private_segment_fixed_size) * CHAR_BIT>(Val))
3333  return OutOfRangeError(ValRange);
3334  KD.private_segment_fixed_size = Val;
3335  } else if (ID == ".amdhsa_user_sgpr_private_segment_buffer") {
3337  KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER,
3338  Val, ValRange);
3339  UserSGPRCount += 4;
3340  } else if (ID == ".amdhsa_user_sgpr_dispatch_ptr") {
3342  KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, Val,
3343  ValRange);
3344  UserSGPRCount += 2;
3345  } else if (ID == ".amdhsa_user_sgpr_queue_ptr") {
3347  KERNEL_CODE_PROPERTY_ENABLE_SGPR_QUEUE_PTR, Val,
3348  ValRange);
3349  UserSGPRCount += 2;
3350  } else if (ID == ".amdhsa_user_sgpr_kernarg_segment_ptr") {
3352  KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR,
3353  Val, ValRange);
3354  UserSGPRCount += 2;
3355  } else if (ID == ".amdhsa_user_sgpr_dispatch_id") {
3357  KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_ID, Val,
3358  ValRange);
3359  UserSGPRCount += 2;
3360  } else if (ID == ".amdhsa_user_sgpr_flat_scratch_init") {
3362  KERNEL_CODE_PROPERTY_ENABLE_SGPR_FLAT_SCRATCH_INIT, Val,
3363  ValRange);
3364  UserSGPRCount += 2;
3365  } else if (ID == ".amdhsa_user_sgpr_private_segment_size") {
3367  KERNEL_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_SIZE,
3368  Val, ValRange);
3369  UserSGPRCount += 1;
3370  } else if (ID == ".amdhsa_system_sgpr_private_segment_wavefront_offset") {
3372  KD.compute_pgm_rsrc2,
3373  COMPUTE_PGM_RSRC2_ENABLE_SGPR_PRIVATE_SEGMENT_WAVEFRONT_OFFSET, Val,
3374  ValRange);
3375  } else if (ID == ".amdhsa_system_sgpr_workgroup_id_x") {
3377  COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, Val,
3378  ValRange);
3379  } else if (ID == ".amdhsa_system_sgpr_workgroup_id_y") {
3381  COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, Val,
3382  ValRange);
3383  } else if (ID == ".amdhsa_system_sgpr_workgroup_id_z") {
3385  COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Z, Val,
3386  ValRange);
3387  } else if (ID == ".amdhsa_system_sgpr_workgroup_info") {
3389  COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_INFO, Val,
3390  ValRange);
3391  } else if (ID == ".amdhsa_system_vgpr_workitem_id") {
3393  COMPUTE_PGM_RSRC2_ENABLE_VGPR_WORKITEM_ID, Val,
3394  ValRange);
3395  } else if (ID == ".amdhsa_next_free_vgpr") {
3396  VGPRRange = ValRange;
3397  NextFreeVGPR = Val;
3398  } else if (ID == ".amdhsa_next_free_sgpr") {
3399  SGPRRange = ValRange;
3400  NextFreeSGPR = Val;
3401  } else if (ID == ".amdhsa_reserve_vcc") {
3402  if (!isUInt<1>(Val))
3403  return OutOfRangeError(ValRange);
3404  ReserveVCC = Val;
3405  } else if (ID == ".amdhsa_reserve_flat_scratch") {
3406  if (IVersion.Major < 7)
3407  return getParser().Error(IDRange.Start, "directive requires gfx7+",
3408  IDRange);
3409  if (!isUInt<1>(Val))
3410  return OutOfRangeError(ValRange);
3411  ReserveFlatScr = Val;
3412  } else if (ID == ".amdhsa_reserve_xnack_mask") {
3413  if (IVersion.Major < 8)
3414  return getParser().Error(IDRange.Start, "directive requires gfx8+",
3415  IDRange);
3416  if (!isUInt<1>(Val))
3417  return OutOfRangeError(ValRange);
3418  ReserveXNACK = Val;
3419  } else if (ID == ".amdhsa_float_round_mode_32") {
3421  COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_32, Val, ValRange);
3422  } else if (ID == ".amdhsa_float_round_mode_16_64") {
3424  COMPUTE_PGM_RSRC1_FLOAT_ROUND_MODE_16_64, Val, ValRange);
3425  } else if (ID == ".amdhsa_float_denorm_mode_32") {
3427  COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_32, Val, ValRange);
3428  } else if (ID == ".amdhsa_float_denorm_mode_16_64") {
3430  COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val,
3431  ValRange);
3432  } else if (ID == ".amdhsa_dx10_clamp") {
3434  COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, Val, ValRange);
3435  } else if (ID == ".amdhsa_ieee_mode") {
3436  PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE,
3437  Val, ValRange);
3438  } else if (ID == ".amdhsa_fp16_overflow") {
3439  if (IVersion.Major < 9)
3440  return getParser().Error(IDRange.Start, "directive requires gfx9+",
3441  IDRange);
3442  PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FP16_OVFL, Val,
3443  ValRange);
3444  } else if (ID == ".amdhsa_workgroup_processor_mode") {
3445  if (IVersion.Major < 10)
3446  return getParser().Error(IDRange.Start, "directive requires gfx10+",
3447  IDRange);
3448  PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE, Val,
3449  ValRange);
3450  } else if (ID == ".amdhsa_memory_ordered") {
3451  if (IVersion.Major < 10)
3452  return getParser().Error(IDRange.Start, "directive requires gfx10+",
3453  IDRange);
3454  PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED, Val,
3455  ValRange);
3456  } else if (ID == ".amdhsa_forward_progress") {
3457  if (IVersion.Major < 10)
3458  return getParser().Error(IDRange.Start, "directive requires gfx10+",
3459  IDRange);
3460  PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS, Val,
3461  ValRange);
3462  } else if (ID == ".amdhsa_exception_fp_ieee_invalid_op") {
3464  KD.compute_pgm_rsrc2,
3465  COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION, Val,
3466  ValRange);
3467  } else if (ID == ".amdhsa_exception_fp_denorm_src") {
3469  COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_FP_DENORMAL_SOURCE,
3470  Val, ValRange);
3471  } else if (ID == ".amdhsa_exception_fp_ieee_div_zero") {
3473  KD.compute_pgm_rsrc2,
3474  COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO, Val,
3475  ValRange);
3476  } else if (ID == ".amdhsa_exception_fp_ieee_overflow") {
3478  COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW,
3479  Val, ValRange);
3480  } else if (ID == ".amdhsa_exception_fp_ieee_underflow") {
3482  COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW,
3483  Val, ValRange);
3484  } else if (ID == ".amdhsa_exception_fp_ieee_inexact") {
3486  COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_IEEE_754_FP_INEXACT,
3487  Val, ValRange);
3488  } else if (ID == ".amdhsa_exception_int_div_zero") {
3490  COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
3491  Val, ValRange);
3492  } else {
3493  return getParser().Error(IDRange.Start,
3494  "unknown .amdhsa_kernel directive", IDRange);
3495  }
3496 
3497 #undef PARSE_BITS_ENTRY
3498  }
3499 
3500  if (Seen.find(".amdhsa_next_free_vgpr") == Seen.end())
3501  return TokError(".amdhsa_next_free_vgpr directive is required");
3502 
3503  if (Seen.find(".amdhsa_next_free_sgpr") == Seen.end())
3504  return TokError(".amdhsa_next_free_sgpr directive is required");
3505 
3506  unsigned VGPRBlocks;
3507  unsigned SGPRBlocks;
3508  if (calculateGPRBlocks(getFeatureBits(), ReserveVCC, ReserveFlatScr,
3509  ReserveXNACK, NextFreeVGPR, VGPRRange, NextFreeSGPR,
3510  SGPRRange, VGPRBlocks, SGPRBlocks))
3511  return true;
3512 
3513  if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT_WIDTH>(
3514  VGPRBlocks))
3515  return OutOfRangeError(VGPRRange);
3517  COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, VGPRBlocks);
3518 
3519  if (!isUInt<COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT_WIDTH>(
3520  SGPRBlocks))
3521  return OutOfRangeError(SGPRRange);
3523  COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT,
3524  SGPRBlocks);
3525 
3526  if (!isUInt<COMPUTE_PGM_RSRC2_USER_SGPR_COUNT_WIDTH>(UserSGPRCount))
3527  return TokError("too many user SGPRs enabled");
3528  AMDHSA_BITS_SET(KD.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT,
3529  UserSGPRCount);
3530 
3531  getTargetStreamer().EmitAmdhsaKernelDescriptor(
3532  getSTI(), KernelName, KD, NextFreeVGPR, NextFreeSGPR, ReserveVCC,
3533  ReserveFlatScr, ReserveXNACK);
3534  return false;
3535 }
3536 
3537 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectVersion() {
3538  uint32_t Major;
3539  uint32_t Minor;
3540 
3541  if (ParseDirectiveMajorMinor(Major, Minor))
3542  return true;
3543 
3544  getTargetStreamer().EmitDirectiveHSACodeObjectVersion(Major, Minor);
3545  return false;
3546 }
3547 
3548 bool AMDGPUAsmParser::ParseDirectiveHSACodeObjectISA() {
3549  uint32_t Major;
3550  uint32_t Minor;
3551  uint32_t Stepping;
3552  StringRef VendorName;
3553  StringRef ArchName;
3554 
3555  // If this directive has no arguments, then use the ISA version for the
3556  // targeted GPU.
3557  if (getLexer().is(AsmToken::EndOfStatement)) {
3558  AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
3559  getTargetStreamer().EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor,
3560  ISA.Stepping,
3561  "AMD", "AMDGPU");
3562  return false;
3563  }
3564 
3565  if (ParseDirectiveMajorMinor(Major, Minor))
3566  return true;
3567 
3568  if (getLexer().isNot(AsmToken::Comma))
3569  return TokError("stepping version number required, comma expected");
3570  Lex();
3571 
3572  if (ParseAsAbsoluteExpression(Stepping))
3573  return TokError("invalid stepping version");
3574 
3575  if (getLexer().isNot(AsmToken::Comma))
3576  return TokError("vendor name required, comma expected");
3577  Lex();
3578 
3579  if (getLexer().isNot(AsmToken::String))
3580  return TokError("invalid vendor name");
3581 
3582  VendorName = getLexer().getTok().getStringContents();
3583  Lex();
3584 
3585  if (getLexer().isNot(AsmToken::Comma))
3586  return TokError("arch name required, comma expected");
3587  Lex();
3588 
3589  if (getLexer().isNot(AsmToken::String))
3590  return TokError("invalid arch name");
3591 
3592  ArchName = getLexer().getTok().getStringContents();
3593  Lex();
3594 
3595  getTargetStreamer().EmitDirectiveHSACodeObjectISA(Major, Minor, Stepping,
3596  VendorName, ArchName);
3597  return false;
3598 }
3599 
3600 bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
3601  amd_kernel_code_t &Header) {
3602  // max_scratch_backing_memory_byte_size is deprecated. Ignore it while parsing
3603  // assembly for backwards compatibility.
3604  if (ID == "max_scratch_backing_memory_byte_size") {
3605  Parser.eatToEndOfStatement();
3606  return false;
3607  }
3608 
3609  SmallString<40> ErrStr;
3610  raw_svector_ostream Err(ErrStr);
3611  if (!parseAmdKernelCodeField(ID, getParser(), Header, Err)) {
3612  return TokError(Err.str());
3613  }
3614  Lex();
3615 
3616  if (ID == "enable_wgp_mode") {
3618  return TokError("enable_wgp_mode=1 is only allowed on GFX10+");
3619  }
3620 
3621  if (ID == "enable_mem_ordered") {
3623  return TokError("enable_mem_ordered=1 is only allowed on GFX10+");
3624  }
3625 
3626  if (ID == "enable_fwd_progress") {
3628  return TokError("enable_fwd_progress=1 is only allowed on GFX10+");
3629  }
3630 
3631  return false;
3632 }
3633 
3634 bool AMDGPUAsmParser::ParseDirectiveAMDKernelCodeT() {
3635  amd_kernel_code_t Header;
3636  AMDGPU::initDefaultAMDKernelCodeT(Header, &getSTI());
3637 
3638  while (true) {
3639  // Lex EndOfStatement. This is in a while loop, because lexing a comment
3640  // will set the current token to EndOfStatement.
3641  while(getLexer().is(AsmToken::EndOfStatement))
3642  Lex();
3643 
3644  if (getLexer().isNot(AsmToken::Identifier))
3645  return TokError("expected value identifier or .end_amd_kernel_code_t");
3646 
3647  StringRef ID = getLexer().getTok().getIdentifier();
3648  Lex();
3649 
3650  if (ID == ".end_amd_kernel_code_t")
3651  break;
3652 
3653  if (ParseAMDKernelCodeTValue(ID, Header))
3654  return true;
3655  }
3656 
3657  getTargetStreamer().EmitAMDKernelCodeT(Header);
3658 
3659  return false;
3660 }
3661 
3662 bool AMDGPUAsmParser::ParseDirectiveAMDGPUHsaKernel() {
3663  if (getLexer().isNot(AsmToken::Identifier))
3664  return TokError("expected symbol name");
3665 
3666  StringRef KernelName = Parser.getTok().getString();
3667 
3668  getTargetStreamer().EmitAMDGPUSymbolType(KernelName,
3670  Lex();
3671  if (!AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI()))
3672  KernelScope.initialize(getContext());
3673  return false;
3674 }
3675 
3676 bool AMDGPUAsmParser::ParseDirectiveISAVersion() {
3677  if (getSTI().getTargetTriple().getArch() != Triple::amdgcn) {
3678  return Error(getParser().getTok().getLoc(),
3679  ".amd_amdgpu_isa directive is not available on non-amdgcn "
3680  "architectures");
3681  }
3682 
3683  auto ISAVersionStringFromASM = getLexer().getTok().getStringContents();
3684 
3685  std::string ISAVersionStringFromSTI;
3686  raw_string_ostream ISAVersionStreamFromSTI(ISAVersionStringFromSTI);
3687  IsaInfo::streamIsaVersion(&getSTI(), ISAVersionStreamFromSTI);
3688 
3689  if (ISAVersionStringFromASM != ISAVersionStreamFromSTI.str()) {
3690  return Error(getParser().getTok().getLoc(),
3691  ".amd_amdgpu_isa directive does not match triple and/or mcpu "
3692  "arguments specified through the command line");
3693  }
3694 
3695  getTargetStreamer().EmitISAVersion(ISAVersionStreamFromSTI.str());
3696  Lex();
3697 
3698  return false;
3699 }
3700 
3701 bool AMDGPUAsmParser::ParseDirectiveHSAMetadata() {
3702  const char *AssemblerDirectiveBegin;
3703  const char *AssemblerDirectiveEnd;
3704  std::tie(AssemblerDirectiveBegin, AssemblerDirectiveEnd) =
3706  ? std::make_tuple(HSAMD::V3::AssemblerDirectiveBegin,
3708  : std::make_tuple(HSAMD::AssemblerDirectiveBegin,
3710 
3711  if (getSTI().getTargetTriple().getOS() != Triple::AMDHSA) {
3712  return Error(getParser().getTok().getLoc(),
3713  (Twine(AssemblerDirectiveBegin) + Twine(" directive is "
3714  "not available on non-amdhsa OSes")).str());
3715  }
3716 
3717  std::string HSAMetadataString;
3718  if (ParseToEndDirective(AssemblerDirectiveBegin, AssemblerDirectiveEnd,
3719  HSAMetadataString))
3720  return true;
3721 
3722  if (IsaInfo::hasCodeObjectV3(&getSTI())) {
3723  if (!getTargetStreamer().EmitHSAMetadataV3(HSAMetadataString))
3724  return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
3725  } else {
3726  if (!getTargetStreamer().EmitHSAMetadataV2(HSAMetadataString))
3727  return Error(getParser().getTok().getLoc(), "invalid HSA metadata");
3728  }
3729 
3730  return false;
3731 }
3732 
3733 /// Common code to parse out a block of text (typically YAML) between start and
3734 /// end directives.
3735 bool AMDGPUAsmParser::ParseToEndDirective(const char *AssemblerDirectiveBegin,
3736  const char *AssemblerDirectiveEnd,
3737  std::string &CollectString) {
3738 
3739  raw_string_ostream CollectStream(CollectString);
3740 
3741  getLexer().setSkipSpace(false);
3742 
3743  bool FoundEnd = false;
3744  while (!getLexer().is(AsmToken::Eof)) {
3745  while (getLexer().is(AsmToken::Space)) {
3746  CollectStream << getLexer().getTok().getString();
3747  Lex();
3748  }
3749 
3750  if (getLexer().is(AsmToken::Identifier)) {
3751  StringRef ID = getLexer().getTok().getIdentifier();
3752  if (ID == AssemblerDirectiveEnd) {
3753  Lex();
3754  FoundEnd = true;
3755  break;
3756  }
3757  }
3758 
3759  CollectStream << Parser.parseStringToEndOfStatement()
3760  << getContext().getAsmInfo()->getSeparatorString();
3761 
3762  Parser.eatToEndOfStatement();
3763  }
3764 
3765  getLexer().setSkipSpace(true);
3766 
3767  if (getLexer().is(AsmToken::Eof) && !FoundEnd) {
3768  return TokError(Twine("expected directive ") +
3769  Twine(AssemblerDirectiveEnd) + Twine(" not found"));
3770  }
3771 
3772  CollectStream.flush();
3773  return false;
3774 }
3775 
3776 /// Parse the assembler directive for new MsgPack-format PAL metadata.
3777 bool AMDGPUAsmParser::ParseDirectivePALMetadataBegin() {
3778  std::string String;
3779  if (ParseToEndDirective(AMDGPU::PALMD::AssemblerDirectiveBegin,
3781  return true;
3782 
3783  auto PALMetadata = getTargetStreamer().getPALMetadata();
3784  if (!PALMetadata->setFromString(String))
3785  return Error(getParser().getTok().getLoc(), "invalid PAL metadata");
3786  return false;
3787 }
3788 
3789 /// Parse the assembler directive for old linear-format PAL metadata.
3790 bool AMDGPUAsmParser::ParseDirectivePALMetadata() {
3791  if (getSTI().getTargetTriple().getOS() != Triple::AMDPAL) {
3792  return Error(getParser().getTok().getLoc(),
3793  (Twine(PALMD::AssemblerDirective) + Twine(" directive is "
3794  "not available on non-amdpal OSes")).str());
3795  }
3796 
3797  auto PALMetadata = getTargetStreamer().getPALMetadata();
3798  PALMetadata->setLegacy();
3799  for (;;) {
3800  uint32_t Key, Value;
3801  if (ParseAsAbsoluteExpression(Key)) {
3802  return TokError(Twine("invalid value in ") +
3804  }
3805  if (getLexer().isNot(AsmToken::Comma)) {
3806  return TokError(Twine("expected an even number of values in ") +
3808  }
3809  Lex();
3810  if (ParseAsAbsoluteExpression(Value)) {
3811  return TokError(Twine("invalid value in ") +
3813  }
3814  PALMetadata->setRegister(Key, Value);
3815  if (getLexer().isNot(AsmToken::Comma))
3816  break;
3817  Lex();
3818  }
3819  return false;
3820 }
3821 
3822 bool AMDGPUAsmParser::ParseDirective(AsmToken DirectiveID) {
3823  StringRef IDVal = DirectiveID.getString();
3824 
3825  if (AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())) {
3826  if (IDVal == ".amdgcn_target")
3827  return ParseDirectiveAMDGCNTarget();
3828 
3829  if (IDVal == ".amdhsa_kernel")
3830  return ParseDirectiveAMDHSAKernel();
3831 
3832  // TODO: Restructure/combine with PAL metadata directive.
3834  return ParseDirectiveHSAMetadata();
3835  } else {
3836  if (IDVal == ".hsa_code_object_version")
3837  return ParseDirectiveHSACodeObjectVersion();
3838 
3839  if (IDVal == ".hsa_code_object_isa")
3840  return ParseDirectiveHSACodeObjectISA();
3841 
3842  if (IDVal == ".amd_kernel_code_t")
3843  return ParseDirectiveAMDKernelCodeT();
3844 
3845  if (IDVal == ".amdgpu_hsa_kernel")
3846  return ParseDirectiveAMDGPUHsaKernel();
3847 
3848  if (IDVal == ".amd_amdgpu_isa")
3849  return ParseDirectiveISAVersion();
3850 
3852  return ParseDirectiveHSAMetadata();
3853  }
3854 
3855  if (IDVal == PALMD::AssemblerDirectiveBegin)
3856  return ParseDirectivePALMetadataBegin();
3857 
3858  if (IDVal == PALMD::AssemblerDirective)
3859  return ParseDirectivePALMetadata();
3860 
3861  return true;
3862 }
3863 
3864 bool AMDGPUAsmParser::subtargetHasRegister(const MCRegisterInfo &MRI,
3865  unsigned RegNo) const {
3866 
3867  for (MCRegAliasIterator R(AMDGPU::TTMP12_TTMP13_TTMP14_TTMP15, &MRI, true);
3868  R.isValid(); ++R) {
3869  if (*R == RegNo)
3870  return isGFX9() || isGFX10();
3871  }
3872 
3873  // GFX10 has 2 more SGPRs 104 and 105.
3874  for (MCRegAliasIterator R(AMDGPU::SGPR104_SGPR105, &MRI, true);
3875  R.isValid(); ++R) {
3876  if (*R == RegNo)
3877  return hasSGPR104_SGPR105();
3878  }
3879 
3880  switch (RegNo) {
3881  case AMDGPU::TBA:
3882  case AMDGPU::TBA_LO:
3883  case AMDGPU::TBA_HI:
3884  case AMDGPU::TMA:
3885  case AMDGPU::TMA_LO:
3886  case AMDGPU::TMA_HI:
3887  return !isGFX9() && !isGFX10();
3888  case AMDGPU::XNACK_MASK:
3889  case AMDGPU::XNACK_MASK_LO:
3890  case AMDGPU::XNACK_MASK_HI:
3891  return !isCI() && !isSI() && !isGFX10() && hasXNACK();
3892  case AMDGPU::SGPR_NULL:
3893  return isGFX10();
3894  default:
3895  break;
3896  }
3897 
3898  if (isInlineValue(RegNo))
3899  return !isCI() && !isSI() && !isVI();
3900 
3901  if (isCI())
3902  return true;
3903 
3904  if (isSI() || isGFX10()) {
3905  // No flat_scr on SI.
3906  // On GFX10 flat scratch is not a valid register operand and can only be
3907  // accessed with s_setreg/s_getreg.
3908  switch (RegNo) {
3909  case AMDGPU::FLAT_SCR:
3910  case AMDGPU::FLAT_SCR_LO:
3911  case AMDGPU::FLAT_SCR_HI:
3912  return false;
3913  default:
3914  return true;
3915  }
3916  }
3917 
3918  // VI only has 102 SGPRs, so make sure we aren't trying to use the 2 more that
3919  // SI/CI have.
3920  for (MCRegAliasIterator R(AMDGPU::SGPR102_SGPR103, &MRI, true);
3921  R.isValid(); ++R) {
3922  if (*R == RegNo)
3923  return hasSGPR102_SGPR103();
3924  }
3925 
3926  return true;
3927 }
3928 
3930 AMDGPUAsmParser::parseOperand(OperandVector &Operands, StringRef Mnemonic,
3931  OperandMode Mode) {
3932  // Try to parse with a custom parser
3933  OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
3934 
3935  // If we successfully parsed the operand or if there as an error parsing,
3936  // we are done.
3937  //
3938  // If we are parsing after we reach EndOfStatement then this means we
3939  // are appending default values to the Operands list. This is only done
3940  // by custom parser, so we shouldn't continue on to the generic parsing.
3941  if (ResTy == MatchOperand_Success || ResTy == MatchOperand_ParseFail ||
3942  getLexer().is(AsmToken::EndOfStatement))
3943  return ResTy;
3944 
3945  if (Mode == OperandMode_NSA && getLexer().is(AsmToken::LBrac)) {
3946  unsigned Prefix = Operands.size();
3947  SMLoc LBraceLoc = getTok().getLoc();
3948  Parser.Lex(); // eat the '['
3949 
3950  for (;;) {
3951  ResTy = parseReg(Operands);
3952  if (ResTy != MatchOperand_Success)
3953  return ResTy;
3954 
3955  if (getLexer().is(AsmToken::RBrac))
3956  break;
3957 
3958  if (getLexer().isNot(AsmToken::Comma))
3959  return MatchOperand_ParseFail;
3960  Parser.Lex();
3961  }
3962 
3963  if (Operands.size() - Prefix > 1) {
3964  Operands.insert(Operands.begin() + Prefix,
3965  AMDGPUOperand::CreateToken(this, "[", LBraceLoc));
3966  Operands.push_back(AMDGPUOperand::CreateToken(this, "]",
3967  getTok().getLoc()));
3968  }
3969 
3970  Parser.Lex(); // eat the ']'
3971  return MatchOperand_Success;
3972  }
3973 
3974  return parseRegOrImm(Operands);
3975 }
3976 
3977 StringRef AMDGPUAsmParser::parseMnemonicSuffix(StringRef Name) {
3978  // Clear any forced encodings from the previous instruction.
3979  setForcedEncodingSize(0);
3980  setForcedDPP(false);
3981  setForcedSDWA(false);
3982 
3983  if (Name.endswith("_e64")) {
3984  setForcedEncodingSize(64);
3985  return Name.substr(0, Name.size() - 4);
3986  } else if (Name.endswith("_e32")) {
3987  setForcedEncodingSize(32);
3988  return Name.substr(0, Name.size() - 4);
3989  } else if (Name.endswith("_dpp")) {
3990  setForcedDPP(true);
3991  return Name.substr(0, Name.size() - 4);
3992  } else if (Name.endswith("_sdwa")) {
3993  setForcedSDWA(true);
3994  return Name.substr(0, Name.size() - 5);
3995  }
3996  return Name;
3997 }
3998 
3999 bool AMDGPUAsmParser::ParseInstruction(ParseInstructionInfo &Info,
4000  StringRef Name,
4001  SMLoc NameLoc, OperandVector &Operands) {
4002  // Add the instruction mnemonic
4003  Name = parseMnemonicSuffix(Name);
4004  Operands.push_back(AMDGPUOperand::CreateToken(this, Name, NameLoc));
4005 
4006  bool IsMIMG = Name.startswith("image_");
4007 
4008  while (!getLexer().is(AsmToken::EndOfStatement)) {
4009  OperandMode Mode = OperandMode_Default;
4010  if (IsMIMG && isGFX10() && Operands.size() == 2)
4011  Mode = OperandMode_NSA;
4012  OperandMatchResultTy Res = parseOperand(Operands, Name, Mode);
4013 
4014  // Eat the comma or space if there is one.
4015  if (getLexer().is(AsmToken::Comma))
4016  Parser.Lex();
4017 
4018  switch (Res) {
4019  case MatchOperand_Success: break;
4021  Error(getLexer().getLoc(), "failed parsing operand.");
4022  while (!getLexer().is(AsmToken::EndOfStatement)) {
4023  Parser.Lex();
4024  }
4025  return true;
4026  case MatchOperand_NoMatch:
4027  Error(getLexer().getLoc(), "not a valid operand.");
4028  while (!getLexer().is(AsmToken::EndOfStatement)) {
4029  Parser.Lex();
4030  }
4031  return true;
4032  }
4033  }
4034 
4035  return false;
4036 }
4037 
4038 //===----------------------------------------------------------------------===//
4039 // Utility functions
4040 //===----------------------------------------------------------------------===//
4041 
4043 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, int64_t &IntVal) {
4044 
4045  if (!trySkipId(Prefix, AsmToken::Colon))
4046  return MatchOperand_NoMatch;
4047 
4048  return parseExpr(IntVal) ? MatchOperand_Success : MatchOperand_ParseFail;
4049 }
4050 
4052 AMDGPUAsmParser::parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
4053  AMDGPUOperand::ImmTy ImmTy,
4054  bool (*ConvertResult)(int64_t&)) {
4055  SMLoc S = getLoc();
4056  int64_t Value = 0;
4057 
4058  OperandMatchResultTy Res = parseIntWithPrefix(Prefix, Value);
4059  if (Res != MatchOperand_Success)
4060  return Res;
4061 
4062  if (ConvertResult && !ConvertResult(Value)) {
4063  Error(S, "invalid " + StringRef(Prefix) + " value.");
4064  }
4065 
4066  Operands.push_back(AMDGPUOperand::CreateImm(this, Value, S, ImmTy));
4067  return MatchOperand_Success;
4068 }
4069 
4070 OperandMatchResultTy AMDGPUAsmParser::parseOperandArrayWithPrefix(
4071  const char *Prefix,
4072  OperandVector &Operands,
4073  AMDGPUOperand::ImmTy ImmTy,
4074  bool (*ConvertResult)(int64_t&)) {
4075  StringRef Name = Parser.getTok().getString();
4076  if (!Name.equals(Prefix))
4077  return MatchOperand_NoMatch;
4078 
4079  Parser.Lex();
4080  if (getLexer().isNot(AsmToken::Colon))
4081  return MatchOperand_ParseFail;
4082 
4083  Parser.Lex();
4084  if (getLexer().isNot(AsmToken::LBrac))
4085  return MatchOperand_ParseFail;
4086  Parser.Lex();
4087 
4088  unsigned Val = 0;
4089  SMLoc S = Parser.getTok().getLoc();
4090 
4091  // FIXME: How to verify the number of elements matches the number of src
4092  // operands?
4093  for (int I = 0; I < 4; ++I) {
4094  if (I != 0) {
4095  if (getLexer().is(AsmToken::RBrac))
4096  break;
4097 
4098  if (getLexer().isNot(AsmToken::Comma))
4099  return MatchOperand_ParseFail;
4100  Parser.Lex();
4101  }
4102 
4103  if (getLexer().isNot(AsmToken::Integer))
4104  return MatchOperand_ParseFail;
4105 
4106  int64_t Op;
4107  if (getParser().parseAbsoluteExpression(Op))
4108  return MatchOperand_ParseFail;
4109 
4110  if (Op != 0 && Op != 1)
4111  return MatchOperand_ParseFail;
4112  Val |= (Op << I);
4113  }
4114 
4115  Parser.Lex();
4116  Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S, ImmTy));
4117  return MatchOperand_Success;
4118 }
4119 
4121 AMDGPUAsmParser::parseNamedBit(const char *Name, OperandVector &Operands,
4122  AMDGPUOperand::ImmTy ImmTy) {
4123  int64_t Bit = 0;
4124  SMLoc S = Parser.getTok().getLoc();
4125 
4126  // We are at the end of the statement, and this is a default argument, so
4127  // use a default value.
4128  if (getLexer().isNot(AsmToken::EndOfStatement)) {
4129  switch(getLexer().getKind()) {
4130  case AsmToken::Identifier: {
4131  StringRef Tok = Parser.getTok().getString();
4132  if (Tok == Name) {
4133  if (Tok == "r128" && isGFX9())
4134  Error(S, "r128 modifier is not supported on this GPU");
4135  if (Tok == "a16" && !isGFX9())
4136  Error(S, "a16 modifier is not supported on this GPU");
4137  Bit = 1;
4138  Parser.Lex();
4139  } else if (Tok.startswith("no") && Tok.endswith(Name)) {
4140  Bit = 0;
4141  Parser.Lex();
4142  } else {
4143  return MatchOperand_NoMatch;
4144  }
4145  break;
4146  }
4147  default:
4148  return MatchOperand_NoMatch;
4149  }
4150  }
4151 
4152  if (!isGFX10() && ImmTy == AMDGPUOperand::ImmTyDLC)
4153  return MatchOperand_ParseFail;
4154 
4155  Operands.push_back(AMDGPUOperand::CreateImm(this, Bit, S, ImmTy));
4156  return MatchOperand_Success;
4157 }
4158 
4160  MCInst& Inst, const OperandVector& Operands,
4161  AMDGPUAsmParser::OptionalImmIndexMap& OptionalIdx,
4162  AMDGPUOperand::ImmTy ImmT,
4163  int64_t Default = 0) {
4164  auto i = OptionalIdx.find(ImmT);
4165  if (i != OptionalIdx.end()) {
4166  unsigned Idx = i->second;
4167  ((AMDGPUOperand &)*Operands[Idx]).addImmOperands(Inst, 1);
4168  } else {
4169  Inst.addOperand(MCOperand::createImm(Default));
4170  }
4171 }
4172 
4174 AMDGPUAsmParser::parseStringWithPrefix(StringRef Prefix, StringRef &Value) {
4175  if (getLexer().isNot(AsmToken::Identifier)) {
4176  return MatchOperand_NoMatch;
4177  }
4178  StringRef Tok = Parser.getTok().getString();
4179  if (Tok != Prefix) {
4180  return MatchOperand_NoMatch;
4181  }
4182 
4183  Parser.Lex();
4184  if (getLexer().isNot(AsmToken::Colon)) {
4185  return MatchOperand_ParseFail;
4186  }
4187 
4188  Parser.Lex();
4189  if (getLexer().isNot(AsmToken::Identifier)) {
4190  return MatchOperand_ParseFail;
4191  }
4192 
4193  Value = Parser.getTok().getString();
4194  return MatchOperand_Success;
4195 }
4196 
4197 // dfmt and nfmt (in a tbuffer instruction) are parsed as one to allow their
4198 // values to live in a joint format operand in the MCInst encoding.
4200 AMDGPUAsmParser::parseDfmtNfmt(OperandVector &Operands) {
4201  SMLoc S = Parser.getTok().getLoc();
4202  int64_t Dfmt = 0, Nfmt = 0;
4203  // dfmt and nfmt can appear in either order, and each is optional.
4204  bool GotDfmt = false, GotNfmt = false;
4205  while (!GotDfmt || !GotNfmt) {
4206  if (!GotDfmt) {
4207  auto Res = parseIntWithPrefix("dfmt", Dfmt);
4208  if (Res != MatchOperand_NoMatch) {
4209  if (Res != MatchOperand_Success)
4210  return Res;
4211  if (Dfmt >= 16) {
4212  Error(Parser.getTok().getLoc(), "out of range dfmt");
4213  return MatchOperand_ParseFail;
4214  }
4215  GotDfmt = true;
4216  Parser.Lex();
4217  continue;
4218  }
4219  }
4220  if (!GotNfmt) {
4221  auto Res = parseIntWithPrefix("nfmt", Nfmt);
4222  if (Res != MatchOperand_NoMatch) {
4223  if (Res != MatchOperand_Success)
4224  return Res;
4225  if (Nfmt >= 8) {
4226  Error(Parser.getTok().getLoc(), "out of range nfmt");
4227  return MatchOperand_ParseFail;
4228  }
4229  GotNfmt = true;
4230  Parser.Lex();
4231  continue;
4232  }
4233  }
4234  break;
4235  }
4236  if (!GotDfmt && !GotNfmt)
4237  return MatchOperand_NoMatch;
4238  auto Format = Dfmt | Nfmt << 4;
4239  Operands.push_back(
4240  AMDGPUOperand::CreateImm(this, Format, S, AMDGPUOperand::ImmTyFORMAT));
4241  return MatchOperand_Success;
4242 }
4243 
4244 //===----------------------------------------------------------------------===//
4245 // ds
4246 //===----------------------------------------------------------------------===//
4247 
4248 void AMDGPUAsmParser::cvtDSOffset01(MCInst &Inst,
4249  const OperandVector &Operands) {
4250  OptionalImmIndexMap OptionalIdx;
4251 
4252  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4253  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4254 
4255  // Add the register arguments
4256  if (Op.isReg()) {
4257  Op.addRegOperands(Inst, 1);
4258  continue;
4259  }
4260 
4261  // Handle optional arguments
4262  OptionalIdx[Op.getImmTy()] = i;
4263  }
4264 
4265  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset0);
4266  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset1);
4267  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
4268 
4269  Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
4270 }
4271 
4272 void AMDGPUAsmParser::cvtDSImpl(MCInst &Inst, const OperandVector &Operands,
4273  bool IsGdsHardcoded) {
4274  OptionalImmIndexMap OptionalIdx;
4275 
4276  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4277  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4278 
4279  // Add the register arguments
4280  if (Op.isReg()) {
4281  Op.addRegOperands(Inst, 1);
4282  continue;
4283  }
4284 
4285  if (Op.isToken() && Op.getToken() == "gds") {
4286  IsGdsHardcoded = true;
4287  continue;
4288  }
4289 
4290  // Handle optional arguments
4291  OptionalIdx[Op.getImmTy()] = i;
4292  }
4293 
4294  AMDGPUOperand::ImmTy OffsetType =
4295  (Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_gfx10 ||
4296  Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_gfx6_gfx7 ||
4297  Inst.getOpcode() == AMDGPU::DS_SWIZZLE_B32_vi) ? AMDGPUOperand::ImmTySwizzle :
4298  AMDGPUOperand::ImmTyOffset;
4299 
4300  addOptionalImmOperand(Inst, Operands, OptionalIdx, OffsetType);
4301 
4302  if (!IsGdsHardcoded) {
4303  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGDS);
4304  }
4305  Inst.addOperand(MCOperand::createReg(AMDGPU::M0)); // m0
4306 }
4307 
4308 void AMDGPUAsmParser::cvtExp(MCInst &Inst, const OperandVector &Operands) {
4309  OptionalImmIndexMap OptionalIdx;
4310 
4311  unsigned OperandIdx[4];
4312  unsigned EnMask = 0;
4313  int SrcIdx = 0;
4314 
4315  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4316  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4317 
4318  // Add the register arguments
4319  if (Op.isReg()) {
4320  assert(SrcIdx < 4);
4321  OperandIdx[SrcIdx] = Inst.size();
4322  Op.addRegOperands(Inst, 1);
4323  ++SrcIdx;
4324  continue;
4325  }
4326 
4327  if (Op.isOff()) {
4328  assert(SrcIdx < 4);
4329  OperandIdx[SrcIdx] = Inst.size();
4330  Inst.addOperand(MCOperand::createReg(AMDGPU::NoRegister));
4331  ++SrcIdx;
4332  continue;
4333  }
4334 
4335  if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyExpTgt) {
4336  Op.addImmOperands(Inst, 1);
4337  continue;
4338  }
4339 
4340  if (Op.isToken() && Op.getToken() == "done")
4341  continue;
4342 
4343  // Handle optional arguments
4344  OptionalIdx[Op.getImmTy()] = i;
4345  }
4346 
4347  assert(SrcIdx == 4);
4348 
4349  bool Compr = false;
4350  if (OptionalIdx.find(AMDGPUOperand::ImmTyExpCompr) != OptionalIdx.end()) {
4351  Compr = true;
4352  Inst.getOperand(OperandIdx[1]) = Inst.getOperand(OperandIdx[2]);
4353  Inst.getOperand(OperandIdx[2]).setReg(AMDGPU::NoRegister);
4354  Inst.getOperand(OperandIdx[3]).setReg(AMDGPU::NoRegister);
4355  }
4356 
4357  for (auto i = 0; i < SrcIdx; ++i) {
4358  if (Inst.getOperand(OperandIdx[i]).getReg() != AMDGPU::NoRegister) {
4359  EnMask |= Compr? (0x3 << i * 2) : (0x1 << i);
4360  }
4361  }
4362 
4363  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpVM);
4364  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyExpCompr);
4365 
4366  Inst.addOperand(MCOperand::createImm(EnMask));
4367 }
4368 
4369 //===----------------------------------------------------------------------===//
4370 // s_waitcnt
4371 //===----------------------------------------------------------------------===//
4372 
4373 static bool
4375  const AMDGPU::IsaVersion ISA,
4376  int64_t &IntVal,
4377  int64_t CntVal,
4378  bool Saturate,
4379  unsigned (*encode)(const IsaVersion &Version, unsigned, unsigned),
4380  unsigned (*decode)(const IsaVersion &Version, unsigned))
4381 {
4382  bool Failed = false;
4383 
4384  IntVal = encode(ISA, IntVal, CntVal);
4385  if (CntVal != decode(ISA, IntVal)) {
4386  if (Saturate) {
4387  IntVal = encode(ISA, IntVal, -1);
4388  } else {
4389  Failed = true;
4390  }
4391  }
4392  return Failed;
4393 }
4394 
4395 bool AMDGPUAsmParser::parseCnt(int64_t &IntVal) {
4396  StringRef CntName = Parser.getTok().getString();
4397  int64_t CntVal;
4398 
4399  Parser.Lex();
4400  if (getLexer().isNot(AsmToken::LParen))
4401  return true;
4402 
4403  Parser.Lex();
4404  if (getLexer().isNot(AsmToken::Integer))
4405  return true;
4406 
4407  SMLoc ValLoc = Parser.getTok().getLoc();
4408  if (getParser().parseAbsoluteExpression(CntVal))
4409  return true;
4410 
4411  AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
4412 
4413  bool Failed = true;
4414  bool Sat = CntName.endswith("_sat");
4415 
4416  if (CntName == "vmcnt" || CntName == "vmcnt_sat") {
4417  Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeVmcnt, decodeVmcnt);
4418  } else if (CntName == "expcnt" || CntName == "expcnt_sat") {
4419  Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeExpcnt, decodeExpcnt);
4420  } else if (CntName == "lgkmcnt" || CntName == "lgkmcnt_sat") {
4421  Failed = encodeCnt(ISA, IntVal, CntVal, Sat, encodeLgkmcnt, decodeLgkmcnt);
4422  }
4423 
4424  if (Failed) {
4425  Error(ValLoc, "too large value for " + CntName);
4426  return true;
4427  }
4428 
4429  if (getLexer().isNot(AsmToken::RParen)) {
4430  return true;
4431  }
4432 
4433  Parser.Lex();
4434  if (getLexer().is(AsmToken::Amp) || getLexer().is(AsmToken::Comma)) {
4435  const AsmToken NextToken = getLexer().peekTok();
4436  if (NextToken.is(AsmToken::Identifier)) {
4437  Parser.Lex();
4438  }
4439  }
4440 
4441  return false;
4442 }
4443 
4445 AMDGPUAsmParser::parseSWaitCntOps(OperandVector &Operands) {
4446  AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(getSTI().getCPU());
4447  int64_t Waitcnt = getWaitcntBitMask(ISA);
4448  SMLoc S = Parser.getTok().getLoc();
4449 
4450  switch(getLexer().getKind()) {
4451  default: return MatchOperand_ParseFail;
4452  case AsmToken::Integer:
4453  // The operand can be an integer value.
4454  if (getParser().parseAbsoluteExpression(Waitcnt))
4455  return MatchOperand_ParseFail;
4456  break;
4457 
4458  case AsmToken::Identifier:
4459  do {
4460  if (parseCnt(Waitcnt))
4461  return MatchOperand_ParseFail;
4462  } while(getLexer().isNot(AsmToken::EndOfStatement));
4463  break;
4464  }
4465  Operands.push_back(AMDGPUOperand::CreateImm(this, Waitcnt, S));
4466  return MatchOperand_Success;
4467 }
4468 
4469 bool AMDGPUAsmParser::parseHwregConstruct(OperandInfoTy &HwReg, int64_t &Offset,
4470  int64_t &Width) {
4471  using namespace llvm::AMDGPU::Hwreg;
4472 
4473  if (Parser.getTok().getString() != "hwreg")
4474  return true;
4475  Parser.Lex();
4476 
4477  if (getLexer().isNot(AsmToken::LParen))
4478  return true;
4479  Parser.Lex();
4480 
4481  if (getLexer().is(AsmToken::Identifier)) {
4482  HwReg.IsSymbolic = true;
4483  HwReg.Id = ID_UNKNOWN_;
4484  const StringRef tok = Parser.getTok().getString();
4485  int Last = ID_SYMBOLIC_LAST_;
4486  if (isSI() || isCI() || isVI())
4487  Last = ID_SYMBOLIC_FIRST_GFX9_;
4488  else if (isGFX9())
4489  Last = ID_SYMBOLIC_FIRST_GFX10_;
4490  for (int i = ID_SYMBOLIC_FIRST_; i < Last; ++i) {
4491  if (tok == IdSymbolic[i]) {
4492  HwReg.Id = i;
4493  break;
4494  }
4495  }
4496  Parser.Lex();
4497  } else {
4498  HwReg.IsSymbolic = false;
4499  if (getLexer().isNot(AsmToken::Integer))
4500  return true;
4501  if (getParser().parseAbsoluteExpression(HwReg.Id))
4502  return true;
4503  }
4504 
4505  if (getLexer().is(AsmToken::RParen)) {
4506  Parser.Lex();
4507  return false;
4508  }
4509 
4510  // optional params
4511  if (getLexer().isNot(AsmToken::Comma))
4512  return true;
4513  Parser.Lex();
4514 
4515  if (getLexer().isNot(AsmToken::Integer))
4516  return true;
4517  if (getParser().parseAbsoluteExpression(Offset))
4518  return true;
4519 
4520  if (getLexer().isNot(AsmToken::Comma))
4521  return true;
4522  Parser.Lex();
4523 
4524  if (getLexer().isNot(AsmToken::Integer))
4525  return true;
4526  if (getParser().parseAbsoluteExpression(Width))
4527  return true;
4528 
4529  if (getLexer().isNot(AsmToken::RParen))
4530  return true;
4531  Parser.Lex();
4532 
4533  return false;
4534 }
4535 
4536 OperandMatchResultTy AMDGPUAsmParser::parseHwreg(OperandVector &Operands) {
4537  using namespace llvm::AMDGPU::Hwreg;
4538 
4539  int64_t Imm16Val = 0;
4540  SMLoc S = Parser.getTok().getLoc();
4541 
4542  switch(getLexer().getKind()) {
4543  default: return MatchOperand_NoMatch;
4544  case AsmToken::Integer:
4545  // The operand can be an integer value.
4546  if (getParser().parseAbsoluteExpression(Imm16Val))
4547  return MatchOperand_NoMatch;
4548  if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
4549  Error(S, "invalid immediate: only 16-bit values are legal");
4550  // Do not return error code, but create an imm operand anyway and proceed
4551  // to the next operand, if any. That avoids unneccessary error messages.
4552  }
4553  break;
4554 
4555  case AsmToken::Identifier: {
4556  OperandInfoTy HwReg(ID_UNKNOWN_);
4557  int64_t Offset = OFFSET_DEFAULT_;
4558  int64_t Width = WIDTH_M1_DEFAULT_ + 1;
4559  if (parseHwregConstruct(HwReg, Offset, Width))
4560  return MatchOperand_ParseFail;
4561  if (HwReg.Id < 0 || !isUInt<ID_WIDTH_>(HwReg.Id)) {
4562  if (HwReg.IsSymbolic)
4563  Error(S, "invalid symbolic name of hardware register");
4564  else
4565  Error(S, "invalid code of hardware register: only 6-bit values are legal");
4566  }
4567  if (Offset < 0 || !isUInt<OFFSET_WIDTH_>(Offset))
4568  Error(S, "invalid bit offset: only 5-bit values are legal");
4569  if ((Width-1) < 0 || !isUInt<WIDTH_M1_WIDTH_>(Width-1))
4570  Error(S, "invalid bitfield width: only values from 1 to 32 are legal");
4571  Imm16Val = (HwReg.Id << ID_SHIFT_) | (Offset << OFFSET_SHIFT_) | ((Width-1) << WIDTH_M1_SHIFT_);
4572  }
4573  break;
4574  }
4575  Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTyHwreg));
4576  return MatchOperand_Success;
4577 }
4578 
4579 bool AMDGPUOperand::isSWaitCnt() const {
4580  return isImm();
4581 }
4582 
4583 bool AMDGPUOperand::isHwreg() const {
4584  return isImmTy(ImmTyHwreg);
4585 }
4586 
4587 bool AMDGPUAsmParser::parseSendMsgConstruct(OperandInfoTy &Msg, OperandInfoTy &Operation, int64_t &StreamId) {
4588  using namespace llvm::AMDGPU::SendMsg;
4589 
4590  if (Parser.getTok().getString() != "sendmsg")
4591  return true;
4592  Parser.Lex();
4593 
4594  if (getLexer().isNot(AsmToken::LParen))
4595  return true;
4596  Parser.Lex();
4597 
4598  if (getLexer().is(AsmToken::Identifier)) {
4599  Msg.IsSymbolic = true;
4600  Msg.Id = ID_UNKNOWN_;
4601  const std::string tok = Parser.getTok().getString();
4602  for (int i = ID_GAPS_FIRST_; i < ID_GAPS_LAST_; ++i) {
4603  switch(i) {
4604  default: continue; // Omit gaps.
4605  case ID_GS_ALLOC_REQ:
4606  if (isSI() || isCI() || isVI())
4607  continue;
4608  break;
4609  case ID_INTERRUPT: case ID_GS: case ID_GS_DONE:
4610  case ID_SYSMSG: break;
4611  }
4612  if (tok == IdSymbolic[i]) {
4613  Msg.Id = i;
4614  break;
4615  }
4616  }
4617  Parser.Lex();
4618  } else {
4619  Msg.IsSymbolic = false;
4620  if (getLexer().isNot(AsmToken::Integer))
4621  return true;
4622  if (getParser().parseAbsoluteExpression(Msg.Id))
4623  return true;
4624  if (getLexer().is(AsmToken::Integer))
4625  if (getParser().parseAbsoluteExpression(Msg.Id))
4626  Msg.Id = ID_UNKNOWN_;
4627  }
4628  if (Msg.Id == ID_UNKNOWN_) // Don't know how to parse the rest.
4629  return false;
4630 
4631  if (!(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG)) {
4632  if (getLexer().isNot(AsmToken::RParen))
4633  return true;
4634  Parser.Lex();
4635  return false;
4636  }
4637 
4638  if (getLexer().isNot(AsmToken::Comma))
4639  return true;
4640  Parser.Lex();
4641 
4642  assert(Msg.Id == ID_GS || Msg.Id == ID_GS_DONE || Msg.Id == ID_SYSMSG);
4643  Operation.Id = ID_UNKNOWN_;
4644  if (getLexer().is(AsmToken::Identifier)) {
4645  Operation.IsSymbolic = true;
4646  const char* const *S = (Msg.Id == ID_SYSMSG) ? OpSysSymbolic : OpGsSymbolic;
4647  const int F = (Msg.Id == ID_SYSMSG) ? OP_SYS_FIRST_ : OP_GS_FIRST_;
4648  const int L = (Msg.Id == ID_SYSMSG) ? OP_SYS_LAST_ : OP_GS_LAST_;
4649  const StringRef Tok = Parser.getTok().getString();
4650  for (int i = F; i < L; ++i) {
4651  if (Tok == S[i]) {
4652  Operation.Id = i;
4653  break;
4654  }
4655  }
4656  Parser.Lex();
4657  } else {
4658  Operation.IsSymbolic = false;
4659  if (getLexer().isNot(AsmToken::Integer))
4660  return true;
4661  if (getParser().parseAbsoluteExpression(Operation.Id))
4662  return true;
4663  }
4664 
4665  if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
4666  // Stream id is optional.
4667  if (getLexer().is(AsmToken::RParen)) {
4668  Parser.Lex();
4669  return false;
4670  }
4671 
4672  if (getLexer().isNot(AsmToken::Comma))
4673  return true;
4674  Parser.Lex();
4675 
4676  if (getLexer().isNot(AsmToken::Integer))
4677  return true;
4678  if (getParser().parseAbsoluteExpression(StreamId))
4679  return true;
4680  }
4681 
4682  if (getLexer().isNot(AsmToken::RParen))
4683  return true;
4684  Parser.Lex();
4685  return false;
4686 }
4687 
4688 OperandMatchResultTy AMDGPUAsmParser::parseInterpSlot(OperandVector &Operands) {
4689  if (getLexer().getKind() != AsmToken::Identifier)
4690  return MatchOperand_NoMatch;
4691 
4692  StringRef Str = Parser.getTok().getString();
4693  int Slot = StringSwitch<int>(Str)
4694  .Case("p10", 0)
4695  .Case("p20", 1)
4696  .Case("p0", 2)
4697  .Default(-1);
4698 
4699  SMLoc S = Parser.getTok().getLoc();
4700  if (Slot == -1)
4701  return MatchOperand_ParseFail;
4702 
4703  Parser.Lex();
4704  Operands.push_back(AMDGPUOperand::CreateImm(this, Slot, S,
4705  AMDGPUOperand::ImmTyInterpSlot));
4706  return MatchOperand_Success;
4707 }
4708 
4709 OperandMatchResultTy AMDGPUAsmParser::parseInterpAttr(OperandVector &Operands) {
4710  if (getLexer().getKind() != AsmToken::Identifier)
4711  return MatchOperand_NoMatch;
4712 
4713  StringRef Str = Parser.getTok().getString();
4714  if (!Str.startswith("attr"))
4715  return MatchOperand_NoMatch;
4716 
4717  StringRef Chan = Str.take_back(2);
4718  int AttrChan = StringSwitch<int>(Chan)
4719  .Case(".x", 0)
4720  .Case(".y", 1)
4721  .Case(".z", 2)
4722  .Case(".w", 3)
4723  .Default(-1);
4724  if (AttrChan == -1)
4725  return MatchOperand_ParseFail;
4726 
4727  Str = Str.drop_back(2).drop_front(4);
4728 
4729  uint8_t Attr;
4730  if (Str.getAsInteger(10, Attr))
4731  return MatchOperand_ParseFail;
4732 
4733  SMLoc S = Parser.getTok().getLoc();
4734  Parser.Lex();
4735  if (Attr > 63) {
4736  Error(S, "out of bounds attr");
4737  return MatchOperand_Success;
4738  }
4739 
4740  SMLoc SChan = SMLoc::getFromPointer(Chan.data());
4741 
4742  Operands.push_back(AMDGPUOperand::CreateImm(this, Attr, S,
4743  AMDGPUOperand::ImmTyInterpAttr));
4744  Operands.push_back(AMDGPUOperand::CreateImm(this, AttrChan, SChan,
4745  AMDGPUOperand::ImmTyAttrChan));
4746  return MatchOperand_Success;
4747 }
4748 
4749 void AMDGPUAsmParser::errorExpTgt() {
4750  Error(Parser.getTok().getLoc(), "invalid exp target");
4751 }
4752 
4753 OperandMatchResultTy AMDGPUAsmParser::parseExpTgtImpl(StringRef Str,
4754  uint8_t &Val) {
4755  if (Str == "null") {
4756  Val = 9;
4757  return MatchOperand_Success;
4758  }
4759 
4760  if (Str.startswith("mrt")) {
4761  Str = Str.drop_front(3);
4762  if (Str == "z") { // == mrtz
4763  Val = 8;
4764  return MatchOperand_Success;
4765  }
4766 
4767  if (Str.getAsInteger(10, Val))
4768  return MatchOperand_ParseFail;
4769 
4770  if (Val > 7)
4771  errorExpTgt();
4772 
4773  return MatchOperand_Success;
4774  }
4775 
4776  if (Str.startswith("pos")) {
4777  Str = Str.drop_front(3);
4778  if (Str.getAsInteger(10, Val))
4779  return MatchOperand_ParseFail;
4780 
4781  if (Val > 4 || (Val == 4 && !isGFX10()))
4782  errorExpTgt();
4783 
4784  Val += 12;
4785  return MatchOperand_Success;
4786  }
4787 
4788  if (isGFX10() && Str == "prim") {
4789  Val = 20;
4790  return MatchOperand_Success;
4791  }
4792 
4793  if (Str.startswith("param")) {
4794  Str = Str.drop_front(5);
4795  if (Str.getAsInteger(10, Val))
4796  return MatchOperand_ParseFail;
4797 
4798  if (Val >= 32)
4799  errorExpTgt();
4800 
4801  Val += 32;
4802  return MatchOperand_Success;
4803  }
4804 
4805  if (Str.startswith("invalid_target_")) {
4806  Str = Str.drop_front(15);
4807  if (Str.getAsInteger(10, Val))
4808  return MatchOperand_ParseFail;
4809 
4810  errorExpTgt();
4811  return MatchOperand_Success;
4812  }
4813 
4814  return MatchOperand_NoMatch;
4815 }
4816 
4817 OperandMatchResultTy AMDGPUAsmParser::parseExpTgt(OperandVector &Operands) {
4818  uint8_t Val;
4819  StringRef Str = Parser.getTok().getString();
4820 
4821  auto Res = parseExpTgtImpl(Str, Val);
4822  if (Res != MatchOperand_Success)
4823  return Res;
4824 
4825  SMLoc S = Parser.getTok().getLoc();
4826  Parser.Lex();
4827 
4828  Operands.push_back(AMDGPUOperand::CreateImm(this, Val, S,
4829  AMDGPUOperand::ImmTyExpTgt));
4830  return MatchOperand_Success;
4831 }
4832 
4834 AMDGPUAsmParser::parseSendMsgOp(OperandVector &Operands) {
4835  using namespace llvm::AMDGPU::SendMsg;
4836 
4837  int64_t Imm16Val = 0;
4838  SMLoc S = Parser.getTok().getLoc();
4839 
4840  switch(getLexer().getKind()) {
4841  default:
4842  return MatchOperand_NoMatch;
4843  case AsmToken::Integer:
4844  // The operand can be an integer value.
4845  if (getParser().parseAbsoluteExpression(Imm16Val))
4846  return MatchOperand_NoMatch;
4847  if (Imm16Val < 0 || !isUInt<16>(Imm16Val)) {
4848  Error(S, "invalid immediate: only 16-bit values are legal");
4849  // Do not return error code, but create an imm operand anyway and proceed
4850  // to the next operand, if any. That avoids unneccessary error messages.
4851  }
4852  break;
4853  case AsmToken::Identifier: {
4854  OperandInfoTy Msg(ID_UNKNOWN_);
4855  OperandInfoTy Operation(OP_UNKNOWN_);
4856  int64_t StreamId = STREAM_ID_DEFAULT_;
4857  if (parseSendMsgConstruct(Msg, Operation, StreamId))
4858  return MatchOperand_ParseFail;
4859  do {
4860  // Validate and encode message ID.
4861  if (! ((ID_INTERRUPT <= Msg.Id && Msg.Id <= ID_GS_DONE)
4862  || (Msg.Id == ID_GS_ALLOC_REQ && !isSI() && !isCI() && !isVI())
4863  || Msg.Id == ID_SYSMSG)) {
4864  if (Msg.IsSymbolic)
4865  Error(S, "invalid/unsupported symbolic name of message");
4866  else
4867  Error(S, "invalid/unsupported code of message");
4868  break;
4869  }
4870  Imm16Val = (Msg.Id << ID_SHIFT_);
4871  // Validate and encode operation ID.
4872  if (Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) {
4873  if (! (OP_GS_FIRST_ <= Operation.Id && Operation.Id < OP_GS_LAST_)) {
4874  if (Operation.IsSymbolic)
4875  Error(S, "invalid symbolic name of GS_OP");
4876  else
4877  Error(S, "invalid code of GS_OP: only 2-bit values are legal");
4878  break;
4879  }
4880  if (Operation.Id == OP_GS_NOP
4881  && Msg.Id != ID_GS_DONE) {
4882  Error(S, "invalid GS_OP: NOP is for GS_DONE only");
4883  break;
4884  }
4885  Imm16Val |= (Operation.Id << OP_SHIFT_);
4886  }
4887  if (Msg.Id == ID_SYSMSG) {
4888  if (! (OP_SYS_FIRST_ <= Operation.Id && Operation.Id < OP_SYS_LAST_)) {
4889  if (Operation.IsSymbolic)
4890  Error(S, "invalid/unsupported symbolic name of SYSMSG_OP");
4891  else
4892  Error(S, "invalid/unsupported code of SYSMSG_OP");
4893  break;
4894  }
4895  Imm16Val |= (Operation.Id << OP_SHIFT_);
4896  }
4897  // Validate and encode stream ID.
4898  if ((Msg.Id == ID_GS || Msg.Id == ID_GS_DONE) && Operation.Id != OP_GS_NOP) {
4899  if (! (STREAM_ID_FIRST_ <= StreamId && StreamId < STREAM_ID_LAST_)) {
4900  Error(S, "invalid stream id: only 2-bit values are legal");
4901  break;
4902  }
4903  Imm16Val |= (StreamId << STREAM_ID_SHIFT_);
4904  }
4905  } while (false);
4906  }
4907  break;
4908  }
4909  Operands.push_back(AMDGPUOperand::CreateImm(this, Imm16Val, S, AMDGPUOperand::ImmTySendMsg));
4910  return MatchOperand_Success;
4911 }
4912 
4913 bool AMDGPUOperand::isSendMsg() const {
4914  return isImmTy(ImmTySendMsg);
4915 }
4916 
4917 //===----------------------------------------------------------------------===//
4918 // parser helpers
4919 //===----------------------------------------------------------------------===//
4920 
4921 bool
4922 AMDGPUAsmParser::isId(const AsmToken &Token, const StringRef Id) const {
4923  return Token.is(AsmToken::Identifier) && Token.getString() == Id;
4924 }
4925 
4926 bool
4927 AMDGPUAsmParser::isId(const StringRef Id) const {
4928  return isId(getToken(), Id);
4929 }
4930 
4931 bool
4932 AMDGPUAsmParser::isToken(const AsmToken::TokenKind Kind) const {
4933  return getTokenKind() == Kind;
4934 }
4935 
4936 bool
4937 AMDGPUAsmParser::trySkipId(const StringRef Id) {
4938  if (isId(Id)) {
4939  lex();
4940  return true;
4941  }
4942  return false;
4943 }
4944 
4945 bool
4946 AMDGPUAsmParser::trySkipId(const StringRef Id, const AsmToken::TokenKind Kind) {
4947  if (isId(Id) && peekToken().is(Kind)) {
4948  lex();
4949  lex();
4950  return true;
4951  }
4952  return false;
4953 }
4954 
4955 bool
4956 AMDGPUAsmParser::trySkipToken(const AsmToken::TokenKind Kind) {
4957  if (isToken(Kind)) {
4958  lex();
4959  return true;
4960  }
4961  return false;
4962 }
4963 
4964 bool
4965 AMDGPUAsmParser::skipToken(const AsmToken::TokenKind Kind,
4966  const StringRef ErrMsg) {
4967  if (!trySkipToken(Kind)) {
4968  Error(getLoc(), ErrMsg);
4969  return false;
4970  }
4971  return true;
4972 }
4973 
4974 bool
4975 AMDGPUAsmParser::parseExpr(int64_t &Imm) {
4976  return !getParser().parseAbsoluteExpression(Imm);
4977 }
4978 
4979 bool
4980 AMDGPUAsmParser::parseString(StringRef &Val, const StringRef ErrMsg) {
4981  if (isToken(AsmToken::String)) {
4982  Val = getToken().getStringContents();
4983  lex();
4984  return true;
4985  } else {
4986  Error(getLoc(), ErrMsg);
4987  return false;
4988  }
4989 }
4990 
4991 AsmToken
4992 AMDGPUAsmParser::getToken() const {
4993  return Parser.getTok();
4994 }
4995 
4996 AsmToken
4997 AMDGPUAsmParser::peekToken() {
4998  return getLexer().peekTok();
4999 }
5000 
5001 void
5002 AMDGPUAsmParser::peekTokens(MutableArrayRef<AsmToken> Tokens) {
5003  auto TokCount = getLexer().peekTokens(Tokens);
5004 
5005  for (auto Idx = TokCount; Idx < Tokens.size(); ++Idx)
5006  Tokens[Idx] = AsmToken(AsmToken::Error, "");
5007 }
5008 
5010 AMDGPUAsmParser::getTokenKind() const {
5011  return getLexer().getKind();
5012 }
5013 
5014 SMLoc
5015 AMDGPUAsmParser::getLoc() const {
5016  return getToken().getLoc();
5017 }
5018 
5019 StringRef
5020 AMDGPUAsmParser::getTokenStr() const {
5021  return getToken().getString();
5022 }
5023 
5024 void
5025 AMDGPUAsmParser::lex() {
5026  Parser.Lex();
5027 }
5028 
5029 //===----------------------------------------------------------------------===//
5030 // swizzle
5031 //===----------------------------------------------------------------------===//
5032 
5034 static unsigned
5035 encodeBitmaskPerm(const unsigned AndMask,
5036  const unsigned OrMask,
5037  const unsigned XorMask) {
5038  using namespace llvm::AMDGPU::Swizzle;
5039 
5040  return BITMASK_PERM_ENC |
5041  (AndMask << BITMASK_AND_SHIFT) |
5042  (OrMask << BITMASK_OR_SHIFT) |
5043  (XorMask << BITMASK_XOR_SHIFT);
5044 }
5045 
5046 bool
5047 AMDGPUAsmParser::parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
5048  const unsigned MinVal,
5049  const unsigned MaxVal,
5050  const StringRef ErrMsg) {
5051  for (unsigned i = 0; i < OpNum; ++i) {
5052  if (!skipToken(AsmToken::Comma, "expected a comma")){
5053  return false;
5054  }
5055  SMLoc ExprLoc = Parser.getTok().getLoc();
5056  if (!parseExpr(Op[i])) {
5057  return false;
5058  }
5059  if (Op[i] < MinVal || Op[i] > MaxVal) {
5060  Error(ExprLoc, ErrMsg);
5061  return false;
5062  }
5063  }
5064 
5065  return true;
5066 }
5067 
5068 bool
5069 AMDGPUAsmParser::parseSwizzleQuadPerm(int64_t &Imm) {
5070  using namespace llvm::AMDGPU::Swizzle;
5071 
5072  int64_t Lane[LANE_NUM];
5073  if (parseSwizzleOperands(LANE_NUM, Lane, 0, LANE_MAX,
5074  "expected a 2-bit lane id")) {
5075  Imm = QUAD_PERM_ENC;
5076  for (unsigned I = 0; I < LANE_NUM; ++I) {
5077  Imm |= Lane[I] << (LANE_SHIFT * I);
5078  }
5079  return true;
5080  }
5081  return false;
5082 }
5083 
5084 bool
5085 AMDGPUAsmParser::parseSwizzleBroadcast(int64_t &Imm) {
5086  using namespace llvm::AMDGPU::Swizzle;
5087 
5088  SMLoc S = Parser.getTok().getLoc();
5089  int64_t GroupSize;
5090  int64_t LaneIdx;
5091 
5092  if (!parseSwizzleOperands(1, &GroupSize,
5093  2, 32,
5094  "group size must be in the interval [2,32]")) {
5095  return false;
5096  }
5097  if (!isPowerOf2_64(GroupSize)) {
5098  Error(S, "group size must be a power of two");
5099  return false;
5100  }
5101  if (parseSwizzleOperands(1, &LaneIdx,
5102  0, GroupSize - 1,
5103  "lane id must be in the interval [0,group size - 1]")) {
5104  Imm = encodeBitmaskPerm(BITMASK_MAX - GroupSize + 1, LaneIdx, 0);
5105  return true;
5106  }
5107  return false;
5108 }
5109 
5110 bool
5111 AMDGPUAsmParser::parseSwizzleReverse(int64_t &Imm) {
5112  using namespace llvm::AMDGPU::Swizzle;
5113 
5114  SMLoc S = Parser.getTok().getLoc();
5115  int64_t GroupSize;
5116 
5117  if (!parseSwizzleOperands(1, &GroupSize,
5118  2, 32, "group size must be in the interval [2,32]")) {
5119  return false;
5120  }
5121  if (!isPowerOf2_64(GroupSize)) {
5122  Error(S, "group size must be a power of two");
5123  return false;
5124  }
5125 
5126  Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize - 1);
5127  return true;
5128 }
5129 
5130 bool
5131 AMDGPUAsmParser::parseSwizzleSwap(int64_t &Imm) {
5132  using namespace llvm::AMDGPU::Swizzle;
5133 
5134  SMLoc S = Parser.getTok().getLoc();
5135  int64_t GroupSize;
5136 
5137  if (!parseSwizzleOperands(1, &GroupSize,
5138  1, 16, "group size must be in the interval [1,16]")) {
5139  return false;
5140  }
5141  if (!isPowerOf2_64(GroupSize)) {
5142  Error(S, "group size must be a power of two");
5143  return false;
5144  }
5145 
5146  Imm = encodeBitmaskPerm(BITMASK_MAX, 0, GroupSize);
5147  return true;
5148 }
5149 
5150 bool
5151 AMDGPUAsmParser::parseSwizzleBitmaskPerm(int64_t &Imm) {
5152  using namespace llvm::AMDGPU::Swizzle;
5153 
5154  if (!skipToken(AsmToken::Comma, "expected a comma")) {
5155  return false;
5156  }
5157 
5158  StringRef Ctl;
5159  SMLoc StrLoc = Parser.getTok().getLoc();
5160  if (!parseString(Ctl)) {
5161  return false;
5162  }
5163  if (Ctl.size() != BITMASK_WIDTH) {
5164  Error(StrLoc, "expected a 5-character mask");
5165  return false;
5166  }
5167 
5168  unsigned AndMask = 0;
5169  unsigned OrMask = 0;
5170  unsigned XorMask = 0;
5171 
5172  for (size_t i = 0; i < Ctl.size(); ++i) {
5173  unsigned Mask = 1 << (BITMASK_WIDTH - 1 - i);
5174  switch(Ctl[i]) {
5175  default:
5176  Error(StrLoc, "invalid mask");
5177  return false;
5178  case '0':
5179  break;
5180  case '1':
5181  OrMask |= Mask;
5182  break;
5183  case 'p':
5184  AndMask |= Mask;
5185  break;
5186  case 'i':
5187  AndMask |= Mask;
5188  XorMask |= Mask;
5189  break;
5190  }
5191  }
5192 
5193  Imm = encodeBitmaskPerm(AndMask, OrMask, XorMask);
5194  return true;
5195 }
5196 
5197 bool
5198 AMDGPUAsmParser::parseSwizzleOffset(int64_t &Imm) {
5199 
5200  SMLoc OffsetLoc = Parser.getTok().getLoc();
5201 
5202  if (!parseExpr(Imm)) {
5203  return false;
5204  }
5205  if (!isUInt<16>(Imm)) {
5206  Error(OffsetLoc, "expected a 16-bit offset");
5207  return false;
5208  }
5209  return true;
5210 }
5211 
5212 bool
5213 AMDGPUAsmParser::parseSwizzleMacro(int64_t &Imm) {
5214  using namespace llvm::AMDGPU::Swizzle;
5215 
5216  if (skipToken(AsmToken::LParen, "expected a left parentheses")) {
5217 
5218  SMLoc ModeLoc = Parser.getTok().getLoc();
5219  bool Ok = false;
5220 
5221  if (trySkipId(IdSymbolic[ID_QUAD_PERM])) {
5222  Ok = parseSwizzleQuadPerm(Imm);
5223  } else if (trySkipId(IdSymbolic[ID_BITMASK_PERM])) {
5224  Ok = parseSwizzleBitmaskPerm(Imm);
5225  } else if (trySkipId(IdSymbolic[ID_BROADCAST])) {
5226  Ok = parseSwizzleBroadcast(Imm);
5227  } else if (trySkipId(IdSymbolic[ID_SWAP])) {
5228  Ok = parseSwizzleSwap(Imm);
5229  } else if (trySkipId(IdSymbolic[ID_REVERSE])) {
5230  Ok = parseSwizzleReverse(Imm);
5231  } else {
5232  Error(ModeLoc, "expected a swizzle mode");
5233  }
5234 
5235  return Ok && skipToken(AsmToken::RParen, "expected a closing parentheses");
5236  }
5237 
5238  return false;
5239 }
5240 
5242 AMDGPUAsmParser::parseSwizzleOp(OperandVector &Operands) {
5243  SMLoc S = Parser.getTok().getLoc();
5244  int64_t Imm = 0;
5245 
5246  if (trySkipId("offset")) {
5247 
5248  bool Ok = false;
5249  if (skipToken(AsmToken::Colon, "expected a colon")) {
5250  if (trySkipId("swizzle")) {
5251  Ok = parseSwizzleMacro(Imm);
5252  } else {
5253  Ok = parseSwizzleOffset(Imm);
5254  }
5255  }
5256 
5257  Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTySwizzle));
5258 
5260  } else {
5261  // Swizzle "offset" operand is optional.
5262  // If it is omitted, try parsing other optional operands.
5263  return parseOptionalOpr(Operands);
5264  }
5265 }
5266 
5267 bool
5268 AMDGPUOperand::isSwizzle() const {
5269  return isImmTy(ImmTySwizzle);
5270 }
5271 
5272 //===----------------------------------------------------------------------===//
5273 // VGPR Index Mode
5274 //===----------------------------------------------------------------------===//
5275 
5276 int64_t AMDGPUAsmParser::parseGPRIdxMacro() {
5277 
5278  using namespace llvm::AMDGPU::VGPRIndexMode;
5279 
5280  if (trySkipToken(AsmToken::RParen)) {
5281  return OFF;
5282  }
5283 
5284  int64_t Imm = 0;
5285 
5286  while (true) {
5287  unsigned Mode = 0;
5288  SMLoc S = Parser.getTok().getLoc();
5289 
5290  for (unsigned ModeId = ID_MIN; ModeId <= ID_MAX; ++ModeId) {
5291  if (trySkipId(IdSymbolic[ModeId])) {
5292  Mode = 1 << ModeId;
5293  break;
5294  }
5295  }
5296 
5297  if (Mode == 0) {
5298  Error(S, (Imm == 0)?
5299  "expected a VGPR index mode or a closing parenthesis" :
5300  "expected a VGPR index mode");
5301  break;
5302  }
5303 
5304  if (Imm & Mode) {
5305  Error(S, "duplicate VGPR index mode");
5306  break;
5307  }
5308  Imm |= Mode;
5309 
5310  if (trySkipToken(AsmToken::RParen))
5311  break;
5312  if (!skipToken(AsmToken::Comma,
5313  "expected a comma or a closing parenthesis"))
5314  break;
5315  }
5316 
5317  return Imm;
5318 }
5319 
5321 AMDGPUAsmParser::parseGPRIdxMode(OperandVector &Operands) {
5322 
5323  int64_t Imm = 0;
5324  SMLoc S = Parser.getTok().getLoc();
5325 
5326  if (getLexer().getKind() == AsmToken::Identifier &&
5327  Parser.getTok().getString() == "gpr_idx" &&
5328  getLexer().peekTok().is(AsmToken::LParen)) {
5329 
5330  Parser.Lex();
5331  Parser.Lex();
5332 
5333  // If parse failed, trigger an error but do not return error code
5334  // to avoid excessive error messages.
5335  Imm = parseGPRIdxMacro();
5336 
5337  } else {
5338  if (getParser().parseAbsoluteExpression(Imm))
5339  return MatchOperand_NoMatch;
5340  if (Imm < 0 || !isUInt<4>(Imm)) {
5341  Error(S, "invalid immediate: only 4-bit values are legal");
5342  }
5343  }
5344 
5345  Operands.push_back(
5346  AMDGPUOperand::CreateImm(this, Imm, S, AMDGPUOperand::ImmTyGprIdxMode));
5347  return MatchOperand_Success;
5348 }
5349 
5350 bool AMDGPUOperand::isGPRIdxMode() const {
5351  return isImmTy(ImmTyGprIdxMode);
5352 }
5353 
5354 //===----------------------------------------------------------------------===//
5355 // sopp branch targets
5356 //===----------------------------------------------------------------------===//
5357 
5359 AMDGPUAsmParser::parseSOppBrTarget(OperandVector &Operands) {
5360  SMLoc S = Parser.getTok().getLoc();
5361 
5362  switch (getLexer().getKind()) {
5363  default: return MatchOperand_ParseFail;
5364  case AsmToken::Integer: {
5365  int64_t Imm;
5366  if (getParser().parseAbsoluteExpression(Imm))
5367  return MatchOperand_ParseFail;
5368  Operands.push_back(AMDGPUOperand::CreateImm(this, Imm, S));
5369  return MatchOperand_Success;
5370  }
5371 
5372  case AsmToken::Identifier:
5373  Operands.push_back(AMDGPUOperand::CreateExpr(this,
5374  MCSymbolRefExpr::create(getContext().getOrCreateSymbol(
5375  Parser.getTok().getString()), getContext()), S));
5376  Parser.Lex();
5377  return MatchOperand_Success;
5378  }
5379 }
5380 
5381 //===----------------------------------------------------------------------===//
5382 // mubuf
5383 //===----------------------------------------------------------------------===//
5384 
5385 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultDLC() const {
5386  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyDLC);
5387 }
5388 
5389 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultGLC() const {
5390  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyGLC);
5391 }
5392 
5393 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSLC() const {
5394  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTySLC);
5395 }
5396 
5397 void AMDGPUAsmParser::cvtMubufImpl(MCInst &Inst,
5398  const OperandVector &Operands,
5399  bool IsAtomic,
5400  bool IsAtomicReturn,
5401  bool IsLds) {
5402  bool IsLdsOpcode = IsLds;
5403  bool HasLdsModifier = false;
5404  OptionalImmIndexMap OptionalIdx;
5405  assert(IsAtomicReturn ? IsAtomic : true);
5406  unsigned FirstOperandIdx = 1;
5407 
5408  for (unsigned i = FirstOperandIdx, e = Operands.size(); i != e; ++i) {
5409  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
5410 
5411  // Add the register arguments
5412  if (Op.isReg()) {
5413  Op.addRegOperands(Inst, 1);
5414  // Insert a tied src for atomic return dst.
5415  // This cannot be postponed as subsequent calls to
5416  // addImmOperands rely on correct number of MC operands.
5417  if (IsAtomicReturn && i == FirstOperandIdx)
5418  Op.addRegOperands(Inst, 1);
5419  continue;
5420  }
5421 
5422  // Handle the case where soffset is an immediate
5423  if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
5424  Op.addImmOperands(Inst, 1);
5425  continue;
5426  }
5427 
5428  HasLdsModifier |= Op.isLDS();
5429 
5430  // Handle tokens like 'offen' which are sometimes hard-coded into the
5431  // asm string. There are no MCInst operands for these.
5432  if (Op.isToken()) {
5433  continue;
5434  }
5435  assert(Op.isImm());
5436 
5437  // Handle optional arguments
5438  OptionalIdx[Op.getImmTy()] = i;
5439  }
5440 
5441  // This is a workaround for an llvm quirk which may result in an
5442  // incorrect instruction selection. Lds and non-lds versions of
5443  // MUBUF instructions are identical except that lds versions
5444  // have mandatory 'lds' modifier. However this modifier follows
5445  // optional modifiers and llvm asm matcher regards this 'lds'
5446  // modifier as an optional one. As a result, an lds version
5447  // of opcode may be selected even if it has no 'lds' modifier.
5448  if (IsLdsOpcode && !HasLdsModifier) {
5449  int NoLdsOpcode = AMDGPU::getMUBUFNoLdsInst(Inst.getOpcode());
5450  if (NoLdsOpcode != -1) { // Got lds version - correct it.
5451  Inst.setOpcode(NoLdsOpcode);
5452  IsLdsOpcode = false;
5453  }
5454  }
5455 
5456  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOffset);
5457  if (!IsAtomic) { // glc is hard-coded.
5458  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
5459  }
5460  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
5461 
5462  if (!IsLdsOpcode) { // tfe is not legal with lds opcodes
5463  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
5464  }
5465 
5466  if (isGFX10())
5467  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDLC);
5468 }
5469 
5470 void AMDGPUAsmParser::cvtMtbuf(MCInst &Inst, const OperandVector &Operands) {
5471  OptionalImmIndexMap OptionalIdx;
5472 
5473  for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
5474  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
5475 
5476  // Add the register arguments
5477  if (Op.isReg()) {
5478  Op.addRegOperands(Inst, 1);
5479  continue;
5480  }
5481 
5482  // Handle the case where soffset is an immediate
5483  if (Op.isImm() && Op.getImmTy() == AMDGPUOperand::ImmTyNone) {
5484  Op.addImmOperands(Inst, 1);
5485  continue;
5486  }
5487 
5488  // Handle tokens like 'offen' which are sometimes hard-coded into the
5489  // asm string. There are no MCInst operands for these.
5490  if (Op.isToken()) {
5491  continue;
5492  }
5493  assert(Op.isImm());
5494 
5495  // Handle optional arguments
5496  OptionalIdx[Op.getImmTy()] = i;
5497  }
5498 
5499  addOptionalImmOperand(Inst, Operands, OptionalIdx,
5500  AMDGPUOperand::ImmTyOffset);
5501  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyFORMAT);
5502  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
5503  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
5504  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
5505 
5506  if (isGFX10())
5507  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDLC);
5508 }
5509 
5510 //===----------------------------------------------------------------------===//
5511 // mimg
5512 //===----------------------------------------------------------------------===//
5513 
5514 void AMDGPUAsmParser::cvtMIMG(MCInst &Inst, const OperandVector &Operands,
5515  bool IsAtomic) {
5516  unsigned I = 1;
5517  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
5518  for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
5519  ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
5520  }
5521 
5522  if (IsAtomic) {
5523  // Add src, same as dst
5524  assert(Desc.getNumDefs() == 1);
5525  ((AMDGPUOperand &)*Operands[I - 1]).addRegOperands(Inst, 1);
5526  }
5527 
5528  OptionalImmIndexMap OptionalIdx;
5529 
5530  for (unsigned E = Operands.size(); I != E; ++I) {
5531  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
5532 
5533  // Add the register arguments
5534  if (Op.isReg()) {
5535  Op.addRegOperands(Inst, 1);
5536  } else if (Op.isImmModifier()) {
5537  OptionalIdx[Op.getImmTy()] = I;
5538  } else if (!Op.isToken()) {
5539  llvm_unreachable("unexpected operand type");
5540  }
5541  }
5542 
5543  bool IsGFX10 = isGFX10();
5544 
5545  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDMask);
5546  if (IsGFX10)
5547  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDim, -1);
5548  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyUNorm);
5549  if (IsGFX10)
5550  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDLC);
5551  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyGLC);
5552  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTySLC);
5553  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyR128A16);
5554  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyTFE);
5555  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyLWE);
5556  if (!IsGFX10)
5557  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyDA);
5558  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyD16);
5559 }
5560 
5561 void AMDGPUAsmParser::cvtMIMGAtomic(MCInst &Inst, const OperandVector &Operands) {
5562  cvtMIMG(Inst, Operands, true);
5563 }
5564 
5565 //===----------------------------------------------------------------------===//
5566 // smrd
5567 //===----------------------------------------------------------------------===//
5568 
5569 bool AMDGPUOperand::isSMRDOffset8() const {
5570  return isImm() && isUInt<8>(getImm());
5571 }
5572 
5573 bool AMDGPUOperand::isSMRDOffset20() const {
5574  return isImm() && isUInt<20>(getImm());
5575 }
5576 
5577 bool AMDGPUOperand::isSMRDLiteralOffset() const {
5578  // 32-bit literals are only supported on CI and we only want to use them
5579  // when the offset is > 8-bits.
5580  return isImm() && !isUInt<8>(getImm()) && isUInt<32>(getImm());
5581 }
5582 
5583 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset8() const {
5584  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
5585 }
5586 
5587 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDOffset20() const {
5588  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
5589 }
5590 
5591 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultSMRDLiteralOffset() const {
5592  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
5593 }
5594 
5595 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetU12() const {
5596  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
5597 }
5598 
5599 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultOffsetS13() const {
5600  return AMDGPUOperand::CreateImm(this, 0, SMLoc(), AMDGPUOperand::ImmTyOffset);
5601 }
5602 
5603 //===----------------------------------------------------------------------===//
5604 // vop3
5605 //===----------------------------------------------------------------------===//
5606 
5607 static bool ConvertOmodMul(int64_t &Mul) {
5608  if (Mul != 1 && Mul != 2 && Mul != 4)
5609  return false;
5610 
5611  Mul >>= 1;
5612  return true;
5613 }
5614 
5615 static bool ConvertOmodDiv(int64_t &Div) {
5616  if (Div == 1) {
5617  Div = 0;
5618  return true;
5619  }
5620 
5621  if (Div == 2) {
5622  Div = 3;
5623  return true;
5624  }
5625 
5626  return false;
5627 }
5628 
5629 static bool ConvertBoundCtrl(int64_t &BoundCtrl) {
5630  if (BoundCtrl == 0) {
5631  BoundCtrl = 1;
5632  return true;
5633  }
5634 
5635  if (BoundCtrl == -1) {
5636  BoundCtrl = 0;
5637  return true;
5638  }
5639 
5640  return false;
5641 }
5642 
5643 // Note: the order in this table matches the order of operands in AsmString.
5644 static const OptionalOperand AMDGPUOptionalOperandTable[] = {
5645  {"offen", AMDGPUOperand::ImmTyOffen, true, nullptr},
5646  {"idxen", AMDGPUOperand::ImmTyIdxen, true, nullptr},
5647  {"addr64", AMDGPUOperand::ImmTyAddr64, true, nullptr},
5648  {"offset0", AMDGPUOperand::ImmTyOffset0, false, nullptr},
5649  {"offset1", AMDGPUOperand::ImmTyOffset1, false, nullptr},
5650  {"gds", AMDGPUOperand::ImmTyGDS, true, nullptr},
5651  {"lds", AMDGPUOperand::ImmTyLDS, true, nullptr},
5652  {"offset", AMDGPUOperand::ImmTyOffset, false, nullptr},
5653  {"inst_offset", AMDGPUOperand::ImmTyInstOffset, false, nullptr},
5654  {"dlc", AMDGPUOperand::ImmTyDLC, true, nullptr},
5655  {"format", AMDGPUOperand::ImmTyFORMAT, false, nullptr},
5656  {"glc", AMDGPUOperand::ImmTyGLC, true, nullptr},
5657  {"slc", AMDGPUOperand::ImmTySLC, true, nullptr},
5658  {"tfe", AMDGPUOperand::ImmTyTFE, true, nullptr},
5659  {"d16", AMDGPUOperand::ImmTyD16, true, nullptr},
5660  {"high", AMDGPUOperand::ImmTyHigh, true, nullptr},
5661  {"clamp", AMDGPUOperand::ImmTyClampSI, true, nullptr},
5662  {"omod", AMDGPUOperand::ImmTyOModSI, false, ConvertOmodMul},
5663  {"unorm", AMDGPUOperand::ImmTyUNorm, true, nullptr},
5664  {"da", AMDGPUOperand::ImmTyDA, true, nullptr},
5665  {"r128", AMDGPUOperand::ImmTyR128A16, true, nullptr},
5666  {"a16", AMDGPUOperand::ImmTyR128A16, true, nullptr},
5667  {"lwe", AMDGPUOperand::ImmTyLWE, true, nullptr},
5668  {"d16", AMDGPUOperand::ImmTyD16, true, nullptr},
5669  {"dmask", AMDGPUOperand::ImmTyDMask, false, nullptr},
5670  {"dim", AMDGPUOperand::ImmTyDim, false, nullptr},
5671  {"row_mask", AMDGPUOperand::ImmTyDppRowMask, false, nullptr},
5672  {"bank_mask", AMDGPUOperand::ImmTyDppBankMask, false, nullptr},
5673  {"bound_ctrl", AMDGPUOperand::ImmTyDppBoundCtrl, false, ConvertBoundCtrl},
5674  {"dst_sel", AMDGPUOperand::ImmTySdwaDstSel, false, nullptr},
5675  {"src0_sel", AMDGPUOperand::ImmTySdwaSrc0Sel, false, nullptr},
5676  {"src1_sel", AMDGPUOperand::ImmTySdwaSrc1Sel, false, nullptr},
5677  {"dst_unused", AMDGPUOperand::ImmTySdwaDstUnused, false, nullptr},
5678  {"compr", AMDGPUOperand::ImmTyExpCompr, true, nullptr },
5679  {"vm", AMDGPUOperand::ImmTyExpVM, true, nullptr},
5680  {"op_sel", AMDGPUOperand::ImmTyOpSel, false, nullptr},
5681  {"op_sel_hi", AMDGPUOperand::ImmTyOpSelHi, false, nullptr},
5682  {"neg_lo", AMDGPUOperand::ImmTyNegLo, false, nullptr},
5683  {"neg_hi", AMDGPUOperand::ImmTyNegHi, false, nullptr}
5684 };
5685 
5686 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOperand(OperandVector &Operands) {
5687  unsigned size = Operands.size();
5688  assert(size > 0);
5689 
5690  OperandMatchResultTy res = parseOptionalOpr(Operands);
5691 
5692  // This is a hack to enable hardcoded mandatory operands which follow
5693  // optional operands.
5694  //
5695  // Current design assumes that all operands after the first optional operand
5696  // are also optional. However implementation of some instructions violates
5697  // this rule (see e.g. flat/global atomic which have hardcoded 'glc' operands).
5698  //
5699  // To alleviate this problem, we have to (implicitly) parse extra operands
5700  // to make sure autogenerated parser of custom operands never hit hardcoded
5701  // mandatory operands.
5702 
5703  if (size == 1 || ((AMDGPUOperand &)*Operands[size - 1]).isRegKind()) {
5704 
5705  // We have parsed the first optional operand.
5706  // Parse as many operands as necessary to skip all mandatory operands.
5707 
5708  for (unsigned i = 0; i < MAX_OPR_LOOKAHEAD; ++i) {
5709  if (res != MatchOperand_Success ||
5710  getLexer().is(AsmToken::EndOfStatement)) break;
5711  if (getLexer().is(AsmToken::Comma)) Parser.Lex();
5712  res = parseOptionalOpr(Operands);
5713  }
5714  }
5715 
5716  return res;
5717 }
5718 
5719 OperandMatchResultTy AMDGPUAsmParser::parseOptionalOpr(OperandVector &Operands) {
5721  for (const OptionalOperand &Op : AMDGPUOptionalOperandTable) {
5722  // try to parse any optional operand here
5723  if (Op.IsBit) {
5724  res = parseNamedBit(Op.Name, Operands, Op.Type);
5725  } else if (Op.Type == AMDGPUOperand::ImmTyOModSI) {
5726  res = parseOModOperand(Operands);
5727  } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstSel ||
5728  Op.Type == AMDGPUOperand::ImmTySdwaSrc0Sel ||
5729  Op.Type == AMDGPUOperand::ImmTySdwaSrc1Sel) {
5730  res = parseSDWASel(Operands, Op.Name, Op.Type);
5731  } else if (Op.Type == AMDGPUOperand::ImmTySdwaDstUnused) {
5732  res = parseSDWADstUnused(Operands);
5733  } else if (Op.Type == AMDGPUOperand::ImmTyOpSel ||
5734  Op.Type == AMDGPUOperand::ImmTyOpSelHi ||
5735  Op.Type == AMDGPUOperand::ImmTyNegLo ||
5736  Op.Type == AMDGPUOperand::ImmTyNegHi) {
5737  res = parseOperandArrayWithPrefix(Op.Name, Operands, Op.Type,
5738  Op.ConvertResult);
5739  } else if (Op.Type == AMDGPUOperand::ImmTyDim) {
5740  res = parseDim(Operands);
5741  } else if (Op.Type == AMDGPUOperand::ImmTyFORMAT && !isGFX10()) {
5742  res = parseDfmtNfmt(Operands);
5743  } else {
5744  res = parseIntWithPrefix(Op.Name, Operands, Op.Type, Op.ConvertResult);
5745  }
5746  if (res != MatchOperand_NoMatch) {
5747  return res;
5748  }
5749  }
5750  return MatchOperand_NoMatch;
5751 }
5752 
5753 OperandMatchResultTy AMDGPUAsmParser::parseOModOperand(OperandVector &Operands) {
5754  StringRef Name = Parser.getTok().getString();
5755  if (Name == "mul") {
5756  return parseIntWithPrefix("mul", Operands,
5757  AMDGPUOperand::ImmTyOModSI, ConvertOmodMul);
5758  }
5759 
5760  if (Name == "div") {
5761  return parseIntWithPrefix("div", Operands,
5762  AMDGPUOperand::ImmTyOModSI, ConvertOmodDiv);
5763  }
5764 
5765  return MatchOperand_NoMatch;
5766 }
5767 
5768 void AMDGPUAsmParser::cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands) {
5769  cvtVOP3P(Inst, Operands);
5770 
5771  int Opc = Inst.getOpcode();
5772 
5773  int SrcNum;
5774  const int Ops[] = { AMDGPU::OpName::src0,
5775  AMDGPU::OpName::src1,
5776  AMDGPU::OpName::src2 };
5777  for (SrcNum = 0;
5778  SrcNum < 3 && AMDGPU::getNamedOperandIdx(Opc, Ops[SrcNum]) != -1;
5779  ++SrcNum);
5780  assert(SrcNum > 0);
5781 
5782  int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
5783  unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
5784 
5785  if ((OpSel & (1 << SrcNum)) != 0) {
5786  int ModIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers);
5787  uint32_t ModVal = Inst.getOperand(ModIdx).getImm();
5788  Inst.getOperand(ModIdx).setImm(ModVal | SISrcMods::DST_OP_SEL);
5789  }
5790 }
5791 
5792 static bool isRegOrImmWithInputMods(const MCInstrDesc &Desc, unsigned OpNum) {
5793  // 1. This operand is input modifiers
5794  return Desc.OpInfo[OpNum].OperandType == AMDGPU::OPERAND_INPUT_MODS
5795  // 2. This is not last operand
5796  && Desc.NumOperands > (OpNum + 1)
5797  // 3. Next operand is register class
5798  && Desc.OpInfo[OpNum + 1].RegClass != -1
5799  // 4. Next register is not tied to any other operand
5800  && Desc.getOperandConstraint(OpNum + 1, MCOI::OperandConstraint::TIED_TO) == -1;
5801 }
5802 
5803 void AMDGPUAsmParser::cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands)
5804 {
5805  OptionalImmIndexMap OptionalIdx;
5806  unsigned Opc = Inst.getOpcode();
5807 
5808  unsigned I = 1;
5809  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
5810  for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
5811  ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
5812  }
5813 
5814  for (unsigned E = Operands.size(); I != E; ++I) {
5815  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
5816  if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
5817  Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
5818  } else if (Op.isInterpSlot() ||
5819  Op.isInterpAttr() ||
5820  Op.isAttrChan()) {
5821  Inst.addOperand(MCOperand::createImm(Op.getImm()));
5822  } else if (Op.isImmModifier()) {
5823  OptionalIdx[Op.getImmTy()] = I;
5824  } else {
5825  llvm_unreachable("unhandled operand type");
5826  }
5827  }
5828 
5829  if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::high) != -1) {
5830  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyHigh);
5831  }
5832 
5833  if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
5834  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
5835  }
5836 
5837  if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
5838  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
5839  }
5840 }
5841 
5842 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands,
5843  OptionalImmIndexMap &OptionalIdx) {
5844  unsigned Opc = Inst.getOpcode();
5845 
5846  unsigned I = 1;
5847  const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
5848  for (unsigned J = 0; J < Desc.getNumDefs(); ++J) {
5849  ((AMDGPUOperand &)*Operands[I++]).addRegOperands(Inst, 1);
5850  }
5851 
5852  if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0_modifiers) != -1) {
5853  // This instruction has src modifiers
5854  for (unsigned E = Operands.size(); I != E; ++I) {
5855  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
5856  if (isRegOrImmWithInputMods(Desc, Inst.getNumOperands())) {
5857  Op.addRegOrImmWithFPInputModsOperands(Inst, 2);
5858  } else if (Op.isImmModifier()) {
5859  OptionalIdx[Op.getImmTy()] = I;
5860  } else if (Op.isRegOrImm()) {
5861  Op.addRegOrImmOperands(Inst, 1);
5862  } else {
5863  llvm_unreachable("unhandled operand type");
5864  }
5865  }
5866  } else {
5867  // No src modifiers
5868  for (unsigned E = Operands.size(); I != E; ++I) {
5869  AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[I]);
5870  if (Op.isMod()) {
5871  OptionalIdx[Op.getImmTy()] = I;
5872  } else {
5873  Op.addRegOrImmOperands(Inst, 1);
5874  }
5875  }
5876  }
5877 
5878  if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp) != -1) {
5879  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyClampSI);
5880  }
5881 
5882  if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::omod) != -1) {
5883  addOptionalImmOperand(Inst, Operands, OptionalIdx, AMDGPUOperand::ImmTyOModSI);
5884  }
5885 
5886  // Special case v_mac_{f16, f32} and v_fmac_{f16, f32} (gfx906/gfx10+):
5887  // it has src2 register operand that is tied to dst operand
5888  // we don't allow modifiers for this operand in assembler so src2_modifiers
5889  // should be 0.
5890  if (Opc == AMDGPU::V_MAC_F32_e64_gfx6_gfx7 ||
5891  Opc == AMDGPU::V_MAC_F32_e64_gfx10 ||
5892  Opc == AMDGPU::V_MAC_F32_e64_vi ||
5893  Opc == AMDGPU::V_MAC_F16_e64_vi ||
5894  Opc == AMDGPU::V_FMAC_F32_e64_gfx10 ||
5895  Opc == AMDGPU::V_FMAC_F32_e64_vi ||
5896  Opc == AMDGPU::V_FMAC_F16_e64_gfx10) {
5897  auto it = Inst.begin();
5898  std::advance(it, AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2_modifiers));
5899  it = Inst.insert(it, MCOperand::createImm(0)); // no modifiers for src2
5900  ++it;
5901  Inst.insert(it, Inst.getOperand(0)); // src2 = dst
5902  }
5903 }
5904 
5905 void AMDGPUAsmParser::cvtVOP3(MCInst &Inst, const OperandVector &Operands) {
5906  OptionalImmIndexMap OptionalIdx;
5907  cvtVOP3(Inst, Operands, OptionalIdx);
5908 }
5909 
5910 void AMDGPUAsmParser::cvtVOP3P(MCInst &Inst,
5911  const OperandVector &Operands) {
5912  OptionalImmIndexMap OptIdx;
5913  const int Opc = Inst.getOpcode();
5914  const MCInstrDesc &Desc = MII.get(Opc);
5915 
5916  const bool IsPacked = (Desc.TSFlags & SIInstrFlags::IsPacked) != 0;
5917 
5918  cvtVOP3(Inst, Operands, OptIdx);
5919 
5920  if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdst_in) != -1) {
5921  assert(!IsPacked);
5922  Inst.addOperand(Inst.getOperand(0));
5923  }
5924 
5925  // FIXME: This is messy. Parse the modifiers as if it was a normal VOP3
5926  // instruction, and then figure out where to actually put the modifiers
5927 
5928  addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSel);
5929 
5930  int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
5931  if (OpSelHiIdx != -1) {
5932  int DefaultVal = IsPacked ? -1 : 0;
5933  addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyOpSelHi,
5934  DefaultVal);
5935  }
5936 
5937  int NegLoIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_lo);
5938  if (NegLoIdx != -1) {
5939  assert(IsPacked);
5940  addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegLo);
5941  addOptionalImmOperand(Inst, Operands, OptIdx, AMDGPUOperand::ImmTyNegHi);
5942  }
5943 
5944  const int Ops[] = { AMDGPU::OpName::src0,
5945  AMDGPU::OpName::src1,
5946  AMDGPU::OpName::src2 };
5947  const int ModOps[] = { AMDGPU::OpName::src0_modifiers,
5948  AMDGPU::OpName::src1_modifiers,
5949  AMDGPU::OpName::src2_modifiers };
5950 
5951  int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
5952 
5953  unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
5954  unsigned OpSelHi = 0;
5955  unsigned NegLo = 0;
5956  unsigned NegHi = 0;
5957 
5958  if (OpSelHiIdx != -1) {
5959  OpSelHi = Inst.getOperand(OpSelHiIdx).getImm();
5960  }
5961 
5962  if (NegLoIdx != -1) {
5963  int NegHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::neg_hi);
5964  NegLo = Inst.getOperand(NegLoIdx).getImm();
5965  NegHi = Inst.getOperand(NegHiIdx).getImm();
5966  }
5967 
5968  for (int J = 0; J < 3; ++J) {
5969  int OpIdx = AMDGPU::getNamedOperandIdx(Opc, Ops[J]);
5970  if (OpIdx == -1)
5971  break;
5972 
5973  uint32_t ModVal = 0;
5974 
5975  if ((OpSel & (1 << J)) != 0)
5976  ModVal |= SISrcMods::OP_SEL_0;
5977 
5978  if ((OpSelHi & (1 << J)) != 0)
5979  ModVal |= SISrcMods::OP_SEL_1;
5980 
5981  if ((NegLo & (1 << J)) != 0)
5982  ModVal |= SISrcMods::NEG;
5983 
5984  if ((NegHi & (1 << J)) != 0)
5985  ModVal |= SISrcMods::NEG_HI;
5986 
5987  int ModIdx = AMDGPU::getNamedOperandIdx(Opc, ModOps[J]);
5988 
5989  Inst.getOperand(ModIdx).setImm(Inst.getOperand(ModIdx).getImm() | ModVal);
5990  }
5991 }
5992 
5993 //===----------------------------------------------------------------------===//
5994 // dpp
5995 //===----------------------------------------------------------------------===//
5996 
5997 bool AMDGPUOperand::isDPPCtrl() const {
5998  using namespace AMDGPU::DPP;
5999 
6000  bool result = isImm() && getImmTy() == ImmTyDppCtrl && isUInt<9>(getImm());
6001  if (result) {
6002  int64_t Imm = getImm();
6003  return (Imm >= DppCtrl::QUAD_PERM_FIRST && Imm <= DppCtrl::QUAD_PERM_LAST) ||
6004  (Imm >= DppCtrl::ROW_SHL_FIRST && Imm <= DppCtrl::ROW_SHL_LAST) ||
6005  (Imm >= DppCtrl::ROW_SHR_FIRST && Imm <= DppCtrl::ROW_SHR_LAST) ||
6006  (Imm >= DppCtrl::ROW_ROR_FIRST && Imm <= DppCtrl::ROW_ROR_LAST) ||
6007  (Imm == DppCtrl::WAVE_SHL1) ||
6008  (Imm == DppCtrl::WAVE_ROL1) ||
6009  (Imm == DppCtrl::WAVE_SHR1) ||
6010  (Imm == DppCtrl::WAVE_ROR1) ||
6011  (Imm == DppCtrl::ROW_MIRROR) ||
6012  (Imm == DppCtrl::ROW_HALF_MIRROR) ||
6013  (Imm == DppCtrl::BCAST15) ||
6014  (Imm == DppCtrl::BCAST31);
6015  }
6016  return false;
6017 }
6018 
6019 bool AMDGPUOperand::isS16Imm() const {
6020  return isImm() && (isInt<16>(getImm()) || isUInt<16>(getImm()));
6021 }
6022 
6023 bool AMDGPUOperand::isU16Imm() const {
6024  return isImm() && isUInt<16>(getImm());
6025 }
6026 
6027 OperandMatchResultTy AMDGPUAsmParser::parseDim(OperandVector &Operands) {
6028  if (!isGFX10())
6029  return MatchOperand_NoMatch;
6030 
6031  SMLoc S = Parser.getTok().getLoc();
6032 
6033  if (getLexer().isNot(AsmToken::Identifier))
6034  return MatchOperand_NoMatch;
6035  if (getLexer().getTok().getString() != "dim")
6036  return MatchOperand_NoMatch;
6037 
6038  Parser.Lex();
6039  if (getLexer().isNot(AsmToken::Colon))
6040  return MatchOperand_ParseFail;
6041 
6042  Parser.Lex();
6043 
6044  // We want to allow "dim:1D" etc., but the initial 1 is tokenized as an
6045  // integer.
6046  std::string Token;
6047  if (getLexer().is(AsmToken::Integer)) {
6048  SMLoc Loc = getLexer().getTok().getEndLoc();
6049  Token = getLexer().getTok().getString();
6050  Parser.Lex();
6051  if (getLexer().getTok().getLoc() != Loc)
6052  return MatchOperand_ParseFail;
6053  }
6054  if (getLexer().isNot(AsmToken::Identifier))
6055  return MatchOperand_ParseFail;
6056  Token += getLexer().getTok().getString();
6057 
6058  StringRef DimId = Token;
6059  if (DimId.startswith("SQ_RSRC_IMG_"))
6060  DimId = DimId.substr(12);
6061 
6062  const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfoByAsmSuffix(DimId);
6063  if (!DimInfo)
6064  return MatchOperand_ParseFail;
6065 
6066  Parser.Lex();
6067 
6068  Operands.push_back(AMDGPUOperand::CreateImm(this, DimInfo->Encoding, S,
6069  AMDGPUOperand::ImmTyDim));
6070  return MatchOperand_Success;
6071 }
6072 
6074 AMDGPUAsmParser::parseDPPCtrl(OperandVector &Operands) {
6075  using namespace AMDGPU::DPP;
6076 
6077  SMLoc S = Parser.getTok().getLoc();
6078  StringRef Prefix;
6079  int64_t Int;
6080 
6081  if (getLexer().getKind() == AsmToken::Identifier) {
6082  Prefix = Parser.getTok().getString();
6083  } else {
6084  return MatchOperand_NoMatch;
6085  }
6086 
6087  if (Prefix == "row_mirror") {
6088  Int = DppCtrl::ROW_MIRROR;
6089  Parser.Lex();
6090  } else if (Prefix == "row_half_mirror") {
6092  Parser.Lex();
6093  } else {
6094  // Check to prevent parseDPPCtrlOps from eating invalid tokens
6095  if (Prefix != "quad_perm"
6096  && Prefix != "row_shl"
6097  && Prefix != "row_shr"
6098  && Prefix != "row_ror"
6099  && Prefix != "wave_shl"
6100  && Prefix != "wave_rol"
6101  && Prefix != "wave_shr"
6102  && Prefix != "wave_ror"
6103  && Prefix != "row_bcast") {
6104  return MatchOperand_NoMatch;
6105  }
6106 
6107  Parser.Lex();
6108  if (getLexer().isNot(AsmToken::Colon))
6109  return MatchOperand_ParseFail;
6110 
6111  if (Prefix == "quad_perm") {
6112  // quad_perm:[%d,%d,%d,%d]
6113  Parser.Lex();
6114  if (getLexer().isNot(AsmToken::LBrac))
6115  return MatchOperand_ParseFail;
6116  Parser.Lex();
6117 
6118  if (getParser().parseAbsoluteExpression(Int) || !(0 <= Int && Int <=3))
6119  return MatchOperand_ParseFail;
6120 
6121  for (int i = 0; i < 3; ++i) {
6122  if (getLexer().isNot(AsmToken::Comma))
6123  return MatchOperand_ParseFail;
6124  Parser.Lex();
6125 
6126  int64_t Temp;
6127  if (getParser().parseAbsoluteExpression(Temp) || !(0 <= Temp && Temp <=3))
6128  return MatchOperand_ParseFail;
6129  const int shift = i*2 + 2;
6130  Int += (Temp << shift);
6131  }
6132 
6133  if (getLexer().isNot(AsmToken::RBrac))
6134  return MatchOperand_ParseFail;
6135  Parser.Lex();
6136  } else {
6137  // sel:%d
6138  Parser.Lex();
6139  if (getParser().parseAbsoluteExpression(Int))
6140  return MatchOperand_ParseFail;
6141 
6142  if (Prefix == "row_shl" && 1 <= Int && Int <= 15) {
6143  Int |= DppCtrl::ROW_SHL0;
6144  } else if (Prefix == "row_shr" && 1 <= Int && Int <= 15) {
6145  Int |= DppCtrl::ROW_SHR0;
6146  } else if (Prefix == "row_ror" && 1 <= Int && Int <= 15) {
6147  Int |= DppCtrl::ROW_ROR0;
6148  } else if (Prefix == "wave_shl" && 1 == Int) {
6149  Int = DppCtrl::WAVE_SHL1;
6150  } else if (Prefix == "wave_rol" && 1 == Int) {
6151  Int = DppCtrl::WAVE_ROL1;
6152  } else if (Prefix == "wave_shr" && 1 == Int) {
6153  Int = DppCtrl::WAVE_SHR1;
6154  } else if (Prefix == "wave_ror" && 1 == Int) {
6155  Int = DppCtrl::WAVE_ROR1;
6156  } else if (Prefix == "row_bcast") {
6157  if (Int == 15) {
6158  Int = DppCtrl::BCAST15;
6159  } else if (Int == 31) {
6160  Int = DppCtrl::BCAST31;
6161  } else {
6162  return MatchOperand_ParseFail;
6163  }
6164  } else {
6165  return MatchOperand_ParseFail;
6166  }
6167  }
6168  }
6169 
6170  Operands.push_back(AMDGPUOperand::CreateImm(this, Int, S, AMDGPUOperand::ImmTyDppCtrl));
6171  return MatchOperand_Success;
6172 }
6173 
6174 AMDGPUOperand::Ptr AMDGPUAsmParser::defaultRowMask() const {
6175  return AMDGPUOperand::CreateImm(this, 0xf,