LLVM 19.0.0git
AMDGPUAsmParser.cpp
Go to the documentation of this file.
1//===- AMDGPUAsmParser.cpp - Parse SI asm to MCInst instructions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDKernelCodeT.h"
14#include "SIDefines.h"
15#include "SIInstrInfo.h"
16#include "SIRegisterInfo.h"
21#include "llvm/ADT/APFloat.h"
23#include "llvm/ADT/StringSet.h"
24#include "llvm/ADT/Twine.h"
27#include "llvm/MC/MCAsmInfo.h"
28#include "llvm/MC/MCContext.h"
29#include "llvm/MC/MCExpr.h"
30#include "llvm/MC/MCInst.h"
31#include "llvm/MC/MCInstrDesc.h"
36#include "llvm/MC/MCSymbol.h"
43#include <optional>
44
45using namespace llvm;
46using namespace llvm::AMDGPU;
47using namespace llvm::amdhsa;
48
49namespace {
50
51class AMDGPUAsmParser;
52
53enum RegisterKind { IS_UNKNOWN, IS_VGPR, IS_SGPR, IS_AGPR, IS_TTMP, IS_SPECIAL };
54
55//===----------------------------------------------------------------------===//
56// Operand
57//===----------------------------------------------------------------------===//
58
59class AMDGPUOperand : public MCParsedAsmOperand {
60 enum KindTy {
61 Token,
62 Immediate,
65 } Kind;
66
67 SMLoc StartLoc, EndLoc;
68 const AMDGPUAsmParser *AsmParser;
69
70public:
71 AMDGPUOperand(KindTy Kind_, const AMDGPUAsmParser *AsmParser_)
72 : Kind(Kind_), AsmParser(AsmParser_) {}
73
74 using Ptr = std::unique_ptr<AMDGPUOperand>;
75
76 struct Modifiers {
77 bool Abs = false;
78 bool Neg = false;
79 bool Sext = false;
80 bool Lit = false;
81
82 bool hasFPModifiers() const { return Abs || Neg; }
83 bool hasIntModifiers() const { return Sext; }
84 bool hasModifiers() const { return hasFPModifiers() || hasIntModifiers(); }
85
86 int64_t getFPModifiersOperand() const {
87 int64_t Operand = 0;
88 Operand |= Abs ? SISrcMods::ABS : 0u;
89 Operand |= Neg ? SISrcMods::NEG : 0u;
90 return Operand;
91 }
92
93 int64_t getIntModifiersOperand() const {
94 int64_t Operand = 0;
95 Operand |= Sext ? SISrcMods::SEXT : 0u;
96 return Operand;
97 }
98
99 int64_t getModifiersOperand() const {
100 assert(!(hasFPModifiers() && hasIntModifiers())
101 && "fp and int modifiers should not be used simultaneously");
102 if (hasFPModifiers()) {
103 return getFPModifiersOperand();
104 } else if (hasIntModifiers()) {
105 return getIntModifiersOperand();
106 } else {
107 return 0;
108 }
109 }
110
111 friend raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods);
112 };
113
114 enum ImmTy {
115 ImmTyNone,
116 ImmTyGDS,
117 ImmTyLDS,
118 ImmTyOffen,
119 ImmTyIdxen,
120 ImmTyAddr64,
121 ImmTyOffset,
122 ImmTyInstOffset,
123 ImmTyOffset0,
124 ImmTyOffset1,
125 ImmTySMEMOffsetMod,
126 ImmTyCPol,
127 ImmTyTFE,
128 ImmTyD16,
129 ImmTyClamp,
130 ImmTyOModSI,
131 ImmTySDWADstSel,
132 ImmTySDWASrc0Sel,
133 ImmTySDWASrc1Sel,
134 ImmTySDWADstUnused,
135 ImmTyDMask,
136 ImmTyDim,
137 ImmTyUNorm,
138 ImmTyDA,
139 ImmTyR128A16,
140 ImmTyA16,
141 ImmTyLWE,
142 ImmTyExpTgt,
143 ImmTyExpCompr,
144 ImmTyExpVM,
145 ImmTyFORMAT,
146 ImmTyHwreg,
147 ImmTyOff,
148 ImmTySendMsg,
149 ImmTyInterpSlot,
150 ImmTyInterpAttr,
151 ImmTyInterpAttrChan,
152 ImmTyOpSel,
153 ImmTyOpSelHi,
154 ImmTyNegLo,
155 ImmTyNegHi,
156 ImmTyIndexKey8bit,
157 ImmTyIndexKey16bit,
158 ImmTyDPP8,
159 ImmTyDppCtrl,
160 ImmTyDppRowMask,
161 ImmTyDppBankMask,
162 ImmTyDppBoundCtrl,
163 ImmTyDppFI,
164 ImmTySwizzle,
165 ImmTyGprIdxMode,
166 ImmTyHigh,
167 ImmTyBLGP,
168 ImmTyCBSZ,
169 ImmTyABID,
170 ImmTyEndpgm,
171 ImmTyWaitVDST,
172 ImmTyWaitEXP,
173 ImmTyWaitVAVDst,
174 ImmTyWaitVMVSrc,
175 ImmTyByteSel,
176 };
177
178 // Immediate operand kind.
179 // It helps to identify the location of an offending operand after an error.
180 // Note that regular literals and mandatory literals (KImm) must be handled
181 // differently. When looking for an offending operand, we should usually
182 // ignore mandatory literals because they are part of the instruction and
183 // cannot be changed. Report location of mandatory operands only for VOPD,
184 // when both OpX and OpY have a KImm and there are no other literals.
185 enum ImmKindTy {
186 ImmKindTyNone,
187 ImmKindTyLiteral,
188 ImmKindTyMandatoryLiteral,
189 ImmKindTyConst,
190 };
191
192private:
193 struct TokOp {
194 const char *Data;
195 unsigned Length;
196 };
197
198 struct ImmOp {
199 int64_t Val;
200 ImmTy Type;
201 bool IsFPImm;
202 mutable ImmKindTy Kind;
203 Modifiers Mods;
204 };
205
206 struct RegOp {
207 unsigned RegNo;
208 Modifiers Mods;
209 };
210
211 union {
212 TokOp Tok;
213 ImmOp Imm;
214 RegOp Reg;
215 const MCExpr *Expr;
216 };
217
218public:
219 bool isToken() const override { return Kind == Token; }
220
221 bool isSymbolRefExpr() const {
222 return isExpr() && Expr && isa<MCSymbolRefExpr>(Expr);
223 }
224
225 bool isImm() const override {
226 return Kind == Immediate;
227 }
228
229 void setImmKindNone() const {
230 assert(isImm());
231 Imm.Kind = ImmKindTyNone;
232 }
233
234 void setImmKindLiteral() const {
235 assert(isImm());
236 Imm.Kind = ImmKindTyLiteral;
237 }
238
239 void setImmKindMandatoryLiteral() const {
240 assert(isImm());
241 Imm.Kind = ImmKindTyMandatoryLiteral;
242 }
243
244 void setImmKindConst() const {
245 assert(isImm());
246 Imm.Kind = ImmKindTyConst;
247 }
248
249 bool IsImmKindLiteral() const {
250 return isImm() && Imm.Kind == ImmKindTyLiteral;
251 }
252
253 bool IsImmKindMandatoryLiteral() const {
254 return isImm() && Imm.Kind == ImmKindTyMandatoryLiteral;
255 }
256
257 bool isImmKindConst() const {
258 return isImm() && Imm.Kind == ImmKindTyConst;
259 }
260
261 bool isInlinableImm(MVT type) const;
262 bool isLiteralImm(MVT type) const;
263
264 bool isRegKind() const {
265 return Kind == Register;
266 }
267
268 bool isReg() const override {
269 return isRegKind() && !hasModifiers();
270 }
271
272 bool isRegOrInline(unsigned RCID, MVT type) const {
273 return isRegClass(RCID) || isInlinableImm(type);
274 }
275
276 bool isRegOrImmWithInputMods(unsigned RCID, MVT type) const {
277 return isRegOrInline(RCID, type) || isLiteralImm(type);
278 }
279
280 bool isRegOrImmWithInt16InputMods() const {
281 return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i16);
282 }
283
284 bool isRegOrImmWithIntT16InputMods() const {
285 return isRegOrImmWithInputMods(AMDGPU::VS_16RegClassID, MVT::i16);
286 }
287
288 bool isRegOrImmWithInt32InputMods() const {
289 return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::i32);
290 }
291
292 bool isRegOrInlineImmWithInt16InputMods() const {
293 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i16);
294 }
295
296 bool isRegOrInlineImmWithInt32InputMods() const {
297 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::i32);
298 }
299
300 bool isRegOrImmWithInt64InputMods() const {
301 return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::i64);
302 }
303
304 bool isRegOrImmWithFP16InputMods() const {
305 return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f16);
306 }
307
308 bool isRegOrImmWithFPT16InputMods() const {
309 return isRegOrImmWithInputMods(AMDGPU::VS_16RegClassID, MVT::f16);
310 }
311
312 bool isRegOrImmWithFP32InputMods() const {
313 return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::f32);
314 }
315
316 bool isRegOrImmWithFP64InputMods() const {
317 return isRegOrImmWithInputMods(AMDGPU::VS_64RegClassID, MVT::f64);
318 }
319
320 template <bool IsFake16> bool isRegOrInlineImmWithFP16InputMods() const {
321 return isRegOrInline(
322 IsFake16 ? AMDGPU::VS_32RegClassID : AMDGPU::VS_16RegClassID, MVT::f16);
323 }
324
325 bool isRegOrInlineImmWithFP32InputMods() const {
326 return isRegOrInline(AMDGPU::VS_32RegClassID, MVT::f32);
327 }
328
329 bool isPackedFP16InputMods() const {
330 return isRegOrImmWithInputMods(AMDGPU::VS_32RegClassID, MVT::v2f16);
331 }
332
333 bool isVReg() const {
334 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
335 isRegClass(AMDGPU::VReg_64RegClassID) ||
336 isRegClass(AMDGPU::VReg_96RegClassID) ||
337 isRegClass(AMDGPU::VReg_128RegClassID) ||
338 isRegClass(AMDGPU::VReg_160RegClassID) ||
339 isRegClass(AMDGPU::VReg_192RegClassID) ||
340 isRegClass(AMDGPU::VReg_256RegClassID) ||
341 isRegClass(AMDGPU::VReg_512RegClassID) ||
342 isRegClass(AMDGPU::VReg_1024RegClassID);
343 }
344
345 bool isVReg32() const {
346 return isRegClass(AMDGPU::VGPR_32RegClassID);
347 }
348
349 bool isVReg32OrOff() const {
350 return isOff() || isVReg32();
351 }
352
353 bool isNull() const {
354 return isRegKind() && getReg() == AMDGPU::SGPR_NULL;
355 }
356
357 bool isVRegWithInputMods() const;
358 template <bool IsFake16> bool isT16VRegWithInputMods() const;
359
360 bool isSDWAOperand(MVT type) const;
361 bool isSDWAFP16Operand() const;
362 bool isSDWAFP32Operand() const;
363 bool isSDWAInt16Operand() const;
364 bool isSDWAInt32Operand() const;
365
366 bool isImmTy(ImmTy ImmT) const {
367 return isImm() && Imm.Type == ImmT;
368 }
369
370 template <ImmTy Ty> bool isImmTy() const { return isImmTy(Ty); }
371
372 bool isImmLiteral() const { return isImmTy(ImmTyNone); }
373
374 bool isImmModifier() const {
375 return isImm() && Imm.Type != ImmTyNone;
376 }
377
378 bool isOModSI() const { return isImmTy(ImmTyOModSI); }
379 bool isDim() const { return isImmTy(ImmTyDim); }
380 bool isR128A16() const { return isImmTy(ImmTyR128A16); }
381 bool isOff() const { return isImmTy(ImmTyOff); }
382 bool isExpTgt() const { return isImmTy(ImmTyExpTgt); }
383 bool isOffen() const { return isImmTy(ImmTyOffen); }
384 bool isIdxen() const { return isImmTy(ImmTyIdxen); }
385 bool isAddr64() const { return isImmTy(ImmTyAddr64); }
386 bool isSMEMOffsetMod() const { return isImmTy(ImmTySMEMOffsetMod); }
387 bool isFlatOffset() const { return isImmTy(ImmTyOffset) || isImmTy(ImmTyInstOffset); }
388 bool isGDS() const { return isImmTy(ImmTyGDS); }
389 bool isLDS() const { return isImmTy(ImmTyLDS); }
390 bool isCPol() const { return isImmTy(ImmTyCPol); }
391 bool isIndexKey8bit() const { return isImmTy(ImmTyIndexKey8bit); }
392 bool isIndexKey16bit() const { return isImmTy(ImmTyIndexKey16bit); }
393 bool isTFE() const { return isImmTy(ImmTyTFE); }
394 bool isFORMAT() const { return isImmTy(ImmTyFORMAT) && isUInt<7>(getImm()); }
395 bool isDppFI() const { return isImmTy(ImmTyDppFI); }
396 bool isSDWADstSel() const { return isImmTy(ImmTySDWADstSel); }
397 bool isSDWASrc0Sel() const { return isImmTy(ImmTySDWASrc0Sel); }
398 bool isSDWASrc1Sel() const { return isImmTy(ImmTySDWASrc1Sel); }
399 bool isSDWADstUnused() const { return isImmTy(ImmTySDWADstUnused); }
400 bool isInterpSlot() const { return isImmTy(ImmTyInterpSlot); }
401 bool isInterpAttr() const { return isImmTy(ImmTyInterpAttr); }
402 bool isInterpAttrChan() const { return isImmTy(ImmTyInterpAttrChan); }
403 bool isOpSel() const { return isImmTy(ImmTyOpSel); }
404 bool isOpSelHi() const { return isImmTy(ImmTyOpSelHi); }
405 bool isNegLo() const { return isImmTy(ImmTyNegLo); }
406 bool isNegHi() const { return isImmTy(ImmTyNegHi); }
407
408 bool isRegOrImm() const {
409 return isReg() || isImm();
410 }
411
412 bool isRegClass(unsigned RCID) const;
413
414 bool isInlineValue() const;
415
416 bool isRegOrInlineNoMods(unsigned RCID, MVT type) const {
417 return isRegOrInline(RCID, type) && !hasModifiers();
418 }
419
420 bool isSCSrcB16() const {
421 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i16);
422 }
423
424 bool isSCSrcV2B16() const {
425 return isSCSrcB16();
426 }
427
428 bool isSCSrc_b32() const {
429 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::i32);
430 }
431
432 bool isSCSrc_b64() const {
433 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::i64);
434 }
435
436 bool isBoolReg() const;
437
438 bool isSCSrcF16() const {
439 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f16);
440 }
441
442 bool isSCSrcV2F16() const {
443 return isSCSrcF16();
444 }
445
446 bool isSCSrcF32() const {
447 return isRegOrInlineNoMods(AMDGPU::SReg_32RegClassID, MVT::f32);
448 }
449
450 bool isSCSrcF64() const {
451 return isRegOrInlineNoMods(AMDGPU::SReg_64RegClassID, MVT::f64);
452 }
453
454 bool isSSrc_b32() const {
455 return isSCSrc_b32() || isLiteralImm(MVT::i32) || isExpr();
456 }
457
458 bool isSSrc_b16() const { return isSCSrcB16() || isLiteralImm(MVT::i16); }
459
460 bool isSSrcV2B16() const {
461 llvm_unreachable("cannot happen");
462 return isSSrc_b16();
463 }
464
465 bool isSSrc_b64() const {
466 // TODO: Find out how SALU supports extension of 32-bit literals to 64 bits.
467 // See isVSrc64().
468 return isSCSrc_b64() || isLiteralImm(MVT::i64);
469 }
470
471 bool isSSrc_f32() const {
472 return isSCSrc_b32() || isLiteralImm(MVT::f32) || isExpr();
473 }
474
475 bool isSSrcF64() const { return isSCSrc_b64() || isLiteralImm(MVT::f64); }
476
477 bool isSSrc_bf16() const { return isSCSrcB16() || isLiteralImm(MVT::bf16); }
478
479 bool isSSrc_f16() const { return isSCSrcB16() || isLiteralImm(MVT::f16); }
480
481 bool isSSrcV2F16() const {
482 llvm_unreachable("cannot happen");
483 return isSSrc_f16();
484 }
485
486 bool isSSrcV2FP32() const {
487 llvm_unreachable("cannot happen");
488 return isSSrc_f32();
489 }
490
491 bool isSCSrcV2FP32() const {
492 llvm_unreachable("cannot happen");
493 return isSCSrcF32();
494 }
495
496 bool isSSrcV2INT32() const {
497 llvm_unreachable("cannot happen");
498 return isSSrc_b32();
499 }
500
501 bool isSCSrcV2INT32() const {
502 llvm_unreachable("cannot happen");
503 return isSCSrc_b32();
504 }
505
506 bool isSSrcOrLds_b32() const {
507 return isRegOrInlineNoMods(AMDGPU::SRegOrLds_32RegClassID, MVT::i32) ||
508 isLiteralImm(MVT::i32) || isExpr();
509 }
510
511 bool isVCSrc_b32() const {
512 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i32);
513 }
514
515 bool isVCSrcB64() const {
516 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::i64);
517 }
518
519 bool isVCSrcTB16() const {
520 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::i16);
521 }
522
523 bool isVCSrcTB16_Lo128() const {
524 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::i16);
525 }
526
527 bool isVCSrcFake16B16_Lo128() const {
528 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::i16);
529 }
530
531 bool isVCSrc_b16() const {
532 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::i16);
533 }
534
535 bool isVCSrc_v2b16() const { return isVCSrc_b16(); }
536
537 bool isVCSrc_f32() const {
538 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f32);
539 }
540
541 bool isVCSrcF64() const {
542 return isRegOrInlineNoMods(AMDGPU::VS_64RegClassID, MVT::f64);
543 }
544
545 bool isVCSrcTBF16() const {
546 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::bf16);
547 }
548
549 bool isVCSrcTF16() const {
550 return isRegOrInlineNoMods(AMDGPU::VS_16RegClassID, MVT::f16);
551 }
552
553 bool isVCSrcTBF16_Lo128() const {
554 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::bf16);
555 }
556
557 bool isVCSrcTF16_Lo128() const {
558 return isRegOrInlineNoMods(AMDGPU::VS_16_Lo128RegClassID, MVT::f16);
559 }
560
561 bool isVCSrcFake16BF16_Lo128() const {
562 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::bf16);
563 }
564
565 bool isVCSrcFake16F16_Lo128() const {
566 return isRegOrInlineNoMods(AMDGPU::VS_32_Lo128RegClassID, MVT::f16);
567 }
568
569 bool isVCSrc_bf16() const {
570 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::bf16);
571 }
572
573 bool isVCSrc_f16() const {
574 return isRegOrInlineNoMods(AMDGPU::VS_32RegClassID, MVT::f16);
575 }
576
577 bool isVCSrc_v2bf16() const { return isVCSrc_bf16(); }
578
579 bool isVCSrc_v2f16() const { return isVCSrc_f16(); }
580
581 bool isVSrc_b32() const {
582 return isVCSrc_f32() || isLiteralImm(MVT::i32) || isExpr();
583 }
584
585 bool isVSrc_b64() const { return isVCSrcF64() || isLiteralImm(MVT::i64); }
586
587 bool isVSrcT_b16() const { return isVCSrcTB16() || isLiteralImm(MVT::i16); }
588
589 bool isVSrcT_b16_Lo128() const {
590 return isVCSrcTB16_Lo128() || isLiteralImm(MVT::i16);
591 }
592
593 bool isVSrcFake16_b16_Lo128() const {
594 return isVCSrcFake16B16_Lo128() || isLiteralImm(MVT::i16);
595 }
596
597 bool isVSrc_b16() const { return isVCSrc_b16() || isLiteralImm(MVT::i16); }
598
599 bool isVSrc_v2b16() const { return isVSrc_b16() || isLiteralImm(MVT::v2i16); }
600
601 bool isVCSrcV2FP32() const {
602 return isVCSrcF64();
603 }
604
605 bool isVSrc_v2f32() const { return isVSrc_f64() || isLiteralImm(MVT::v2f32); }
606
607 bool isVCSrcV2INT32() const {
608 return isVCSrcB64();
609 }
610
611 bool isVSrc_v2b32() const { return isVSrc_b64() || isLiteralImm(MVT::v2i32); }
612
613 bool isVSrc_f32() const {
614 return isVCSrc_f32() || isLiteralImm(MVT::f32) || isExpr();
615 }
616
617 bool isVSrc_f64() const { return isVCSrcF64() || isLiteralImm(MVT::f64); }
618
619 bool isVSrcT_bf16() const { return isVCSrcTBF16() || isLiteralImm(MVT::bf16); }
620
621 bool isVSrcT_f16() const { return isVCSrcTF16() || isLiteralImm(MVT::f16); }
622
623 bool isVSrcT_bf16_Lo128() const {
624 return isVCSrcTBF16_Lo128() || isLiteralImm(MVT::bf16);
625 }
626
627 bool isVSrcT_f16_Lo128() const {
628 return isVCSrcTF16_Lo128() || isLiteralImm(MVT::f16);
629 }
630
631 bool isVSrcFake16_bf16_Lo128() const {
632 return isVCSrcFake16BF16_Lo128() || isLiteralImm(MVT::bf16);
633 }
634
635 bool isVSrcFake16_f16_Lo128() const {
636 return isVCSrcFake16F16_Lo128() || isLiteralImm(MVT::f16);
637 }
638
639 bool isVSrc_bf16() const { return isVCSrc_bf16() || isLiteralImm(MVT::bf16); }
640
641 bool isVSrc_f16() const { return isVCSrc_f16() || isLiteralImm(MVT::f16); }
642
643 bool isVSrc_v2bf16() const {
644 return isVSrc_bf16() || isLiteralImm(MVT::v2bf16);
645 }
646
647 bool isVSrc_v2f16() const { return isVSrc_f16() || isLiteralImm(MVT::v2f16); }
648
649 bool isVISrcB32() const {
650 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i32);
651 }
652
653 bool isVISrcB16() const {
654 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::i16);
655 }
656
657 bool isVISrcV2B16() const {
658 return isVISrcB16();
659 }
660
661 bool isVISrcF32() const {
662 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f32);
663 }
664
665 bool isVISrcF16() const {
666 return isRegOrInlineNoMods(AMDGPU::VGPR_32RegClassID, MVT::f16);
667 }
668
669 bool isVISrcV2F16() const {
670 return isVISrcF16() || isVISrcB32();
671 }
672
673 bool isVISrc_64_bf16() const {
674 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::bf16);
675 }
676
677 bool isVISrc_64_f16() const {
678 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f16);
679 }
680
681 bool isVISrc_64_b32() const {
682 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
683 }
684
685 bool isVISrc_64B64() const {
686 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i64);
687 }
688
689 bool isVISrc_64_f64() const {
690 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f64);
691 }
692
693 bool isVISrc_64V2FP32() const {
694 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::f32);
695 }
696
697 bool isVISrc_64V2INT32() const {
698 return isRegOrInlineNoMods(AMDGPU::VReg_64RegClassID, MVT::i32);
699 }
700
701 bool isVISrc_256_b32() const {
702 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
703 }
704
705 bool isVISrc_256_f32() const {
706 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
707 }
708
709 bool isVISrc_256B64() const {
710 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i64);
711 }
712
713 bool isVISrc_256_f64() const {
714 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f64);
715 }
716
717 bool isVISrc_128B16() const {
718 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i16);
719 }
720
721 bool isVISrc_128V2B16() const {
722 return isVISrc_128B16();
723 }
724
725 bool isVISrc_128_b32() const {
726 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::i32);
727 }
728
729 bool isVISrc_128_f32() const {
730 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f32);
731 }
732
733 bool isVISrc_256V2FP32() const {
734 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::f32);
735 }
736
737 bool isVISrc_256V2INT32() const {
738 return isRegOrInlineNoMods(AMDGPU::VReg_256RegClassID, MVT::i32);
739 }
740
741 bool isVISrc_512_b32() const {
742 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i32);
743 }
744
745 bool isVISrc_512B16() const {
746 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::i16);
747 }
748
749 bool isVISrc_512V2B16() const {
750 return isVISrc_512B16();
751 }
752
753 bool isVISrc_512_f32() const {
754 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f32);
755 }
756
757 bool isVISrc_512F16() const {
758 return isRegOrInlineNoMods(AMDGPU::VReg_512RegClassID, MVT::f16);
759 }
760
761 bool isVISrc_512V2F16() const {
762 return isVISrc_512F16() || isVISrc_512_b32();
763 }
764
765 bool isVISrc_1024_b32() const {
766 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i32);
767 }
768
769 bool isVISrc_1024B16() const {
770 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::i16);
771 }
772
773 bool isVISrc_1024V2B16() const {
774 return isVISrc_1024B16();
775 }
776
777 bool isVISrc_1024_f32() const {
778 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f32);
779 }
780
781 bool isVISrc_1024F16() const {
782 return isRegOrInlineNoMods(AMDGPU::VReg_1024RegClassID, MVT::f16);
783 }
784
785 bool isVISrc_1024V2F16() const {
786 return isVISrc_1024F16() || isVISrc_1024_b32();
787 }
788
789 bool isAISrcB32() const {
790 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i32);
791 }
792
793 bool isAISrcB16() const {
794 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::i16);
795 }
796
797 bool isAISrcV2B16() const {
798 return isAISrcB16();
799 }
800
801 bool isAISrcF32() const {
802 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f32);
803 }
804
805 bool isAISrcF16() const {
806 return isRegOrInlineNoMods(AMDGPU::AGPR_32RegClassID, MVT::f16);
807 }
808
809 bool isAISrcV2F16() const {
810 return isAISrcF16() || isAISrcB32();
811 }
812
813 bool isAISrc_64B64() const {
814 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::i64);
815 }
816
817 bool isAISrc_64_f64() const {
818 return isRegOrInlineNoMods(AMDGPU::AReg_64RegClassID, MVT::f64);
819 }
820
821 bool isAISrc_128_b32() const {
822 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i32);
823 }
824
825 bool isAISrc_128B16() const {
826 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::i16);
827 }
828
829 bool isAISrc_128V2B16() const {
830 return isAISrc_128B16();
831 }
832
833 bool isAISrc_128_f32() const {
834 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f32);
835 }
836
837 bool isAISrc_128F16() const {
838 return isRegOrInlineNoMods(AMDGPU::AReg_128RegClassID, MVT::f16);
839 }
840
841 bool isAISrc_128V2F16() const {
842 return isAISrc_128F16() || isAISrc_128_b32();
843 }
844
845 bool isVISrc_128_bf16() const {
846 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::bf16);
847 }
848
849 bool isVISrc_128_f16() const {
850 return isRegOrInlineNoMods(AMDGPU::VReg_128RegClassID, MVT::f16);
851 }
852
853 bool isVISrc_128V2F16() const {
854 return isVISrc_128_f16() || isVISrc_128_b32();
855 }
856
857 bool isAISrc_256B64() const {
858 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::i64);
859 }
860
861 bool isAISrc_256_f64() const {
862 return isRegOrInlineNoMods(AMDGPU::AReg_256RegClassID, MVT::f64);
863 }
864
865 bool isAISrc_512_b32() const {
866 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i32);
867 }
868
869 bool isAISrc_512B16() const {
870 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::i16);
871 }
872
873 bool isAISrc_512V2B16() const {
874 return isAISrc_512B16();
875 }
876
877 bool isAISrc_512_f32() const {
878 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f32);
879 }
880
881 bool isAISrc_512F16() const {
882 return isRegOrInlineNoMods(AMDGPU::AReg_512RegClassID, MVT::f16);
883 }
884
885 bool isAISrc_512V2F16() const {
886 return isAISrc_512F16() || isAISrc_512_b32();
887 }
888
889 bool isAISrc_1024_b32() const {
890 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i32);
891 }
892
893 bool isAISrc_1024B16() const {
894 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::i16);
895 }
896
897 bool isAISrc_1024V2B16() const {
898 return isAISrc_1024B16();
899 }
900
901 bool isAISrc_1024_f32() const {
902 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f32);
903 }
904
905 bool isAISrc_1024F16() const {
906 return isRegOrInlineNoMods(AMDGPU::AReg_1024RegClassID, MVT::f16);
907 }
908
909 bool isAISrc_1024V2F16() const {
910 return isAISrc_1024F16() || isAISrc_1024_b32();
911 }
912
913 bool isKImmFP32() const {
914 return isLiteralImm(MVT::f32);
915 }
916
917 bool isKImmFP16() const {
918 return isLiteralImm(MVT::f16);
919 }
920
921 bool isMem() const override {
922 return false;
923 }
924
925 bool isExpr() const {
926 return Kind == Expression;
927 }
928
929 bool isSOPPBrTarget() const { return isExpr() || isImm(); }
930
931 bool isSWaitCnt() const;
932 bool isDepCtr() const;
933 bool isSDelayALU() const;
934 bool isHwreg() const;
935 bool isSendMsg() const;
936 bool isSplitBarrier() const;
937 bool isSwizzle() const;
938 bool isSMRDOffset8() const;
939 bool isSMEMOffset() const;
940 bool isSMRDLiteralOffset() const;
941 bool isDPP8() const;
942 bool isDPPCtrl() const;
943 bool isBLGP() const;
944 bool isGPRIdxMode() const;
945 bool isS16Imm() const;
946 bool isU16Imm() const;
947 bool isEndpgm() const;
948
949 auto getPredicate(std::function<bool(const AMDGPUOperand &Op)> P) const {
950 return std::bind(P, *this);
951 }
952
953 StringRef getToken() const {
954 assert(isToken());
955 return StringRef(Tok.Data, Tok.Length);
956 }
957
958 int64_t getImm() const {
959 assert(isImm());
960 return Imm.Val;
961 }
962
963 void setImm(int64_t Val) {
964 assert(isImm());
965 Imm.Val = Val;
966 }
967
968 ImmTy getImmTy() const {
969 assert(isImm());
970 return Imm.Type;
971 }
972
973 MCRegister getReg() const override {
974 assert(isRegKind());
975 return Reg.RegNo;
976 }
977
978 SMLoc getStartLoc() const override {
979 return StartLoc;
980 }
981
982 SMLoc getEndLoc() const override {
983 return EndLoc;
984 }
985
986 SMRange getLocRange() const {
987 return SMRange(StartLoc, EndLoc);
988 }
989
990 Modifiers getModifiers() const {
991 assert(isRegKind() || isImmTy(ImmTyNone));
992 return isRegKind() ? Reg.Mods : Imm.Mods;
993 }
994
995 void setModifiers(Modifiers Mods) {
996 assert(isRegKind() || isImmTy(ImmTyNone));
997 if (isRegKind())
998 Reg.Mods = Mods;
999 else
1000 Imm.Mods = Mods;
1001 }
1002
1003 bool hasModifiers() const {
1004 return getModifiers().hasModifiers();
1005 }
1006
1007 bool hasFPModifiers() const {
1008 return getModifiers().hasFPModifiers();
1009 }
1010
1011 bool hasIntModifiers() const {
1012 return getModifiers().hasIntModifiers();
1013 }
1014
1015 uint64_t applyInputFPModifiers(uint64_t Val, unsigned Size) const;
1016
1017 void addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers = true) const;
1018
1019 void addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const;
1020
1021 void addRegOperands(MCInst &Inst, unsigned N) const;
1022
1023 void addRegOrImmOperands(MCInst &Inst, unsigned N) const {
1024 if (isRegKind())
1025 addRegOperands(Inst, N);
1026 else
1027 addImmOperands(Inst, N);
1028 }
1029
1030 void addRegOrImmWithInputModsOperands(MCInst &Inst, unsigned N) const {
1031 Modifiers Mods = getModifiers();
1032 Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
1033 if (isRegKind()) {
1034 addRegOperands(Inst, N);
1035 } else {
1036 addImmOperands(Inst, N, false);
1037 }
1038 }
1039
1040 void addRegOrImmWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
1041 assert(!hasIntModifiers());
1042 addRegOrImmWithInputModsOperands(Inst, N);
1043 }
1044
1045 void addRegOrImmWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
1046 assert(!hasFPModifiers());
1047 addRegOrImmWithInputModsOperands(Inst, N);
1048 }
1049
1050 void addRegWithInputModsOperands(MCInst &Inst, unsigned N) const {
1051 Modifiers Mods = getModifiers();
1052 Inst.addOperand(MCOperand::createImm(Mods.getModifiersOperand()));
1053 assert(isRegKind());
1054 addRegOperands(Inst, N);
1055 }
1056
1057 void addRegWithFPInputModsOperands(MCInst &Inst, unsigned N) const {
1058 assert(!hasIntModifiers());
1059 addRegWithInputModsOperands(Inst, N);
1060 }
1061
1062 void addRegWithIntInputModsOperands(MCInst &Inst, unsigned N) const {
1063 assert(!hasFPModifiers());
1064 addRegWithInputModsOperands(Inst, N);
1065 }
1066
1067 static void printImmTy(raw_ostream& OS, ImmTy Type) {
1068 // clang-format off
1069 switch (Type) {
1070 case ImmTyNone: OS << "None"; break;
1071 case ImmTyGDS: OS << "GDS"; break;
1072 case ImmTyLDS: OS << "LDS"; break;
1073 case ImmTyOffen: OS << "Offen"; break;
1074 case ImmTyIdxen: OS << "Idxen"; break;
1075 case ImmTyAddr64: OS << "Addr64"; break;
1076 case ImmTyOffset: OS << "Offset"; break;
1077 case ImmTyInstOffset: OS << "InstOffset"; break;
1078 case ImmTyOffset0: OS << "Offset0"; break;
1079 case ImmTyOffset1: OS << "Offset1"; break;
1080 case ImmTySMEMOffsetMod: OS << "SMEMOffsetMod"; break;
1081 case ImmTyCPol: OS << "CPol"; break;
1082 case ImmTyIndexKey8bit: OS << "index_key"; break;
1083 case ImmTyIndexKey16bit: OS << "index_key"; break;
1084 case ImmTyTFE: OS << "TFE"; break;
1085 case ImmTyD16: OS << "D16"; break;
1086 case ImmTyFORMAT: OS << "FORMAT"; break;
1087 case ImmTyClamp: OS << "Clamp"; break;
1088 case ImmTyOModSI: OS << "OModSI"; break;
1089 case ImmTyDPP8: OS << "DPP8"; break;
1090 case ImmTyDppCtrl: OS << "DppCtrl"; break;
1091 case ImmTyDppRowMask: OS << "DppRowMask"; break;
1092 case ImmTyDppBankMask: OS << "DppBankMask"; break;
1093 case ImmTyDppBoundCtrl: OS << "DppBoundCtrl"; break;
1094 case ImmTyDppFI: OS << "DppFI"; break;
1095 case ImmTySDWADstSel: OS << "SDWADstSel"; break;
1096 case ImmTySDWASrc0Sel: OS << "SDWASrc0Sel"; break;
1097 case ImmTySDWASrc1Sel: OS << "SDWASrc1Sel"; break;
1098 case ImmTySDWADstUnused: OS << "SDWADstUnused"; break;
1099 case ImmTyDMask: OS << "DMask"; break;
1100 case ImmTyDim: OS << "Dim"; break;
1101 case ImmTyUNorm: OS << "UNorm"; break;
1102 case ImmTyDA: OS << "DA"; break;
1103 case ImmTyR128A16: OS << "R128A16"; break;
1104 case ImmTyA16: OS << "A16"; break;
1105 case ImmTyLWE: OS << "LWE"; break;
1106 case ImmTyOff: OS << "Off"; break;
1107 case ImmTyExpTgt: OS << "ExpTgt"; break;
1108 case ImmTyExpCompr: OS << "ExpCompr"; break;
1109 case ImmTyExpVM: OS << "ExpVM"; break;
1110 case ImmTyHwreg: OS << "Hwreg"; break;
1111 case ImmTySendMsg: OS << "SendMsg"; break;
1112 case ImmTyInterpSlot: OS << "InterpSlot"; break;
1113 case ImmTyInterpAttr: OS << "InterpAttr"; break;
1114 case ImmTyInterpAttrChan: OS << "InterpAttrChan"; break;
1115 case ImmTyOpSel: OS << "OpSel"; break;
1116 case ImmTyOpSelHi: OS << "OpSelHi"; break;
1117 case ImmTyNegLo: OS << "NegLo"; break;
1118 case ImmTyNegHi: OS << "NegHi"; break;
1119 case ImmTySwizzle: OS << "Swizzle"; break;
1120 case ImmTyGprIdxMode: OS << "GprIdxMode"; break;
1121 case ImmTyHigh: OS << "High"; break;
1122 case ImmTyBLGP: OS << "BLGP"; break;
1123 case ImmTyCBSZ: OS << "CBSZ"; break;
1124 case ImmTyABID: OS << "ABID"; break;
1125 case ImmTyEndpgm: OS << "Endpgm"; break;
1126 case ImmTyWaitVDST: OS << "WaitVDST"; break;
1127 case ImmTyWaitEXP: OS << "WaitEXP"; break;
1128 case ImmTyWaitVAVDst: OS << "WaitVAVDst"; break;
1129 case ImmTyWaitVMVSrc: OS << "WaitVMVSrc"; break;
1130 case ImmTyByteSel: OS << "ByteSel" ; break;
1131 }
1132 // clang-format on
1133 }
1134
1135 void print(raw_ostream &OS) const override {
1136 switch (Kind) {
1137 case Register:
1138 OS << "<register " << getReg() << " mods: " << Reg.Mods << '>';
1139 break;
1140 case Immediate:
1141 OS << '<' << getImm();
1142 if (getImmTy() != ImmTyNone) {
1143 OS << " type: "; printImmTy(OS, getImmTy());
1144 }
1145 OS << " mods: " << Imm.Mods << '>';
1146 break;
1147 case Token:
1148 OS << '\'' << getToken() << '\'';
1149 break;
1150 case Expression:
1151 OS << "<expr " << *Expr << '>';
1152 break;
1153 }
1154 }
1155
1156 static AMDGPUOperand::Ptr CreateImm(const AMDGPUAsmParser *AsmParser,
1157 int64_t Val, SMLoc Loc,
1158 ImmTy Type = ImmTyNone,
1159 bool IsFPImm = false) {
1160 auto Op = std::make_unique<AMDGPUOperand>(Immediate, AsmParser);
1161 Op->Imm.Val = Val;
1162 Op->Imm.IsFPImm = IsFPImm;
1163 Op->Imm.Kind = ImmKindTyNone;
1164 Op->Imm.Type = Type;
1165 Op->Imm.Mods = Modifiers();
1166 Op->StartLoc = Loc;
1167 Op->EndLoc = Loc;
1168 return Op;
1169 }
1170
1171 static AMDGPUOperand::Ptr CreateToken(const AMDGPUAsmParser *AsmParser,
1172 StringRef Str, SMLoc Loc,
1173 bool HasExplicitEncodingSize = true) {
1174 auto Res = std::make_unique<AMDGPUOperand>(Token, AsmParser);
1175 Res->Tok.Data = Str.data();
1176 Res->Tok.Length = Str.size();
1177 Res->StartLoc = Loc;
1178 Res->EndLoc = Loc;
1179 return Res;
1180 }
1181
1182 static AMDGPUOperand::Ptr CreateReg(const AMDGPUAsmParser *AsmParser,
1183 unsigned RegNo, SMLoc S,
1184 SMLoc E) {
1185 auto Op = std::make_unique<AMDGPUOperand>(Register, AsmParser);
1186 Op->Reg.RegNo = RegNo;
1187 Op->Reg.Mods = Modifiers();
1188 Op->StartLoc = S;
1189 Op->EndLoc = E;
1190 return Op;
1191 }
1192
1193 static AMDGPUOperand::Ptr CreateExpr(const AMDGPUAsmParser *AsmParser,
1194 const class MCExpr *Expr, SMLoc S) {
1195 auto Op = std::make_unique<AMDGPUOperand>(Expression, AsmParser);
1196 Op->Expr = Expr;
1197 Op->StartLoc = S;
1198 Op->EndLoc = S;
1199 return Op;
1200 }
1201};
1202
1203raw_ostream &operator <<(raw_ostream &OS, AMDGPUOperand::Modifiers Mods) {
1204 OS << "abs:" << Mods.Abs << " neg: " << Mods.Neg << " sext:" << Mods.Sext;
1205 return OS;
1206}
1207
1208//===----------------------------------------------------------------------===//
1209// AsmParser
1210//===----------------------------------------------------------------------===//
1211
1212// Holds info related to the current kernel, e.g. count of SGPRs used.
1213// Kernel scope begins at .amdgpu_hsa_kernel directive, ends at next
1214// .amdgpu_hsa_kernel or at EOF.
1215class KernelScopeInfo {
1216 int SgprIndexUnusedMin = -1;
1217 int VgprIndexUnusedMin = -1;
1218 int AgprIndexUnusedMin = -1;
1219 MCContext *Ctx = nullptr;
1220 MCSubtargetInfo const *MSTI = nullptr;
1221
1222 void usesSgprAt(int i) {
1223 if (i >= SgprIndexUnusedMin) {
1224 SgprIndexUnusedMin = ++i;
1225 if (Ctx) {
1226 MCSymbol* const Sym =
1227 Ctx->getOrCreateSymbol(Twine(".kernel.sgpr_count"));
1228 Sym->setVariableValue(MCConstantExpr::create(SgprIndexUnusedMin, *Ctx));
1229 }
1230 }
1231 }
1232
1233 void usesVgprAt(int i) {
1234 if (i >= VgprIndexUnusedMin) {
1235 VgprIndexUnusedMin = ++i;
1236 if (Ctx) {
1237 MCSymbol* const Sym =
1238 Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
1239 int totalVGPR = getTotalNumVGPRs(isGFX90A(*MSTI), AgprIndexUnusedMin,
1240 VgprIndexUnusedMin);
1241 Sym->setVariableValue(MCConstantExpr::create(totalVGPR, *Ctx));
1242 }
1243 }
1244 }
1245
1246 void usesAgprAt(int i) {
1247 // Instruction will error in AMDGPUAsmParser::MatchAndEmitInstruction
1248 if (!hasMAIInsts(*MSTI))
1249 return;
1250
1251 if (i >= AgprIndexUnusedMin) {
1252 AgprIndexUnusedMin = ++i;
1253 if (Ctx) {
1254 MCSymbol* const Sym =
1255 Ctx->getOrCreateSymbol(Twine(".kernel.agpr_count"));
1256 Sym->setVariableValue(MCConstantExpr::create(AgprIndexUnusedMin, *Ctx));
1257
1258 // Also update vgpr_count (dependent on agpr_count for gfx908/gfx90a)
1259 MCSymbol* const vSym =
1260 Ctx->getOrCreateSymbol(Twine(".kernel.vgpr_count"));
1261 int totalVGPR = getTotalNumVGPRs(isGFX90A(*MSTI), AgprIndexUnusedMin,
1262 VgprIndexUnusedMin);
1263 vSym->setVariableValue(MCConstantExpr::create(totalVGPR, *Ctx));
1264 }
1265 }
1266 }
1267
1268public:
1269 KernelScopeInfo() = default;
1270
1271 void initialize(MCContext &Context) {
1272 Ctx = &Context;
1273 MSTI = Ctx->getSubtargetInfo();
1274
1275 usesSgprAt(SgprIndexUnusedMin = -1);
1276 usesVgprAt(VgprIndexUnusedMin = -1);
1277 if (hasMAIInsts(*MSTI)) {
1278 usesAgprAt(AgprIndexUnusedMin = -1);
1279 }
1280 }
1281
1282 void usesRegister(RegisterKind RegKind, unsigned DwordRegIndex,
1283 unsigned RegWidth) {
1284 switch (RegKind) {
1285 case IS_SGPR:
1286 usesSgprAt(DwordRegIndex + divideCeil(RegWidth, 32) - 1);
1287 break;
1288 case IS_AGPR:
1289 usesAgprAt(DwordRegIndex + divideCeil(RegWidth, 32) - 1);
1290 break;
1291 case IS_VGPR:
1292 usesVgprAt(DwordRegIndex + divideCeil(RegWidth, 32) - 1);
1293 break;
1294 default:
1295 break;
1296 }
1297 }
1298};
1299
1300class AMDGPUAsmParser : public MCTargetAsmParser {
1301 MCAsmParser &Parser;
1302
1303 unsigned ForcedEncodingSize = 0;
1304 bool ForcedDPP = false;
1305 bool ForcedSDWA = false;
1306 KernelScopeInfo KernelScope;
1307
1308 /// @name Auto-generated Match Functions
1309 /// {
1310
1311#define GET_ASSEMBLER_HEADER
1312#include "AMDGPUGenAsmMatcher.inc"
1313
1314 /// }
1315
1316private:
1317 bool ParseAsAbsoluteExpression(uint32_t &Ret);
1318 bool OutOfRangeError(SMRange Range);
1319 /// Calculate VGPR/SGPR blocks required for given target, reserved
1320 /// registers, and user-specified NextFreeXGPR values.
1321 ///
1322 /// \param Features [in] Target features, used for bug corrections.
1323 /// \param VCCUsed [in] Whether VCC special SGPR is reserved.
1324 /// \param FlatScrUsed [in] Whether FLAT_SCRATCH special SGPR is reserved.
1325 /// \param XNACKUsed [in] Whether XNACK_MASK special SGPR is reserved.
1326 /// \param EnableWavefrontSize32 [in] Value of ENABLE_WAVEFRONT_SIZE32 kernel
1327 /// descriptor field, if valid.
1328 /// \param NextFreeVGPR [in] Max VGPR number referenced, plus one.
1329 /// \param VGPRRange [in] Token range, used for VGPR diagnostics.
1330 /// \param NextFreeSGPR [in] Max SGPR number referenced, plus one.
1331 /// \param SGPRRange [in] Token range, used for SGPR diagnostics.
1332 /// \param VGPRBlocks [out] Result VGPR block count.
1333 /// \param SGPRBlocks [out] Result SGPR block count.
1334 bool calculateGPRBlocks(const FeatureBitset &Features, bool VCCUsed,
1335 bool FlatScrUsed, bool XNACKUsed,
1336 std::optional<bool> EnableWavefrontSize32,
1337 unsigned NextFreeVGPR, SMRange VGPRRange,
1338 unsigned NextFreeSGPR, SMRange SGPRRange,
1339 unsigned &VGPRBlocks, unsigned &SGPRBlocks);
1340 bool ParseDirectiveAMDGCNTarget();
1341 bool ParseDirectiveAMDHSACodeObjectVersion();
1342 bool ParseDirectiveAMDHSAKernel();
1343 bool ParseAMDKernelCodeTValue(StringRef ID, AMDGPUMCKernelCodeT &Header);
1344 bool ParseDirectiveAMDKernelCodeT();
1345 // TODO: Possibly make subtargetHasRegister const.
1346 bool subtargetHasRegister(const MCRegisterInfo &MRI, unsigned RegNo);
1347 bool ParseDirectiveAMDGPUHsaKernel();
1348
1349 bool ParseDirectiveISAVersion();
1350 bool ParseDirectiveHSAMetadata();
1351 bool ParseDirectivePALMetadataBegin();
1352 bool ParseDirectivePALMetadata();
1353 bool ParseDirectiveAMDGPULDS();
1354
1355 /// Common code to parse out a block of text (typically YAML) between start and
1356 /// end directives.
1357 bool ParseToEndDirective(const char *AssemblerDirectiveBegin,
1358 const char *AssemblerDirectiveEnd,
1359 std::string &CollectString);
1360
1361 bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
1362 RegisterKind RegKind, unsigned Reg1, SMLoc Loc);
1363 bool ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1364 unsigned &RegNum, unsigned &RegWidth,
1365 bool RestoreOnFailure = false);
1366 bool ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
1367 unsigned &RegNum, unsigned &RegWidth,
1369 unsigned ParseRegularReg(RegisterKind &RegKind, unsigned &RegNum,
1370 unsigned &RegWidth,
1372 unsigned ParseSpecialReg(RegisterKind &RegKind, unsigned &RegNum,
1373 unsigned &RegWidth,
1375 unsigned ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
1376 unsigned &RegWidth, SmallVectorImpl<AsmToken> &Tokens);
1377 bool ParseRegRange(unsigned& Num, unsigned& Width);
1378 unsigned getRegularReg(RegisterKind RegKind, unsigned RegNum, unsigned SubReg,
1379 unsigned RegWidth, SMLoc Loc);
1380
1381 bool isRegister();
1382 bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const;
1383 std::optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
1384 void initializeGprCountSymbol(RegisterKind RegKind);
1385 bool updateGprCountSymbols(RegisterKind RegKind, unsigned DwordRegIndex,
1386 unsigned RegWidth);
1387 void cvtMubufImpl(MCInst &Inst, const OperandVector &Operands,
1388 bool IsAtomic);
1389
1390public:
1391 enum OperandMode {
1392 OperandMode_Default,
1393 OperandMode_NSA,
1394 };
1395
1396 using OptionalImmIndexMap = std::map<AMDGPUOperand::ImmTy, unsigned>;
1397
1398 AMDGPUAsmParser(const MCSubtargetInfo &STI, MCAsmParser &_Parser,
1399 const MCInstrInfo &MII,
1400 const MCTargetOptions &Options)
1401 : MCTargetAsmParser(Options, STI, MII), Parser(_Parser) {
1403
1404 if (getFeatureBits().none()) {
1405 // Set default features.
1406 copySTI().ToggleFeature("southern-islands");
1407 }
1408
1409 setAvailableFeatures(ComputeAvailableFeatures(getFeatureBits()));
1410
1411 {
1412 // TODO: make those pre-defined variables read-only.
1413 // Currently there is none suitable machinery in the core llvm-mc for this.
1414 // MCSymbol::isRedefinable is intended for another purpose, and
1415 // AsmParser::parseDirectiveSet() cannot be specialized for specific target.
1417 MCContext &Ctx = getContext();
1418 if (ISA.Major >= 6 && isHsaAbi(getSTI())) {
1419 MCSymbol *Sym =
1420 Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_number"));
1421 Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
1422 Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_minor"));
1423 Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
1424 Sym = Ctx.getOrCreateSymbol(Twine(".amdgcn.gfx_generation_stepping"));
1425 Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
1426 } else {
1427 MCSymbol *Sym =
1428 Ctx.getOrCreateSymbol(Twine(".option.machine_version_major"));
1429 Sym->setVariableValue(MCConstantExpr::create(ISA.Major, Ctx));
1430 Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_minor"));
1431 Sym->setVariableValue(MCConstantExpr::create(ISA.Minor, Ctx));
1432 Sym = Ctx.getOrCreateSymbol(Twine(".option.machine_version_stepping"));
1433 Sym->setVariableValue(MCConstantExpr::create(ISA.Stepping, Ctx));
1434 }
1435 if (ISA.Major >= 6 && isHsaAbi(getSTI())) {
1436 initializeGprCountSymbol(IS_VGPR);
1437 initializeGprCountSymbol(IS_SGPR);
1438 } else
1439 KernelScope.initialize(getContext());
1440 }
1441 }
1442
1443 bool hasMIMG_R128() const {
1444 return AMDGPU::hasMIMG_R128(getSTI());
1445 }
1446
1447 bool hasPackedD16() const {
1448 return AMDGPU::hasPackedD16(getSTI());
1449 }
1450
1451 bool hasA16() const { return AMDGPU::hasA16(getSTI()); }
1452
1453 bool hasG16() const { return AMDGPU::hasG16(getSTI()); }
1454
1455 bool hasGDS() const { return AMDGPU::hasGDS(getSTI()); }
1456
1457 bool isSI() const {
1458 return AMDGPU::isSI(getSTI());
1459 }
1460
1461 bool isCI() const {
1462 return AMDGPU::isCI(getSTI());
1463 }
1464
1465 bool isVI() const {
1466 return AMDGPU::isVI(getSTI());
1467 }
1468
1469 bool isGFX9() const {
1470 return AMDGPU::isGFX9(getSTI());
1471 }
1472
1473 // TODO: isGFX90A is also true for GFX940. We need to clean it.
1474 bool isGFX90A() const {
1475 return AMDGPU::isGFX90A(getSTI());
1476 }
1477
1478 bool isGFX940() const {
1479 return AMDGPU::isGFX940(getSTI());
1480 }
1481
1482 bool isGFX9Plus() const {
1483 return AMDGPU::isGFX9Plus(getSTI());
1484 }
1485
1486 bool isGFX10() const {
1487 return AMDGPU::isGFX10(getSTI());
1488 }
1489
1490 bool isGFX10Plus() const { return AMDGPU::isGFX10Plus(getSTI()); }
1491
1492 bool isGFX11() const {
1493 return AMDGPU::isGFX11(getSTI());
1494 }
1495
1496 bool isGFX11Plus() const {
1497 return AMDGPU::isGFX11Plus(getSTI());
1498 }
1499
1500 bool isGFX12() const { return AMDGPU::isGFX12(getSTI()); }
1501
1502 bool isGFX12Plus() const { return AMDGPU::isGFX12Plus(getSTI()); }
1503
1504 bool isGFX10_AEncoding() const { return AMDGPU::isGFX10_AEncoding(getSTI()); }
1505
1506 bool isGFX10_BEncoding() const {
1508 }
1509
1510 bool hasInv2PiInlineImm() const {
1511 return getFeatureBits()[AMDGPU::FeatureInv2PiInlineImm];
1512 }
1513
1514 bool hasFlatOffsets() const {
1515 return getFeatureBits()[AMDGPU::FeatureFlatInstOffsets];
1516 }
1517
1518 bool hasArchitectedFlatScratch() const {
1519 return getFeatureBits()[AMDGPU::FeatureArchitectedFlatScratch];
1520 }
1521
1522 bool hasSGPR102_SGPR103() const {
1523 return !isVI() && !isGFX9();
1524 }
1525
1526 bool hasSGPR104_SGPR105() const { return isGFX10Plus(); }
1527
1528 bool hasIntClamp() const {
1529 return getFeatureBits()[AMDGPU::FeatureIntClamp];
1530 }
1531
1532 bool hasPartialNSAEncoding() const {
1533 return getFeatureBits()[AMDGPU::FeaturePartialNSAEncoding];
1534 }
1535
1536 unsigned getNSAMaxSize(bool HasSampler = false) const {
1537 return AMDGPU::getNSAMaxSize(getSTI(), HasSampler);
1538 }
1539
1540 unsigned getMaxNumUserSGPRs() const {
1542 }
1543
1544 bool hasKernargPreload() const { return AMDGPU::hasKernargPreload(getSTI()); }
1545
1546 AMDGPUTargetStreamer &getTargetStreamer() {
1548 return static_cast<AMDGPUTargetStreamer &>(TS);
1549 }
1550
1551 const MCRegisterInfo *getMRI() const {
1552 // We need this const_cast because for some reason getContext() is not const
1553 // in MCAsmParser.
1554 return const_cast<AMDGPUAsmParser*>(this)->getContext().getRegisterInfo();
1555 }
1556
1557 const MCInstrInfo *getMII() const {
1558 return &MII;
1559 }
1560
1561 const FeatureBitset &getFeatureBits() const {
1562 return getSTI().getFeatureBits();
1563 }
1564
1565 void setForcedEncodingSize(unsigned Size) { ForcedEncodingSize = Size; }
1566 void setForcedDPP(bool ForceDPP_) { ForcedDPP = ForceDPP_; }
1567 void setForcedSDWA(bool ForceSDWA_) { ForcedSDWA = ForceSDWA_; }
1568
1569 unsigned getForcedEncodingSize() const { return ForcedEncodingSize; }
1570 bool isForcedVOP3() const { return ForcedEncodingSize == 64; }
1571 bool isForcedDPP() const { return ForcedDPP; }
1572 bool isForcedSDWA() const { return ForcedSDWA; }
1573 ArrayRef<unsigned> getMatchedVariants() const;
1574 StringRef getMatchedVariantName() const;
1575
1576 std::unique_ptr<AMDGPUOperand> parseRegister(bool RestoreOnFailure = false);
1577 bool ParseRegister(MCRegister &RegNo, SMLoc &StartLoc, SMLoc &EndLoc,
1578 bool RestoreOnFailure);
1579 bool parseRegister(MCRegister &Reg, SMLoc &StartLoc, SMLoc &EndLoc) override;
1581 SMLoc &EndLoc) override;
1582 unsigned checkTargetMatchPredicate(MCInst &Inst) override;
1584 unsigned Kind) override;
1585 bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
1588 bool MatchingInlineAsm) override;
1589 bool ParseDirective(AsmToken DirectiveID) override;
1590 ParseStatus parseOperand(OperandVector &Operands, StringRef Mnemonic,
1591 OperandMode Mode = OperandMode_Default);
1592 StringRef parseMnemonicSuffix(StringRef Name);
1594 SMLoc NameLoc, OperandVector &Operands) override;
1595 //bool ProcessInstruction(MCInst &Inst);
1596
1598
1599 ParseStatus parseIntWithPrefix(const char *Prefix, int64_t &Int);
1600
1602 parseIntWithPrefix(const char *Prefix, OperandVector &Operands,
1603 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1604 std::function<bool(int64_t &)> ConvertResult = nullptr);
1605
1606 ParseStatus parseOperandArrayWithPrefix(
1607 const char *Prefix, OperandVector &Operands,
1608 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone,
1609 bool (*ConvertResult)(int64_t &) = nullptr);
1610
1612 parseNamedBit(StringRef Name, OperandVector &Operands,
1613 AMDGPUOperand::ImmTy ImmTy = AMDGPUOperand::ImmTyNone);
1614 unsigned getCPolKind(StringRef Id, StringRef Mnemo, bool &Disabling) const;
1616 ParseStatus parseScope(OperandVector &Operands, int64_t &Scope);
1617 ParseStatus parseTH(OperandVector &Operands, int64_t &TH);
1618 ParseStatus parseStringWithPrefix(StringRef Prefix, StringRef &Value,
1619 SMLoc &StringLoc);
1620
1621 bool isModifier();
1622 bool isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1623 bool isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1624 bool isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const;
1625 bool isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const;
1626 bool parseSP3NegModifier();
1627 ParseStatus parseImm(OperandVector &Operands, bool HasSP3AbsModifier = false,
1628 bool HasLit = false);
1630 ParseStatus parseRegOrImm(OperandVector &Operands, bool HasSP3AbsMod = false,
1631 bool HasLit = false);
1632 ParseStatus parseRegOrImmWithFPInputMods(OperandVector &Operands,
1633 bool AllowImm = true);
1634 ParseStatus parseRegOrImmWithIntInputMods(OperandVector &Operands,
1635 bool AllowImm = true);
1636 ParseStatus parseRegWithFPInputMods(OperandVector &Operands);
1637 ParseStatus parseRegWithIntInputMods(OperandVector &Operands);
1638 ParseStatus parseVReg32OrOff(OperandVector &Operands);
1639 ParseStatus tryParseIndexKey(OperandVector &Operands,
1640 AMDGPUOperand::ImmTy ImmTy);
1641 ParseStatus parseIndexKey8bit(OperandVector &Operands);
1642 ParseStatus parseIndexKey16bit(OperandVector &Operands);
1643
1644 ParseStatus parseDfmtNfmt(int64_t &Format);
1645 ParseStatus parseUfmt(int64_t &Format);
1646 ParseStatus parseSymbolicSplitFormat(StringRef FormatStr, SMLoc Loc,
1647 int64_t &Format);
1648 ParseStatus parseSymbolicUnifiedFormat(StringRef FormatStr, SMLoc Loc,
1649 int64_t &Format);
1650 ParseStatus parseFORMAT(OperandVector &Operands);
1651 ParseStatus parseSymbolicOrNumericFormat(int64_t &Format);
1652 ParseStatus parseNumericFormat(int64_t &Format);
1653 ParseStatus parseFlatOffset(OperandVector &Operands);
1654 ParseStatus parseR128A16(OperandVector &Operands);
1656 bool tryParseFmt(const char *Pref, int64_t MaxVal, int64_t &Val);
1657 bool matchDfmtNfmt(int64_t &Dfmt, int64_t &Nfmt, StringRef FormatStr, SMLoc Loc);
1658
1659 void cvtExp(MCInst &Inst, const OperandVector &Operands);
1660
1661 bool parseCnt(int64_t &IntVal);
1662 ParseStatus parseSWaitCnt(OperandVector &Operands);
1663
1664 bool parseDepCtr(int64_t &IntVal, unsigned &Mask);
1665 void depCtrError(SMLoc Loc, int ErrorId, StringRef DepCtrName);
1666 ParseStatus parseDepCtr(OperandVector &Operands);
1667
1668 bool parseDelay(int64_t &Delay);
1669 ParseStatus parseSDelayALU(OperandVector &Operands);
1670
1671 ParseStatus parseHwreg(OperandVector &Operands);
1672
1673private:
1674 struct OperandInfoTy {
1675 SMLoc Loc;
1676 int64_t Val;
1677 bool IsSymbolic = false;
1678 bool IsDefined = false;
1679
1680 OperandInfoTy(int64_t Val) : Val(Val) {}
1681 };
1682
1683 struct StructuredOpField : OperandInfoTy {
1686 unsigned Width;
1687 bool IsDefined = false;
1688
1689 StructuredOpField(StringLiteral Id, StringLiteral Desc, unsigned Width,
1690 int64_t Default)
1691 : OperandInfoTy(Default), Id(Id), Desc(Desc), Width(Width) {}
1692 virtual ~StructuredOpField() = default;
1693
1694 bool Error(AMDGPUAsmParser &Parser, const Twine &Err) const {
1695 Parser.Error(Loc, "invalid " + Desc + ": " + Err);
1696 return false;
1697 }
1698
1699 virtual bool validate(AMDGPUAsmParser &Parser) const {
1700 if (IsSymbolic && Val == OPR_ID_UNSUPPORTED)
1701 return Error(Parser, "not supported on this GPU");
1702 if (!isUIntN(Width, Val))
1703 return Error(Parser, "only " + Twine(Width) + "-bit values are legal");
1704 return true;
1705 }
1706 };
1707
1708 ParseStatus parseStructuredOpFields(ArrayRef<StructuredOpField *> Fields);
1709 bool validateStructuredOpFields(ArrayRef<const StructuredOpField *> Fields);
1710
1711 bool parseSendMsgBody(OperandInfoTy &Msg, OperandInfoTy &Op, OperandInfoTy &Stream);
1712 bool validateSendMsg(const OperandInfoTy &Msg,
1713 const OperandInfoTy &Op,
1714 const OperandInfoTy &Stream);
1715
1716 ParseStatus parseHwregFunc(OperandInfoTy &HwReg, OperandInfoTy &Offset,
1717 OperandInfoTy &Width);
1718
1719 SMLoc getFlatOffsetLoc(const OperandVector &Operands) const;
1720 SMLoc getSMEMOffsetLoc(const OperandVector &Operands) const;
1721 SMLoc getBLGPLoc(const OperandVector &Operands) const;
1722
1723 SMLoc getOperandLoc(std::function<bool(const AMDGPUOperand&)> Test,
1724 const OperandVector &Operands) const;
1725 SMLoc getImmLoc(AMDGPUOperand::ImmTy Type, const OperandVector &Operands) const;
1726 SMLoc getRegLoc(unsigned Reg, const OperandVector &Operands) const;
1727 SMLoc getLitLoc(const OperandVector &Operands,
1728 bool SearchMandatoryLiterals = false) const;
1729 SMLoc getMandatoryLitLoc(const OperandVector &Operands) const;
1730 SMLoc getConstLoc(const OperandVector &Operands) const;
1731 SMLoc getInstLoc(const OperandVector &Operands) const;
1732
1733 bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc, const OperandVector &Operands);
1734 bool validateOffset(const MCInst &Inst, const OperandVector &Operands);
1735 bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands);
1736 bool validateSMEMOffset(const MCInst &Inst, const OperandVector &Operands);
1737 bool validateSOPLiteral(const MCInst &Inst) const;
1738 bool validateConstantBusLimitations(const MCInst &Inst, const OperandVector &Operands);
1739 bool validateVOPDRegBankConstraints(const MCInst &Inst,
1740 const OperandVector &Operands);
1741 bool validateIntClampSupported(const MCInst &Inst);
1742 bool validateMIMGAtomicDMask(const MCInst &Inst);
1743 bool validateMIMGGatherDMask(const MCInst &Inst);
1744 bool validateMovrels(const MCInst &Inst, const OperandVector &Operands);
1745 bool validateMIMGDataSize(const MCInst &Inst, const SMLoc &IDLoc);
1746 bool validateMIMGAddrSize(const MCInst &Inst, const SMLoc &IDLoc);
1747 bool validateMIMGD16(const MCInst &Inst);
1748 bool validateMIMGMSAA(const MCInst &Inst);
1749 bool validateOpSel(const MCInst &Inst);
1750 bool validateNeg(const MCInst &Inst, int OpName);
1751 bool validateDPP(const MCInst &Inst, const OperandVector &Operands);
1752 bool validateVccOperand(unsigned Reg) const;
1753 bool validateVOPLiteral(const MCInst &Inst, const OperandVector &Operands);
1754 bool validateMAIAccWrite(const MCInst &Inst, const OperandVector &Operands);
1755 bool validateMAISrc2(const MCInst &Inst, const OperandVector &Operands);
1756 bool validateMFMA(const MCInst &Inst, const OperandVector &Operands);
1757 bool validateAGPRLdSt(const MCInst &Inst) const;
1758 bool validateVGPRAlign(const MCInst &Inst) const;
1759 bool validateBLGP(const MCInst &Inst, const OperandVector &Operands);
1760 bool validateDS(const MCInst &Inst, const OperandVector &Operands);
1761 bool validateGWS(const MCInst &Inst, const OperandVector &Operands);
1762 bool validateDivScale(const MCInst &Inst);
1763 bool validateWaitCnt(const MCInst &Inst, const OperandVector &Operands);
1764 bool validateCoherencyBits(const MCInst &Inst, const OperandVector &Operands,
1765 const SMLoc &IDLoc);
1766 bool validateTHAndScopeBits(const MCInst &Inst, const OperandVector &Operands,
1767 const unsigned CPol);
1768 bool validateExeczVcczOperands(const OperandVector &Operands);
1769 bool validateTFE(const MCInst &Inst, const OperandVector &Operands);
1770 std::optional<StringRef> validateLdsDirect(const MCInst &Inst);
1771 unsigned getConstantBusLimit(unsigned Opcode) const;
1772 bool usesConstantBus(const MCInst &Inst, unsigned OpIdx);
1773 bool isInlineConstant(const MCInst &Inst, unsigned OpIdx) const;
1774 unsigned findImplicitSGPRReadInVOP(const MCInst &Inst) const;
1775
1776 bool isSupportedMnemo(StringRef Mnemo,
1777 const FeatureBitset &FBS);
1778 bool isSupportedMnemo(StringRef Mnemo,
1779 const FeatureBitset &FBS,
1780 ArrayRef<unsigned> Variants);
1781 bool checkUnsupportedInstruction(StringRef Name, const SMLoc &IDLoc);
1782
1783 bool isId(const StringRef Id) const;
1784 bool isId(const AsmToken &Token, const StringRef Id) const;
1785 bool isToken(const AsmToken::TokenKind Kind) const;
1786 StringRef getId() const;
1787 bool trySkipId(const StringRef Id);
1788 bool trySkipId(const StringRef Pref, const StringRef Id);
1789 bool trySkipId(const StringRef Id, const AsmToken::TokenKind Kind);
1790 bool trySkipToken(const AsmToken::TokenKind Kind);
1791 bool skipToken(const AsmToken::TokenKind Kind, const StringRef ErrMsg);
1792 bool parseString(StringRef &Val, const StringRef ErrMsg = "expected a string");
1793 bool parseId(StringRef &Val, const StringRef ErrMsg = "");
1794
1795 void peekTokens(MutableArrayRef<AsmToken> Tokens);
1796 AsmToken::TokenKind getTokenKind() const;
1797 bool parseExpr(int64_t &Imm, StringRef Expected = "");
1799 StringRef getTokenStr() const;
1800 AsmToken peekToken(bool ShouldSkipSpace = true);
1801 AsmToken getToken() const;
1802 SMLoc getLoc() const;
1803 void lex();
1804
1805public:
1806 void onBeginOfFile() override;
1807 bool parsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc) override;
1808
1809 ParseStatus parseCustomOperand(OperandVector &Operands, unsigned MCK);
1810
1811 ParseStatus parseExpTgt(OperandVector &Operands);
1812 ParseStatus parseSendMsg(OperandVector &Operands);
1813 ParseStatus parseInterpSlot(OperandVector &Operands);
1814 ParseStatus parseInterpAttr(OperandVector &Operands);
1815 ParseStatus parseSOPPBrTarget(OperandVector &Operands);
1816 ParseStatus parseBoolReg(OperandVector &Operands);
1817
1818 bool parseSwizzleOperand(int64_t &Op,
1819 const unsigned MinVal,
1820 const unsigned MaxVal,
1821 const StringRef ErrMsg,
1822 SMLoc &Loc);
1823 bool parseSwizzleOperands(const unsigned OpNum, int64_t* Op,
1824 const unsigned MinVal,
1825 const unsigned MaxVal,
1826 const StringRef ErrMsg);
1827 ParseStatus parseSwizzle(OperandVector &Operands);
1828 bool parseSwizzleOffset(int64_t &Imm);
1829 bool parseSwizzleMacro(int64_t &Imm);
1830 bool parseSwizzleQuadPerm(int64_t &Imm);
1831 bool parseSwizzleBitmaskPerm(int64_t &Imm);
1832 bool parseSwizzleBroadcast(int64_t &Imm);
1833 bool parseSwizzleSwap(int64_t &Imm);
1834 bool parseSwizzleReverse(int64_t &Imm);
1835
1836 ParseStatus parseGPRIdxMode(OperandVector &Operands);
1837 int64_t parseGPRIdxMacro();
1838
1839 void cvtMubuf(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, false); }
1840 void cvtMubufAtomic(MCInst &Inst, const OperandVector &Operands) { cvtMubufImpl(Inst, Operands, true); }
1841
1842 ParseStatus parseOModSI(OperandVector &Operands);
1843
1844 void cvtVOP3(MCInst &Inst, const OperandVector &Operands,
1845 OptionalImmIndexMap &OptionalIdx);
1846 void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands);
1847 void cvtVOP3(MCInst &Inst, const OperandVector &Operands);
1848 void cvtVOP3P(MCInst &Inst, const OperandVector &Operands);
1849 void cvtSWMMAC(MCInst &Inst, const OperandVector &Operands);
1850
1851 void cvtVOPD(MCInst &Inst, const OperandVector &Operands);
1852 void cvtVOP3OpSel(MCInst &Inst, const OperandVector &Operands,
1853 OptionalImmIndexMap &OptionalIdx);
1854 void cvtVOP3P(MCInst &Inst, const OperandVector &Operands,
1855 OptionalImmIndexMap &OptionalIdx);
1856
1857 void cvtVOP3Interp(MCInst &Inst, const OperandVector &Operands);
1858 void cvtVINTERP(MCInst &Inst, const OperandVector &Operands);
1859
1860 bool parseDimId(unsigned &Encoding);
1862 bool convertDppBoundCtrl(int64_t &BoundCtrl);
1864 ParseStatus parseDPPCtrl(OperandVector &Operands);
1865 bool isSupportedDPPCtrl(StringRef Ctrl, const OperandVector &Operands);
1866 int64_t parseDPPCtrlSel(StringRef Ctrl);
1867 int64_t parseDPPCtrlPerm();
1868 void cvtDPP(MCInst &Inst, const OperandVector &Operands, bool IsDPP8 = false);
1869 void cvtDPP8(MCInst &Inst, const OperandVector &Operands) {
1870 cvtDPP(Inst, Operands, true);
1871 }
1872 void cvtVOP3DPP(MCInst &Inst, const OperandVector &Operands,
1873 bool IsDPP8 = false);
1874 void cvtVOP3DPP8(MCInst &Inst, const OperandVector &Operands) {
1875 cvtVOP3DPP(Inst, Operands, true);
1876 }
1877
1878 ParseStatus parseSDWASel(OperandVector &Operands, StringRef Prefix,
1879 AMDGPUOperand::ImmTy Type);
1880 ParseStatus parseSDWADstUnused(OperandVector &Operands);
1881 void cvtSdwaVOP1(MCInst &Inst, const OperandVector &Operands);
1882 void cvtSdwaVOP2(MCInst &Inst, const OperandVector &Operands);
1883 void cvtSdwaVOP2b(MCInst &Inst, const OperandVector &Operands);
1884 void cvtSdwaVOP2e(MCInst &Inst, const OperandVector &Operands);
1885 void cvtSdwaVOPC(MCInst &Inst, const OperandVector &Operands);
1886 void cvtSDWA(MCInst &Inst, const OperandVector &Operands,
1887 uint64_t BasicInstType,
1888 bool SkipDstVcc = false,
1889 bool SkipSrcVcc = false);
1890
1891 ParseStatus parseEndpgm(OperandVector &Operands);
1892
1894};
1895
1896} // end anonymous namespace
1897
1898// May be called with integer type with equivalent bitwidth.
1899static const fltSemantics *getFltSemantics(unsigned Size) {
1900 switch (Size) {
1901 case 4:
1902 return &APFloat::IEEEsingle();
1903 case 8:
1904 return &APFloat::IEEEdouble();
1905 case 2:
1906 return &APFloat::IEEEhalf();
1907 default:
1908 llvm_unreachable("unsupported fp type");
1909 }
1910}
1911
1913 return getFltSemantics(VT.getSizeInBits() / 8);
1914}
1915
1917 switch (OperandType) {
1918 // When floating-point immediate is used as operand of type i16, the 32-bit
1919 // representation of the constant truncated to the 16 LSBs should be used.
1939 return &APFloat::IEEEsingle();
1945 return &APFloat::IEEEdouble();
1954 return &APFloat::IEEEhalf();
1962 return &APFloat::BFloat();
1963 default:
1964 llvm_unreachable("unsupported fp type");
1965 }
1966}
1967
1968//===----------------------------------------------------------------------===//
1969// Operand
1970//===----------------------------------------------------------------------===//
1971
1972static bool canLosslesslyConvertToFPType(APFloat &FPLiteral, MVT VT) {
1973 bool Lost;
1974
1975 // Convert literal to single precision
1977 APFloat::rmNearestTiesToEven,
1978 &Lost);
1979 // We allow precision lost but not overflow or underflow
1980 if (Status != APFloat::opOK &&
1981 Lost &&
1982 ((Status & APFloat::opOverflow) != 0 ||
1983 (Status & APFloat::opUnderflow) != 0)) {
1984 return false;
1985 }
1986
1987 return true;
1988}
1989
1990static bool isSafeTruncation(int64_t Val, unsigned Size) {
1991 return isUIntN(Size, Val) || isIntN(Size, Val);
1992}
1993
1994static bool isInlineableLiteralOp16(int64_t Val, MVT VT, bool HasInv2Pi) {
1995 if (VT.getScalarType() == MVT::i16)
1996 return isInlinableLiteral32(Val, HasInv2Pi);
1997
1998 if (VT.getScalarType() == MVT::f16)
1999 return AMDGPU::isInlinableLiteralFP16(Val, HasInv2Pi);
2000
2001 assert(VT.getScalarType() == MVT::bf16);
2002
2003 return AMDGPU::isInlinableLiteralBF16(Val, HasInv2Pi);
2004}
2005
2006bool AMDGPUOperand::isInlinableImm(MVT type) const {
2007
2008 // This is a hack to enable named inline values like
2009 // shared_base with both 32-bit and 64-bit operands.
2010 // Note that these values are defined as
2011 // 32-bit operands only.
2012 if (isInlineValue()) {
2013 return true;
2014 }
2015
2016 if (!isImmTy(ImmTyNone)) {
2017 // Only plain immediates are inlinable (e.g. "clamp" attribute is not)
2018 return false;
2019 }
2020 // TODO: We should avoid using host float here. It would be better to
2021 // check the float bit values which is what a few other places do.
2022 // We've had bot failures before due to weird NaN support on mips hosts.
2023
2024 APInt Literal(64, Imm.Val);
2025
2026 if (Imm.IsFPImm) { // We got fp literal token
2027 if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
2029 AsmParser->hasInv2PiInlineImm());
2030 }
2031
2032 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
2033 if (!canLosslesslyConvertToFPType(FPLiteral, type))
2034 return false;
2035
2036 if (type.getScalarSizeInBits() == 16) {
2037 bool Lost = false;
2038 switch (type.getScalarType().SimpleTy) {
2039 default:
2040 llvm_unreachable("unknown 16-bit type");
2041 case MVT::bf16:
2042 FPLiteral.convert(APFloatBase::BFloat(), APFloat::rmNearestTiesToEven,
2043 &Lost);
2044 break;
2045 case MVT::f16:
2046 FPLiteral.convert(APFloatBase::IEEEhalf(), APFloat::rmNearestTiesToEven,
2047 &Lost);
2048 break;
2049 case MVT::i16:
2050 FPLiteral.convert(APFloatBase::IEEEsingle(),
2051 APFloat::rmNearestTiesToEven, &Lost);
2052 break;
2053 }
2054 // We need to use 32-bit representation here because when a floating-point
2055 // inline constant is used as an i16 operand, its 32-bit representation
2056 // representation will be used. We will need the 32-bit value to check if
2057 // it is FP inline constant.
2058 uint32_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2059 return isInlineableLiteralOp16(ImmVal, type,
2060 AsmParser->hasInv2PiInlineImm());
2061 }
2062
2063 // Check if single precision literal is inlinable
2065 static_cast<int32_t>(FPLiteral.bitcastToAPInt().getZExtValue()),
2066 AsmParser->hasInv2PiInlineImm());
2067 }
2068
2069 // We got int literal token.
2070 if (type == MVT::f64 || type == MVT::i64) { // Expected 64-bit operand
2072 AsmParser->hasInv2PiInlineImm());
2073 }
2074
2075 if (!isSafeTruncation(Imm.Val, type.getScalarSizeInBits())) {
2076 return false;
2077 }
2078
2079 if (type.getScalarSizeInBits() == 16) {
2081 static_cast<int16_t>(Literal.getLoBits(16).getSExtValue()),
2082 type, AsmParser->hasInv2PiInlineImm());
2083 }
2084
2086 static_cast<int32_t>(Literal.getLoBits(32).getZExtValue()),
2087 AsmParser->hasInv2PiInlineImm());
2088}
2089
2090bool AMDGPUOperand::isLiteralImm(MVT type) const {
2091 // Check that this immediate can be added as literal
2092 if (!isImmTy(ImmTyNone)) {
2093 return false;
2094 }
2095
2096 if (!Imm.IsFPImm) {
2097 // We got int literal token.
2098
2099 if (type == MVT::f64 && hasFPModifiers()) {
2100 // Cannot apply fp modifiers to int literals preserving the same semantics
2101 // for VOP1/2/C and VOP3 because of integer truncation. To avoid ambiguity,
2102 // disable these cases.
2103 return false;
2104 }
2105
2106 unsigned Size = type.getSizeInBits();
2107 if (Size == 64)
2108 Size = 32;
2109
2110 // FIXME: 64-bit operands can zero extend, sign extend, or pad zeroes for FP
2111 // types.
2112 return isSafeTruncation(Imm.Val, Size);
2113 }
2114
2115 // We got fp literal token
2116 if (type == MVT::f64) { // Expected 64-bit fp operand
2117 // We would set low 64-bits of literal to zeroes but we accept this literals
2118 return true;
2119 }
2120
2121 if (type == MVT::i64) { // Expected 64-bit int operand
2122 // We don't allow fp literals in 64-bit integer instructions. It is
2123 // unclear how we should encode them.
2124 return false;
2125 }
2126
2127 // We allow fp literals with f16x2 operands assuming that the specified
2128 // literal goes into the lower half and the upper half is zero. We also
2129 // require that the literal may be losslessly converted to f16.
2130 //
2131 // For i16x2 operands, we assume that the specified literal is encoded as a
2132 // single-precision float. This is pretty odd, but it matches SP3 and what
2133 // happens in hardware.
2134 MVT ExpectedType = (type == MVT::v2f16) ? MVT::f16
2135 : (type == MVT::v2i16) ? MVT::f32
2136 : (type == MVT::v2f32) ? MVT::f32
2137 : type;
2138
2139 APFloat FPLiteral(APFloat::IEEEdouble(), APInt(64, Imm.Val));
2140 return canLosslesslyConvertToFPType(FPLiteral, ExpectedType);
2141}
2142
2143bool AMDGPUOperand::isRegClass(unsigned RCID) const {
2144 return isRegKind() && AsmParser->getMRI()->getRegClass(RCID).contains(getReg());
2145}
2146
2147bool AMDGPUOperand::isVRegWithInputMods() const {
2148 return isRegClass(AMDGPU::VGPR_32RegClassID) ||
2149 // GFX90A allows DPP on 64-bit operands.
2150 (isRegClass(AMDGPU::VReg_64RegClassID) &&
2151 AsmParser->getFeatureBits()[AMDGPU::FeatureDPALU_DPP]);
2152}
2153
2154template <bool IsFake16> bool AMDGPUOperand::isT16VRegWithInputMods() const {
2155 return isRegClass(IsFake16 ? AMDGPU::VGPR_32_Lo128RegClassID
2156 : AMDGPU::VGPR_16_Lo128RegClassID);
2157}
2158
2159bool AMDGPUOperand::isSDWAOperand(MVT type) const {
2160 if (AsmParser->isVI())
2161 return isVReg32();
2162 else if (AsmParser->isGFX9Plus())
2163 return isRegClass(AMDGPU::VS_32RegClassID) || isInlinableImm(type);
2164 else
2165 return false;
2166}
2167
2168bool AMDGPUOperand::isSDWAFP16Operand() const {
2169 return isSDWAOperand(MVT::f16);
2170}
2171
2172bool AMDGPUOperand::isSDWAFP32Operand() const {
2173 return isSDWAOperand(MVT::f32);
2174}
2175
2176bool AMDGPUOperand::isSDWAInt16Operand() const {
2177 return isSDWAOperand(MVT::i16);
2178}
2179
2180bool AMDGPUOperand::isSDWAInt32Operand() const {
2181 return isSDWAOperand(MVT::i32);
2182}
2183
2184bool AMDGPUOperand::isBoolReg() const {
2185 auto FB = AsmParser->getFeatureBits();
2186 return isReg() && ((FB[AMDGPU::FeatureWavefrontSize64] && isSCSrc_b64()) ||
2187 (FB[AMDGPU::FeatureWavefrontSize32] && isSCSrc_b32()));
2188}
2189
2190uint64_t AMDGPUOperand::applyInputFPModifiers(uint64_t Val, unsigned Size) const
2191{
2192 assert(isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
2193 assert(Size == 2 || Size == 4 || Size == 8);
2194
2195 const uint64_t FpSignMask = (1ULL << (Size * 8 - 1));
2196
2197 if (Imm.Mods.Abs) {
2198 Val &= ~FpSignMask;
2199 }
2200 if (Imm.Mods.Neg) {
2201 Val ^= FpSignMask;
2202 }
2203
2204 return Val;
2205}
2206
2207void AMDGPUOperand::addImmOperands(MCInst &Inst, unsigned N, bool ApplyModifiers) const {
2208 if (isExpr()) {
2210 return;
2211 }
2212
2213 if (AMDGPU::isSISrcOperand(AsmParser->getMII()->get(Inst.getOpcode()),
2214 Inst.getNumOperands())) {
2215 addLiteralImmOperand(Inst, Imm.Val,
2216 ApplyModifiers &
2217 isImmTy(ImmTyNone) && Imm.Mods.hasFPModifiers());
2218 } else {
2219 assert(!isImmTy(ImmTyNone) || !hasModifiers());
2221 setImmKindNone();
2222 }
2223}
2224
2225void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyModifiers) const {
2226 const auto& InstDesc = AsmParser->getMII()->get(Inst.getOpcode());
2227 auto OpNum = Inst.getNumOperands();
2228 // Check that this operand accepts literals
2229 assert(AMDGPU::isSISrcOperand(InstDesc, OpNum));
2230
2231 if (ApplyModifiers) {
2232 assert(AMDGPU::isSISrcFPOperand(InstDesc, OpNum));
2233 const unsigned Size = Imm.IsFPImm ? sizeof(double) : getOperandSize(InstDesc, OpNum);
2234 Val = applyInputFPModifiers(Val, Size);
2235 }
2236
2237 APInt Literal(64, Val);
2238 uint8_t OpTy = InstDesc.operands()[OpNum].OperandType;
2239
2240 if (Imm.IsFPImm) { // We got fp literal token
2241 switch (OpTy) {
2247 if (AMDGPU::isInlinableLiteral64(Literal.getZExtValue(),
2248 AsmParser->hasInv2PiInlineImm())) {
2249 Inst.addOperand(MCOperand::createImm(Literal.getZExtValue()));
2250 setImmKindConst();
2251 return;
2252 }
2253
2254 // Non-inlineable
2255 if (AMDGPU::isSISrcFPOperand(InstDesc, OpNum)) { // Expected 64-bit fp operand
2256 // For fp operands we check if low 32 bits are zeros
2257 if (Literal.getLoBits(32) != 0) {
2258 const_cast<AMDGPUAsmParser *>(AsmParser)->Warning(Inst.getLoc(),
2259 "Can't encode literal as exact 64-bit floating-point operand. "
2260 "Low 32-bits will be set to zero");
2261 Val &= 0xffffffff00000000u;
2262 }
2263
2265 setImmKindLiteral();
2266 return;
2267 }
2268
2269 // We don't allow fp literals in 64-bit integer instructions. It is
2270 // unclear how we should encode them. This case should be checked earlier
2271 // in predicate methods (isLiteralImm())
2272 llvm_unreachable("fp literal in 64-bit integer instruction.");
2273
2281 if (AsmParser->hasInv2PiInlineImm() && Literal == 0x3fc45f306725feed) {
2282 // This is the 1/(2*pi) which is going to be truncated to bf16 with the
2283 // loss of precision. The constant represents ideomatic fp32 value of
2284 // 1/(2*pi) = 0.15915494 since bf16 is in fact fp32 with cleared low 16
2285 // bits. Prevent rounding below.
2286 Inst.addOperand(MCOperand::createImm(0x3e22));
2287 setImmKindLiteral();
2288 return;
2289 }
2290 [[fallthrough]];
2291
2319 bool lost;
2320 APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
2321 // Convert literal to single precision
2322 FPLiteral.convert(*getOpFltSemantics(OpTy),
2323 APFloat::rmNearestTiesToEven, &lost);
2324 // We allow precision lost but not overflow or underflow. This should be
2325 // checked earlier in isLiteralImm()
2326
2327 uint64_t ImmVal = FPLiteral.bitcastToAPInt().getZExtValue();
2328 Inst.addOperand(MCOperand::createImm(ImmVal));
2329 if (OpTy == AMDGPU::OPERAND_KIMM32 || OpTy == AMDGPU::OPERAND_KIMM16) {
2330 setImmKindMandatoryLiteral();
2331 } else {
2332 setImmKindLiteral();
2333 }
2334 return;
2335 }
2336 default:
2337 llvm_unreachable("invalid operand size");
2338 }
2339
2340 return;
2341 }
2342
2343 // We got int literal token.
2344 // Only sign extend inline immediates.
2345 switch (OpTy) {
2361 if (isSafeTruncation(Val, 32) &&
2362 AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
2363 AsmParser->hasInv2PiInlineImm())) {
2365 setImmKindConst();
2366 return;
2367 }
2368
2369 Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
2370 setImmKindLiteral();
2371 return;
2372
2378 if (AMDGPU::isInlinableLiteral64(Val, AsmParser->hasInv2PiInlineImm())) {
2380 setImmKindConst();
2381 return;
2382 }
2383
2384 Val = AMDGPU::isSISrcFPOperand(InstDesc, OpNum) ? (uint64_t)Val << 32
2385 : Lo_32(Val);
2386
2388 setImmKindLiteral();
2389 return;
2390
2394 if (isSafeTruncation(Val, 16) &&
2395 AMDGPU::isInlinableIntLiteral(static_cast<int16_t>(Val))) {
2396 Inst.addOperand(MCOperand::createImm(Val & 0xffffffff));
2397 setImmKindConst();
2398 return;
2399 }
2400
2401 Inst.addOperand(MCOperand::createImm(Val & 0xffff));
2402 setImmKindLiteral();
2403 return;
2404
2409 if (isSafeTruncation(Val, 16) &&
2410 AMDGPU::isInlinableLiteralFP16(static_cast<int16_t>(Val),
2411 AsmParser->hasInv2PiInlineImm())) {
2413 setImmKindConst();
2414 return;
2415 }
2416
2417 Inst.addOperand(MCOperand::createImm(Val & 0xffff));
2418 setImmKindLiteral();
2419 return;
2420
2425 if (isSafeTruncation(Val, 16) &&
2426 AMDGPU::isInlinableLiteralBF16(static_cast<int16_t>(Val),
2427 AsmParser->hasInv2PiInlineImm())) {
2429 setImmKindConst();
2430 return;
2431 }
2432
2433 Inst.addOperand(MCOperand::createImm(Val & 0xffff));
2434 setImmKindLiteral();
2435 return;
2436
2439 assert(isSafeTruncation(Val, 16));
2440 assert(AMDGPU::isInlinableIntLiteral(static_cast<int16_t>(Val)));
2442 return;
2443 }
2446 assert(isSafeTruncation(Val, 16));
2447 assert(AMDGPU::isInlinableLiteralFP16(static_cast<int16_t>(Val),
2448 AsmParser->hasInv2PiInlineImm()));
2449
2451 return;
2452 }
2453
2456 assert(isSafeTruncation(Val, 16));
2457 assert(AMDGPU::isInlinableLiteralBF16(static_cast<int16_t>(Val),
2458 AsmParser->hasInv2PiInlineImm()));
2459
2461 return;
2462 }
2463
2465 Inst.addOperand(MCOperand::createImm(Literal.getLoBits(32).getZExtValue()));
2466 setImmKindMandatoryLiteral();
2467 return;
2469 Inst.addOperand(MCOperand::createImm(Literal.getLoBits(16).getZExtValue()));
2470 setImmKindMandatoryLiteral();
2471 return;
2472 default:
2473 llvm_unreachable("invalid operand size");
2474 }
2475}
2476
2477void AMDGPUOperand::addRegOperands(MCInst &Inst, unsigned N) const {
2478 Inst.addOperand(MCOperand::createReg(AMDGPU::getMCReg(getReg(), AsmParser->getSTI())));
2479}
2480
2481bool AMDGPUOperand::isInlineValue() const {
2482 return isRegKind() && ::isInlineValue(getReg());
2483}
2484
2485//===----------------------------------------------------------------------===//
2486// AsmParser
2487//===----------------------------------------------------------------------===//
2488
2489static int getRegClass(RegisterKind Is, unsigned RegWidth) {
2490 if (Is == IS_VGPR) {
2491 switch (RegWidth) {
2492 default: return -1;
2493 case 32:
2494 return AMDGPU::VGPR_32RegClassID;
2495 case 64:
2496 return AMDGPU::VReg_64RegClassID;
2497 case 96:
2498 return AMDGPU::VReg_96RegClassID;
2499 case 128:
2500 return AMDGPU::VReg_128RegClassID;
2501 case 160:
2502 return AMDGPU::VReg_160RegClassID;
2503 case 192:
2504 return AMDGPU::VReg_192RegClassID;
2505 case 224:
2506 return AMDGPU::VReg_224RegClassID;
2507 case 256:
2508 return AMDGPU::VReg_256RegClassID;
2509 case 288:
2510 return AMDGPU::VReg_288RegClassID;
2511 case 320:
2512 return AMDGPU::VReg_320RegClassID;
2513 case 352:
2514 return AMDGPU::VReg_352RegClassID;
2515 case 384:
2516 return AMDGPU::VReg_384RegClassID;
2517 case 512:
2518 return AMDGPU::VReg_512RegClassID;
2519 case 1024:
2520 return AMDGPU::VReg_1024RegClassID;
2521 }
2522 } else if (Is == IS_TTMP) {
2523 switch (RegWidth) {
2524 default: return -1;
2525 case 32:
2526 return AMDGPU::TTMP_32RegClassID;
2527 case 64:
2528 return AMDGPU::TTMP_64RegClassID;
2529 case 128:
2530 return AMDGPU::TTMP_128RegClassID;
2531 case 256:
2532 return AMDGPU::TTMP_256RegClassID;
2533 case 512:
2534 return AMDGPU::TTMP_512RegClassID;
2535 }
2536 } else if (Is == IS_SGPR) {
2537 switch (RegWidth) {
2538 default: return -1;
2539 case 32:
2540 return AMDGPU::SGPR_32RegClassID;
2541 case 64:
2542 return AMDGPU::SGPR_64RegClassID;
2543 case 96:
2544 return AMDGPU::SGPR_96RegClassID;
2545 case 128:
2546 return AMDGPU::SGPR_128RegClassID;
2547 case 160:
2548 return AMDGPU::SGPR_160RegClassID;
2549 case 192:
2550 return AMDGPU::SGPR_192RegClassID;
2551 case 224:
2552 return AMDGPU::SGPR_224RegClassID;
2553 case 256:
2554 return AMDGPU::SGPR_256RegClassID;
2555 case 288:
2556 return AMDGPU::SGPR_288RegClassID;
2557 case 320:
2558 return AMDGPU::SGPR_320RegClassID;
2559 case 352:
2560 return AMDGPU::SGPR_352RegClassID;
2561 case 384:
2562 return AMDGPU::SGPR_384RegClassID;
2563 case 512:
2564 return AMDGPU::SGPR_512RegClassID;
2565 }
2566 } else if (Is == IS_AGPR) {
2567 switch (RegWidth) {
2568 default: return -1;
2569 case 32:
2570 return AMDGPU::AGPR_32RegClassID;
2571 case 64:
2572 return AMDGPU::AReg_64RegClassID;
2573 case 96:
2574 return AMDGPU::AReg_96RegClassID;
2575 case 128:
2576 return AMDGPU::AReg_128RegClassID;
2577 case 160:
2578 return AMDGPU::AReg_160RegClassID;
2579 case 192:
2580 return AMDGPU::AReg_192RegClassID;
2581 case 224:
2582 return AMDGPU::AReg_224RegClassID;
2583 case 256:
2584 return AMDGPU::AReg_256RegClassID;
2585 case 288:
2586 return AMDGPU::AReg_288RegClassID;
2587 case 320:
2588 return AMDGPU::AReg_320RegClassID;
2589 case 352:
2590 return AMDGPU::AReg_352RegClassID;
2591 case 384:
2592 return AMDGPU::AReg_384RegClassID;
2593 case 512:
2594 return AMDGPU::AReg_512RegClassID;
2595 case 1024:
2596 return AMDGPU::AReg_1024RegClassID;
2597 }
2598 }
2599 return -1;
2600}
2601
2604 .Case("exec", AMDGPU::EXEC)
2605 .Case("vcc", AMDGPU::VCC)
2606 .Case("flat_scratch", AMDGPU::FLAT_SCR)
2607 .Case("xnack_mask", AMDGPU::XNACK_MASK)
2608 .Case("shared_base", AMDGPU::SRC_SHARED_BASE)
2609 .Case("src_shared_base", AMDGPU::SRC_SHARED_BASE)
2610 .Case("shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2611 .Case("src_shared_limit", AMDGPU::SRC_SHARED_LIMIT)
2612 .Case("private_base", AMDGPU::SRC_PRIVATE_BASE)
2613 .Case("src_private_base", AMDGPU::SRC_PRIVATE_BASE)
2614 .Case("private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2615 .Case("src_private_limit", AMDGPU::SRC_PRIVATE_LIMIT)
2616 .Case("pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2617 .Case("src_pops_exiting_wave_id", AMDGPU::SRC_POPS_EXITING_WAVE_ID)
2618 .Case("lds_direct", AMDGPU::LDS_DIRECT)
2619 .Case("src_lds_direct", AMDGPU::LDS_DIRECT)
2620 .Case("m0", AMDGPU::M0)
2621 .Case("vccz", AMDGPU::SRC_VCCZ)
2622 .Case("src_vccz", AMDGPU::SRC_VCCZ)
2623 .Case("execz", AMDGPU::SRC_EXECZ)
2624 .Case("src_execz", AMDGPU::SRC_EXECZ)
2625 .Case("scc", AMDGPU::SRC_SCC)
2626 .Case("src_scc", AMDGPU::SRC_SCC)
2627 .Case("tba", AMDGPU::TBA)
2628 .Case("tma", AMDGPU::TMA)
2629 .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
2630 .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
2631 .Case("xnack_mask_lo", AMDGPU::XNACK_MASK_LO)
2632 .Case("xnack_mask_hi", AMDGPU::XNACK_MASK_HI)
2633 .Case("vcc_lo", AMDGPU::VCC_LO)
2634 .Case("vcc_hi", AMDGPU::VCC_HI)
2635 .Case("exec_lo", AMDGPU::EXEC_LO)
2636 .Case("exec_hi", AMDGPU::EXEC_HI)
2637 .Case("tma_lo", AMDGPU::TMA_LO)
2638 .Case("tma_hi", AMDGPU::TMA_HI)
2639 .Case("tba_lo", AMDGPU::TBA_LO)
2640 .Case("tba_hi", AMDGPU::TBA_HI)
2641 .Case("pc", AMDGPU::PC_REG)
2642 .Case("null", AMDGPU::SGPR_NULL)
2643 .Default(AMDGPU::NoRegister);
2644}
2645
2646bool AMDGPUAsmParser::ParseRegister(MCRegister &RegNo, SMLoc &StartLoc,
2647 SMLoc &EndLoc, bool RestoreOnFailure) {
2648 auto R = parseRegister();
2649 if (!R) return true;
2650 assert(R->isReg());
2651 RegNo = R->getReg();
2652 StartLoc = R->getStartLoc();
2653 EndLoc = R->getEndLoc();
2654 return false;
2655}
2656
2657bool AMDGPUAsmParser::parseRegister(MCRegister &Reg, SMLoc &StartLoc,
2658 SMLoc &EndLoc) {
2659 return ParseRegister(Reg, StartLoc, EndLoc, /*RestoreOnFailure=*/false);
2660}
2661
2662ParseStatus AMDGPUAsmParser::tryParseRegister(MCRegister &Reg, SMLoc &StartLoc,
2663 SMLoc &EndLoc) {
2664 bool Result = ParseRegister(Reg, StartLoc, EndLoc, /*RestoreOnFailure=*/true);
2665 bool PendingErrors = getParser().hasPendingError();
2666 getParser().clearPendingErrors();
2667 if (PendingErrors)
2668 return ParseStatus::Failure;
2669 if (Result)
2670 return ParseStatus::NoMatch;
2671 return ParseStatus::Success;
2672}
2673
2674bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
2675 RegisterKind RegKind, unsigned Reg1,
2676 SMLoc Loc) {
2677 switch (RegKind) {
2678 case IS_SPECIAL:
2679 if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
2680 Reg = AMDGPU::EXEC;
2681 RegWidth = 64;
2682 return true;
2683 }
2684 if (Reg == AMDGPU::FLAT_SCR_LO && Reg1 == AMDGPU::FLAT_SCR_HI) {
2685 Reg = AMDGPU::FLAT_SCR;
2686 RegWidth = 64;
2687 return true;
2688 }
2689 if (Reg == AMDGPU::XNACK_MASK_LO && Reg1 == AMDGPU::XNACK_MASK_HI) {
2690 Reg = AMDGPU::XNACK_MASK;
2691 RegWidth = 64;
2692 return true;
2693 }
2694 if (Reg == AMDGPU::VCC_LO && Reg1 == AMDGPU::VCC_HI) {
2695 Reg = AMDGPU::VCC;
2696 RegWidth = 64;
2697 return true;
2698 }
2699 if (Reg == AMDGPU::TBA_LO && Reg1 == AMDGPU::TBA_HI) {
2700 Reg = AMDGPU::TBA;
2701 RegWidth = 64;
2702 return true;
2703 }
2704 if (Reg == AMDGPU::TMA_LO && Reg1 == AMDGPU::TMA_HI) {
2705 Reg = AMDGPU::TMA;
2706 RegWidth = 64;
2707 return true;
2708 }
2709 Error(Loc, "register does not fit in the list");
2710 return false;
2711 case IS_VGPR:
2712 case IS_SGPR:
2713 case IS_AGPR:
2714 case IS_TTMP:
2715 if (Reg1 != Reg + RegWidth / 32) {
2716 Error(Loc, "registers in a list must have consecutive indices");
2717 return false;
2718 }
2719 RegWidth += 32;
2720 return true;
2721 default:
2722 llvm_unreachable("unexpected register kind");
2723 }
2724}
2725
2726struct RegInfo {
2728 RegisterKind Kind;
2729};
2730
2731static constexpr RegInfo RegularRegisters[] = {
2732 {{"v"}, IS_VGPR},
2733 {{"s"}, IS_SGPR},
2734 {{"ttmp"}, IS_TTMP},
2735 {{"acc"}, IS_AGPR},
2736 {{"a"}, IS_AGPR},
2737};
2738
2739static bool isRegularReg(RegisterKind Kind) {
2740 return Kind == IS_VGPR ||
2741 Kind == IS_SGPR ||
2742 Kind == IS_TTMP ||
2743 Kind == IS_AGPR;
2744}
2745
2747 for (const RegInfo &Reg : RegularRegisters)
2748 if (Str.starts_with(Reg.Name))
2749 return &Reg;
2750 return nullptr;
2751}
2752
2753static bool getRegNum(StringRef Str, unsigned& Num) {
2754 return !Str.getAsInteger(10, Num);
2755}
2756
2757bool
2758AMDGPUAsmParser::isRegister(const AsmToken &Token,
2759 const AsmToken &NextToken) const {
2760
2761 // A list of consecutive registers: [s0,s1,s2,s3]
2762 if (Token.is(AsmToken::LBrac))
2763 return true;
2764
2765 if (!Token.is(AsmToken::Identifier))
2766 return false;
2767
2768 // A single register like s0 or a range of registers like s[0:1]
2769
2770 StringRef Str = Token.getString();
2771 const RegInfo *Reg = getRegularRegInfo(Str);
2772 if (Reg) {
2773 StringRef RegName = Reg->Name;
2774 StringRef RegSuffix = Str.substr(RegName.size());
2775 if (!RegSuffix.empty()) {
2776 RegSuffix.consume_back(".l");
2777 RegSuffix.consume_back(".h");
2778 unsigned Num;
2779 // A single register with an index: rXX
2780 if (getRegNum(RegSuffix, Num))
2781 return true;
2782 } else {
2783 // A range of registers: r[XX:YY].
2784 if (NextToken.is(AsmToken::LBrac))
2785 return true;
2786 }
2787 }
2788
2789 return getSpecialRegForName(Str) != AMDGPU::NoRegister;
2790}
2791
2792bool
2793AMDGPUAsmParser::isRegister()
2794{
2795 return isRegister(getToken(), peekToken());
2796}
2797
2798unsigned AMDGPUAsmParser::getRegularReg(RegisterKind RegKind, unsigned RegNum,
2799 unsigned SubReg, unsigned RegWidth,
2800 SMLoc Loc) {
2801 assert(isRegularReg(RegKind));
2802
2803 unsigned AlignSize = 1;
2804 if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
2805 // SGPR and TTMP registers must be aligned.
2806 // Max required alignment is 4 dwords.
2807 AlignSize = std::min(llvm::bit_ceil(RegWidth / 32), 4u);
2808 }
2809
2810 if (RegNum % AlignSize != 0) {
2811 Error(Loc, "invalid register alignment");
2812 return AMDGPU::NoRegister;
2813 }
2814
2815 unsigned RegIdx = RegNum / AlignSize;
2816 int RCID = getRegClass(RegKind, RegWidth);
2817 if (RCID == -1) {
2818 Error(Loc, "invalid or unsupported register size");
2819 return AMDGPU::NoRegister;
2820 }
2821
2822 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
2823 const MCRegisterClass RC = TRI->getRegClass(RCID);
2824 if (RegIdx >= RC.getNumRegs()) {
2825 Error(Loc, "register index is out of range");
2826 return AMDGPU::NoRegister;
2827 }
2828
2829 unsigned Reg = RC.getRegister(RegIdx);
2830
2831 if (SubReg) {
2832 Reg = TRI->getSubReg(Reg, SubReg);
2833
2834 // Currently all regular registers have their .l and .h subregisters, so
2835 // we should never need to generate an error here.
2836 assert(Reg && "Invalid subregister!");
2837 }
2838
2839 return Reg;
2840}
2841
2842bool AMDGPUAsmParser::ParseRegRange(unsigned &Num, unsigned &RegWidth) {
2843 int64_t RegLo, RegHi;
2844 if (!skipToken(AsmToken::LBrac, "missing register index"))
2845 return false;
2846
2847 SMLoc FirstIdxLoc = getLoc();
2848 SMLoc SecondIdxLoc;
2849
2850 if (!parseExpr(RegLo))
2851 return false;
2852
2853 if (trySkipToken(AsmToken::Colon)) {
2854 SecondIdxLoc = getLoc();
2855 if (!parseExpr(RegHi))
2856 return false;
2857 } else {
2858 RegHi = RegLo;
2859 }
2860
2861 if (!skipToken(AsmToken::RBrac, "expected a closing square bracket"))
2862 return false;
2863
2864 if (!isUInt<32>(RegLo)) {
2865 Error(FirstIdxLoc, "invalid register index");
2866 return false;
2867 }
2868
2869 if (!isUInt<32>(RegHi)) {
2870 Error(SecondIdxLoc, "invalid register index");
2871 return false;
2872 }
2873
2874 if (RegLo > RegHi) {
2875 Error(FirstIdxLoc, "first register index should not exceed second index");
2876 return false;
2877 }
2878
2879 Num = static_cast<unsigned>(RegLo);
2880 RegWidth = 32 * ((RegHi - RegLo) + 1);
2881 return true;
2882}
2883
2884unsigned AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
2885 unsigned &RegNum, unsigned &RegWidth,
2886 SmallVectorImpl<AsmToken> &Tokens) {
2887 assert(isToken(AsmToken::Identifier));
2888 unsigned Reg = getSpecialRegForName(getTokenStr());
2889 if (Reg) {
2890 RegNum = 0;
2891 RegWidth = 32;
2892 RegKind = IS_SPECIAL;
2893 Tokens.push_back(getToken());
2894 lex(); // skip register name
2895 }
2896 return Reg;
2897}
2898
2899unsigned AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
2900 unsigned &RegNum, unsigned &RegWidth,
2901 SmallVectorImpl<AsmToken> &Tokens) {
2902 assert(isToken(AsmToken::Identifier));
2903 StringRef RegName = getTokenStr();
2904 auto Loc = getLoc();
2905
2906 const RegInfo *RI = getRegularRegInfo(RegName);
2907 if (!RI) {
2908 Error(Loc, "invalid register name");
2909 return AMDGPU::NoRegister;
2910 }
2911
2912 Tokens.push_back(getToken());
2913 lex(); // skip register name
2914
2915 RegKind = RI->Kind;
2916 StringRef RegSuffix = RegName.substr(RI->Name.size());
2917 unsigned SubReg = NoSubRegister;
2918 if (!RegSuffix.empty()) {
2919 // We don't know the opcode till we are done parsing, so we don't know if
2920 // registers should be 16 or 32 bit. It is therefore mandatory to put .l or
2921 // .h to correctly specify 16 bit registers. We also can't determine class
2922 // VGPR_16_Lo128 or VGPR_16, so always parse them as VGPR_16.
2923 if (RegSuffix.consume_back(".l"))
2924 SubReg = AMDGPU::lo16;
2925 else if (RegSuffix.consume_back(".h"))
2926 SubReg = AMDGPU::hi16;
2927
2928 // Single 32-bit register: vXX.
2929 if (!getRegNum(RegSuffix, RegNum)) {
2930 Error(Loc, "invalid register index");
2931 return AMDGPU::NoRegister;
2932 }
2933 RegWidth = 32;
2934 } else {
2935 // Range of registers: v[XX:YY]. ":YY" is optional.
2936 if (!ParseRegRange(RegNum, RegWidth))
2937 return AMDGPU::NoRegister;
2938 }
2939
2940 return getRegularReg(RegKind, RegNum, SubReg, RegWidth, Loc);
2941}
2942
2943unsigned AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind, unsigned &RegNum,
2944 unsigned &RegWidth,
2945 SmallVectorImpl<AsmToken> &Tokens) {
2946 unsigned Reg = AMDGPU::NoRegister;
2947 auto ListLoc = getLoc();
2948
2949 if (!skipToken(AsmToken::LBrac,
2950 "expected a register or a list of registers")) {
2951 return AMDGPU::NoRegister;
2952 }
2953
2954 // List of consecutive registers, e.g.: [s0,s1,s2,s3]
2955
2956 auto Loc = getLoc();
2957 if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
2958 return AMDGPU::NoRegister;
2959 if (RegWidth != 32) {
2960 Error(Loc, "expected a single 32-bit register");
2961 return AMDGPU::NoRegister;
2962 }
2963
2964 for (; trySkipToken(AsmToken::Comma); ) {
2965 RegisterKind NextRegKind;
2966 unsigned NextReg, NextRegNum, NextRegWidth;
2967 Loc = getLoc();
2968
2969 if (!ParseAMDGPURegister(NextRegKind, NextReg,
2970 NextRegNum, NextRegWidth,
2971 Tokens)) {
2972 return AMDGPU::NoRegister;
2973 }
2974 if (NextRegWidth != 32) {
2975 Error(Loc, "expected a single 32-bit register");
2976 return AMDGPU::NoRegister;
2977 }
2978 if (NextRegKind != RegKind) {
2979 Error(Loc, "registers in a list must be of the same kind");
2980 return AMDGPU::NoRegister;
2981 }
2982 if (!AddNextRegisterToList(Reg, RegWidth, RegKind, NextReg, Loc))
2983 return AMDGPU::NoRegister;
2984 }
2985
2986 if (!skipToken(AsmToken::RBrac,
2987 "expected a comma or a closing square bracket")) {
2988 return AMDGPU::NoRegister;
2989 }
2990
2991 if (isRegularReg(RegKind))
2992 Reg = getRegularReg(RegKind, RegNum, NoSubRegister, RegWidth, ListLoc);
2993
2994 return Reg;
2995}
2996
2997bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
2998 unsigned &RegNum, unsigned &RegWidth,
2999 SmallVectorImpl<AsmToken> &Tokens) {
3000 auto Loc = getLoc();
3001 Reg = AMDGPU::NoRegister;
3002
3003 if (isToken(AsmToken::Identifier)) {
3004 Reg = ParseSpecialReg(RegKind, RegNum, RegWidth, Tokens);
3005 if (Reg == AMDGPU::NoRegister)
3006 Reg = ParseRegularReg(RegKind, RegNum, RegWidth, Tokens);
3007 } else {
3008 Reg = ParseRegList(RegKind, RegNum, RegWidth, Tokens);
3009 }
3010
3011 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3012 if (Reg == AMDGPU::NoRegister) {
3013 assert(Parser.hasPendingError());
3014 return false;
3015 }
3016
3017 if (!subtargetHasRegister(*TRI, Reg)) {
3018 if (Reg == AMDGPU::SGPR_NULL) {
3019 Error(Loc, "'null' operand is not supported on this GPU");
3020 } else {
3021 Error(Loc, "register not available on this GPU");
3022 }
3023 return false;
3024 }
3025
3026 return true;
3027}
3028
3029bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
3030 unsigned &RegNum, unsigned &RegWidth,
3031 bool RestoreOnFailure /*=false*/) {
3032 Reg = AMDGPU::NoRegister;
3033
3035 if (ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, Tokens)) {
3036 if (RestoreOnFailure) {
3037 while (!Tokens.empty()) {
3038 getLexer().UnLex(Tokens.pop_back_val());
3039 }
3040 }
3041 return true;
3042 }
3043 return false;
3044}
3045
3046std::optional<StringRef>
3047AMDGPUAsmParser::getGprCountSymbolName(RegisterKind RegKind) {
3048 switch (RegKind) {
3049 case IS_VGPR:
3050 return StringRef(".amdgcn.next_free_vgpr");
3051 case IS_SGPR:
3052 return StringRef(".amdgcn.next_free_sgpr");
3053 default:
3054 return std::nullopt;
3055 }
3056}
3057
3058void AMDGPUAsmParser::initializeGprCountSymbol(RegisterKind RegKind) {
3059 auto SymbolName = getGprCountSymbolName(RegKind);
3060 assert(SymbolName && "initializing invalid register kind");
3061 MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
3062 Sym->setVariableValue(MCConstantExpr::create(0, getContext()));
3063}
3064
3065bool AMDGPUAsmParser::updateGprCountSymbols(RegisterKind RegKind,
3066 unsigned DwordRegIndex,
3067 unsigned RegWidth) {
3068 // Symbols are only defined for GCN targets
3069 if (AMDGPU::getIsaVersion(getSTI().getCPU()).Major < 6)
3070 return true;
3071
3072 auto SymbolName = getGprCountSymbolName(RegKind);
3073 if (!SymbolName)
3074 return true;
3075 MCSymbol *Sym = getContext().getOrCreateSymbol(*SymbolName);
3076
3077 int64_t NewMax = DwordRegIndex + divideCeil(RegWidth, 32) - 1;
3078 int64_t OldCount;
3079
3080 if (!Sym->isVariable())
3081 return !Error(getLoc(),
3082 ".amdgcn.next_free_{v,s}gpr symbols must be variable");
3083 if (!Sym->getVariableValue(false)->evaluateAsAbsolute(OldCount))
3084 return !Error(
3085 getLoc(),
3086 ".amdgcn.next_free_{v,s}gpr symbols must be absolute expressions");
3087
3088 if (OldCount <= NewMax)
3089 Sym->setVariableValue(MCConstantExpr::create(NewMax + 1, getContext()));
3090
3091 return true;
3092}
3093
3094std::unique_ptr<AMDGPUOperand>
3095AMDGPUAsmParser::parseRegister(bool RestoreOnFailure) {
3096 const auto &Tok = getToken();
3097 SMLoc StartLoc = Tok.getLoc();
3098 SMLoc EndLoc = Tok.getEndLoc();
3099 RegisterKind RegKind;
3100 unsigned Reg, RegNum, RegWidth;
3101
3102 if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
3103 return nullptr;
3104 }
3105 if (isHsaAbi(getSTI())) {
3106 if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
3107 return nullptr;
3108 } else
3109 KernelScope.usesRegister(RegKind, RegNum, RegWidth);
3110 return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc);
3111}
3112
3113ParseStatus AMDGPUAsmParser::parseImm(OperandVector &Operands,
3114 bool HasSP3AbsModifier, bool HasLit) {
3115 // TODO: add syntactic sugar for 1/(2*PI)
3116
3117 if (isRegister())
3118 return ParseStatus::NoMatch;
3119 assert(!isModifier());
3120
3121 if (!HasLit) {
3122 HasLit = trySkipId("lit");
3123 if (HasLit) {
3124 if (!skipToken(AsmToken::LParen, "expected left paren after lit"))
3125 return ParseStatus::Failure;
3126 ParseStatus S = parseImm(Operands, HasSP3AbsModifier, HasLit);
3127 if (S.isSuccess() &&
3128 !skipToken(AsmToken::RParen, "expected closing parentheses"))
3129 return ParseStatus::Failure;
3130 return S;
3131 }
3132 }
3133
3134 const auto& Tok = getToken();
3135 const auto& NextTok = peekToken();
3136 bool IsReal = Tok.is(AsmToken::Real);
3137 SMLoc S = getLoc();
3138 bool Negate = false;
3139
3140 if (!IsReal && Tok.is(AsmToken::Minus) && NextTok.is(AsmToken::Real)) {
3141 lex();
3142 IsReal = true;
3143 Negate = true;
3144 }
3145
3146 AMDGPUOperand::Modifiers Mods;
3147 Mods.Lit = HasLit;
3148
3149 if (IsReal) {
3150 // Floating-point expressions are not supported.
3151 // Can only allow floating-point literals with an
3152 // optional sign.
3153
3154 StringRef Num = getTokenStr();
3155 lex();
3156
3157 APFloat RealVal(APFloat::IEEEdouble());
3158 auto roundMode = APFloat::rmNearestTiesToEven;
3159 if (errorToBool(RealVal.convertFromString(Num, roundMode).takeError()))
3160 return ParseStatus::Failure;
3161 if (Negate)
3162 RealVal.changeSign();
3163
3164 Operands.push_back(
3165 AMDGPUOperand::CreateImm(this, RealVal.bitcastToAPInt().getZExtValue(), S,
3166 AMDGPUOperand::ImmTyNone, true));
3167 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3168 Op.setModifiers(Mods);
3169
3170 return ParseStatus::Success;
3171
3172 } else {
3173 int64_t IntVal;
3174 const MCExpr *Expr;
3175 SMLoc S = getLoc();
3176
3177 if (HasSP3AbsModifier) {
3178 // This is a workaround for handling expressions
3179 // as arguments of SP3 'abs' modifier, for example:
3180 // |1.0|
3181 // |-1|
3182 // |1+x|
3183 // This syntax is not compatible with syntax of standard
3184 // MC expressions (due to the trailing '|').
3185 SMLoc EndLoc;
3186 if (getParser().parsePrimaryExpr(Expr, EndLoc, nullptr))
3187 return ParseStatus::Failure;
3188 } else {
3189 if (Parser.parseExpression(Expr))
3190 return ParseStatus::Failure;
3191 }
3192
3193 if (Expr->evaluateAsAbsolute(IntVal)) {
3194 Operands.push_back(AMDGPUOperand::CreateImm(this, IntVal, S));
3195 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3196 Op.setModifiers(Mods);
3197 } else {
3198 if (HasLit)
3199 return ParseStatus::NoMatch;
3200 Operands.push_back(AMDGPUOperand::CreateExpr(this, Expr, S));
3201 }
3202
3203 return ParseStatus::Success;
3204 }
3205
3206 return ParseStatus::NoMatch;
3207}
3208
3209ParseStatus AMDGPUAsmParser::parseReg(OperandVector &Operands) {
3210 if (!isRegister())
3211 return ParseStatus::NoMatch;
3212
3213 if (auto R = parseRegister()) {
3214 assert(R->isReg());
3215 Operands.push_back(std::move(R));
3216 return ParseStatus::Success;
3217 }
3218 return ParseStatus::Failure;
3219}
3220
3221ParseStatus AMDGPUAsmParser::parseRegOrImm(OperandVector &Operands,
3222 bool HasSP3AbsMod, bool HasLit) {
3223 ParseStatus Res = parseReg(Operands);
3224 if (!Res.isNoMatch())
3225 return Res;
3226 if (isModifier())
3227 return ParseStatus::NoMatch;
3228 return parseImm(Operands, HasSP3AbsMod, HasLit);
3229}
3230
3231bool
3232AMDGPUAsmParser::isNamedOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
3233 if (Token.is(AsmToken::Identifier) && NextToken.is(AsmToken::LParen)) {
3234 const auto &str = Token.getString();
3235 return str == "abs" || str == "neg" || str == "sext";
3236 }
3237 return false;
3238}
3239
3240bool
3241AMDGPUAsmParser::isOpcodeModifierWithVal(const AsmToken &Token, const AsmToken &NextToken) const {
3242 return Token.is(AsmToken::Identifier) && NextToken.is(AsmToken::Colon);
3243}
3244
3245bool
3246AMDGPUAsmParser::isOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
3247 return isNamedOperandModifier(Token, NextToken) || Token.is(AsmToken::Pipe);
3248}
3249
3250bool
3251AMDGPUAsmParser::isRegOrOperandModifier(const AsmToken &Token, const AsmToken &NextToken) const {
3252 return isRegister(Token, NextToken) || isOperandModifier(Token, NextToken);
3253}
3254
3255// Check if this is an operand modifier or an opcode modifier
3256// which may look like an expression but it is not. We should
3257// avoid parsing these modifiers as expressions. Currently
3258// recognized sequences are:
3259// |...|
3260// abs(...)
3261// neg(...)
3262// sext(...)
3263// -reg
3264// -|...|
3265// -abs(...)
3266// name:...
3267//
3268bool
3269AMDGPUAsmParser::isModifier() {
3270
3271 AsmToken Tok = getToken();
3272 AsmToken NextToken[2];
3273 peekTokens(NextToken);
3274
3275 return isOperandModifier(Tok, NextToken[0]) ||
3276 (Tok.is(AsmToken::Minus) && isRegOrOperandModifier(NextToken[0], NextToken[1])) ||
3277 isOpcodeModifierWithVal(Tok, NextToken[0]);
3278}
3279
3280// Check if the current token is an SP3 'neg' modifier.
3281// Currently this modifier is allowed in the following context:
3282//
3283// 1. Before a register, e.g. "-v0", "-v[...]" or "-[v0,v1]".
3284// 2. Before an 'abs' modifier: -abs(...)
3285// 3. Before an SP3 'abs' modifier: -|...|
3286//
3287// In all other cases "-" is handled as a part
3288// of an expression that follows the sign.
3289//
3290// Note: When "-" is followed by an integer literal,
3291// this is interpreted as integer negation rather
3292// than a floating-point NEG modifier applied to N.
3293// Beside being contr-intuitive, such use of floating-point
3294// NEG modifier would have resulted in different meaning
3295// of integer literals used with VOP1/2/C and VOP3,
3296// for example:
3297// v_exp_f32_e32 v5, -1 // VOP1: src0 = 0xFFFFFFFF
3298// v_exp_f32_e64 v5, -1 // VOP3: src0 = 0x80000001
3299// Negative fp literals with preceding "-" are
3300// handled likewise for uniformity
3301//
3302bool
3303AMDGPUAsmParser::parseSP3NegModifier() {
3304
3305 AsmToken NextToken[2];
3306 peekTokens(NextToken);
3307
3308 if (isToken(AsmToken::Minus) &&
3309 (isRegister(NextToken[0], NextToken[1]) ||
3310 NextToken[0].is(AsmToken::Pipe) ||
3311 isId(NextToken[0], "abs"))) {
3312 lex();
3313 return true;
3314 }
3315
3316 return false;
3317}
3318
3320AMDGPUAsmParser::parseRegOrImmWithFPInputMods(OperandVector &Operands,
3321 bool AllowImm) {
3322 bool Neg, SP3Neg;
3323 bool Abs, SP3Abs;
3324 bool Lit;
3325 SMLoc Loc;
3326
3327 // Disable ambiguous constructs like '--1' etc. Should use neg(-1) instead.
3328 if (isToken(AsmToken::Minus) && peekToken().is(AsmToken::Minus))
3329 return Error(getLoc(), "invalid syntax, expected 'neg' modifier");
3330
3331 SP3Neg = parseSP3NegModifier();
3332
3333 Loc = getLoc();
3334 Neg = trySkipId("neg");
3335 if (Neg && SP3Neg)
3336 return Error(Loc, "expected register or immediate");
3337 if (Neg && !skipToken(AsmToken::LParen, "expected left paren after neg"))
3338 return ParseStatus::Failure;
3339
3340 Abs = trySkipId("abs");
3341 if (Abs && !skipToken(AsmToken::LParen, "expected left paren after abs"))
3342 return ParseStatus::Failure;
3343
3344 Lit = trySkipId("lit");
3345 if (Lit && !skipToken(AsmToken::LParen, "expected left paren after lit"))
3346 return ParseStatus::Failure;
3347
3348 Loc = getLoc();
3349 SP3Abs = trySkipToken(AsmToken::Pipe);
3350 if (Abs && SP3Abs)
3351 return Error(Loc, "expected register or immediate");
3352
3353 ParseStatus Res;
3354 if (AllowImm) {
3355 Res = parseRegOrImm(Operands, SP3Abs, Lit);
3356 } else {
3357 Res = parseReg(Operands);
3358 }
3359 if (!Res.isSuccess())
3360 return (SP3Neg || Neg || SP3Abs || Abs || Lit) ? ParseStatus::Failure : Res;
3361
3362 if (Lit && !Operands.back()->isImm())
3363 Error(Loc, "expected immediate with lit modifier");
3364
3365 if (SP3Abs && !skipToken(AsmToken::Pipe, "expected vertical bar"))
3366 return ParseStatus::Failure;
3367 if (Abs && !skipToken(AsmToken::RParen, "expected closing parentheses"))
3368 return ParseStatus::Failure;
3369 if (Neg && !skipToken(AsmToken::RParen, "expected closing parentheses"))
3370 return ParseStatus::Failure;
3371 if (Lit && !skipToken(AsmToken::RParen, "expected closing parentheses"))
3372 return ParseStatus::Failure;
3373
3374 AMDGPUOperand::Modifiers Mods;
3375 Mods.Abs = Abs || SP3Abs;
3376 Mods.Neg = Neg || SP3Neg;
3377 Mods.Lit = Lit;
3378
3379 if (Mods.hasFPModifiers() || Lit) {
3380 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3381 if (Op.isExpr())
3382 return Error(Op.getStartLoc(), "expected an absolute expression");
3383 Op.setModifiers(Mods);
3384 }
3385 return ParseStatus::Success;
3386}
3387
3389AMDGPUAsmParser::parseRegOrImmWithIntInputMods(OperandVector &Operands,
3390 bool AllowImm) {
3391 bool Sext = trySkipId("sext");
3392 if (Sext && !skipToken(AsmToken::LParen, "expected left paren after sext"))
3393 return ParseStatus::Failure;
3394
3395 ParseStatus Res;
3396 if (AllowImm) {
3397 Res = parseRegOrImm(Operands);
3398 } else {
3399 Res = parseReg(Operands);
3400 }
3401 if (!Res.isSuccess())
3402 return Sext ? ParseStatus::Failure : Res;
3403
3404 if (Sext && !skipToken(AsmToken::RParen, "expected closing parentheses"))
3405 return ParseStatus::Failure;
3406
3407 AMDGPUOperand::Modifiers Mods;
3408 Mods.Sext = Sext;
3409
3410 if (Mods.hasIntModifiers()) {
3411 AMDGPUOperand &Op = static_cast<AMDGPUOperand &>(*Operands.back());
3412 if (Op.isExpr())
3413 return Error(Op.getStartLoc(), "expected an absolute expression");
3414 Op.setModifiers(Mods);
3415 }
3416
3417 return ParseStatus::Success;
3418}
3419
3420ParseStatus AMDGPUAsmParser::parseRegWithFPInputMods(OperandVector &Operands) {
3421 return parseRegOrImmWithFPInputMods(Operands, false);
3422}
3423
3424ParseStatus AMDGPUAsmParser::parseRegWithIntInputMods(OperandVector &Operands) {
3425 return parseRegOrImmWithIntInputMods(Operands, false);
3426}
3427
3428ParseStatus AMDGPUAsmParser::parseVReg32OrOff(OperandVector &Operands) {
3429 auto Loc = getLoc();
3430 if (trySkipId("off")) {
3431 Operands.push_back(AMDGPUOperand::CreateImm(this, 0, Loc,
3432 AMDGPUOperand::ImmTyOff, false));
3433 return ParseStatus::Success;
3434 }
3435
3436 if (!isRegister())
3437 return ParseStatus::NoMatch;
3438
3439 std::unique_ptr<AMDGPUOperand> Reg = parseRegister();
3440 if (Reg) {
3441 Operands.push_back(std::move(Reg));
3442 return ParseStatus::Success;
3443 }
3444
3445 return ParseStatus::Failure;
3446}
3447
3448unsigned AMDGPUAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
3449 uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
3450
3451 if ((getForcedEncodingSize() == 32 && (TSFlags & SIInstrFlags::VOP3)) ||
3452 (getForcedEncodingSize() == 64 && !(TSFlags & SIInstrFlags::VOP3)) ||
3453 (isForcedDPP() && !(TSFlags & SIInstrFlags::DPP)) ||
3454 (isForcedSDWA() && !(TSFlags & SIInstrFlags::SDWA)) )
3455 return Match_InvalidOperand;
3456
3457 if (Inst.getOpcode() == AMDGPU::V_MAC_F32_sdwa_vi ||
3458 Inst.getOpcode() == AMDGPU::V_MAC_F16_sdwa_vi) {
3459 // v_mac_f32/16 allow only dst_sel == DWORD;
3460 auto OpNum =
3461 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::dst_sel);
3462 const auto &Op = Inst.getOperand(OpNum);
3463 if (!Op.isImm() || Op.getImm() != AMDGPU::SDWA::SdwaSel::DWORD) {
3464 return Match_InvalidOperand;
3465 }
3466 }
3467
3468 return Match_Success;
3469}
3470
3472 static const unsigned Variants[] = {
3476 };
3477
3478 return ArrayRef(Variants);
3479}
3480
3481// What asm variants we should check
3482ArrayRef<unsigned> AMDGPUAsmParser::getMatchedVariants() const {
3483 if (isForcedDPP() && isForcedVOP3()) {
3484 static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3_DPP};
3485 return ArrayRef(Variants);
3486 }
3487 if (getForcedEncodingSize() == 32) {
3488 static const unsigned Variants[] = {AMDGPUAsmVariants::DEFAULT};
3489 return ArrayRef(Variants);
3490 }
3491
3492 if (isForcedVOP3()) {
3493 static const unsigned Variants[] = {AMDGPUAsmVariants::VOP3};
3494 return ArrayRef(Variants);
3495 }
3496
3497 if (isForcedSDWA()) {
3498 static const unsigned Variants[] = {AMDGPUAsmVariants::SDWA,
3500 return ArrayRef(Variants);
3501 }
3502
3503 if (isForcedDPP()) {
3504 static const unsigned Variants[] = {AMDGPUAsmVariants::DPP};
3505 return ArrayRef(Variants);
3506 }
3507
3508 return getAllVariants();
3509}
3510
3511StringRef AMDGPUAsmParser::getMatchedVariantName() const {
3512 if (isForcedDPP() && isForcedVOP3())
3513 return "e64_dpp";
3514
3515 if (getForcedEncodingSize() == 32)
3516 return "e32";
3517
3518 if (isForcedVOP3())
3519 return "e64";
3520
3521 if (isForcedSDWA())
3522 return "sdwa";
3523
3524 if (isForcedDPP())
3525 return "dpp";
3526
3527 return "";
3528}
3529
3530unsigned AMDGPUAsmParser::findImplicitSGPRReadInVOP(const MCInst &Inst) const {
3531 const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3532 for (MCPhysReg Reg : Desc.implicit_uses()) {
3533 switch (Reg) {
3534 case AMDGPU::FLAT_SCR:
3535 case AMDGPU::VCC:
3536 case AMDGPU::VCC_LO:
3537 case AMDGPU::VCC_HI:
3538 case AMDGPU::M0:
3539 return Reg;
3540 default:
3541 break;
3542 }
3543 }
3544 return AMDGPU::NoRegister;
3545}
3546
3547// NB: This code is correct only when used to check constant
3548// bus limitations because GFX7 support no f16 inline constants.
3549// Note that there are no cases when a GFX7 opcode violates
3550// constant bus limitations due to the use of an f16 constant.
3551bool AMDGPUAsmParser::isInlineConstant(const MCInst &Inst,
3552 unsigned OpIdx) const {
3553 const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
3554
3555 if (!AMDGPU::isSISrcOperand(Desc, OpIdx) ||
3556 AMDGPU::isKImmOperand(Desc, OpIdx)) {
3557 return false;
3558 }
3559
3560 const MCOperand &MO = Inst.getOperand(OpIdx);
3561
3562 int64_t Val = MO.getImm();
3563 auto OpSize = AMDGPU::getOperandSize(Desc, OpIdx);
3564
3565 switch (OpSize) { // expected operand size
3566 case 8:
3567 return AMDGPU::isInlinableLiteral64(Val, hasInv2PiInlineImm());
3568 case 4:
3569 return AMDGPU::isInlinableLiteral32(Val, hasInv2PiInlineImm());
3570 case 2: {
3571 const unsigned OperandType = Desc.operands()[OpIdx].OperandType;
3575 return AMDGPU::isInlinableLiteralI16(Val, hasInv2PiInlineImm());
3576
3581
3586
3591
3596 return AMDGPU::isInlinableLiteralFP16(Val, hasInv2PiInlineImm());
3597
3602 return AMDGPU::isInlinableLiteralBF16(Val, hasInv2PiInlineImm());
3603
3604 llvm_unreachable("invalid operand type");
3605 }
3606 default:
3607 llvm_unreachable("invalid operand size");
3608 }
3609}
3610
3611unsigned AMDGPUAsmParser::getConstantBusLimit(unsigned Opcode) const {
3612 if (!isGFX10Plus())
3613 return 1;
3614
3615 switch (Opcode) {
3616 // 64-bit shift instructions can use only one scalar value input
3617 case AMDGPU::V_LSHLREV_B64_e64:
3618 case AMDGPU::V_LSHLREV_B64_gfx10:
3619 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
3620 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
3621 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
3622 case AMDGPU::V_LSHRREV_B64_e64:
3623 case AMDGPU::V_LSHRREV_B64_gfx10:
3624 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
3625 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
3626 case AMDGPU::V_ASHRREV_I64_e64:
3627 case AMDGPU::V_ASHRREV_I64_gfx10:
3628 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
3629 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
3630 case AMDGPU::V_LSHL_B64_e64:
3631 case AMDGPU::V_LSHR_B64_e64:
3632 case AMDGPU::V_ASHR_I64_e64:
3633 return 1;
3634 default:
3635 return 2;
3636 }
3637}
3638
3639constexpr unsigned MAX_SRC_OPERANDS_NUM = 6;
3641
3642// Get regular operand indices in the same order as specified
3643// in the instruction (but append mandatory literals to the end).
3645 bool AddMandatoryLiterals = false) {
3646
3647 int16_t ImmIdx =
3648 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::imm) : -1;
3649
3650 if (isVOPD(Opcode)) {
3651 int16_t ImmDeferredIdx =
3652 AddMandatoryLiterals ? getNamedOperandIdx(Opcode, OpName::immDeferred)
3653 : -1;
3654
3655 return {getNamedOperandIdx(Opcode, OpName::src0X),
3656 getNamedOperandIdx(Opcode, OpName::vsrc1X),
3657 getNamedOperandIdx(Opcode, OpName::src0Y),
3658 getNamedOperandIdx(Opcode, OpName::vsrc1Y),
3659 ImmDeferredIdx,
3660 ImmIdx};
3661 }
3662
3663 return {getNamedOperandIdx(Opcode, OpName::src0),
3664 getNamedOperandIdx(Opcode, OpName::src1),
3665 getNamedOperandIdx(Opcode, OpName::src2), ImmIdx};
3666}
3667
3668bool AMDGPUAsmParser::usesConstantBus(const MCInst &Inst, unsigned OpIdx) {
3669 const MCOperand &MO = Inst.getOperand(OpIdx);
3670 if (MO.isImm()) {
3671 return !isInlineConstant(Inst, OpIdx);
3672 } else if (MO.isReg()) {
3673 auto Reg = MO.getReg();
3674 if (!Reg) {
3675 return false;
3676 }
3677 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3678 auto PReg = mc2PseudoReg(Reg);
3679 return isSGPR(PReg, TRI) && PReg != SGPR_NULL;
3680 } else {
3681 return true;
3682 }
3683}
3684
3685// Based on the comment for `AMDGPUInstructionSelector::selectWritelane`:
3686// Writelane is special in that it can use SGPR and M0 (which would normally
3687// count as using the constant bus twice - but in this case it is allowed since
3688// the lane selector doesn't count as a use of the constant bus). However, it is
3689// still required to abide by the 1 SGPR rule.
3690static bool checkWriteLane(const MCInst &Inst) {
3691 const unsigned Opcode = Inst.getOpcode();
3692 if (Opcode != V_WRITELANE_B32_gfx6_gfx7 && Opcode != V_WRITELANE_B32_vi)
3693 return false;
3694 const MCOperand &LaneSelOp = Inst.getOperand(2);
3695 if (!LaneSelOp.isReg())
3696 return false;
3697 auto LaneSelReg = mc2PseudoReg(LaneSelOp.getReg());
3698 return LaneSelReg == M0 || LaneSelReg == M0_gfxpre11;
3699}
3700
3701bool AMDGPUAsmParser::validateConstantBusLimitations(
3702 const MCInst &Inst, const OperandVector &Operands) {
3703 const unsigned Opcode = Inst.getOpcode();
3704 const MCInstrDesc &Desc = MII.get(Opcode);
3705 unsigned LastSGPR = AMDGPU::NoRegister;
3706 unsigned ConstantBusUseCount = 0;
3707 unsigned NumLiterals = 0;
3708 unsigned LiteralSize;
3709
3710 if (!(Desc.TSFlags &
3713 !isVOPD(Opcode))
3714 return true;
3715
3716 if (checkWriteLane(Inst))
3717 return true;
3718
3719 // Check special imm operands (used by madmk, etc)
3720 if (AMDGPU::hasNamedOperand(Opcode, AMDGPU::OpName::imm)) {
3721 ++NumLiterals;
3722 LiteralSize = 4;
3723 }
3724
3725 SmallDenseSet<unsigned> SGPRsUsed;
3726 unsigned SGPRUsed = findImplicitSGPRReadInVOP(Inst);
3727 if (SGPRUsed != AMDGPU::NoRegister) {
3728 SGPRsUsed.insert(SGPRUsed);
3729 ++ConstantBusUseCount;
3730 }
3731
3732 OperandIndices OpIndices = getSrcOperandIndices(Opcode);
3733
3734 for (int OpIdx : OpIndices) {
3735 if (OpIdx == -1)
3736 continue;
3737
3738 const MCOperand &MO = Inst.getOperand(OpIdx);
3739 if (usesConstantBus(Inst, OpIdx)) {
3740 if (MO.isReg()) {
3741 LastSGPR = mc2PseudoReg(MO.getReg());
3742 // Pairs of registers with a partial intersections like these
3743 // s0, s[0:1]
3744 // flat_scratch_lo, flat_scratch
3745 // flat_scratch_lo, flat_scratch_hi
3746 // are theoretically valid but they are disabled anyway.
3747 // Note that this code mimics SIInstrInfo::verifyInstruction
3748 if (SGPRsUsed.insert(LastSGPR).second) {
3749 ++ConstantBusUseCount;
3750 }
3751 } else { // Expression or a literal
3752
3753 if (Desc.operands()[OpIdx].OperandType == MCOI::OPERAND_IMMEDIATE)
3754 continue; // special operand like VINTERP attr_chan
3755
3756 // An instruction may use only one literal.
3757 // This has been validated on the previous step.
3758 // See validateVOPLiteral.
3759 // This literal may be used as more than one operand.
3760 // If all these operands are of the same size,
3761 // this literal counts as one scalar value.
3762 // Otherwise it counts as 2 scalar values.
3763 // See "GFX10 Shader Programming", section 3.6.2.3.
3764
3765 unsigned Size = AMDGPU::getOperandSize(Desc, OpIdx);
3766 if (Size < 4)
3767 Size = 4;
3768
3769 if (NumLiterals == 0) {
3770 NumLiterals = 1;
3771 LiteralSize = Size;
3772 } else if (LiteralSize != Size) {
3773 NumLiterals = 2;
3774 }
3775 }
3776 }
3777 }
3778 ConstantBusUseCount += NumLiterals;
3779
3780 if (ConstantBusUseCount <= getConstantBusLimit(Opcode))
3781 return true;
3782
3783 SMLoc LitLoc = getLitLoc(Operands);
3784 SMLoc RegLoc = getRegLoc(LastSGPR, Operands);
3785 SMLoc Loc = (LitLoc.getPointer() < RegLoc.getPointer()) ? RegLoc : LitLoc;
3786 Error(Loc, "invalid operand (violates constant bus restrictions)");
3787 return false;
3788}
3789
3790bool AMDGPUAsmParser::validateVOPDRegBankConstraints(
3791 const MCInst &Inst, const OperandVector &Operands) {
3792
3793 const unsigned Opcode = Inst.getOpcode();
3794 if (!isVOPD(Opcode))
3795 return true;
3796
3797 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
3798
3799 auto getVRegIdx = [&](unsigned, unsigned OperandIdx) {
3800 const MCOperand &Opr = Inst.getOperand(OperandIdx);
3801 return (Opr.isReg() && !isSGPR(mc2PseudoReg(Opr.getReg()), TRI))
3802 ? Opr.getReg()
3804 };
3805
3806 // On GFX12 if both OpX and OpY are V_MOV_B32 then OPY uses SRC2 source-cache.
3807 bool SkipSrc = Opcode == AMDGPU::V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12;
3808
3809 const auto &InstInfo = getVOPDInstInfo(Opcode, &MII);
3810 auto InvalidCompOprIdx =
3811 InstInfo.getInvalidCompOperandIndex(getVRegIdx, SkipSrc);
3812 if (!InvalidCompOprIdx)
3813 return true;
3814
3815 auto CompOprIdx = *InvalidCompOprIdx;
3816 auto ParsedIdx =
3817 std::max(InstInfo[VOPD::X].getIndexInParsedOperands(CompOprIdx),
3818 InstInfo[VOPD::Y].getIndexInParsedOperands(CompOprIdx));
3819 assert(ParsedIdx > 0 && ParsedIdx < Operands.size());
3820
3821 auto Loc = ((AMDGPUOperand &)*Operands[ParsedIdx]).getStartLoc();
3822 if (CompOprIdx == VOPD::Component::DST) {
3823 Error(Loc, "one dst register must be even and the other odd");
3824 } else {
3825 auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM;
3826 Error(Loc, Twine("src") + Twine(CompSrcIdx) +
3827 " operands must use different VGPR banks");
3828 }
3829
3830 return false;
3831}
3832
3833bool AMDGPUAsmParser::validateIntClampSupported(const MCInst &Inst) {
3834
3835 const unsigned Opc = Inst.getOpcode();
3836 const MCInstrDesc &Desc = MII.get(Opc);
3837
3838 if ((Desc.TSFlags & SIInstrFlags::IntClamp) != 0 && !hasIntClamp()) {
3839 int ClampIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::clamp);
3840 assert(ClampIdx != -1);
3841 return Inst.getOperand(ClampIdx).getImm() == 0;
3842 }
3843
3844 return true;
3845}
3846
3849
3850bool AMDGPUAsmParser::validateMIMGDataSize(const MCInst &Inst,
3851 const SMLoc &IDLoc) {
3852
3853 const unsigned Opc = Inst.getOpcode();
3854 const MCInstrDesc &Desc = MII.get(Opc);
3855
3856 if ((Desc.TSFlags & MIMGFlags) == 0)
3857 return true;
3858
3859 int VDataIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vdata);
3860 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
3861 int TFEIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::tfe);
3862
3863 assert(VDataIdx != -1);
3864
3865 if ((DMaskIdx == -1 || TFEIdx == -1) && isGFX10_AEncoding()) // intersect_ray
3866 return true;
3867
3868 unsigned VDataSize = AMDGPU::getRegOperandSize(getMRI(), Desc, VDataIdx);
3869 unsigned TFESize = (TFEIdx != -1 && Inst.getOperand(TFEIdx).getImm()) ? 1 : 0;
3870 unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
3871 if (DMask == 0)
3872 DMask = 1;
3873
3874 bool IsPackedD16 = false;
3875 unsigned DataSize =
3876 (Desc.TSFlags & SIInstrFlags::Gather4) ? 4 : llvm::popcount(DMask);
3877 if (hasPackedD16()) {
3878 int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
3879 IsPackedD16 = D16Idx >= 0;
3880 if (IsPackedD16 && Inst.getOperand(D16Idx).getImm())
3881 DataSize = (DataSize + 1) / 2;
3882 }
3883
3884 if ((VDataSize / 4) == DataSize + TFESize)
3885 return true;
3886
3887 StringRef Modifiers;
3888 if (isGFX90A())
3889 Modifiers = IsPackedD16 ? "dmask and d16" : "dmask";
3890 else
3891 Modifiers = IsPackedD16 ? "dmask, d16 and tfe" : "dmask and tfe";
3892
3893 Error(IDLoc, Twine("image data size does not match ") + Modifiers);
3894 return false;
3895}
3896
3897bool AMDGPUAsmParser::validateMIMGAddrSize(const MCInst &Inst,
3898 const SMLoc &IDLoc) {
3899 const unsigned Opc = Inst.getOpcode();
3900 const MCInstrDesc &Desc = MII.get(Opc);
3901
3902 if ((Desc.TSFlags & MIMGFlags) == 0 || !isGFX10Plus())
3903 return true;
3904
3906
3907 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
3909 int VAddr0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr0);
3910 int RSrcOpName = (Desc.TSFlags & SIInstrFlags::MIMG) ? AMDGPU::OpName::srsrc
3911 : AMDGPU::OpName::rsrc;
3912 int SrsrcIdx = AMDGPU::getNamedOperandIdx(Opc, RSrcOpName);
3913 int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
3914 int A16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::a16);
3915
3916 assert(VAddr0Idx != -1);
3917 assert(SrsrcIdx != -1);
3918 assert(SrsrcIdx > VAddr0Idx);
3919
3920 bool IsA16 = (A16Idx != -1 && Inst.getOperand(A16Idx).getImm());
3921 if (BaseOpcode->BVH) {
3922 if (IsA16 == BaseOpcode->A16)
3923 return true;
3924 Error(IDLoc, "image address size does not match a16");
3925 return false;
3926 }
3927
3928 unsigned Dim = Inst.getOperand(DimIdx).getImm();
3930 bool IsNSA = SrsrcIdx - VAddr0Idx > 1;
3931 unsigned ActualAddrSize =
3932 IsNSA ? SrsrcIdx - VAddr0Idx
3933 : AMDGPU::getRegOperandSize(getMRI(), Desc, VAddr0Idx) / 4;
3934
3935 unsigned ExpectedAddrSize =
3936 AMDGPU::getAddrSizeMIMGOp(BaseOpcode, DimInfo, IsA16, hasG16());
3937
3938 if (IsNSA) {
3939 if (hasPartialNSAEncoding() &&
3940 ExpectedAddrSize >
3942 int VAddrLastIdx = SrsrcIdx - 1;
3943 unsigned VAddrLastSize =
3944 AMDGPU::getRegOperandSize(getMRI(), Desc, VAddrLastIdx) / 4;
3945
3946 ActualAddrSize = VAddrLastIdx - VAddr0Idx + VAddrLastSize;
3947 }
3948 } else {
3949 if (ExpectedAddrSize > 12)
3950 ExpectedAddrSize = 16;
3951
3952 // Allow oversized 8 VGPR vaddr when only 5/6/7 VGPRs are required.
3953 // This provides backward compatibility for assembly created
3954 // before 160b/192b/224b types were directly supported.
3955 if (ActualAddrSize == 8 && (ExpectedAddrSize >= 5 && ExpectedAddrSize <= 7))
3956 return true;
3957 }
3958
3959 if (ActualAddrSize == ExpectedAddrSize)
3960 return true;
3961
3962 Error(IDLoc, "image address size does not match dim and a16");
3963 return false;
3964}
3965
3966bool AMDGPUAsmParser::validateMIMGAtomicDMask(const MCInst &Inst) {
3967
3968 const unsigned Opc = Inst.getOpcode();
3969 const MCInstrDesc &Desc = MII.get(Opc);
3970
3971 if ((Desc.TSFlags & MIMGFlags) == 0)
3972 return true;
3973 if (!Desc.mayLoad() || !Desc.mayStore())
3974 return true; // Not atomic
3975
3976 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
3977 unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
3978
3979 // This is an incomplete check because image_atomic_cmpswap
3980 // may only use 0x3 and 0xf while other atomic operations
3981 // may use 0x1 and 0x3. However these limitations are
3982 // verified when we check that dmask matches dst size.
3983 return DMask == 0x1 || DMask == 0x3 || DMask == 0xf;
3984}
3985
3986bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) {
3987
3988 const unsigned Opc = Inst.getOpcode();
3989 const MCInstrDesc &Desc = MII.get(Opc);
3990
3991 if ((Desc.TSFlags & SIInstrFlags::Gather4) == 0)
3992 return true;
3993
3994 int DMaskIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dmask);
3995 unsigned DMask = Inst.getOperand(DMaskIdx).getImm() & 0xf;
3996
3997 // GATHER4 instructions use dmask in a different fashion compared to
3998 // other MIMG instructions. The only useful DMASK values are
3999 // 1=red, 2=green, 4=blue, 8=alpha. (e.g. 1 returns
4000 // (red,red,red,red) etc.) The ISA document doesn't mention
4001 // this.
4002 return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8;
4003}
4004
4005bool AMDGPUAsmParser::validateMIMGMSAA(const MCInst &Inst) {
4006 const unsigned Opc = Inst.getOpcode();
4007 const MCInstrDesc &Desc = MII.get(Opc);
4008
4009 if ((Desc.TSFlags & MIMGFlags) == 0)
4010 return true;
4011
4013 const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode =
4015
4016 if (!BaseOpcode->MSAA)
4017 return true;
4018
4019 int DimIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dim);
4020 assert(DimIdx != -1);
4021
4022 unsigned Dim = Inst.getOperand(DimIdx).getImm();
4024
4025 return DimInfo->MSAA;
4026}
4027
4028static bool IsMovrelsSDWAOpcode(const unsigned Opcode)
4029{
4030 switch (Opcode) {
4031 case AMDGPU::V_MOVRELS_B32_sdwa_gfx10:
4032 case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10:
4033 case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10:
4034 return true;
4035 default:
4036 return false;
4037 }
4038}
4039
4040// movrels* opcodes should only allow VGPRS as src0.
4041// This is specified in .td description for vop1/vop3,
4042// but sdwa is handled differently. See isSDWAOperand.
4043bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst,
4044 const OperandVector &Operands) {
4045
4046 const unsigned Opc = Inst.getOpcode();
4047 const MCInstrDesc &Desc = MII.get(Opc);
4048
4049 if ((Desc.TSFlags & SIInstrFlags::SDWA) == 0 || !IsMovrelsSDWAOpcode(Opc))
4050 return true;
4051
4052 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
4053 assert(Src0Idx != -1);
4054
4055 SMLoc ErrLoc;
4056 const MCOperand &Src0 = Inst.getOperand(Src0Idx);
4057 if (Src0.isReg()) {
4058 auto Reg = mc2PseudoReg(Src0.getReg());
4059 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4060 if (!isSGPR(Reg, TRI))
4061 return true;
4062 ErrLoc = getRegLoc(Reg, Operands);
4063 } else {
4064 ErrLoc = getConstLoc(Operands);
4065 }
4066
4067 Error(ErrLoc, "source operand must be a VGPR");
4068 return false;
4069}
4070
4071bool AMDGPUAsmParser::validateMAIAccWrite(const MCInst &Inst,
4072 const OperandVector &Operands) {
4073
4074 const unsigned Opc = Inst.getOpcode();
4075
4076 if (Opc != AMDGPU::V_ACCVGPR_WRITE_B32_vi)
4077 return true;
4078
4079 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0);
4080 assert(Src0Idx != -1);
4081
4082 const MCOperand &Src0 = Inst.getOperand(Src0Idx);
4083 if (!Src0.isReg())
4084 return true;
4085
4086 auto Reg = mc2PseudoReg(Src0.getReg());
4087 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4088 if (!isGFX90A() && isSGPR(Reg, TRI)) {
4089 Error(getRegLoc(Reg, Operands),
4090 "source operand must be either a VGPR or an inline constant");
4091 return false;
4092 }
4093
4094 return true;
4095}
4096
4097bool AMDGPUAsmParser::validateMAISrc2(const MCInst &Inst,
4098 const OperandVector &Operands) {
4099 unsigned Opcode = Inst.getOpcode();
4100 const MCInstrDesc &Desc = MII.get(Opcode);
4101
4102 if (!(Desc.TSFlags & SIInstrFlags::IsMAI) ||
4103 !getFeatureBits()[FeatureMFMAInlineLiteralBug])
4104 return true;
4105
4106 const int Src2Idx = getNamedOperandIdx(Opcode, OpName::src2);
4107 if (Src2Idx == -1)
4108 return true;
4109
4110 if (Inst.getOperand(Src2Idx).isImm() && isInlineConstant(Inst, Src2Idx)) {
4111 Error(getConstLoc(Operands),
4112 "inline constants are not allowed for this operand");
4113 return false;
4114 }
4115
4116 return true;
4117}
4118
4119bool AMDGPUAsmParser::validateMFMA(const MCInst &Inst,
4120 const OperandVector &Operands) {
4121 const unsigned Opc = Inst.getOpcode();
4122 const MCInstrDesc &Desc = MII.get(Opc);
4123
4124 if ((Desc.TSFlags & SIInstrFlags::IsMAI) == 0)
4125 return true;
4126
4127 const int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
4128 if (Src2Idx == -1)
4129 return true;
4130
4131 const MCOperand &Src2 = Inst.getOperand(Src2Idx);
4132 if (!Src2.isReg())
4133 return true;
4134
4135 MCRegister Src2Reg = Src2.getReg();
4136 MCRegister DstReg = Inst.getOperand(0).getReg();
4137 if (Src2Reg == DstReg)
4138 return true;
4139
4140 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4141 if (TRI->getRegClass(Desc.operands()[0].RegClass).getSizeInBits() <= 128)
4142 return true;
4143
4144 if (TRI->regsOverlap(Src2Reg, DstReg)) {
4145 Error(getRegLoc(mc2PseudoReg(Src2Reg), Operands),
4146 "source 2 operand must not partially overlap with dst");
4147 return false;
4148 }
4149
4150 return true;
4151}
4152
4153bool AMDGPUAsmParser::validateDivScale(const MCInst &Inst) {
4154 switch (Inst.getOpcode()) {
4155 default:
4156 return true;
4157 case V_DIV_SCALE_F32_gfx6_gfx7:
4158 case V_DIV_SCALE_F32_vi:
4159 case V_DIV_SCALE_F32_gfx10:
4160 case V_DIV_SCALE_F64_gfx6_gfx7:
4161 case V_DIV_SCALE_F64_vi:
4162 case V_DIV_SCALE_F64_gfx10:
4163 break;
4164 }
4165
4166 // TODO: Check that src0 = src1 or src2.
4167
4168 for (auto Name : {AMDGPU::OpName::src0_modifiers,
4169 AMDGPU::OpName::src2_modifiers,
4170 AMDGPU::OpName::src2_modifiers}) {
4172 .getImm() &
4174 return false;
4175 }
4176 }
4177
4178 return true;
4179}
4180
4181bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) {
4182
4183 const unsigned Opc = Inst.getOpcode();
4184 const MCInstrDesc &Desc = MII.get(Opc);
4185
4186 if ((Desc.TSFlags & MIMGFlags) == 0)
4187 return true;
4188
4189 int D16Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::d16);
4190 if (D16Idx >= 0 && Inst.getOperand(D16Idx).getImm()) {
4191 if (isCI() || isSI())
4192 return false;
4193 }
4194
4195 return true;
4196}
4197
4198static bool IsRevOpcode(const unsigned Opcode)
4199{
4200 switch (Opcode) {
4201 case AMDGPU::V_SUBREV_F32_e32:
4202 case AMDGPU::V_SUBREV_F32_e64:
4203 case AMDGPU::V_SUBREV_F32_e32_gfx10:
4204 case AMDGPU::V_SUBREV_F32_e32_gfx6_gfx7:
4205 case AMDGPU::V_SUBREV_F32_e32_vi:
4206 case AMDGPU::V_SUBREV_F32_e64_gfx10:
4207 case AMDGPU::V_SUBREV_F32_e64_gfx6_gfx7:
4208 case AMDGPU::V_SUBREV_F32_e64_vi:
4209
4210 case AMDGPU::V_SUBREV_CO_U32_e32:
4211 case AMDGPU::V_SUBREV_CO_U32_e64:
4212 case AMDGPU::V_SUBREV_I32_e32_gfx6_gfx7:
4213 case AMDGPU::V_SUBREV_I32_e64_gfx6_gfx7:
4214
4215 case AMDGPU::V_SUBBREV_U32_e32:
4216 case AMDGPU::V_SUBBREV_U32_e64:
4217 case AMDGPU::V_SUBBREV_U32_e32_gfx6_gfx7:
4218 case AMDGPU::V_SUBBREV_U32_e32_vi:
4219 case AMDGPU::V_SUBBREV_U32_e64_gfx6_gfx7:
4220 case AMDGPU::V_SUBBREV_U32_e64_vi:
4221
4222 case AMDGPU::V_SUBREV_U32_e32:
4223 case AMDGPU::V_SUBREV_U32_e64:
4224 case AMDGPU::V_SUBREV_U32_e32_gfx9:
4225 case AMDGPU::V_SUBREV_U32_e32_vi:
4226 case AMDGPU::V_SUBREV_U32_e64_gfx9:
4227 case AMDGPU::V_SUBREV_U32_e64_vi:
4228
4229 case AMDGPU::V_SUBREV_F16_e32:
4230 case AMDGPU::V_SUBREV_F16_e64:
4231 case AMDGPU::V_SUBREV_F16_e32_gfx10:
4232 case AMDGPU::V_SUBREV_F16_e32_vi:
4233 case AMDGPU::V_SUBREV_F16_e64_gfx10:
4234 case AMDGPU::V_SUBREV_F16_e64_vi:
4235
4236 case AMDGPU::V_SUBREV_U16_e32:
4237 case AMDGPU::V_SUBREV_U16_e64:
4238 case AMDGPU::V_SUBREV_U16_e32_vi:
4239 case AMDGPU::V_SUBREV_U16_e64_vi:
4240
4241 case AMDGPU::V_SUBREV_CO_U32_e32_gfx9:
4242 case AMDGPU::V_SUBREV_CO_U32_e64_gfx10:
4243 case AMDGPU::V_SUBREV_CO_U32_e64_gfx9:
4244
4245 case AMDGPU::V_SUBBREV_CO_U32_e32_gfx9:
4246 case AMDGPU::V_SUBBREV_CO_U32_e64_gfx9:
4247
4248 case AMDGPU::V_SUBREV_NC_U32_e32_gfx10:
4249 case AMDGPU::V_SUBREV_NC_U32_e64_gfx10:
4250
4251 case AMDGPU::V_SUBREV_CO_CI_U32_e32_gfx10:
4252 case AMDGPU::V_SUBREV_CO_CI_U32_e64_gfx10:
4253
4254 case AMDGPU::V_LSHRREV_B32_e32:
4255 case AMDGPU::V_LSHRREV_B32_e64:
4256 case AMDGPU::V_LSHRREV_B32_e32_gfx6_gfx7:
4257 case AMDGPU::V_LSHRREV_B32_e64_gfx6_gfx7:
4258 case AMDGPU::V_LSHRREV_B32_e32_vi:
4259 case AMDGPU::V_LSHRREV_B32_e64_vi:
4260 case AMDGPU::V_LSHRREV_B32_e32_gfx10:
4261 case AMDGPU::V_LSHRREV_B32_e64_gfx10:
4262
4263 case AMDGPU::V_ASHRREV_I32_e32:
4264 case AMDGPU::V_ASHRREV_I32_e64:
4265 case AMDGPU::V_ASHRREV_I32_e32_gfx10:
4266 case AMDGPU::V_ASHRREV_I32_e32_gfx6_gfx7:
4267 case AMDGPU::V_ASHRREV_I32_e32_vi:
4268 case AMDGPU::V_ASHRREV_I32_e64_gfx10:
4269 case AMDGPU::V_ASHRREV_I32_e64_gfx6_gfx7:
4270 case AMDGPU::V_ASHRREV_I32_e64_vi:
4271
4272 case AMDGPU::V_LSHLREV_B32_e32:
4273 case AMDGPU::V_LSHLREV_B32_e64:
4274 case AMDGPU::V_LSHLREV_B32_e32_gfx10:
4275 case AMDGPU::V_LSHLREV_B32_e32_gfx6_gfx7:
4276 case AMDGPU::V_LSHLREV_B32_e32_vi:
4277 case AMDGPU::V_LSHLREV_B32_e64_gfx10:
4278 case AMDGPU::V_LSHLREV_B32_e64_gfx6_gfx7:
4279 case AMDGPU::V_LSHLREV_B32_e64_vi:
4280
4281 case AMDGPU::V_LSHLREV_B16_e32:
4282 case AMDGPU::V_LSHLREV_B16_e64:
4283 case AMDGPU::V_LSHLREV_B16_e32_vi:
4284 case AMDGPU::V_LSHLREV_B16_e64_vi:
4285 case AMDGPU::V_LSHLREV_B16_gfx10:
4286
4287 case AMDGPU::V_LSHRREV_B16_e32:
4288 case AMDGPU::V_LSHRREV_B16_e64:
4289 case AMDGPU::V_LSHRREV_B16_e32_vi:
4290 case AMDGPU::V_LSHRREV_B16_e64_vi:
4291 case AMDGPU::V_LSHRREV_B16_gfx10:
4292
4293 case AMDGPU::V_ASHRREV_I16_e32:
4294 case AMDGPU::V_ASHRREV_I16_e64:
4295 case AMDGPU::V_ASHRREV_I16_e32_vi:
4296 case AMDGPU::V_ASHRREV_I16_e64_vi:
4297 case AMDGPU::V_ASHRREV_I16_gfx10:
4298
4299 case AMDGPU::V_LSHLREV_B64_e64:
4300 case AMDGPU::V_LSHLREV_B64_gfx10:
4301 case AMDGPU::V_LSHLREV_B64_vi:
4302
4303 case AMDGPU::V_LSHRREV_B64_e64:
4304 case AMDGPU::V_LSHRREV_B64_gfx10:
4305 case AMDGPU::V_LSHRREV_B64_vi:
4306
4307 case AMDGPU::V_ASHRREV_I64_e64:
4308 case AMDGPU::V_ASHRREV_I64_gfx10:
4309 case AMDGPU::V_ASHRREV_I64_vi:
4310
4311 case AMDGPU::V_PK_LSHLREV_B16:
4312 case AMDGPU::V_PK_LSHLREV_B16_gfx10:
4313 case AMDGPU::V_PK_LSHLREV_B16_vi:
4314
4315 case AMDGPU::V_PK_LSHRREV_B16:
4316 case AMDGPU::V_PK_LSHRREV_B16_gfx10:
4317 case AMDGPU::V_PK_LSHRREV_B16_vi:
4318 case AMDGPU::V_PK_ASHRREV_I16:
4319 case AMDGPU::V_PK_ASHRREV_I16_gfx10:
4320 case AMDGPU::V_PK_ASHRREV_I16_vi:
4321 return true;
4322 default:
4323 return false;
4324 }
4325}
4326
4327std::optional<StringRef>
4328AMDGPUAsmParser::validateLdsDirect(const MCInst &Inst) {
4329
4330 using namespace SIInstrFlags;
4331 const unsigned Opcode = Inst.getOpcode();
4332 const MCInstrDesc &Desc = MII.get(Opcode);
4333
4334 // lds_direct register is defined so that it can be used
4335 // with 9-bit operands only. Ignore encodings which do not accept these.
4336 const auto Enc = VOP1 | VOP2 | VOP3 | VOPC | VOP3P | SIInstrFlags::SDWA;
4337 if ((Desc.TSFlags & Enc) == 0)
4338 return std::nullopt;
4339
4340 for (auto SrcName : {OpName::src0, OpName::src1, OpName::src2}) {
4341 auto SrcIdx = getNamedOperandIdx(Opcode, SrcName);
4342 if (SrcIdx == -1)
4343 break;
4344 const auto &Src = Inst.getOperand(SrcIdx);
4345 if (Src.isReg() && Src.getReg() == LDS_DIRECT) {
4346
4347 if (isGFX90A() || isGFX11Plus())
4348 return StringRef("lds_direct is not supported on this GPU");
4349
4350 if (IsRevOpcode(Opcode) || (Desc.TSFlags & SIInstrFlags::SDWA))
4351 return StringRef("lds_direct cannot be used with this instruction");
4352
4353 if (SrcName != OpName::src0)
4354 return StringRef("lds_direct may be used as src0 only");
4355 }
4356 }
4357
4358 return std::nullopt;
4359}
4360
4361SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const {
4362 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4363 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4364 if (Op.isFlatOffset())
4365 return Op.getStartLoc();
4366 }
4367 return getLoc();
4368}
4369
4370bool AMDGPUAsmParser::validateOffset(const MCInst &Inst,
4371 const OperandVector &Operands) {
4372 auto Opcode = Inst.getOpcode();
4373 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4374 if (OpNum == -1)
4375 return true;
4376
4377 uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4378 if ((TSFlags & SIInstrFlags::FLAT))
4379 return validateFlatOffset(Inst, Operands);
4380
4381 if ((TSFlags & SIInstrFlags::SMRD))
4382 return validateSMEMOffset(Inst, Operands);
4383
4384 const auto &Op = Inst.getOperand(OpNum);
4385 if (isGFX12Plus() &&
4386 (TSFlags & (SIInstrFlags::MUBUF | SIInstrFlags::MTBUF))) {
4387 const unsigned OffsetSize = 24;
4388 if (!isIntN(OffsetSize, Op.getImm())) {
4389 Error(getFlatOffsetLoc(Operands),
4390 Twine("expected a ") + Twine(OffsetSize) + "-bit signed offset");
4391 return false;
4392 }
4393 } else {
4394 const unsigned OffsetSize = 16;
4395 if (!isUIntN(OffsetSize, Op.getImm())) {
4396 Error(getFlatOffsetLoc(Operands),
4397 Twine("expected a ") + Twine(OffsetSize) + "-bit unsigned offset");
4398 return false;
4399 }
4400 }
4401 return true;
4402}
4403
4404bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst,
4405 const OperandVector &Operands) {
4406 uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4407 if ((TSFlags & SIInstrFlags::FLAT) == 0)
4408 return true;
4409
4410 auto Opcode = Inst.getOpcode();
4411 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4412 assert(OpNum != -1);
4413
4414 const auto &Op = Inst.getOperand(OpNum);
4415 if (!hasFlatOffsets() && Op.getImm() != 0) {
4416 Error(getFlatOffsetLoc(Operands),
4417 "flat offset modifier is not supported on this GPU");
4418 return false;
4419 }
4420
4421 // For pre-GFX12 FLAT instructions the offset must be positive;
4422 // MSB is ignored and forced to zero.
4423 unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(getSTI());
4424 bool AllowNegative =
4426 isGFX12Plus();
4427 if (!isIntN(OffsetSize, Op.getImm()) || (!AllowNegative && Op.getImm() < 0)) {
4428 Error(getFlatOffsetLoc(Operands),
4429 Twine("expected a ") +
4430 (AllowNegative ? Twine(OffsetSize) + "-bit signed offset"
4431 : Twine(OffsetSize - 1) + "-bit unsigned offset"));
4432 return false;
4433 }
4434
4435 return true;
4436}
4437
4438SMLoc AMDGPUAsmParser::getSMEMOffsetLoc(const OperandVector &Operands) const {
4439 // Start with second operand because SMEM Offset cannot be dst or src0.
4440 for (unsigned i = 2, e = Operands.size(); i != e; ++i) {
4441 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4442 if (Op.isSMEMOffset() || Op.isSMEMOffsetMod())
4443 return Op.getStartLoc();
4444 }
4445 return getLoc();
4446}
4447
4448bool AMDGPUAsmParser::validateSMEMOffset(const MCInst &Inst,
4449 const OperandVector &Operands) {
4450 if (isCI() || isSI())
4451 return true;
4452
4453 uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4454 if ((TSFlags & SIInstrFlags::SMRD) == 0)
4455 return true;
4456
4457 auto Opcode = Inst.getOpcode();
4458 auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
4459 if (OpNum == -1)
4460 return true;
4461
4462 const auto &Op = Inst.getOperand(OpNum);
4463 if (!Op.isImm())
4464 return true;
4465
4466 uint64_t Offset = Op.getImm();
4467 bool IsBuffer = AMDGPU::getSMEMIsBuffer(Opcode);
4470 return true;
4471
4472 Error(getSMEMOffsetLoc(Operands),
4473 isGFX12Plus() ? "expected a 24-bit signed offset"
4474 : (isVI() || IsBuffer) ? "expected a 20-bit unsigned offset"
4475 : "expected a 21-bit signed offset");
4476
4477 return false;
4478}
4479
4480bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
4481 unsigned Opcode = Inst.getOpcode();
4482 const MCInstrDesc &Desc = MII.get(Opcode);
4483 if (!(Desc.TSFlags & (SIInstrFlags::SOP2 | SIInstrFlags::SOPC)))
4484 return true;
4485
4486 const int Src0Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src0);
4487 const int Src1Idx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::src1);
4488
4489 const int OpIndices[] = { Src0Idx, Src1Idx };
4490
4491 unsigned NumExprs = 0;
4492 unsigned NumLiterals = 0;
4494
4495 for (int OpIdx : OpIndices) {
4496 if (OpIdx == -1) break;
4497
4498 const MCOperand &MO = Inst.getOperand(OpIdx);
4499 // Exclude special imm operands (like that used by s_set_gpr_idx_on)
4500 if (AMDGPU::isSISrcOperand(Desc, OpIdx)) {
4501 if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
4502 uint32_t Value = static_cast<uint32_t>(MO.getImm());
4503 if (NumLiterals == 0 || LiteralValue != Value) {
4505 ++NumLiterals;
4506 }
4507 } else if (MO.isExpr()) {
4508 ++NumExprs;
4509 }
4510 }
4511 }
4512
4513 return NumLiterals + NumExprs <= 1;
4514}
4515
4516bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
4517 const unsigned Opc = Inst.getOpcode();
4518 if (isPermlane16(Opc)) {
4519 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4520 unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4521
4522 if (OpSel & ~3)
4523 return false;
4524 }
4525
4526 uint64_t TSFlags = MII.get(Opc).TSFlags;
4527
4528 if (isGFX940() && (TSFlags & SIInstrFlags::IsDOT)) {
4529 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4530 if (OpSelIdx != -1) {
4531 if (Inst.getOperand(OpSelIdx).getImm() != 0)
4532 return false;
4533 }
4534 int OpSelHiIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel_hi);
4535 if (OpSelHiIdx != -1) {
4536 if (Inst.getOperand(OpSelHiIdx).getImm() != -1)
4537 return false;
4538 }
4539 }
4540
4541 // op_sel[0:1] must be 0 for v_dot2_bf16_bf16 and v_dot2_f16_f16 (VOP3 Dot).
4542 if (isGFX11Plus() && (TSFlags & SIInstrFlags::IsDOT) &&
4543 (TSFlags & SIInstrFlags::VOP3) && !(TSFlags & SIInstrFlags::VOP3P)) {
4544 int OpSelIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::op_sel);
4545 unsigned OpSel = Inst.getOperand(OpSelIdx).getImm();
4546 if (OpSel & 3)
4547 return false;
4548 }
4549
4550 return true;
4551}
4552
4553bool AMDGPUAsmParser::validateNeg(const MCInst &Inst, int OpName) {
4554 assert(OpName == AMDGPU::OpName::neg_lo || OpName == AMDGPU::OpName::neg_hi);
4555
4556 const unsigned Opc = Inst.getOpcode();
4557 uint64_t TSFlags = MII.get(Opc).TSFlags;
4558
4559 // v_dot4 fp8/bf8 neg_lo/neg_hi not allowed on src0 and src1 (allowed on src2)
4560 // v_wmma iu4/iu8 neg_lo not allowed on src2 (allowed on src0, src1)
4561 // v_swmmac f16/bf16 neg_lo/neg_hi not allowed on src2 (allowed on src0, src1)
4562 // other wmma/swmmac instructions don't have neg_lo/neg_hi operand.
4563 if (!(TSFlags & SIInstrFlags::IsDOT) && !(TSFlags & SIInstrFlags::IsWMMA) &&
4564 !(TSFlags & SIInstrFlags::IsSWMMAC))
4565 return true;
4566
4567 int NegIdx = AMDGPU::getNamedOperandIdx(Opc, OpName);
4568 if (NegIdx == -1)
4569 return true;
4570
4571 unsigned Neg = Inst.getOperand(NegIdx).getImm();
4572
4573 // Instructions that have neg_lo or neg_hi operand but neg modifier is allowed
4574 // on some src operands but not allowed on other.
4575 // It is convenient that such instructions don't have src_modifiers operand
4576 // for src operands that don't allow neg because they also don't allow opsel.
4577
4578 int SrcMods[3] = {AMDGPU::OpName::src0_modifiers,
4579 AMDGPU::OpName::src1_modifiers,
4580 AMDGPU::OpName::src2_modifiers};
4581
4582 for (unsigned i = 0; i < 3; ++i) {
4583 if (!AMDGPU::hasNamedOperand(Opc, SrcMods[i])) {
4584 if (Neg & (1 << i))
4585 return false;
4586 }
4587 }
4588
4589 return true;
4590}
4591
4592bool AMDGPUAsmParser::validateDPP(const MCInst &Inst,
4593 const OperandVector &Operands) {
4594 const unsigned Opc = Inst.getOpcode();
4595 int DppCtrlIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dpp_ctrl);
4596 if (DppCtrlIdx >= 0) {
4597 unsigned DppCtrl = Inst.getOperand(DppCtrlIdx).getImm();
4598
4599 if (!AMDGPU::isLegalDPALU_DPPControl(DppCtrl) &&
4600 AMDGPU::isDPALU_DPP(MII.get(Opc))) {
4601 // DP ALU DPP is supported for row_newbcast only on GFX9*
4602 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyDppCtrl, Operands);
4603 Error(S, "DP ALU dpp only supports row_newbcast");
4604 return false;
4605 }
4606 }
4607
4608 int Dpp8Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::dpp8);
4609 bool IsDPP = DppCtrlIdx >= 0 || Dpp8Idx >= 0;
4610
4611 if (IsDPP && !hasDPPSrc1SGPR(getSTI())) {
4612 int Src1Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src1);
4613 if (Src1Idx >= 0) {
4614 const MCOperand &Src1 = Inst.getOperand(Src1Idx);
4615 const MCRegisterInfo *TRI = getContext().getRegisterInfo();
4616 if (Src1.isReg() && isSGPR(mc2PseudoReg(Src1.getReg()), TRI)) {
4617 auto Reg = mc2PseudoReg(Inst.getOperand(Src1Idx).getReg());
4618 SMLoc S = getRegLoc(Reg, Operands);
4619 Error(S, "invalid operand for instruction");
4620 return false;
4621 }
4622 if (Src1.isImm()) {
4623 Error(getInstLoc(Operands),
4624 "src1 immediate operand invalid for instruction");
4625 return false;
4626 }
4627 }
4628 }
4629
4630 return true;
4631}
4632
4633// Check if VCC register matches wavefront size
4634bool AMDGPUAsmParser::validateVccOperand(unsigned Reg) const {
4635 auto FB = getFeatureBits();
4636 return (FB[AMDGPU::FeatureWavefrontSize64] && Reg == AMDGPU::VCC) ||
4637 (FB[AMDGPU::FeatureWavefrontSize32] && Reg == AMDGPU::VCC_LO);
4638}
4639
4640// One unique literal can be used. VOP3 literal is only allowed in GFX10+
4641bool AMDGPUAsmParser::validateVOPLiteral(const MCInst &Inst,
4642 const OperandVector &Operands) {
4643 unsigned Opcode = Inst.getOpcode();
4644 const MCInstrDesc &Desc = MII.get(Opcode);
4645 bool HasMandatoryLiteral = getNamedOperandIdx(Opcode, OpName::imm) != -1;
4646 if (!(Desc.TSFlags & (SIInstrFlags::VOP3 | SIInstrFlags::VOP3P)) &&
4647 !HasMandatoryLiteral && !isVOPD(Opcode))
4648 return true;
4649
4650 OperandIndices OpIndices = getSrcOperandIndices(Opcode, HasMandatoryLiteral);
4651
4652 unsigned NumExprs = 0;
4653 unsigned NumLiterals = 0;
4655
4656 for (int OpIdx : OpIndices) {
4657 if (OpIdx == -1)
4658 continue;
4659
4660 const MCOperand &MO = Inst.getOperand(OpIdx);
4661 if (!MO.isImm() && !MO.isExpr())
4662 continue;
4663 if (!isSISrcOperand(Desc, OpIdx))
4664 continue;
4665
4666 if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
4667 uint64_t Value = static_cast<uint64_t>(MO.getImm());
4668 bool IsFP64 = AMDGPU::isSISrcFPOperand(Desc, OpIdx) &&
4669 AMDGPU::getOperandSize(Desc.operands()[OpIdx]) == 8;
4670 bool IsValid32Op = AMDGPU::isValid32BitLiteral(Value, IsFP64);
4671
4672 if (!IsValid32Op && !isInt<32>(Value) && !isUInt<32>(Value)) {
4673 Error(getLitLoc(Operands), "invalid operand for instruction");
4674 return false;
4675 }
4676
4677 if (IsFP64 && IsValid32Op)
4678 Value = Hi_32(Value);
4679
4680 if (NumLiterals == 0 || LiteralValue != Value) {
4682 ++NumLiterals;
4683 }
4684 } else if (MO.isExpr()) {
4685 ++NumExprs;
4686 }
4687 }
4688 NumLiterals += NumExprs;
4689
4690 if (!NumLiterals)
4691 return true;
4692
4693 if (!HasMandatoryLiteral && !getFeatureBits()[FeatureVOP3Literal]) {
4694 Error(getLitLoc(Operands), "literal operands are not supported");
4695 return false;
4696 }
4697
4698 if (NumLiterals > 1) {
4699 Error(getLitLoc(Operands, true), "only one unique literal operand is allowed");
4700 return false;
4701 }
4702
4703 return true;
4704}
4705
4706// Returns -1 if not a register, 0 if VGPR and 1 if AGPR.
4707static int IsAGPROperand(const MCInst &Inst, uint16_t NameIdx,
4708 const MCRegisterInfo *MRI) {
4709 int OpIdx = AMDGPU::getNamedOperandIdx(Inst.getOpcode(), NameIdx);
4710 if (OpIdx < 0)
4711 return -1;
4712
4713 const MCOperand &Op = Inst.getOperand(OpIdx);
4714 if (!Op.isReg())
4715 return -1;
4716
4717 unsigned Sub = MRI->getSubReg(Op.getReg(), AMDGPU::sub0);
4718 auto Reg = Sub ? Sub : Op.getReg();
4719 const MCRegisterClass &AGPR32 = MRI->getRegClass(AMDGPU::AGPR_32RegClassID);
4720 return AGPR32.contains(Reg) ? 1 : 0;
4721}
4722
4723bool AMDGPUAsmParser::validateAGPRLdSt(const MCInst &Inst) const {
4724 uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4725 if ((TSFlags & (SIInstrFlags::FLAT | SIInstrFlags::MUBUF |
4727 SIInstrFlags::DS)) == 0)
4728 return true;
4729
4730 uint16_t DataNameIdx = (TSFlags & SIInstrFlags::DS) ? AMDGPU::OpName::data0
4731 : AMDGPU::OpName::vdata;
4732
4733 const MCRegisterInfo *MRI = getMRI();
4734 int DstAreg = IsAGPROperand(Inst, AMDGPU::OpName::vdst, MRI);
4735 int DataAreg = IsAGPROperand(Inst, DataNameIdx, MRI);
4736
4737 if ((TSFlags & SIInstrFlags::DS) && DataAreg >= 0) {
4738 int Data2Areg = IsAGPROperand(Inst, AMDGPU::OpName::data1, MRI);
4739 if (Data2Areg >= 0 && Data2Areg != DataAreg)
4740 return false;
4741 }
4742
4743 auto FB = getFeatureBits();
4744 if (FB[AMDGPU::FeatureGFX90AInsts]) {
4745 if (DataAreg < 0 || DstAreg < 0)
4746 return true;
4747 return DstAreg == DataAreg;
4748 }
4749
4750 return DstAreg < 1 && DataAreg < 1;
4751}
4752
4753bool AMDGPUAsmParser::validateVGPRAlign(const MCInst &Inst) const {
4754 auto FB = getFeatureBits();
4755 if (!FB[AMDGPU::FeatureGFX90AInsts])
4756 return true;
4757
4758 const MCRegisterInfo *MRI = getMRI();
4759 const MCRegisterClass &VGPR32 = MRI->getRegClass(AMDGPU::VGPR_32RegClassID);
4760 const MCRegisterClass &AGPR32 = MRI->getRegClass(AMDGPU::AGPR_32RegClassID);
4761 for (unsigned I = 0, E = Inst.getNumOperands(); I != E; ++I) {
4762 const MCOperand &Op = Inst.getOperand(I);
4763 if (!Op.isReg())
4764 continue;
4765
4766 unsigned Sub = MRI->getSubReg(Op.getReg(), AMDGPU::sub0);
4767 if (!Sub)
4768 continue;
4769
4770 if (VGPR32.contains(Sub) && ((Sub - AMDGPU::VGPR0) & 1))
4771 return false;
4772 if (AGPR32.contains(Sub) && ((Sub - AMDGPU::AGPR0) & 1))
4773 return false;
4774 }
4775
4776 return true;
4777}
4778
4779SMLoc AMDGPUAsmParser::getBLGPLoc(const OperandVector &Operands) const {
4780 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
4781 AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]);
4782 if (Op.isBLGP())
4783 return Op.getStartLoc();
4784 }
4785 return SMLoc();
4786}
4787
4788bool AMDGPUAsmParser::validateBLGP(const MCInst &Inst,
4789 const OperandVector &Operands) {
4790 unsigned Opc = Inst.getOpcode();
4791 int BlgpIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::blgp);
4792 if (BlgpIdx == -1)
4793 return true;
4794 SMLoc BLGPLoc = getBLGPLoc(Operands);
4795 if (!BLGPLoc.isValid())
4796 return true;
4797 bool IsNeg = StringRef(BLGPLoc.getPointer()).starts_with("neg:");
4798 auto FB = getFeatureBits();
4799 bool UsesNeg = false;
4800 if (FB[AMDGPU::FeatureGFX940Insts]) {
4801 switch (Opc) {
4802 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_acd:
4803 case AMDGPU::V_MFMA_F64_16X16X4F64_gfx940_vcd:
4804 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_acd:
4805 case AMDGPU::V_MFMA_F64_4X4X4F64_gfx940_vcd:
4806 UsesNeg = true;
4807 }
4808 }
4809
4810 if (IsNeg == UsesNeg)
4811 return true;
4812
4813 Error(BLGPLoc,
4814 UsesNeg ? "invalid modifier: blgp is not supported"
4815 : "invalid modifier: neg is not supported");
4816
4817 return false;
4818}
4819
4820bool AMDGPUAsmParser::validateWaitCnt(const MCInst &Inst,
4821 const OperandVector &Operands) {
4822 if (!isGFX11Plus())
4823 return true;
4824
4825 unsigned Opc = Inst.getOpcode();
4826 if (Opc != AMDGPU::S_WAITCNT_EXPCNT_gfx11 &&
4827 Opc != AMDGPU::S_WAITCNT_LGKMCNT_gfx11 &&
4828 Opc != AMDGPU::S_WAITCNT_VMCNT_gfx11 &&
4829 Opc != AMDGPU::S_WAITCNT_VSCNT_gfx11)
4830 return true;
4831
4832 int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::sdst);
4833 assert(Src0Idx >= 0 && Inst.getOperand(Src0Idx).isReg());
4834 auto Reg = mc2PseudoReg(Inst.getOperand(Src0Idx).getReg());
4835 if (Reg == AMDGPU::SGPR_NULL)
4836 return true;
4837
4838 SMLoc RegLoc = getRegLoc(Reg, Operands);
4839 Error(RegLoc, "src0 must be null");
4840 return false;
4841}
4842
4843bool AMDGPUAsmParser::validateDS(const MCInst &Inst,
4844 const OperandVector &Operands) {
4845 uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4846 if ((TSFlags & SIInstrFlags::DS) == 0)
4847 return true;
4848 if (TSFlags & SIInstrFlags::GWS)
4849 return validateGWS(Inst, Operands);
4850 // Only validate GDS for non-GWS instructions.
4851 if (hasGDS())
4852 return true;
4853 int GDSIdx =
4854 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::gds);
4855 if (GDSIdx < 0)
4856 return true;
4857 unsigned GDS = Inst.getOperand(GDSIdx).getImm();
4858 if (GDS) {
4859 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyGDS, Operands);
4860 Error(S, "gds modifier is not supported on this GPU");
4861 return false;
4862 }
4863 return true;
4864}
4865
4866// gfx90a has an undocumented limitation:
4867// DS_GWS opcodes must use even aligned registers.
4868bool AMDGPUAsmParser::validateGWS(const MCInst &Inst,
4869 const OperandVector &Operands) {
4870 if (!getFeatureBits()[AMDGPU::FeatureGFX90AInsts])
4871 return true;
4872
4873 int Opc = Inst.getOpcode();
4874 if (Opc != AMDGPU::DS_GWS_INIT_vi && Opc != AMDGPU::DS_GWS_BARRIER_vi &&
4875 Opc != AMDGPU::DS_GWS_SEMA_BR_vi)
4876 return true;
4877
4878 const MCRegisterInfo *MRI = getMRI();
4879 const MCRegisterClass &VGPR32 = MRI->getRegClass(AMDGPU::VGPR_32RegClassID);
4880 int Data0Pos =
4881 AMDGPU::getNamedOperandIdx(Inst.getOpcode(), AMDGPU::OpName::data0);
4882 assert(Data0Pos != -1);
4883 auto Reg = Inst.getOperand(Data0Pos).getReg();
4884 auto RegIdx = Reg - (VGPR32.contains(Reg) ? AMDGPU::VGPR0 : AMDGPU::AGPR0);
4885 if (RegIdx & 1) {
4886 SMLoc RegLoc = getRegLoc(Reg, Operands);
4887 Error(RegLoc, "vgpr must be even aligned");
4888 return false;
4889 }
4890
4891 return true;
4892}
4893
4894bool AMDGPUAsmParser::validateCoherencyBits(const MCInst &Inst,
4895 const OperandVector &Operands,
4896 const SMLoc &IDLoc) {
4897 int CPolPos = AMDGPU::getNamedOperandIdx(Inst.getOpcode(),
4898 AMDGPU::OpName::cpol);
4899 if (CPolPos == -1)
4900 return true;
4901
4902 unsigned CPol = Inst.getOperand(CPolPos).getImm();
4903
4904 if (isGFX12Plus())
4905 return validateTHAndScopeBits(Inst, Operands, CPol);
4906
4907 uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
4908 if (TSFlags & SIInstrFlags::SMRD) {
4909 if (CPol && (isSI() || isCI())) {
4910 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
4911 Error(S, "cache policy is not supported for SMRD instructions");
4912 return false;
4913 }
4914 if (CPol & ~(AMDGPU::CPol::GLC | AMDGPU::CPol::DLC)) {
4915 Error(IDLoc, "invalid cache policy for SMEM instruction");
4916 return false;
4917 }
4918 }
4919
4920 if (isGFX90A() && !isGFX940() && (CPol & CPol::SCC)) {
4921 const uint64_t AllowSCCModifier = SIInstrFlags::MUBUF |
4924 if (!(TSFlags & AllowSCCModifier)) {
4925 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
4926 StringRef CStr(S.getPointer());
4927 S = SMLoc::getFromPointer(&CStr.data()[CStr.find("scc")]);
4928 Error(S,
4929 "scc modifier is not supported for this instruction on this GPU");
4930 return false;
4931 }
4932 }
4933
4935 return true;
4936
4937 if (TSFlags & SIInstrFlags::IsAtomicRet) {
4938 if (!(TSFlags & SIInstrFlags::MIMG) && !(CPol & CPol::GLC)) {
4939 Error(IDLoc, isGFX940() ? "instruction must use sc0"
4940 : "instruction must use glc");
4941 return false;
4942 }
4943 } else {
4944 if (CPol & CPol::GLC) {
4945 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
4946 StringRef CStr(S.getPointer());
4948 &CStr.data()[CStr.find(isGFX940() ? "sc0" : "glc")]);
4949 Error(S, isGFX940() ? "instruction must not use sc0"
4950 : "instruction must not use glc");
4951 return false;
4952 }
4953 }
4954
4955 return true;
4956}
4957
4958bool AMDGPUAsmParser::validateTHAndScopeBits(const MCInst &Inst,
4959 const OperandVector &Operands,
4960 const unsigned CPol) {
4961 const unsigned TH = CPol & AMDGPU::CPol::TH;
4962 const unsigned Scope = CPol & AMDGPU::CPol::SCOPE;
4963
4964 const unsigned Opcode = Inst.getOpcode();
4965 const MCInstrDesc &TID = MII.get(Opcode);
4966
4967 auto PrintError = [&](StringRef Msg) {
4968 SMLoc S = getImmLoc(AMDGPUOperand::ImmTyCPol, Operands);
4969 Error(S, Msg);
4970 return false;
4971 };
4972
4973 if ((TID.TSFlags & SIInstrFlags::IsAtomicRet) &&
4976 return PrintError("instruction must use th:TH_ATOMIC_RETURN");
4977
4978 if (TH == 0)
4979 return true;
4980
4981 if ((TID.TSFlags & SIInstrFlags::SMRD) &&
4982 ((TH == AMDGPU::CPol::TH_NT_RT) || (TH == AMDGPU::CPol::TH_RT_NT) ||
4983 (TH == AMDGPU::CPol::TH_NT_HT)))
4984 return PrintError("invalid th value for SMEM instruction");
4985
4986 if (TH == AMDGPU::CPol::TH_BYPASS) {
4987 if ((Scope != AMDGPU::CPol::SCOPE_SYS &&
4989 (Scope == AMDGPU::CPol::SCOPE_SYS &&
4991 return PrintError("scope and th combination is not valid");
4992 }
4993
4994 bool IsStore = TID.mayStore();
4995 bool IsAtomic =
4997
4998 if (IsAtomic) {
4999 if (!(CPol & AMDGPU::CPol::TH_TYPE_ATOMIC))
5000 return PrintError("invalid th value for atomic instructions");
5001 } else if (IsStore) {
5002 if (!(CPol & AMDGPU::CPol::TH_TYPE_STORE))
5003 return PrintError("invalid th value for store instructions");
5004 } else {
5005 if (!(CPol & AMDGPU::CPol::TH_TYPE_LOAD))
5006 return PrintError("invalid th value for load instructions");
5007 }
5008
5009 return true;
5010}
5011
5012bool AMDGPUAsmParser::validateExeczVcczOperands(const OperandVector &Operands) {
5013 if (!isGFX11Plus())
5014 return true;
5015 for (auto &Operand : Operands) {
5016 if (!Operand->isReg())
5017 continue;
5018 unsigned Reg = Operand->getReg();
5019 if (Reg == SRC_EXECZ || Reg == SRC_VCCZ) {
5020 Error(getRegLoc(Reg, Operands),
5021 "execz and vccz are not supported on this GPU");
5022 return false;
5023 }
5024 }
5025 return true;
5026}
5027
5028bool AMDGPUAsmParser::validateTFE(const MCInst &Inst,
5029 const OperandVector &Operands) {
5030 const MCInstrDesc &Desc = MII.get(Inst.getOpcode());
5031 if (Desc.mayStore() &&
5033 SMLoc Loc = getImmLoc(AMDGPUOperand::ImmTyTFE, Operands);
5034 if (Loc != getInstLoc(Operands)) {
5035 Error(Loc, "TFE modifier has no meaning for store instructions");
5036 return false;
5037 }
5038 }
5039
5040 return true;
5041}
5042
5043bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
5044 const SMLoc &IDLoc,
5045 const OperandVector &Operands) {
5046 if (auto ErrMsg = validateLdsDirect(Inst)) {
5047 Error(getRegLoc(LDS_DIRECT, Operands), *ErrMsg);
5048 return false;
5049 }
5050 if (!validateSOPLiteral(Inst)) {
5051 Error(getLitLoc(Operands),
5052 "only one unique literal operand is allowed");
5053 return false;
5054 }
5055 if (!validateVOPLiteral(Inst, Operands)) {
5056 return false;
5057 }
5058 if (!validateConstantBusLimitations(Inst, Operands)) {
5059 return false;
5060 }
5061 if (!validateVOPDRegBankConstraints(Inst, Operands)) {
5062 return false;
5063 }
5064 if (!validateIntClampSupported(Inst)) {
5065 Error(getImmLoc(AMDGPUOperand::ImmTyClamp, Operands),
5066 "integer clamping is not supported on this GPU");
5067 return false;
5068 }
5069 if (!validateOpSel(Inst)) {
5070 Error(getImmLoc(AMDGPUOperand::ImmTyOpSel, Operands),
5071 "invalid op_sel operand");
5072 return false;
5073 }
5074 if (!validateNeg(Inst, AMDGPU::OpName::neg_lo)) {
5075 Error(getImmLoc(AMDGPUOperand::ImmTyNegLo, Operands),
5076 "invalid neg_lo operand");
5077 return false;
5078 }
5079 if (!validateNeg(Inst, AMDGPU::OpName::neg_hi)) {
5080 Error(getImmLoc(AMDGPUOperand::ImmTyNegHi, Operands),
5081 "invalid neg_hi operand");
5082 return false;
5083 }
5084 if (!validateDPP(Inst, Operands)) {
5085 return false;
5086 }
5087 // For MUBUF/MTBUF d16 is a part of opcode, so there is nothing to validate.
5088 if (!validateMIMGD16(Inst)) {
5089 Error(getImmLoc(AMDGPUOperand::ImmTyD16, Operands),
5090 "d16 modifier is not supported on this GPU");
5091 return false;
5092 }
5093 if (!validateMIMGMSAA(Inst)) {
5094 Error(getImmLoc(AMDGPUOperand::ImmTyDim, Operands),
5095 "invalid dim; must be MSAA type");
5096 return false;
5097 }
5098 if (!validateMIMGDataSize(Inst, IDLoc)) {
5099 return false;
5100 }
5101 if (!validateMIMGAddrSize(Inst, IDLoc))
5102 return false;
5103 if (!validateMIMGAtomicDMask(Inst)) {
5104 Error(getImmLoc(AMDGPUOperand::ImmTyDMask, Operands),
5105 "invalid atomic image dmask");
5106 return false;
5107 }
5108 if (!validateMIMGGatherDMask(Inst)) {
5109 Error(getImmLoc(AMDGPUOperand::ImmTyDMask, Operands),
5110 "invalid image_gather dmask: only one bit must be set");
5111 return false;
5112 }
5113 if (!validateMovrels(Inst, Operands)) {
5114 return false;
5115 }
5116 if (!validateOffset(Inst, Operands)) {
5117 return false;
5118 }
5119 if (!validateMAIAccWrite(Inst, Operands)) {
5120 return false;
5121 }
5122 if (!validateMAISrc2(Inst, Operands)) {
5123 return false;
5124 }
5125 if (!validateMFMA(Inst, Operands)) {
5126 return false;
5127 }
5128 if (!validateCoherencyBits(Inst, Operands, IDLoc)) {
5129 return false;
5130 }
5131
5132 if (!validateAGPRLdSt(Inst)) {
5133 Error(IDLoc, getFeatureBits()[AMDGPU::FeatureGFX90AInsts]
5134 ? "invalid register class: data and dst should be all VGPR or AGPR"
5135 : "invalid register class: agpr loads and stores not supported on this GPU"
5136 );
5137 return false;
5138 }
5139 if (!validateVGPRAlign(Inst)) {
5140 Error(IDLoc,
5141 "invalid register class: vgpr tuples must be 64 bit aligned");
5142 return false;
5143 }
5144 if (!validateDS(Inst, Operands)) {
5145 return false;
5146 }
5147
5148 if (!validateBLGP(Inst, Operands)) {
5149 return false;
5150 }
5151
5152 if (!validateDivScale(Inst)) {
5153 Error(IDLoc, "ABS not allowed in VOP3B instructions");
5154 return false;
5155 }
5156 if (!validateWaitCnt(Inst, Operands)) {
5157 return false;
5158 }
5159 if (!validateExeczVcczOperands(Operands)) {
5160 return false;
5161 }
5162 if (!validateTFE(Inst, Operands)) {
5163 return false;
5164 }
5165
5166 return true;
5167}
5168
5170 const FeatureBitset &FBS,
5171 unsigned VariantID = 0);
5172
5173static bool AMDGPUCheckMnemonic(StringRef Mnemonic,
5174 const FeatureBitset &AvailableFeatures,
5175 unsigned VariantID);
5176
5177bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5178 const FeatureBitset &FBS) {
5179 return isSupportedMnemo(Mnemo, FBS, getAllVariants());
5180}
5181
5182bool AMDGPUAsmParser::isSupportedMnemo(StringRef Mnemo,
5183 const FeatureBitset &FBS,
5184 ArrayRef<unsigned> Variants) {
5185 for (auto Variant : Variants) {
5186 if (AMDGPUCheckMnemonic(Mnemo, FBS, Variant))
5187 return true;
5188 }
5189
5190 return false;
5191}
5192
5193bool AMDGPUAsmParser::checkUnsupportedInstruction(StringRef Mnemo,
5194 const SMLoc &IDLoc) {
5195 FeatureBitset FBS = ComputeAvailableFeatures(getFeatureBits());
5196
5197 // Check if requested instruction variant is supported.
5198 if (isSupportedMnemo(Mnemo, FBS, getMatchedVariants()))
5199 return false;
5200
5201 // This instruction is not supported.
5202 // Clear any other pending errors because they are no longer relevant.
5203 getParser().clearPendingErrors();
5204
5205 // Requested instruction variant is not supported.
5206 // Check if any other variants are supported.
5207 StringRef VariantName = getMatchedVariantName();
5208 if (!VariantName.empty() && isSupportedMnemo(Mnemo, FBS)) {
5209 return Error(IDLoc,
5210 Twine(VariantName,
5211 " variant of this instruction is not supported"));
5212 }
5213
5214 // Check if this instruction may be used with a different wavesize.
52